apify_scheduler 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 51bf7810d570a6b2fb56a8fa8a9897f4ef09ef72
+  data.tar.gz: 81fad9bc1450907abe21caf16de8e4e345125390
+SHA512:
+  metadata.gz: 611a01092d6e1c01b8360f8482ccf9192108c80ce115db067f4674dddeb99368264930d3df512bdace186d6f199934e1a7929c242786c405afca25fedafc4825
+  data.tar.gz: a7bf58a72c8e41dd730e7574ae15082f5c86da4d91dc03623cf5beaeeb9a9d50070a70e83ca1f1a0a9e46e26531aa5f9eded6556f9e7fe43c13c02958b83a358

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2015 YOURNAME
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,3 @@
+= Scheduler
+
+This project rocks and uses MIT-LICENSE.

data/Rakefile

@@ -0,0 +1,34 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Apify Scheduler'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+APP_RAKEFILE = File.expand_path("../test/dummy/Rakefile", __FILE__)
+load 'rails/tasks/engine.rake'
+
+
+
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+
+task default: :test

data/app/assets/javascripts/scheduler/application.js

@@ -0,0 +1,19 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or vendor/assets/javascripts of plugins, if any, can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file.
+//
+// Read Sprockets README (https://github.com/sstephenson/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require jquery
+//= require jquery_ujs
+
+
+$('.message .close').on('click', function() {
+  $(this).closest('.message').fadeOut();
+});

data/app/assets/stylesheets/scheduler/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or vendor/assets/stylesheets of plugins, if any, can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any styles
+ * defined in the other CSS/SCSS files in this directory. It is generally better to create a new
+ * file per style scope.
+ *
+ *= require_self
+ *
+*/

data/app/controllers/apify/scheduler/application_controller.rb

@@ -0,0 +1,6 @@
+module Apify
+  module Scheduler
+    class ApplicationController < ActionController::Base
+    end
+  end
+end

data/app/controllers/apify/scheduler/histories_controller.rb

@@ -0,0 +1,24 @@
+module Apify::Scheduler
+  class HistoriesController < ApplicationController
+    layout 'apify/scheduler/application'
+    before_action :set_history, only: [:destroy]
+
+    # GET /histories
+    def index
+      @unit = Unit.find(params[:unit_id])
+      @histories = @unit.histories.order('created_at DESC')
+    end
+
+    # DELETE /histories/1
+    def destroy
+      @history.destroy
+      redirect_to :back, notice: 'History was successfully destroyed.'
+    end
+
+    private
+      # Use callbacks to share common setup or constraints between actions.
+      def set_history
+        @history = History.find(params[:id])
+      end
+  end
+end

data/app/controllers/apify/scheduler/servers_controller.rb

@@ -0,0 +1,71 @@
+module Apify
+  module Scheduler
+    class ServersController < ApplicationController
+      layout 'apify/scheduler/application'
+      before_action :set_server, only: [:show, :edit, :update, :destroy, :test]
+
+      # GET /servers
+      def index
+        @servers = Apify::Scheduler::Server.all
+      end
+
+      # GET /servers/1
+      def show
+      end
+
+      # GET /servers/new
+      def new
+        @server = Apify::Scheduler::Server.new
+      end
+
+      # GET /servers/1/edit
+      def edit
+      end
+
+      def test
+        response  = @server.test_api_key
+        message   = response['message']
+        flash_key = response['status'] == '1' ? :notice : :error
+
+        redirect_to servers_path, flash: { flash_key => message }
+      end
+
+      # POST /servers
+      def create
+        @server = Apify::Scheduler::Server.new(server_params)
+
+        if @server.save
+          redirect_to @server, notice: 'Server was successfully created.'
+        else
+          render :new
+        end
+      end
+
+      # PATCH/PUT /servers/1
+      def update
+        if @server.update(server_params)
+          redirect_to @server, notice: 'Server was successfully updated.'
+        else
+          render :edit
+        end
+      end
+
+      # DELETE /servers/1
+      def destroy
+        @server.destroy
+        redirect_to servers_url, notice: 'Server was successfully destroyed.'
+      end
+
+      private
+        # Use callbacks to share common setup or constraints between actions.
+        def set_server
+          @server = Apify::Scheduler::Server.find(params[:id])
+        end
+
+        # Only allow a trusted parameter "white list" through.
+        def server_params
+          params.require(:server).permit(:name, :url, :description, :api_key)
+        end
+    end
+  end
+end

data/app/controllers/apify/scheduler/units_controller.rb

@@ -0,0 +1,77 @@
+module Apify
+  module Scheduler
+    class UnitsController < ApplicationController
+
+      layout 'apify/scheduler/application'
+      before_action :set_unit, only: [:show, :edit, :update, :destroy, :perform]
+
+
+      # GET /units
+      def index
+        @units = Apify::Scheduler::Unit.all
+      end
+
+      # GET /units/1
+      def show
+      end
+
+      # GET /units/new
+      def new
+        @unit = Apify::Scheduler::Unit.new
+      end
+
+      # GET /units/1/edit
+      def edit
+      end
+
+      # POST /units
+      def create
+        @unit = Apify::Scheduler::Unit.new(unit_params)
+
+        if @unit.save
+          redirect_to @unit, notice: 'Unit was successfully created.'
+        else
+          render :new
+        end
+      end
+
+      # PATCH/PUT /units/1
+      def update
+        if @unit.update(unit_params)
+          redirect_to @unit, notice: 'Unit was successfully updated.'
+        else
+          render :edit
+        end
+      end
+
+      # POST /units/1/perform
+      def perform
+        worker = @unit.enqueue
+        message   = worker ? "Successfully added #{@unit.name} unit to queue. Refresh page to see progress." : "Error occured while queuing #{@unit.name}. Try again"
+        flash_key = worker ? :notice : :error
+        redirect_to units_path, flash: { flash_key => message }
+      end
+
+      # DELETE /units/1
+      def destroy
+        @unit.destroy
+        redirect_to units_url, notice: 'Unit was successfully destroyed.'
+      end
+
+      private
+        # Use callbacks to share common setup or constraints between actions.
+        def set_unit
+          @unit = Apify::Scheduler::Unit.find(params[:id])
+        end
+
+        # Only allow a trusted parameter "white list" through.
+        def unit_params
+          params.require(:unit).permit( :name, :description, :pattern,
+                                        :processes, :delay, :destination,
+                                        :apify_scheduler_server_id, :apify_scheduler_frequency_period_id,
+                                        :frequency_quantity, :at)
+        end
+
+    end
+  end
+end

data/app/helpers/apify/scheduler/application_helper.rb

@@ -0,0 +1,6 @@
+module Apify
+  module Scheduler
+    module ApplicationHelper
+    end
+  end
+end

data/app/helpers/apify/scheduler/histories_helper.rb

@@ -0,0 +1,4 @@
+module Apify::Scheduler
+  module HistoriesHelper
+  end
+end

data/app/helpers/apify/scheduler/servers_helper.rb

@@ -0,0 +1,4 @@
+module Apify::Scheduler
+  module ServersHelper
+  end
+end

data/app/helpers/apify/scheduler/units_helper.rb

@@ -0,0 +1,4 @@
+module Apify::Scheduler
+  module UnitsHelper
+  end
+end

data/app/jobs/unit_performer_job.rb

@@ -0,0 +1,10 @@
+class UnitPerformerJob
+  @queue = :units
+  def self.perform(unit_id, history_id)
+    unit      = Apify::Scheduler::Unit.find(unit_id)
+    history   = Apify::Scheduler::History.find(history_id)
+    history.update(queued: true)
+    response  = unit.perform
+    history.update finished_at: Time.zone.now, response_body: response
+  end
+end

data/app/models/apify/scheduler/frequency_period.rb

@@ -0,0 +1,5 @@
+module Apify::Scheduler
+  class FrequencyPeriod < ActiveRecord::Base
+
+  end
+end

data/app/models/apify/scheduler/history.rb

@@ -0,0 +1,9 @@
+module Apify::Scheduler
+  class History < ActiveRecord::Base
+    belongs_to :unit, class_name: 'Apify::Scheduler::Unit', foreign_key: :apify_scheduler_unit_id
+
+    def currently_working?
+      Resque.working.map{ |w| w.job['payload']['args'].last }.include? self.id
+    end
+  end
+end

data/app/models/apify/scheduler/server.rb

@@ -0,0 +1,12 @@
+module Apify::Scheduler
+  class Server < ActiveRecord::Base
+    validates :name, :url, :api_key, presence: true
+    has_many :units, foreign_key: :apify_scheduler_server_id
+
+    def test_api_key
+      test_url = "#{url.chomp('/')}/test-api-key?apify_secret=#{api_key}"
+      response = RestClient.post(test_url, {})
+      JSON.parse(response)
+    end
+  end
+end

data/app/models/apify/scheduler/unit.rb

@@ -0,0 +1,45 @@
+module Apify::Scheduler
+  class Unit < ActiveRecord::Base
+    require 'rest_client'
+    require 'pry'
+    validates :name, :pattern, :destination, :server, presence: true
+    belongs_to :server, class_name: 'Apify::Scheduler::Server', foreign_key: :apify_scheduler_server_id
+    belongs_to :frequency_period, class_name: 'Apify::Scheduler::FrequencyPeriod', foreign_key: :apify_scheduler_frequency_period_id
+
+    has_many :histories, foreign_key: :apify_scheduler_unit_id
+
+    def currently_working?
+      Resque.working.map{ |w| w.job['payload']['args'].first }.include? self.id
+    end
+
+    def enqueue
+      history = histories.create
+      Resque.enqueue(UnitPerformerJob, self.id, history.id)
+    end
+
+    def perform
+      begin
+        json = pattern
+        server_query = ["processes=#{processes}", "delay=#{delay}", "apify_secret=#{server.api_key}"].join('&')
+        server_crawler_url = [server.url.chomp('/'), '/crawler'].join
+        server_url = [server_crawler_url, server_query].join('?')
+
+        request_resource = RestClient::Resource.new( server_url, timeout: -1, open_timeout: -1 )
+        request = request_resource.post json, {:content_type => :json, :accept => :json}
+
+        # TODO: record download attempts
+        request = JSON.parse(request).to_json
+
+        response_resource = RestClient::Resource.new( destination, timeout: -1, open_timeout: -1 )
+        response_resource.post request, {:content_type => :json, :accept => :json}
+      #rescue ActiveRecord, JSON, RestClient, Parallel, Errno, Errno::EPIPE, Errno::PIPE
+      rescue Exception => e
+        { success: 0, message: e.message}.to_json
+      end
+    end
+
+    def frequency
+      frequency_quantity.send(frequency_period.name.pluralize)
+    end
+  end
+end

data/app/views/apify/scheduler/histories/_history.html.erb

@@ -0,0 +1,6 @@
+<div class='ui segment'>
+  <div class='ui column right aligned'>
+    <%= link_to raw('<i class="remove circle icon red"></i>'), history_path(history), method: :delete, 'data-confirm' => "Are you sure?" %>
+  </div>
+  <%= render 'apify/scheduler/shared/short_history', history: history %>
+</div>

data/app/views/apify/scheduler/histories/index.html.erb

@@ -0,0 +1,14 @@
+<div class='ui grid'>
+  <div class='ui column'>
+    <div class='ui segment'>
+
+      <h1 class='ui header'>
+        <%= @unit.name %>
+        Histories
+      </h1>
+
+      <%= render @histories %>
+
+    </div>
+  </div>
+</div>

data/app/views/apify/scheduler/servers/_form.html.erb

@@ -0,0 +1,48 @@
+<div class='ui grid'>
+  <div class='column'>
+    <div class='ui segment form'>
+
+      <%= form_for(@server) do |f| %>
+        <% if @server.errors.any? %>
+          <div id="error_explanation">
+            <h2><%= pluralize(@server.errors.count, "error") %> prohibited this server from being saved:</h2>
+
+            <ul>
+            <% @server.errors.full_messages.each do |message| %>
+              <li><%= message %></li>
+            <% end %>
+            </ul>
+          </div>
+        <% end %>
+
+        <div class="field">
+          <div class='ui input'>
+            <%= f.label :name %><br>
+            <%= f.text_field :name %>
+          </div>
+        </div>
+        <div class="field">
+          <div class='ui input'>
+            <%= f.label :url %><br>
+            <%= f.text_field :url %>
+          </div>
+        </div>
+        <div class="field">
+          <div class='ui input'>
+            <%= f.label :api_key %><br>
+            <%= f.text_field :api_key %>
+          </div>
+        </div>
+        <div class="field">
+          <div class='ui input'>
+            <%= f.label :description %><br>
+            <%= f.text_area :description %>
+          </div>
+        </div>
+        <div class="right aligned column">
+          <%= f.submit (@server.persisted? ? 'Update Server' : 'Create Server'), class: 'ui blue submit button' %>
+        </div>
+      <% end %>
+    </div>
+  </div>
+</div>