answersengine 0.10.1 → 0.10.2

data/lib/answersengine/client/job_output.rb

@@ -1,19 +0,0 @@
-module AnswersEngine
-  module Client
-    class JobOutput < AnswersEngine::Client::Base
-      def find(job_id, collection, id)
-        self.class.get("/jobs/#{job_id}/output/collections/#{collection}/records/#{id}", @options)
-      end
-
-      def all(job_id, collection = 'default')
-
-        self.class.get("/jobs/#{job_id}/output/collections/#{collection}/records", @options)
-      end
-
-      def collections(job_id)
-        self.class.get("/jobs/#{job_id}/output/collections", @options)
-      end
-    end
-  end
-end
-

data/lib/answersengine/client/job_page.rb

@@ -1,58 +0,0 @@
-module AnswersEngine
-  module Client
-    class JobPage < AnswersEngine::Client::Base
-      def find(job_id, gid)
-        self.class.get("/jobs/#{job_id}/pages/#{gid}", @options)
-      end
-
-      def all(job_id, opts={})
-        params = @options.merge(opts)
-        self.class.get("/jobs/#{job_id}/pages", params)
-      end
-
-      def update(job_id, gid, opts={})
-        body = {}
-        body[:page_type] = opts[:page_type] if opts[:page_type]
-        body[:priority] = opts[:priority] if opts[:priority]
-        body[:vars] = opts[:vars] if opts[:vars]
-
-        params = @options.merge({body: body.to_json})
-
-        self.class.put("/jobs/#{job_id}/pages/#{gid}", params)
-      end
-
-      def enqueue(job_id, method, url, opts={})
-        body = {}
-        body[:method] =  method != "" ? method : "GET"
-        body[:url] =  url
-        body[:page_type] = opts[:page_type] if opts[:page_type]
-        body[:priority] = opts[:priority] if opts[:priority]
-        body[:fetch_type] = opts[:fetch_type] if opts[:fetch_type]
-        body[:body] = opts[:body] if opts[:body]
-        body[:headers] = opts[:headers] if opts[:headers]
-        body[:vars] = opts[:vars] if opts[:vars]
-        body[:force_fetch] = opts[:force_fetch] if opts[:force_fetch]
-        body[:freshness] = opts[:freshness] if opts[:freshness]
-        body[:ua_type] = opts[:ua_type] if opts[:ua_type]
-        body[:no_redirect] = opts[:no_redirect] if opts[:no_redirect]
-        body[:cookie] = opts[:cookie] if opts[:cookie]
-
-        params = @options.merge({body: body.to_json})
-
-        self.class.post("/jobs/#{job_id}/pages", params)
-      end
-
-      def parsing_update(job_id, gid, opts={})
-        body = {}
-        body[:outputs] = opts.fetch(:outputs) {[]}
-        body[:pages] = opts.fetch(:pages) {[]}
-        body[:parsing_status] = opts.fetch(:parsing_status){ nil }
-        body[:log_error] = opts[:log_error] if opts[:log_error]
-
-        params = @options.merge({body: body.to_json})
-
-        self.class.put("/jobs/#{job_id}/pages/#{gid}/parsing_update", params)
-      end
-    end
-  end
-end

data/lib/answersengine/client/job_stat.rb

@@ -1,16 +0,0 @@
-module AnswersEngine
-  module Client
-    class JobStat < AnswersEngine::Client::Base
-
-      def job_current_stats(job_id)
-        self.class.get("/jobs/#{job_id}/stats/current", @options)
-      end
-
-      def scraper_job_current_stats(scraper_name)
-        self.class.get("/scrapers/#{scraper_name}/current_job/stats/current", @options)
-      end
-
-    end
-  end
-end
-

data/lib/answersengine/client/scraper.rb

@@ -1,57 +0,0 @@
-module AnswersEngine
-  module Client
-    class Scraper < AnswersEngine::Client::Base
-
-      def find(scraper_name)
-        self.class.get("/scrapers/#{scraper_name}", @options)
-      end
-
-      def all(opts={})
-        params = @options.merge opts
-        self.class.get("/scrapers", params)
-      end
-
-      def create(scraper_name, git_repository, opts={})
-        body = {}
-        body[:name] = scraper_name
-        body[:git_repository] = git_repository
-        body[:git_branch] = opts[:branch] || opts[:git_branch] || "master" if opts[:branch] || opts[:git_branch]
-        body[:freshness_type] = opts[:freshness_type] if opts[:freshness_type]
-        body[:force_fetch] = opts[:force_fetch] if opts[:force_fetch]
-        body[:standard_worker_count] = opts[:workers] || opts[:standard_worker_count] if opts[:workers] || opts[:standard_worker_count]
-        body[:browser_worker_count] = opts[:browsers] || opts[:browser_worker_count] if opts[:browsers] || opts[:browser_worker_count]
-        body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
-        body[:disable_scheduler] = opts[:disable_scheduler] if opts[:disable_scheduler]
-        body[:cancel_current_job] = opts[:cancel_current_job] if opts[:cancel_current_job]
-        body[:schedule] = opts[:schedule] if opts[:schedule]
-        body[:timezone] = opts[:timezone] if opts[:timezone]
-        params = @options.merge({body: body.to_json})
-        self.class.post("/scrapers", params)
-      end
-
-      def update(scraper_name, opts={})
-        body = {}
-        body[:name] = opts[:name] if opts[:name]
-        body[:git_repository] = opts[:repo] || opts[:git_repository] if opts[:repo] || opts[:git_repository]
-        body[:git_branch] = opts[:branch] || opts[:git_branch] if opts[:branch] || opts[:git_branch]
-        body[:freshness_type] = opts[:freshness_type] if opts[:freshness_type]
-        body[:force_fetch] = opts[:force_fetch] if opts.has_key?("force_fetch") || opts.has_key?(:force_fetch)
-        body[:standard_worker_count] = opts[:workers] || opts[:standard_worker_count] if opts[:workers] || opts[:standard_worker_count]
-        body[:browser_worker_count] = opts[:browsers] || opts[:browser_worker_count] if opts[:browsers] || opts[:browser_worker_count]
-        body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
-        body[:disable_scheduler] = opts[:disable_scheduler] if opts.has_key?("disable_scheduler") || opts.has_key?(:disable_scheduler)
-        body[:cancel_current_job] = opts[:cancel_current_job] if opts.has_key?("cancel_current_job") || opts.has_key?(:cancel_current_job)
-        body[:schedule] = opts[:schedule] if opts[:schedule]
-        body[:timezone] = opts[:timezone] if opts[:timezone]
-        params = @options.merge({body: body.to_json})
-
-        self.class.put("/scrapers/#{scraper_name}", params)
-      end
-
-      def delete(scraper_name, opts={})
-        params = @options.merge(opts)
-        self.class.delete("/scrapers/#{scraper_name}", params)
-      end
-    end
-  end
-end

data/lib/answersengine/client/scraper_deployment.rb

@@ -1,18 +0,0 @@
-module AnswersEngine
-  module Client
-    class ScraperDeployment < AnswersEngine::Client::Base
-
-      def all(scraper_name, opts={})
-        params = @options.merge(opts)
-        self.class.get("/scrapers/#{scraper_name}/deployments", params)
-      end
-
-
-      def deploy(scraper_name, opts={})
-        params = @options.merge(opts)
-        self.class.post("/scrapers/#{scraper_name}/deployments", params)
-      end
-
-    end
-  end
-end

data/lib/answersengine/client/scraper_export.rb

@@ -1,22 +0,0 @@
-module AnswersEngine
-  module Client
-    class ScraperExport < AnswersEngine::Client::Base
-      def all(scraper_name, opts={})
-        params = @options.merge(opts)
-        self.class.get("/scrapers/#{scraper_name}/exports", params)
-      end
-
-      def find(export_id)
-        self.class.get("/scrapers/exports/#{export_id}", @options)
-      end
-
-      def create(scraper_name, exporter_name)
-        self.class.post("/scrapers/#{scraper_name}/exports/#{exporter_name}", @options)
-      end
-
-      def download(export_id)
-        self.class.get("/scrapers/exports/#{export_id}/download", @options)
-      end
-    end
-  end
-end

data/lib/answersengine/client/scraper_exporter.rb

@@ -1,14 +0,0 @@
-module AnswersEngine
-  module Client
-    class ScraperExporter < AnswersEngine::Client::Base
-      def all(scraper_name, opts={})
-        params = @options.merge(opts)
-        self.class.get("/scrapers/#{scraper_name}/exporters", params)
-      end
-
-      def find(scraper_name, exporter_name)
-        self.class.get("/scrapers/#{scraper_name}/exporters/#{exporter_name}", @options)
-      end
-    end
-  end
-end

data/lib/answersengine/client/scraper_finisher.rb

@@ -1,16 +0,0 @@
-module AnswersEngine
-  module Client
-    class ScraperFinisher < AnswersEngine::Client::Base
-      # Reset finisher on a scraper's current job.
-      #
-      # @param [String] scraper_name Scraper name.
-      # @param [Hash] opts ({}) API custom parameters.
-      #
-      # @return [HTTParty::Response]
-      def reset(scraper_name, opts={})
-        params = @options.merge(opts)
-        self.class.put("/scrapers/#{scraper_name}/current_job/finisher/reset", params)
-      end
-    end
-  end
-end

data/lib/answersengine/client/scraper_job.rb

@@ -1,49 +0,0 @@
-module AnswersEngine
-  module Client
-    class ScraperJob < AnswersEngine::Client::Base
-      def all(scraper_name, opts={})
-        params = @options.merge(opts)
-        self.class.get("/scrapers/#{scraper_name}/jobs", params)
-      end
-
-      def create(scraper_name, opts={})
-        body = {}
-        body[:standard_worker_count] = opts[:workers] if opts[:workers]
-        body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
-        body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
-        params = @options.merge({body: body.to_json})
-        self.class.post("/scrapers/#{scraper_name}/jobs", params)
-      end
-
-      def find(scraper_name)
-        self.class.get("/scrapers/#{scraper_name}/current_job", @options)
-      end
-
-      def update(scraper_name, opts={})
-        body = {}
-        body[:status] = opts[:status] if opts[:status]
-        body[:standard_worker_count] = opts[:workers] if opts[:workers]
-        body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
-        body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
-        params = @options.merge({body: body.to_json})
-
-        self.class.put("/scrapers/#{scraper_name}/current_job", params)
-      end
-
-      def cancel(scraper_name, opts={})
-        opts[:status] = 'cancelled'
-        update(scraper_name, opts)
-      end
-
-      def resume(scraper_name, opts={})
-        opts[:status] = 'active'
-        update(scraper_name, opts)
-      end
-
-      def pause(scraper_name, opts={})
-        opts[:status] = 'paused'
-        update(scraper_name, opts)
-      end
-    end
-  end
-end

data/lib/answersengine/client/scraper_job_output.rb

@@ -1,19 +0,0 @@
-module AnswersEngine
-  module Client
-    class ScraperJobOutput < AnswersEngine::Client::Base
-      def find(scraper_name, collection, id)
-        self.class.get("/scrapers/#{scraper_name}/current_job/output/collections/#{collection}/records/#{id}", @options)
-      end
-
-      def all(scraper_name, collection = 'default')
-
-        self.class.get("/scrapers/#{scraper_name}/current_job/output/collections/#{collection}/records", @options)
-      end
-
-      def collections(scraper_name)
-        self.class.get("/scrapers/#{scraper_name}/current_job/output/collections", @options)
-      end
-    end
-  end
-end
-

data/lib/answersengine/client/scraper_job_page.rb

@@ -1,67 +0,0 @@
-module AnswersEngine
-  module Client
-    class ScraperJobPage < AnswersEngine::Client::Base
-      def find(scraper_name, gid)
-        self.class.get("/scrapers/#{scraper_name}/current_job/pages/#{gid}", @options)
-      end
-
-      def all(scraper_name, opts={})
-        params = @options.merge(opts)
-        self.class.get("/scrapers/#{scraper_name}/current_job/pages", params)
-      end
-
-      def update(scraper_name, gid, opts={})
-        body = {}
-        body[:page_type] = opts[:page_type] if opts[:page_type]
-        body[:priority] = opts[:priority] if opts[:priority]
-        body[:vars] = opts[:vars] if opts[:vars]
-
-        params = @options.merge({body: body.to_json})
-
-        self.class.put("/scrapers/#{scraper_name}/current_job/pages/#{gid}", params)
-      end
-
-      def refetch(scraper_name, opts={})
-        params = @options.merge(opts)
-        self.class.put("/scrapers/#{scraper_name}/current_job/pages/refetch", params)
-      end
-
-      def refetch_by_job(job_id, opts={})
-        params = @options.merge(opts)
-        self.class.put("/jobs/#{job_id}/pages/refetch", params)
-      end
-
-      def reparse(scraper_name, opts={})
-        params = @options.merge(opts)
-        self.class.put("/scrapers/#{scraper_name}/current_job/pages/reparse", params)
-      end
-
-      def reparse_by_job(job_id, opts={})
-        params = @options.merge(opts)
-        self.class.put("/jobs/#{job_id}/pages/reparse", params)
-      end
-
-      def enqueue(scraper_name, method, url, opts={})
-        body = {}
-        body[:method] =  method != "" ? method : "GET"
-        body[:url] =  url
-        body[:page_type] = opts[:page_type] if opts[:page_type]
-        body[:priority] = opts[:priority] if opts[:priority]
-        body[:fetch_type] = opts[:fetch_type] if opts[:fetch_type]
-        body[:body] = opts[:body] if opts[:body]
-        body[:headers] = opts[:headers] if opts[:headers]
-        body[:vars] = opts[:vars] if opts[:vars]
-        body[:force_fetch] = opts[:force_fetch] if opts[:force_fetch]
-        body[:freshness] = opts[:freshness] if opts[:freshness]
-        body[:ua_type] = opts[:ua_type] if opts[:ua_type]
-        body[:no_redirect] = opts[:no_redirect] if opts[:no_redirect]
-        body[:cookie] = opts[:cookie] if opts[:cookie]
-
-        params = @options.merge({body: body.to_json})
-
-        self.class.post("/scrapers/#{scraper_name}/current_job/pages", params)
-      end
-
-    end
-  end
-end

data/lib/answersengine/client/scraper_job_var.rb

@@ -1,28 +0,0 @@
-module AnswersEngine
-  module Client
-    class ScraperJobVar < AnswersEngine::Client::Base
-
-      def find(scraper_name, var_name)
-        self.class.get("/scrapers/#{scraper_name}/current_job/vars/#{var_name}", @options)
-      end
-
-      def all(scraper_name, opts={})
-        params = @options.merge opts
-        self.class.get("/scrapers/#{scraper_name}/current_job/vars", params)
-      end
-
-      def set(scraper_name, var_name, value, opts={})
-        body = {}
-        body[:value] = value
-        body[:secret] = opts[:secret] if opts[:secret]
-        params = @options.merge({body: body.to_json})
-        self.class.put("/scrapers/#{scraper_name}/current_job/vars/#{var_name}", params)
-      end
-
-      def unset(scraper_name, var_name, opts={})
-        params = @options.merge(opts)
-        self.class.delete("/scrapers/#{scraper_name}/current_job/vars/#{var_name}", params)
-      end
-    end
-  end
-end

data/lib/answersengine/client/scraper_var.rb

@@ -1,28 +0,0 @@
-module AnswersEngine
-  module Client
-    class ScraperVar < AnswersEngine::Client::Base
-
-      def find(scraper_name, var_name)
-        self.class.get("/scrapers/#{scraper_name}/vars/#{var_name}", @options)
-      end
-
-      def all(scraper_name, opts={})
-        params = @options.merge opts
-        self.class.get("/scrapers/#{scraper_name}/vars", params)
-      end
-
-      def set(scraper_name, var_name, value, opts={})
-        body = {}
-        body[:value] = value
-        body[:secret] = opts[:secret] if opts[:secret]
-        params = @options.merge({body: body.to_json})
-        self.class.put("/scrapers/#{scraper_name}/vars/#{var_name}", params)
-      end
-
-      def unset(scraper_name, var_name, opts={})
-        params = @options.merge(opts)
-        self.class.delete("/scrapers/#{scraper_name}/vars/#{var_name}", params)
-      end
-    end
-  end
-end

data/lib/answersengine/plugin.rb

@@ -1,6 +0,0 @@
-require 'answersengine/plugin/context_exposer'
-
-module AnswersEngine
-  module Plugin
-  end
-end

data/lib/answersengine/plugin/context_exposer.rb

@@ -1,55 +0,0 @@
-module AnswersEngine
-  module Plugin
-    module ContextExposer
-      def self.exposed_methods
-        raise NotImplementedError.new('Specify methods exposed to isolated env')
-      end
-
-      def exposed_methods
-        self.class.exposed_methods
-      end
-
-      # Create lambda to retrieve a variable or call instance method
-      def var_or_proc vars, key
-        myself = self # Avoid stack overflow
-        return lambda{vars[key]} if vars.has_key?(key)
-        lambda{|*args| myself.send(key, *args)}
-      end
-
-      def exposed_env vars
-        keys = exposed_methods + vars.keys
-        Hash[keys.uniq.map{|key|[key, var_or_proc(vars, key)]}]
-      end
-
-      def expose_to object, env
-        metaclass = class << object; self; end
-        env.each do |key, block|
-          metaclass.send(:define_method, key, block)
-        end
-        object
-      end
-
-      # Create isolated context object from self
-      def create_context vars = {}
-        create_top_object_script = '(
-          lambda do
-            object = Object.new
-            metaclass = class << object
-              define_method(:context_binding){binding}
-            end
-            object
-          end
-        ).call'
-        object = TOPLEVEL_BINDING.eval(create_top_object_script)
-        env = exposed_env(vars)
-        expose_to object, env
-        object
-      end
-
-      # Create an isolated binding
-      def isolated_binding vars = {}
-        create_context(vars).context_binding
-      end
-    end
-  end
-end