answersengine 0.2.33

data/lib/answersengine/client/base.rb

@@ -0,0 +1,50 @@
+require 'httparty'
+
+module AnswersEngine
+  module Client
+    class Base
+      include HTTParty
+      base_uri(ENV['ANSWERSENGINE_API_URL'].nil? ? 'https://fetch.answersengine.com/api/v1' : ENV['ANSWERSENGINE_API_URL'])
+
+      def self.env_auth_token
+        ENV['ANSWERSENGINE_TOKEN']
+      end
+
+      def auth_token
+        @auth_token ||= self.class.env_auth_token
+      end
+
+      def auth_token= value
+        @auth_token = value
+      end
+
+      def initialize(opts={})
+        self.auth_token = opts[:auth_token] unless opts[:auth_token].nil?
+        @options = { headers: {
+          "Authorization" => "Bearer #{auth_token}",
+          "Content-Type" => "application/json",
+          }}
+
+        query = {}
+        query[:p] = opts[:page] if opts[:page]
+        query[:pp] = opts[:per_page] if opts[:per_page]
+        query[:fetchfail] = opts[:fetch_fail] if opts[:fetch_fail]
+        query[:parsefail] = opts[:parse_fail] if opts[:parse_fail]
+        query[:page_type] = opts[:page_type] if opts[:page_type]
+        query[:gid] = opts[:gid] if opts[:gid]
+
+        if opts[:query]
+          if opts[:query].is_a?(Hash)
+            query[:q] = opts[:query].to_json
+          elsif opts[:query].is_a?(String)
+            query[:q] = JSON.parse(opts[:query]).to_json
+          end
+        end
+
+        unless query.empty?
+          @options.merge!(query: query)
+        end
+      end
+    end
+  end
+end

data/lib/answersengine/client/export.rb

@@ -0,0 +1,10 @@
+module AnswersEngine
+  module Client
+    class Export < AnswersEngine::Client::Base
+      def all(opts={})
+        self.class.get("/scrapers/exports", @options)
+      end
+    end
+  end
+end
+

data/lib/answersengine/client/global_page.rb

@@ -0,0 +1,18 @@
+module AnswersEngine
+  module Client
+    class GlobalPage < AnswersEngine::Client::Base
+      def find(gid)
+        self.class.get("/global_pages/#{gid}", @options)
+      end
+
+      def find_content(gid)
+        self.class.get("/global_pages/#{gid}/content", @options)
+      end
+
+      def find_failed_content(gid)
+        self.class.get("/global_pages/#{gid}/failed_content", @options)
+      end
+    end
+  end
+end
+

data/lib/answersengine/client/job.rb

@@ -0,0 +1,53 @@
+module AnswersEngine
+  module Client
+    class Job < AnswersEngine::Client::Base
+      def all(opts={})
+        self.class.get("/jobs", @options)
+      end
+
+      def find(job_id)
+        self.class.get("/jobs/#{job_id}", @options)
+      end
+
+      def update(job_id, opts={})
+        body = {}
+        body[:status] = opts[:status] if opts[:status]
+        body[:standard_worker_count] = opts[:workers] if opts[:workers]
+        body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
+        @options.merge!({body: body.to_json})
+
+        self.class.put("/jobs/#{job_id}", @options)
+      end
+
+      def cancel(job_id, opts={})
+        opts[:status] = 'cancelled'
+        update(job_id, opts)
+      end
+
+      def resume(job_id, opts={})
+        opts[:status] = 'active'
+        update(job_id, opts)
+      end
+
+      def pause(job_id, opts={})
+        opts[:status] = 'paused'
+        update(job_id, opts)
+      end
+
+      def seeding_update(job_id, opts={})
+        body = {}
+        body[:outputs] = opts.fetch(:outputs) {[]}
+        body[:pages] = opts.fetch(:pages) {[]}
+        body[:seeding_status] = opts.fetch(:seeding_status){ nil }
+        body[:log_error] = opts[:log_error] if opts[:log_error]
+        
+        @options.merge!({body: body.to_json})
+
+        self.class.put("/jobs/#{job_id}/seeding_update", @options)
+      end
+
+    end
+
+  end
+end
+

data/lib/answersengine/client/job_export.rb

@@ -0,0 +1,10 @@
+module AnswersEngine
+  module Client
+    class JobExport < AnswersEngine::Client::Base
+      def create(job_id, exporter_name)
+        self.class.post("/jobs/#{job_id}/exports/#{exporter_name}", @options)
+      end
+    end
+  end
+end
+

data/lib/answersengine/client/job_log.rb

@@ -0,0 +1,27 @@
+module AnswersEngine
+  module Client
+    class JobLog < AnswersEngine::Client::Base
+      def all_job_page_log(job_id, gid, opts={})
+        @options.merge!(opts)
+        self.class.get("/jobs/#{job_id}/pages/#{gid}/log", @options)
+      end
+
+      def scraper_all_job_page_log(scraper_name, gid, opts={})
+        @options.merge!(opts)
+        self.class.get("/scrapers/#{scraper_name}/current_job/pages/#{gid}/log", @options)
+      end
+
+      def all_job_log(job_id, opts={})
+        @options.merge!(opts)
+        self.class.get("/jobs/#{job_id}/log", @options)
+      end
+
+      def scraper_all_job_log(scraper_name, opts={})
+        @options.merge!(opts)
+        self.class.get("/scrapers/#{scraper_name}/current_job/log", @options)
+      end
+
+    end
+  end
+end
+

data/lib/answersengine/client/job_output.rb

@@ -0,0 +1,19 @@
+module AnswersEngine
+  module Client
+    class JobOutput < AnswersEngine::Client::Base
+      def find(job_id, collection, id)
+        self.class.get("/jobs/#{job_id}/output/collections/#{collection}/records/#{id}", @options)
+      end
+
+      def all(job_id, collection = 'default')
+
+        self.class.get("/jobs/#{job_id}/output/collections/#{collection}/records", @options)
+      end
+
+      def collections(job_id)
+        self.class.get("/jobs/#{job_id}/output/collections", @options)
+      end
+    end
+  end
+end
+

data/lib/answersengine/client/job_page.rb

@@ -0,0 +1,62 @@
+module AnswersEngine
+  module Client
+    class JobPage < AnswersEngine::Client::Base
+      def find(job_id, gid)
+        self.class.get("/jobs/#{job_id}/pages/#{gid}", @options)
+      end
+
+      def all(job_id, opts={})
+        self.class.get("/jobs/#{job_id}/pages", @options)
+      end
+
+      def update(job_id, gid, opts={})
+        body = {}        
+        body[:page_type] = opts[:page_type] if opts[:page_type]
+        body[:priority] = opts[:priority] if opts[:priority]
+        body[:vars] = opts[:vars] if opts[:vars]
+        
+        @options.merge!({body: body.to_json})
+
+        self.class.put("/jobs/#{job_id}/pages/#{gid}", @options)
+      end
+
+      def reset(job_id, gid, opts={})
+        self.class.put("/jobs/#{job_id}/pages/#{gid}/reset", @options)
+      end
+
+      def enqueue(job_id, method, url, opts={})
+        body = {}        
+        body[:method] =  method != "" ? method : "GET"
+        body[:url] =  url
+        body[:page_type] = opts[:page_type] if opts[:page_type]
+        body[:priority] = opts[:priority] if opts[:priority]
+        body[:fetch_type] = opts[:fetch_type] if opts[:fetch_type]
+        body[:body] = opts[:body] if opts[:body]
+        body[:headers] = opts[:headers] if opts[:headers]
+        body[:vars] = opts[:vars] if opts[:vars]
+        body[:force_fetch] = opts[:force_fetch] if opts[:force_fetch]
+        body[:freshness] = opts[:freshness] if opts[:freshness]
+        body[:ua_type] = opts[:ua_type] if opts[:ua_type]
+        body[:no_redirect] = opts[:no_redirect] if opts[:no_redirect]
+        body[:cookie] = opts[:cookie] if opts[:cookie]
+        
+        @options.merge!({body: body.to_json})
+
+        self.class.post("/jobs/#{job_id}/pages", @options)
+      end
+
+      def parsing_update(job_id, gid, opts={})
+        body = {}
+        body[:outputs] = opts.fetch(:outputs) {[]}
+        body[:pages] = opts.fetch(:pages) {[]}
+        body[:parsing_status] = opts.fetch(:parsing_status){ nil }
+        body[:log_error] = opts[:log_error] if opts[:log_error]
+        
+        @options.merge!({body: body.to_json})
+
+        self.class.put("/jobs/#{job_id}/pages/#{gid}/parsing_update", @options)
+      end
+    end
+  end
+end
+

data/lib/answersengine/client/job_stat.rb

@@ -0,0 +1,16 @@
+module AnswersEngine
+  module Client
+    class JobStat < AnswersEngine::Client::Base
+
+      def job_current_stats(job_id)
+        self.class.get("/jobs/#{job_id}/stats/current", @options)
+      end
+
+      def scraper_job_current_stats(scraper_name)
+        self.class.get("/scrapers/#{scraper_name}/current_job/stats/current", @options)
+      end
+
+    end
+  end
+end
+

data/lib/answersengine/client/scraper.rb

@@ -0,0 +1,54 @@
+module AnswersEngine
+  module Client
+    class Scraper < AnswersEngine::Client::Base
+
+      def find(scraper_name)
+        self.class.get("/scrapers/#{scraper_name}", @options)
+      end
+
+      def all(opts={})
+        self.class.get("/scrapers", @options)
+      end
+
+      def create(scraper_name, git_repository, opts={})
+        body = {
+            name: scraper_name,
+            git_repository: git_repository,
+            git_branch: opts[:branch] ? opts[:branch] : "master"}
+
+        body[:freshness_type] = opts[:freshness_type] if opts[:freshness_type]
+        body[:force_fetch] = opts[:force_fetch] if opts[:force_fetch]
+        body[:standard_worker_count] = opts[:workers] if opts[:workers]
+        body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
+        body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
+        body[:disable_scheduler] = opts[:disable_scheduler] if opts[:disable_scheduler]
+        body[:cancel_current_job] = opts[:cancel_current_job] if opts[:cancel_current_job]
+        body[:schedule] = opts[:schedule] if opts[:schedule]
+        body[:timezone] = opts[:timezone] if opts[:timezone]
+        @options.merge!({body: body.to_json})
+        self.class.post("/scrapers", @options)
+      end
+
+      def update(scraper_name, opts={})
+        body = {}
+
+        body[:name] = opts[:name] if opts[:name]
+        body[:git_repository] = opts[:repo] if opts[:repo]
+        body[:git_branch] = opts[:branch] if opts[:branch]
+        body[:freshness_type] = opts[:freshness_type] if opts[:freshness_type]
+        body[:force_fetch] = opts[:force_fetch] if opts.has_key?("force_fetch")
+        body[:standard_worker_count] = opts[:workers] if opts[:workers]
+        body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
+        body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
+        body[:disable_scheduler] = opts[:disable_scheduler] if opts.has_key?("disable_scheduler")
+        body[:cancel_current_job] = opts[:cancel_current_job] if opts.has_key?("cancel_current_job")
+        body[:schedule] = opts[:schedule] if opts[:schedule]
+        body[:timezone] = opts[:timezone] if opts[:timezone]
+        @options.merge!({body: body.to_json})
+
+        self.class.put("/scrapers/#{scraper_name}", @options)
+      end
+    end
+  end
+end
+

data/lib/answersengine/client/scraper_deployment.rb

@@ -0,0 +1,17 @@
+module AnswersEngine
+  module Client
+    class ScraperDeployment < AnswersEngine::Client::Base
+
+      def all(scraper_name, opts={})
+        self.class.get("/scrapers/#{scraper_name}/deployments", @options)
+      end
+
+
+      def deploy(scraper_name, opts={})
+        self.class.post("/scrapers/#{scraper_name}/deployments", @options)
+      end
+
+    end
+  end
+end
+

data/lib/answersengine/client/scraper_export.rb

@@ -0,0 +1,22 @@
+module AnswersEngine
+  module Client
+    class ScraperExport < AnswersEngine::Client::Base
+      def all(scraper_name, opts={})
+        self.class.get("/scrapers/#{scraper_name}/exports", @options)
+      end
+
+      def find(export_id)
+        self.class.get("/scrapers/exports/#{export_id}", @options)
+      end
+
+      def create(scraper_name, exporter_name)
+        self.class.post("/scrapers/#{scraper_name}/exports/#{exporter_name}", @options)
+      end
+
+      def download(export_id)
+        self.class.get("/scrapers/exports/#{export_id}/download", @options)
+      end
+    end
+  end
+end
+

data/lib/answersengine/client/scraper_exporter.rb

@@ -0,0 +1,14 @@
+module AnswersEngine
+  module Client
+    class ScraperExporter < AnswersEngine::Client::Base
+      def all(scraper_name, opts={})
+        self.class.get("/scrapers/#{scraper_name}/exporters", @options)
+      end
+
+      def find(scraper_name, exporter_name)
+        self.class.get("/scrapers/#{scraper_name}/exporters/#{exporter_name}", @options)
+      end
+    end
+  end
+end
+

data/lib/answersengine/client/scraper_job.rb

@@ -0,0 +1,49 @@
+module AnswersEngine
+  module Client
+    class ScraperJob < AnswersEngine::Client::Base
+      def all(scraper_name, opts={})
+        self.class.get("/scrapers/#{scraper_name}/jobs", @options)
+      end
+
+      def create(scraper_name, opts={})
+        body = {}
+        body[:standard_worker_count] = opts[:workers] if opts[:workers]
+        body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
+        body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
+        @options.merge!({body: body.to_json})
+        self.class.post("/scrapers/#{scraper_name}/jobs", @options)
+      end
+
+      def find(scraper_name)
+        self.class.get("/scrapers/#{scraper_name}/current_job", @options)
+      end
+
+      def update(scraper_name, opts={})
+        body = {}
+        body[:status] = opts[:status] if opts[:status]
+        body[:standard_worker_count] = opts[:workers] if opts[:workers]
+        body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
+        body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
+        @options.merge!({body: body.to_json})
+
+        self.class.put("/scrapers/#{scraper_name}/current_job", @options)
+      end
+
+      def cancel(scraper_name, opts={})
+        opts[:status] = 'cancelled'
+        update(scraper_name, opts)
+      end
+
+      def resume(scraper_name, opts={})
+        opts[:status] = 'active'
+        update(scraper_name, opts)
+      end
+
+      def pause(scraper_name, opts={})
+        opts[:status] = 'paused'
+        update(scraper_name, opts)
+      end
+    end
+  end
+end
+