datahen 0.10.4

data/lib/datahen/client/job_log.rb

@@ -0,0 +1,26 @@
+module Datahen
+  module Client
+    class JobLog < Datahen::Client::Base
+      def all_job_page_log(job_id, gid, opts={})
+        params = @options.merge(opts)
+        self.class.get("/jobs/#{job_id}/pages/#{gid}/log", params)
+      end
+
+      def scraper_all_job_page_log(scraper_name, gid, opts={})
+        params = @options.merge(opts)
+        self.class.get("/scrapers/#{scraper_name}/current_job/pages/#{gid}/log", params)
+      end
+
+      def all_job_log(job_id, opts={})
+        params = @options.merge(opts)
+        self.class.get("/jobs/#{job_id}/log", params)
+      end
+
+      def scraper_all_job_log(scraper_name, opts={})
+        params = @options.merge(opts)
+        self.class.get("/scrapers/#{scraper_name}/current_job/log", params)
+      end
+
+    end
+  end
+end

data/lib/datahen/client/job_output.rb

@@ -0,0 +1,19 @@
+module Datahen
+  module Client
+    class JobOutput < Datahen::Client::Base
+      def find(job_id, collection, id)
+        self.class.get("/jobs/#{job_id}/output/collections/#{collection}/records/#{id}", @options)
+      end
+
+      def all(job_id, collection = 'default')
+
+        self.class.get("/jobs/#{job_id}/output/collections/#{collection}/records", @options)
+      end
+
+      def collections(job_id)
+        self.class.get("/jobs/#{job_id}/output/collections", @options)
+      end
+    end
+  end
+end
+

data/lib/datahen/client/job_page.rb

@@ -0,0 +1,58 @@
+module Datahen
+  module Client
+    class JobPage < Datahen::Client::Base
+      def find(job_id, gid)
+        self.class.get("/jobs/#{job_id}/pages/#{gid}", @options)
+      end
+
+      def all(job_id, opts={})
+        params = @options.merge(opts)
+        self.class.get("/jobs/#{job_id}/pages", params)
+      end
+
+      def update(job_id, gid, opts={})
+        body = {}
+        body[:page_type] = opts[:page_type] if opts[:page_type]
+        body[:priority] = opts[:priority] if opts[:priority]
+        body[:vars] = opts[:vars] if opts[:vars]
+
+        params = @options.merge({body: body.to_json})
+
+        self.class.put("/jobs/#{job_id}/pages/#{gid}", params)
+      end
+
+      def enqueue(job_id, method, url, opts={})
+        body = {}
+        body[:method] =  method != "" ? method : "GET"
+        body[:url] =  url
+        body[:page_type] = opts[:page_type] if opts[:page_type]
+        body[:priority] = opts[:priority] if opts[:priority]
+        body[:fetch_type] = opts[:fetch_type] if opts[:fetch_type]
+        body[:body] = opts[:body] if opts[:body]
+        body[:headers] = opts[:headers] if opts[:headers]
+        body[:vars] = opts[:vars] if opts[:vars]
+        body[:force_fetch] = opts[:force_fetch] if opts[:force_fetch]
+        body[:freshness] = opts[:freshness] if opts[:freshness]
+        body[:ua_type] = opts[:ua_type] if opts[:ua_type]
+        body[:no_redirect] = opts[:no_redirect] if opts[:no_redirect]
+        body[:cookie] = opts[:cookie] if opts[:cookie]
+
+        params = @options.merge({body: body.to_json})
+
+        self.class.post("/jobs/#{job_id}/pages", params)
+      end
+
+      def parsing_update(job_id, gid, opts={})
+        body = {}
+        body[:outputs] = opts.fetch(:outputs) {[]}
+        body[:pages] = opts.fetch(:pages) {[]}
+        body[:parsing_status] = opts.fetch(:parsing_status){ nil }
+        body[:log_error] = opts[:log_error] if opts[:log_error]
+
+        params = @options.merge({body: body.to_json})
+
+        self.class.put("/jobs/#{job_id}/pages/#{gid}/parsing_update", params)
+      end
+    end
+  end
+end

data/lib/datahen/client/job_stat.rb

@@ -0,0 +1,16 @@
+module Datahen
+  module Client
+    class JobStat < Datahen::Client::Base
+
+      def job_current_stats(job_id)
+        self.class.get("/jobs/#{job_id}/stats/current", @options)
+      end
+
+      def scraper_job_current_stats(scraper_name)
+        self.class.get("/scrapers/#{scraper_name}/current_job/stats/current", @options)
+      end
+
+    end
+  end
+end
+

data/lib/datahen/client/scraper.rb

@@ -0,0 +1,57 @@
+module Datahen
+  module Client
+    class Scraper < Datahen::Client::Base
+
+      def find(scraper_name)
+        self.class.get("/scrapers/#{scraper_name}", @options)
+      end
+
+      def all(opts={})
+        params = @options.merge opts
+        self.class.get("/scrapers", params)
+      end
+
+      def create(scraper_name, git_repository, opts={})
+        body = {}
+        body[:name] = scraper_name
+        body[:git_repository] = git_repository
+        body[:git_branch] = opts[:branch] || opts[:git_branch] || "master" if opts[:branch] || opts[:git_branch]
+        body[:freshness_type] = opts[:freshness_type] if opts[:freshness_type]
+        body[:force_fetch] = opts[:force_fetch] if opts[:force_fetch]
+        body[:standard_worker_count] = opts[:workers] || opts[:standard_worker_count] if opts[:workers] || opts[:standard_worker_count]
+        body[:browser_worker_count] = opts[:browsers] || opts[:browser_worker_count] if opts[:browsers] || opts[:browser_worker_count]
+        body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
+        body[:disable_scheduler] = opts[:disable_scheduler] if opts[:disable_scheduler]
+        body[:cancel_current_job] = opts[:cancel_current_job] if opts[:cancel_current_job]
+        body[:schedule] = opts[:schedule] if opts[:schedule]
+        body[:timezone] = opts[:timezone] if opts[:timezone]
+        params = @options.merge({body: body.to_json})
+        self.class.post("/scrapers", params)
+      end
+
+      def update(scraper_name, opts={})
+        body = {}
+        body[:name] = opts[:name] if opts[:name]
+        body[:git_repository] = opts[:repo] || opts[:git_repository] if opts[:repo] || opts[:git_repository]
+        body[:git_branch] = opts[:branch] || opts[:git_branch] if opts[:branch] || opts[:git_branch]
+        body[:freshness_type] = opts[:freshness_type] if opts[:freshness_type]
+        body[:force_fetch] = opts[:force_fetch] if opts.has_key?("force_fetch") || opts.has_key?(:force_fetch)
+        body[:standard_worker_count] = opts[:workers] || opts[:standard_worker_count] if opts[:workers] || opts[:standard_worker_count]
+        body[:browser_worker_count] = opts[:browsers] || opts[:browser_worker_count] if opts[:browsers] || opts[:browser_worker_count]
+        body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
+        body[:disable_scheduler] = opts[:disable_scheduler] if opts.has_key?("disable_scheduler") || opts.has_key?(:disable_scheduler)
+        body[:cancel_current_job] = opts[:cancel_current_job] if opts.has_key?("cancel_current_job") || opts.has_key?(:cancel_current_job)
+        body[:schedule] = opts[:schedule] if opts[:schedule]
+        body[:timezone] = opts[:timezone] if opts[:timezone]
+        params = @options.merge({body: body.to_json})
+
+        self.class.put("/scrapers/#{scraper_name}", params)
+      end
+
+      def delete(scraper_name, opts={})
+        params = @options.merge(opts)
+        self.class.delete("/scrapers/#{scraper_name}", params)
+      end
+    end
+  end
+end

data/lib/datahen/client/scraper_deployment.rb

@@ -0,0 +1,18 @@
+module Datahen
+  module Client
+    class ScraperDeployment < Datahen::Client::Base
+
+      def all(scraper_name, opts={})
+        params = @options.merge(opts)
+        self.class.get("/scrapers/#{scraper_name}/deployments", params)
+      end
+
+
+      def deploy(scraper_name, opts={})
+        params = @options.merge(opts)
+        self.class.post("/scrapers/#{scraper_name}/deployments", params)
+      end
+
+    end
+  end
+end

data/lib/datahen/client/scraper_export.rb

@@ -0,0 +1,22 @@
+module Datahen
+  module Client
+    class ScraperExport < Datahen::Client::Base
+      def all(scraper_name, opts={})
+        params = @options.merge(opts)
+        self.class.get("/scrapers/#{scraper_name}/exports", params)
+      end
+
+      def find(export_id)
+        self.class.get("/scrapers/exports/#{export_id}", @options)
+      end
+
+      def create(scraper_name, exporter_name)
+        self.class.post("/scrapers/#{scraper_name}/exports/#{exporter_name}", @options)
+      end
+
+      def download(export_id)
+        self.class.get("/scrapers/exports/#{export_id}/download", @options)
+      end
+    end
+  end
+end

data/lib/datahen/client/scraper_exporter.rb

@@ -0,0 +1,14 @@
+module Datahen
+  module Client
+    class ScraperExporter < Datahen::Client::Base
+      def all(scraper_name, opts={})
+        params = @options.merge(opts)
+        self.class.get("/scrapers/#{scraper_name}/exporters", params)
+      end
+
+      def find(scraper_name, exporter_name)
+        self.class.get("/scrapers/#{scraper_name}/exporters/#{exporter_name}", @options)
+      end
+    end
+  end
+end

data/lib/datahen/client/scraper_finisher.rb

@@ -0,0 +1,16 @@
+module Datahen
+  module Client
+    class ScraperFinisher < Datahen::Client::Base
+      # Reset finisher on a scraper's current job.
+      #
+      # @param [String] scraper_name Scraper name.
+      # @param [Hash] opts ({}) API custom parameters.
+      #
+      # @return [HTTParty::Response]
+      def reset(scraper_name, opts={})
+        params = @options.merge(opts)
+        self.class.put("/scrapers/#{scraper_name}/current_job/finisher/reset", params)
+      end
+    end
+  end
+end

data/lib/datahen/client/scraper_job.rb

@@ -0,0 +1,49 @@
+module Datahen
+  module Client
+    class ScraperJob < Datahen::Client::Base
+      def all(scraper_name, opts={})
+        params = @options.merge(opts)
+        self.class.get("/scrapers/#{scraper_name}/jobs", params)
+      end
+
+      def create(scraper_name, opts={})
+        body = {}
+        body[:standard_worker_count] = opts[:workers] if opts[:workers]
+        body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
+        body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
+        params = @options.merge({body: body.to_json})
+        self.class.post("/scrapers/#{scraper_name}/jobs", params)
+      end
+
+      def find(scraper_name)
+        self.class.get("/scrapers/#{scraper_name}/current_job", @options)
+      end
+
+      def update(scraper_name, opts={})
+        body = {}
+        body[:status] = opts[:status] if opts[:status]
+        body[:standard_worker_count] = opts[:workers] if opts[:workers]
+        body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
+        body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
+        params = @options.merge({body: body.to_json})
+
+        self.class.put("/scrapers/#{scraper_name}/current_job", params)
+      end
+
+      def cancel(scraper_name, opts={})
+        opts[:status] = 'cancelled'
+        update(scraper_name, opts)
+      end
+
+      def resume(scraper_name, opts={})
+        opts[:status] = 'active'
+        update(scraper_name, opts)
+      end
+
+      def pause(scraper_name, opts={})
+        opts[:status] = 'paused'
+        update(scraper_name, opts)
+      end
+    end
+  end
+end

data/lib/datahen/client/scraper_job_output.rb

@@ -0,0 +1,19 @@
+module Datahen
+  module Client
+    class ScraperJobOutput < Datahen::Client::Base
+      def find(scraper_name, collection, id)
+        self.class.get("/scrapers/#{scraper_name}/current_job/output/collections/#{collection}/records/#{id}", @options)
+      end
+
+      def all(scraper_name, collection = 'default')
+
+        self.class.get("/scrapers/#{scraper_name}/current_job/output/collections/#{collection}/records", @options)
+      end
+
+      def collections(scraper_name)
+        self.class.get("/scrapers/#{scraper_name}/current_job/output/collections", @options)
+      end
+    end
+  end
+end
+

data/lib/datahen/client/scraper_job_page.rb

@@ -0,0 +1,67 @@
+module Datahen
+  module Client
+    class ScraperJobPage < Datahen::Client::Base
+      def find(scraper_name, gid)
+        self.class.get("/scrapers/#{scraper_name}/current_job/pages/#{gid}", @options)
+      end
+
+      def all(scraper_name, opts={})
+        params = @options.merge(opts)
+        self.class.get("/scrapers/#{scraper_name}/current_job/pages", params)
+      end
+
+      def update(scraper_name, gid, opts={})
+        body = {}
+        body[:page_type] = opts[:page_type] if opts[:page_type]
+        body[:priority] = opts[:priority] if opts[:priority]
+        body[:vars] = opts[:vars] if opts[:vars]
+
+        params = @options.merge({body: body.to_json})
+
+        self.class.put("/scrapers/#{scraper_name}/current_job/pages/#{gid}", params)
+      end
+
+      def refetch(scraper_name, opts={})
+        params = @options.merge(opts)
+        self.class.put("/scrapers/#{scraper_name}/current_job/pages/refetch", params)
+      end
+
+      def refetch_by_job(job_id, opts={})
+        params = @options.merge(opts)
+        self.class.put("/jobs/#{job_id}/pages/refetch", params)
+      end
+
+      def reparse(scraper_name, opts={})
+        params = @options.merge(opts)
+        self.class.put("/scrapers/#{scraper_name}/current_job/pages/reparse", params)
+      end
+
+      def reparse_by_job(job_id, opts={})
+        params = @options.merge(opts)
+        self.class.put("/jobs/#{job_id}/pages/reparse", params)
+      end
+
+      def enqueue(scraper_name, method, url, opts={})
+        body = {}
+        body[:method] =  method != "" ? method : "GET"
+        body[:url] =  url
+        body[:page_type] = opts[:page_type] if opts[:page_type]
+        body[:priority] = opts[:priority] if opts[:priority]
+        body[:fetch_type] = opts[:fetch_type] if opts[:fetch_type]
+        body[:body] = opts[:body] if opts[:body]
+        body[:headers] = opts[:headers] if opts[:headers]
+        body[:vars] = opts[:vars] if opts[:vars]
+        body[:force_fetch] = opts[:force_fetch] if opts[:force_fetch]
+        body[:freshness] = opts[:freshness] if opts[:freshness]
+        body[:ua_type] = opts[:ua_type] if opts[:ua_type]
+        body[:no_redirect] = opts[:no_redirect] if opts[:no_redirect]
+        body[:cookie] = opts[:cookie] if opts[:cookie]
+
+        params = @options.merge({body: body.to_json})
+
+        self.class.post("/scrapers/#{scraper_name}/current_job/pages", params)
+      end
+
+    end
+  end
+end

data/lib/datahen/client/scraper_job_var.rb

@@ -0,0 +1,28 @@
+module Datahen
+  module Client
+    class ScraperJobVar < Datahen::Client::Base
+
+      def find(scraper_name, var_name)
+        self.class.get("/scrapers/#{scraper_name}/current_job/vars/#{var_name}", @options)
+      end
+
+      def all(scraper_name, opts={})
+        params = @options.merge opts
+        self.class.get("/scrapers/#{scraper_name}/current_job/vars", params)
+      end
+
+      def set(scraper_name, var_name, value, opts={})
+        body = {}
+        body[:value] = value
+        body[:secret] = opts[:secret] if opts[:secret]
+        params = @options.merge({body: body.to_json})
+        self.class.put("/scrapers/#{scraper_name}/current_job/vars/#{var_name}", params)
+      end
+
+      def unset(scraper_name, var_name, opts={})
+        params = @options.merge(opts)
+        self.class.delete("/scrapers/#{scraper_name}/current_job/vars/#{var_name}", params)
+      end
+    end
+  end
+end

data/lib/datahen/client/scraper_var.rb

@@ -0,0 +1,28 @@
+module Datahen
+  module Client
+    class ScraperVar < Datahen::Client::Base
+
+      def find(scraper_name, var_name)
+        self.class.get("/scrapers/#{scraper_name}/vars/#{var_name}", @options)
+      end
+
+      def all(scraper_name, opts={})
+        params = @options.merge opts
+        self.class.get("/scrapers/#{scraper_name}/vars", params)
+      end
+
+      def set(scraper_name, var_name, value, opts={})
+        body = {}
+        body[:value] = value
+        body[:secret] = opts[:secret] if opts[:secret]
+        params = @options.merge({body: body.to_json})
+        self.class.put("/scrapers/#{scraper_name}/vars/#{var_name}", params)
+      end
+
+      def unset(scraper_name, var_name, opts={})
+        params = @options.merge(opts)
+        self.class.delete("/scrapers/#{scraper_name}/vars/#{var_name}", params)
+      end
+    end
+  end
+end