datahen 0.10.4

data/lib/datahen/cli/scraper_var.rb

@@ -0,0 +1,48 @@
+module Datahen
+  class CLI < Thor
+    class ScraperVar < Thor
+
+      package_name "scraper var"
+      def self.banner(command, namespace = nil, subcommand = false)
+        "#{basename} #{@package_name} #{command.usage}"
+      end
+
+      desc "list <scraper_name>", "List environment variables on the scraper"
+      long_desc <<-LONGDESC
+        List all environment variables on the scraper.
+      LONGDESC
+      option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
+      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
+      def list(scraper_name)
+        client = Client::ScraperVar.new(options)
+        puts "#{client.all(scraper_name)}"
+      end
+
+      desc "set <scraper_name> <var_name> <value>", "Set an environment var on the scraper"
+      long_desc <<-LONGDESC
+          Creates an environment variable\x5
+          <var_name>: Var name can only consist of alphabets, numbers, underscores. Name must be unique to your scraper, otherwise it will be overwritten.\x5
+          <value>: Value of variable.\x5
+          LONGDESC
+      option :secret, type: :boolean, desc: 'Set true to make it decrypt the value. Default: false' 
+      def set(scraper_name, var_name, value)
+        # puts "options #{options}"
+        client = Client::ScraperVar.new(options)
+        puts "#{client.set(scraper_name, var_name, value, options)}"
+      end
+
+      desc "show <scraper_name> <var_name>", "Show an environment variable on the scraper"
+      def show(scraper_name, var_name)
+        client = Client::ScraperVar.new(options)
+        puts "#{client.find(scraper_name, var_name)}"
+      end
+
+      desc "unset <scraper_name> <var_name>", "Deletes an environment variable on the scraper"
+      def unset(scraper_name, var_name)
+        client = Client::ScraperVar.new(options)
+        puts "#{client.unset(scraper_name, var_name)}"
+      end
+    end
+  end
+
+end

data/lib/datahen/cli/seeder.rb

@@ -0,0 +1,40 @@
+module Datahen
+  class CLI < Thor
+    class Seeder < Thor
+      desc "try <scraper_name> <seeder_file>", "Tries a seeder file"
+      long_desc <<-LONGDESC
+            Takes a seeder script and tries to execute it without saving anything.\x5
+            <seeder_file>: Seeder script file will be executed.\x5
+          LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      def try_seed(scraper_name, seeder_file)
+        if options[:job]
+          job_id = options[:job]
+        else
+          job = Client::ScraperJob.new(options).find(scraper_name)
+          job_id = job['id']
+        end
+        
+        puts Datahen::Scraper::Seeder.exec_seeder(seeder_file, job_id, false)
+      end
+
+      desc "exec <scraper_name> <seeder_file>", "Executes a seeder script onto a scraper's current job."
+      long_desc <<-LONGDESC
+            Takes a seeder script and execute it against a job and enqueues the pages into the scraper's current job\x5
+            <seeder_file>: Seeder script file that will be executed on the scraper's current job.\x5
+          LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      def exec_parse(scraper_name, seeder_file)
+        if options[:job]
+          job_id = options[:job]
+        else
+          job = Client::ScraperJob.new(options).find(scraper_name)
+          job_id = job['id']
+        end
+
+        puts Datahen::Scraper::Seeder.exec_seeder(seeder_file, job_id, true)
+      end
+    end
+  end
+
+end

data/lib/datahen/client.rb

@@ -0,0 +1,29 @@
+require "datahen/client/base"
+require "datahen/client/auth_token"
+require "datahen/client/deploy_key"
+require 'datahen/client/export'
+require "datahen/client/scraper"
+require "datahen/client/scraper_deployment"
+require "datahen/client/scraper_job_output"
+require "datahen/client/scraper_job_page"
+require "datahen/client/scraper_exporter"
+require "datahen/client/scraper_export"
+require "datahen/client/scraper_job"
+require "datahen/client/scraper_finisher"
+require 'datahen/client/job_export'
+require "datahen/client/job"
+require "datahen/client/job_log"
+require "datahen/client/global_page"
+require "datahen/client/job_page"
+require "datahen/client/job_output"
+require "datahen/client/job_stat"
+require "datahen/client/backblaze_content"
+require "datahen/client/env_var"
+require "datahen/client/scraper_var"
+require "datahen/client/scraper_job_var"
+
+
+module Datahen
+  module Client
+  end
+end

data/lib/datahen/client/auth_token.rb

@@ -0,0 +1,50 @@
+module Datahen
+  module Client
+    class AuthToken < Datahen::Client::Base
+
+      def find(token)
+        self.class.get("/auth_tokens/#{token}", @options)
+      end
+
+      def all(opts={})
+        params = @options.merge(opts)
+        self.class.get("/auth_tokens", params)
+      end
+
+      def create(role, description, opts={})
+        body = {
+            role: role,
+            description: description}
+
+        params = @options.merge({body: body.to_json})
+        self.class.post("/auth_tokens", params)
+      end
+
+      def create_on_account(account_id, role, description)
+        body = {
+            role: role,
+            description: description}
+
+        params = @options.merge({body: body.to_json})
+        self.class.post("/accounts/#{account_id}/auth_tokens", params)
+      end
+
+      def update(token, role, description="", opts={})
+        body = {}
+
+        body[:role] = role
+        body[:description] = description if description.present?
+        params = @options.merge({body: body.to_json})
+
+        self.class.put("/auth_tokens/#{token}", params)
+      end
+
+      def delete(token, opts={})
+        body = {}
+        params = @options.merge({body: body.to_json})
+
+        self.class.delete("/auth_tokens/#{token}", params)
+      end
+    end
+  end
+end

data/lib/datahen/client/backblaze_content.rb

@@ -0,0 +1,45 @@
+require 'zlib'
+require 'httparty'
+
+module Datahen
+  module Client
+    class BackblazeContent
+      include HTTParty    
+      
+      def get_content(url)
+        self.class.get(url, format: :plain)
+      end
+
+      def get_gunzipped_content(url)
+        # Zlib.gunzip(get_content(url))
+        gunzip(get_content(url))
+      end
+
+      def gunzip(string)
+        sio = StringIO.new(string)
+        gz = Zlib::GzipReader.new(sio, encoding: Encoding::ASCII_8BIT)
+        _content = ""
+        begin 
+          _content = gz.read
+        rescue => e
+          # if unexpected eof error, then readchar until error, and ignore it
+          if e.to_s == 'unexpected end of file'
+            begin 
+              while !gz.eof?
+                _content += gz.readchar
+              end
+            rescue => e
+              puts "Ignored Zlib error: #{e.to_s}"
+            end
+          else 
+            raise e
+          end
+        end
+
+        return _content
+      ensure
+        gz.close if gz.respond_to?(:close)
+      end
+    end
+  end
+end

data/lib/datahen/client/base.rb

@@ -0,0 +1,69 @@
+require 'httparty'
+
+module Datahen
+  module Client
+    class Base
+      include HTTParty
+
+      def self.env_auth_token
+        ENV['DATAHEN_TOKEN']
+      end
+
+      def self.env_ignore_ssl
+        ENV['DATAHEN_IGNORE_SSL'].to_s.strip == '1'
+      end
+
+      def env_api_url
+        ENV['DATAHEN_API_URL'].nil? ? 'https://app.datahen.com/api/v1' : ENV['DATAHEN_API_URL']
+      end
+
+      def ignore_ssl
+        return @ignore_ssl unless @ignore_ssl.nil?
+        @ignore_ssl = self.class.env_ignore_ssl
+        @ignore_ssl
+      end
+
+      def auth_token
+        @auth_token ||= self.class.env_auth_token
+      end
+
+      def auth_token= value
+        @auth_token = value
+      end
+
+      def initialize(opts={})
+        @ignore_ssl = opts[:ignore_ssl]
+        self.class.base_uri(env_api_url)
+        self.auth_token = opts[:auth_token] unless opts[:auth_token].nil?
+        @options = {
+          headers: {
+            "Authorization" => "Bearer #{auth_token}",
+            "Content-Type" => "application/json",
+          },
+          verify: !ignore_ssl
+        }
+
+        query = {}
+        query[:p] = opts[:page] if opts[:page]
+        query[:pp] = opts[:per_page] if opts[:per_page]
+        query[:fetchfail] = opts[:fetch_fail] if opts[:fetch_fail]
+        query[:parsefail] = opts[:parse_fail] if opts[:parse_fail]
+        query[:status] = opts[:status] if opts[:status]
+        query[:page_type] = opts[:page_type] if opts[:page_type]
+        query[:gid] = opts[:gid] if opts[:gid]
+
+        if opts[:query]
+          if opts[:query].is_a?(Hash)
+            query[:q] = opts[:query].to_json
+          elsif opts[:query].is_a?(String)
+            query[:q] = JSON.parse(opts[:query]).to_json
+          end
+        end
+
+        unless query.empty?
+          @options.merge!(query: query)
+        end
+      end
+    end
+  end
+end

data/lib/datahen/client/deploy_key.rb

@@ -0,0 +1,21 @@
+module Datahen
+  module Client
+    class DeployKey < Datahen::Client::Base
+
+      def find(opts={})
+        params = @options.merge(opts)
+        self.class.get("/deploy_key", params)
+      end
+
+      def create(opts={})
+        params = @options.merge(opts)
+        self.class.post("/deploy_key", params)
+      end
+
+      def delete(opts={})
+        params = @options.merge(opts)
+        self.class.delete("/deploy_key", params)
+      end
+    end
+  end
+end

data/lib/datahen/client/env_var.rb

@@ -0,0 +1,28 @@
+module Datahen
+  module Client
+    class EnvVar < Datahen::Client::Base
+
+      def find(name)
+        self.class.get("/env_vars/#{name}", @options)
+      end
+
+      def all(opts={})
+        params = @options.merge opts
+        self.class.get("/env_vars", params)
+      end
+
+      def set(name, value, opts={})
+        body = {}
+        body[:value] = value
+        body[:secret] = opts[:secret] if opts[:secret]
+        params = @options.merge({body: body.to_json})
+        self.class.put("/env_vars/#{name}", params)
+      end
+
+      def unset(name, opts={})
+        params = @options.merge(opts)
+        self.class.delete("/env_vars/#{name}", params)
+      end
+    end
+  end
+end

data/lib/datahen/client/export.rb

@@ -0,0 +1,10 @@
+module Datahen
+  module Client
+    class Export < Datahen::Client::Base
+      def all(opts={})
+        params = @options.merge(opts)
+        self.class.get("/scrapers/exports", params)
+      end
+    end
+  end
+end

data/lib/datahen/client/global_page.rb

@@ -0,0 +1,18 @@
+module Datahen
+  module Client
+    class GlobalPage < Datahen::Client::Base
+      def find(gid)
+        self.class.get("/global_pages/#{gid}", @options)
+      end
+
+      def find_content(gid)
+        self.class.get("/global_pages/#{gid}/content", @options)
+      end
+
+      def find_failed_content(gid)
+        self.class.get("/global_pages/#{gid}/failed_content", @options)
+      end
+    end
+  end
+end
+

data/lib/datahen/client/job.rb

@@ -0,0 +1,64 @@
+module Datahen
+  module Client
+    class Job < Datahen::Client::Base
+      def all(opts={})
+        params = @options.merge(opts)
+        self.class.get("/jobs", params)
+      end
+
+      def find(job_id)
+        self.class.get("/jobs/#{job_id}", @options)
+      end
+
+      def update(job_id, opts={})
+        body = {}
+        body[:status] = opts[:status] if opts[:status]
+        body[:standard_worker_count] = opts[:workers] if opts[:workers]
+        body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
+        params = @options.merge({body: body.to_json})
+
+        self.class.put("/jobs/#{job_id}", params)
+      end
+
+      def cancel(job_id, opts={})
+        opts[:status] = 'cancelled'
+        update(job_id, opts)
+      end
+
+      def resume(job_id, opts={})
+        opts[:status] = 'active'
+        update(job_id, opts)
+      end
+
+      def pause(job_id, opts={})
+        opts[:status] = 'paused'
+        update(job_id, opts)
+      end
+
+      def seeding_update(job_id, opts={})
+        body = {}
+        body[:outputs] = opts.fetch(:outputs) {[]}
+        body[:pages] = opts.fetch(:pages) {[]}
+        body[:seeding_status] = opts.fetch(:seeding_status){ nil }
+        body[:log_error] = opts[:log_error] if opts[:log_error]
+
+        params = @options.merge({body: body.to_json})
+
+        self.class.put("/jobs/#{job_id}/seeding_update", params)
+      end
+
+      def finisher_update(job_id, opts={})
+        body = {}
+        body[:outputs] = opts.fetch(:outputs) {[]}
+        body[:finisher_status] = opts.fetch(:finisher_status){ nil }
+        body[:log_error] = opts[:log_error] if opts[:log_error]
+
+        params = @options.merge({body: body.to_json})
+
+        self.class.put("/jobs/#{job_id}/finisher_update", params)
+      end
+
+    end
+
+  end
+end

data/lib/datahen/client/job_export.rb

@@ -0,0 +1,10 @@
+module Datahen
+  module Client
+    class JobExport < Datahen::Client::Base
+      def create(job_id, exporter_name)
+        self.class.post("/jobs/#{job_id}/exports/#{exporter_name}", @options)
+      end
+    end
+  end
+end
+