answersengine 0.10.1 → 0.10.2

data/lib/answersengine/cli/scraper_var.rb

@@ -1,48 +0,0 @@
-module AnswersEngine
-  class CLI < Thor
-    class ScraperVar < Thor
-
-      package_name "scraper var"
-      def self.banner(command, namespace = nil, subcommand = false)
-        "#{basename} #{@package_name} #{command.usage}"
-      end
-
-      desc "list <scraper_name>", "List environment variables on the scraper"
-      long_desc <<-LONGDESC
-        List all environment variables on the scraper.
-      LONGDESC
-      option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
-      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
-      def list(scraper_name)
-        client = Client::ScraperVar.new(options)
-        puts "#{client.all(scraper_name)}"
-      end
-
-      desc "set <scraper_name> <var_name> <value>", "Set an environment var on the scraper"
-      long_desc <<-LONGDESC
-          Creates an environment variable\x5
-          <var_name>: Var name can only consist of alphabets, numbers, underscores. Name must be unique to your scraper, otherwise it will be overwritten.\x5
-          <value>: Value of variable.\x5
-          LONGDESC
-      option :secret, type: :boolean, desc: 'Set true to make it decrypt the value. Default: false' 
-      def set(scraper_name, var_name, value)
-        # puts "options #{options}"
-        client = Client::ScraperVar.new(options)
-        puts "#{client.set(scraper_name, var_name, value, options)}"
-      end
-
-      desc "show <scraper_name> <var_name>", "Show an environment variable on the scraper"
-      def show(scraper_name, var_name)
-        client = Client::ScraperVar.new(options)
-        puts "#{client.find(scraper_name, var_name)}"
-      end
-
-      desc "unset <scraper_name> <var_name>", "Deletes an environment variable on the scraper"
-      def unset(scraper_name, var_name)
-        client = Client::ScraperVar.new(options)
-        puts "#{client.unset(scraper_name, var_name)}"
-      end
-    end
-  end
-
-end

data/lib/answersengine/cli/seeder.rb

@@ -1,40 +0,0 @@
-module AnswersEngine
-  class CLI < Thor
-    class Seeder < Thor
-      desc "try <scraper_name> <seeder_file>", "Tries a seeder file"
-      long_desc <<-LONGDESC
-            Takes a seeder script and tries to execute it without saving anything.\x5
-            <seeder_file>: Seeder script file will be executed.\x5
-          LONGDESC
-      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
-      def try_seed(scraper_name, seeder_file)
-        if options[:job]
-          job_id = options[:job]
-        else
-          job = Client::ScraperJob.new(options).find(scraper_name)
-          job_id = job['id']
-        end
-        
-        puts AnswersEngine::Scraper::Seeder.exec_seeder(seeder_file, job_id, false)
-      end
-
-      desc "exec <scraper_name> <seeder_file>", "Executes a seeder script onto a scraper's current job."
-      long_desc <<-LONGDESC
-            Takes a seeder script and execute it against a job and enqueues the pages into the scraper's current job\x5
-            <seeder_file>: Seeder script file that will be executed on the scraper's current job.\x5
-          LONGDESC
-      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
-      def exec_parse(scraper_name, seeder_file)
-        if options[:job]
-          job_id = options[:job]
-        else
-          job = Client::ScraperJob.new(options).find(scraper_name)
-          job_id = job['id']
-        end
-
-        puts AnswersEngine::Scraper::Seeder.exec_seeder(seeder_file, job_id, true)
-      end
-    end
-  end
-
-end

data/lib/answersengine/client.rb

@@ -1,29 +0,0 @@
-require "answersengine/client/base"
-require "answersengine/client/auth_token"
-require "answersengine/client/deploy_key"
-require 'answersengine/client/export'
-require "answersengine/client/scraper"
-require "answersengine/client/scraper_deployment"
-require "answersengine/client/scraper_job_output"
-require "answersengine/client/scraper_job_page"
-require "answersengine/client/scraper_exporter"
-require "answersengine/client/scraper_export"
-require "answersengine/client/scraper_job"
-require "answersengine/client/scraper_finisher"
-require 'answersengine/client/job_export'
-require "answersengine/client/job"
-require "answersengine/client/job_log"
-require "answersengine/client/global_page"
-require "answersengine/client/job_page"
-require "answersengine/client/job_output"
-require "answersengine/client/job_stat"
-require "answersengine/client/backblaze_content"
-require "answersengine/client/env_var"
-require "answersengine/client/scraper_var"
-require "answersengine/client/scraper_job_var"
-
-
-module AnswersEngine
-  module Client
-  end
-end

data/lib/answersengine/client/auth_token.rb

@@ -1,50 +0,0 @@
-module AnswersEngine
-  module Client
-    class AuthToken < AnswersEngine::Client::Base
-
-      def find(token)
-        self.class.get("/auth_tokens/#{token}", @options)
-      end
-
-      def all(opts={})
-        params = @options.merge(opts)
-        self.class.get("/auth_tokens", params)
-      end
-
-      def create(role, description, opts={})
-        body = {
-            role: role,
-            description: description}
-
-        params = @options.merge({body: body.to_json})
-        self.class.post("/auth_tokens", params)
-      end
-
-      def create_on_account(account_id, role, description)
-        body = {
-            role: role,
-            description: description}
-
-        params = @options.merge({body: body.to_json})
-        self.class.post("/accounts/#{account_id}/auth_tokens", params)
-      end
-
-      def update(token, role, description="", opts={})
-        body = {}
-
-        body[:role] = role
-        body[:description] = description if description.present?
-        params = @options.merge({body: body.to_json})
-
-        self.class.put("/auth_tokens/#{token}", params)
-      end
-
-      def delete(token, opts={})
-        body = {}
-        params = @options.merge({body: body.to_json})
-
-        self.class.delete("/auth_tokens/#{token}", params)
-      end
-    end
-  end
-end

data/lib/answersengine/client/backblaze_content.rb

@@ -1,45 +0,0 @@
-require 'zlib'
-require 'httparty'
-
-module AnswersEngine
-  module Client
-    class BackblazeContent
-      include HTTParty    
-      
-      def get_content(url)
-        self.class.get(url, format: :plain)
-      end
-
-      def get_gunzipped_content(url)
-        # Zlib.gunzip(get_content(url))
-        gunzip(get_content(url))
-      end
-
-      def gunzip(string)
-        sio = StringIO.new(string)
-        gz = Zlib::GzipReader.new(sio, encoding: Encoding::ASCII_8BIT)
-        _content = ""
-        begin 
-          _content = gz.read
-        rescue => e
-          # if unexpected eof error, then readchar until error, and ignore it
-          if e.to_s == 'unexpected end of file'
-            begin 
-              while !gz.eof?
-                _content += gz.readchar
-              end
-            rescue => e
-              puts "Ignored Zlib error: #{e.to_s}"
-            end
-          else 
-            raise e
-          end
-        end
-
-        return _content
-      ensure
-        gz.close if gz.respond_to?(:close)
-      end
-    end
-  end
-end

data/lib/answersengine/client/base.rb

@@ -1,55 +0,0 @@
-require 'httparty'
-
-module AnswersEngine
-  module Client
-    class Base
-      include HTTParty
-
-      def self.env_auth_token
-        ENV['ANSWERSENGINE_TOKEN']
-      end
-
-      def env_api_url
-        ENV['ANSWERSENGINE_API_URL'].nil? ? 'https://fetch.answersengine.com/api/v1' : ENV['ANSWERSENGINE_API_URL']
-      end
-
-      def auth_token
-        @auth_token ||= self.class.env_auth_token
-      end
-
-      def auth_token= value
-        @auth_token = value
-      end
-
-      def initialize(opts={})
-        self.class.base_uri(env_api_url)
-        self.auth_token = opts[:auth_token] unless opts[:auth_token].nil?
-        @options = { headers: {
-          "Authorization" => "Bearer #{auth_token}",
-          "Content-Type" => "application/json",
-          }}
-
-        query = {}
-        query[:p] = opts[:page] if opts[:page]
-        query[:pp] = opts[:per_page] if opts[:per_page]
-        query[:fetchfail] = opts[:fetch_fail] if opts[:fetch_fail]
-        query[:parsefail] = opts[:parse_fail] if opts[:parse_fail]
-        query[:status] = opts[:status] if opts[:status]
-        query[:page_type] = opts[:page_type] if opts[:page_type]
-        query[:gid] = opts[:gid] if opts[:gid]
-
-        if opts[:query]
-          if opts[:query].is_a?(Hash)
-            query[:q] = opts[:query].to_json
-          elsif opts[:query].is_a?(String)
-            query[:q] = JSON.parse(opts[:query]).to_json
-          end
-        end
-
-        unless query.empty?
-          @options.merge!(query: query)
-        end
-      end
-    end
-  end
-end

data/lib/answersengine/client/deploy_key.rb

@@ -1,21 +0,0 @@
-module AnswersEngine
-  module Client
-    class DeployKey < AnswersEngine::Client::Base
-
-      def find(opts={})
-        params = @options.merge(opts)
-        self.class.get("/deploy_key", params)
-      end
-
-      def create(opts={})
-        params = @options.merge(opts)
-        self.class.post("/deploy_key", params)
-      end
-
-      def delete(opts={})
-        params = @options.merge(opts)
-        self.class.delete("/deploy_key", params)
-      end
-    end
-  end
-end

data/lib/answersengine/client/env_var.rb

@@ -1,28 +0,0 @@
-module AnswersEngine
-  module Client
-    class EnvVar < AnswersEngine::Client::Base
-
-      def find(name)
-        self.class.get("/env_vars/#{name}", @options)
-      end
-
-      def all(opts={})
-        params = @options.merge opts
-        self.class.get("/env_vars", params)
-      end
-
-      def set(name, value, opts={})
-        body = {}
-        body[:value] = value
-        body[:secret] = opts[:secret] if opts[:secret]
-        params = @options.merge({body: body.to_json})
-        self.class.put("/env_vars/#{name}", params)
-      end
-
-      def unset(name, opts={})
-        params = @options.merge(opts)
-        self.class.delete("/env_vars/#{name}", params)
-      end
-    end
-  end
-end

data/lib/answersengine/client/export.rb

@@ -1,10 +0,0 @@
-module AnswersEngine
-  module Client
-    class Export < AnswersEngine::Client::Base
-      def all(opts={})
-        params = @options.merge(opts)
-        self.class.get("/scrapers/exports", params)
-      end
-    end
-  end
-end

data/lib/answersengine/client/global_page.rb

@@ -1,18 +0,0 @@
-module AnswersEngine
-  module Client
-    class GlobalPage < AnswersEngine::Client::Base
-      def find(gid)
-        self.class.get("/global_pages/#{gid}", @options)
-      end
-
-      def find_content(gid)
-        self.class.get("/global_pages/#{gid}/content", @options)
-      end
-
-      def find_failed_content(gid)
-        self.class.get("/global_pages/#{gid}/failed_content", @options)
-      end
-    end
-  end
-end
-

data/lib/answersengine/client/job.rb

@@ -1,64 +0,0 @@
-module AnswersEngine
-  module Client
-    class Job < AnswersEngine::Client::Base
-      def all(opts={})
-        params = @options.merge(opts)
-        self.class.get("/jobs", params)
-      end
-
-      def find(job_id)
-        self.class.get("/jobs/#{job_id}", @options)
-      end
-
-      def update(job_id, opts={})
-        body = {}
-        body[:status] = opts[:status] if opts[:status]
-        body[:standard_worker_count] = opts[:workers] if opts[:workers]
-        body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
-        params = @options.merge({body: body.to_json})
-
-        self.class.put("/jobs/#{job_id}", params)
-      end
-
-      def cancel(job_id, opts={})
-        opts[:status] = 'cancelled'
-        update(job_id, opts)
-      end
-
-      def resume(job_id, opts={})
-        opts[:status] = 'active'
-        update(job_id, opts)
-      end
-
-      def pause(job_id, opts={})
-        opts[:status] = 'paused'
-        update(job_id, opts)
-      end
-
-      def seeding_update(job_id, opts={})
-        body = {}
-        body[:outputs] = opts.fetch(:outputs) {[]}
-        body[:pages] = opts.fetch(:pages) {[]}
-        body[:seeding_status] = opts.fetch(:seeding_status){ nil }
-        body[:log_error] = opts[:log_error] if opts[:log_error]
-
-        params = @options.merge({body: body.to_json})
-
-        self.class.put("/jobs/#{job_id}/seeding_update", params)
-      end
-
-      def finisher_update(job_id, opts={})
-        body = {}
-        body[:outputs] = opts.fetch(:outputs) {[]}
-        body[:finisher_status] = opts.fetch(:finisher_status){ nil }
-        body[:log_error] = opts[:log_error] if opts[:log_error]
-
-        params = @options.merge({body: body.to_json})
-
-        self.class.put("/jobs/#{job_id}/finisher_update", params)
-      end
-
-    end
-
-  end
-end

data/lib/answersengine/client/job_export.rb

@@ -1,10 +0,0 @@
-module AnswersEngine
-  module Client
-    class JobExport < AnswersEngine::Client::Base
-      def create(job_id, exporter_name)
-        self.class.post("/jobs/#{job_id}/exports/#{exporter_name}", @options)
-      end
-    end
-  end
-end
-

data/lib/answersengine/client/job_log.rb

@@ -1,26 +0,0 @@
-module AnswersEngine
-  module Client
-    class JobLog < AnswersEngine::Client::Base
-      def all_job_page_log(job_id, gid, opts={})
-        params = @options.merge(opts)
-        self.class.get("/jobs/#{job_id}/pages/#{gid}/log", params)
-      end
-
-      def scraper_all_job_page_log(scraper_name, gid, opts={})
-        params = @options.merge(opts)
-        self.class.get("/scrapers/#{scraper_name}/current_job/pages/#{gid}/log", params)
-      end
-
-      def all_job_log(job_id, opts={})
-        params = @options.merge(opts)
-        self.class.get("/jobs/#{job_id}/log", params)
-      end
-
-      def scraper_all_job_log(scraper_name, opts={})
-        params = @options.merge(opts)
-        self.class.get("/scrapers/#{scraper_name}/current_job/log", params)
-      end
-
-    end
-  end
-end