answersengine 0.2.33

data/lib/answersengine/cli/scraper_export.rb

@@ -0,0 +1,51 @@
+module AnswersEngine
+  class CLI < Thor
+    class ScraperExport < Thor
+      package_name "scraper export"
+      def self.banner(command, namespace = nil, subcommand = false)
+        "#{basename} #{@package_name} #{command.usage}"
+      end
+
+      desc "show <export_id>", "Show a scraper's export"
+      def show(export_id)
+        client = Client::ScraperExport.new(options)
+        puts "#{client.find(export_id)}"
+      end
+
+
+      desc "list", "Gets a list of exports"
+      long_desc <<-LONGDESC
+        List exports.
+      LONGDESC
+      option :scraper_name, :aliases => :s, type: :string, desc: 'Filter by a specific scraper_name'
+      option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
+      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
+      def list()
+        if options[:scraper_name]
+          client = Client::ScraperExport.new(options)
+          puts "#{client.all(options[:scraper_name])}"
+        else
+          client = Client::Export.new(options)
+          puts "#{client.all}"
+        end
+      end
+
+      desc "download <export_id>", "Download the exported file"
+      def download(export_id)
+        client = Client::ScraperExport.new(options)
+        result = JSON.parse(client.download(export_id).to_s)
+        
+        if result['signed_url']
+          puts "Download url: \"#{result['signed_url']}\""
+          `open "#{result['signed_url']}"`
+        else
+          puts "Exported file does not exist"
+        end        
+      end
+
+
+
+    end
+  end
+
+end

data/lib/answersengine/cli/scraper_exporter.rb

@@ -0,0 +1,40 @@
+module AnswersEngine
+  class CLI < Thor
+    class ScraperExporter < Thor
+      package_name "scraper exporter"
+      def self.banner(command, namespace = nil, subcommand = false)
+        "#{basename} #{@package_name} #{command.usage}"
+      end
+
+      desc "show <scraper_name> <exporter_name>", "Show a scraper's exporter"
+      def show(scraper_name, exporter_name)
+        client = Client::ScraperExporter.new(options)
+        puts "#{client.find(scraper_name, exporter_name)}"
+      end
+
+      desc "start <scraper_name> <exporter_name>", "Starts an export"
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      def start(scraper_name, exporter_name)
+        if options[:job]
+          client = Client::JobExport.new(options)
+          puts "#{client.create(options[:job], exporter_name)}"
+        else
+          client = Client::ScraperExport.new(options)
+          puts "#{client.create(scraper_name, exporter_name)}"
+        end
+      end
+
+      desc "list <scraper_name>", "gets a list of exporters on a scraper"
+      long_desc <<-LONGDESC
+        List exporters on a scraper.
+      LONGDESC
+      option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
+      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
+      def list(scraper_name)
+        client = Client::ScraperExporter.new(options)
+        puts "#{client.all(scraper_name)}"
+      end
+    end
+  end
+
+end

data/lib/answersengine/cli/scraper_job.rb

@@ -0,0 +1,71 @@
+module AnswersEngine
+  class CLI < Thor
+    class ScraperJob < Thor
+      package_name "scraper job"
+      def self.banner(command, namespace = nil, subcommand = false)
+        "#{basename} #{@package_name} #{command.usage}"
+      end
+
+      desc "show <scraper_name>", "Show a scraper's current job"
+      def show(scraper_name)
+        client = Client::ScraperJob.new(options)
+        puts "#{client.find(scraper_name)}"
+      end
+
+
+      desc "list <scraper_name>", "gets a list of jobs on a scraper"
+      long_desc <<-LONGDESC
+        List jobs on a scraper.
+      LONGDESC
+      option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
+      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
+      def list(scraper_name)
+        client = Client::ScraperJob.new(options)
+        puts "#{client.all(scraper_name)}"
+      end
+
+
+      desc "cancel <scraper_name>", "cancels a scraper's current job"
+      long_desc <<-LONGDESC
+        Cancels a scraper's current job
+      LONGDESC
+      def cancel(scraper_name)
+        client = Client::ScraperJob.new(options)
+        puts "#{client.cancel(scraper_name)}"
+      end
+
+      desc "resume <scraper_name>", "resumes a scraper's current job"
+      long_desc <<-LONGDESC
+        Resumes a scraper's current job
+      LONGDESC
+      def resume(scraper_name)
+        client = Client::ScraperJob.new(options)
+        puts "#{client.resume(scraper_name)}"
+      end
+
+      desc "pause <scraper_name>", "pauses a scraper's current job"
+      long_desc <<-LONGDESC
+        pauses a scraper's current job
+      LONGDESC
+      def pause(scraper_name)
+        client = Client::ScraperJob.new(options)
+        puts "#{client.pause(scraper_name)}"
+      end
+
+
+      desc "update <scraper_name>", "updates a scraper's current job"
+      long_desc <<-LONGDESC
+        Updates a scraper's current job.
+      LONGDESC
+      option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Scraper job must be restarted(paused then resumed, or cancelled then resumed) for it to take effect. Default: 1. '
+      option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Scraper job must be restarted(paused then resumed, or cancelled then resumed) for it to take effect. Default: 0. '
+      option :proxy_type, desc: 'Set the Proxy type. Default: standard'
+      def update(scraper_name)
+        client = Client::ScraperJob.new(options)
+        puts "#{client.update(scraper_name, options)}"
+      end
+
+    end
+  end
+
+end

data/lib/answersengine/cli/scraper_page.rb

@@ -0,0 +1,200 @@
+module AnswersEngine
+  class CLI < Thor
+    class ScraperPage < Thor
+
+      package_name "scraper page"
+      def self.banner(command, namespace = nil, subcommand = false)
+        "#{basename} #{@package_name} #{command.usage}"
+      end
+
+      desc "list <scraper_name>", "List Pages on a scraper's current job"
+      long_desc <<-LONGDESC
+        List all pages in a scraper's current job.\x5
+      LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :page_type, :aliases => :t, type: :string, desc: 'Filter by page_type'
+      option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
+      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
+      option :fetch_fail, type: :boolean, desc: 'Returns only pages that fails fetching.'
+      option :parse_fail, type: :boolean, desc: 'Returns only pages that fails parsing.'
+      def list(scraper_name)
+        if options[:job]
+          client = Client::JobPage.new(options)
+          puts "#{client.all(options[:job])}"
+        else
+          client = Client::ScraperJobPage.new(options)
+          puts "#{client.all(scraper_name)}"
+        end
+      end
+
+      desc "add <scraper_name> <url>", "Enqueues a page to a scraper's current job"
+      long_desc <<-LONGDESC
+          Enqueues a page to a scraper's current job\x5
+          LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :method, :aliases => :m, type: :string, desc: 'Set request method. Default: GET'
+      option :headers, :aliases => :H, type: :string, banner: :JSON, desc: 'Set request headers. Must be in json format. i.e: {"Foo":"bar"} '
+      option :cookie, :aliases => :c, type: :string, desc: 'Set request cookie.'
+      option :vars, :aliases => :v, type: :string, banner: :JSON, desc: 'Set user-defined page variables. Must be in json format. i.e: {"Foo":"bar"}'
+      option :page_type, :aliases => :t, desc: 'Set page type'
+      option :priority, type: :numeric, desc: 'Set fetch priority. The higher the value, the sooner the page gets fetched. Default: 0'
+      option :fetch_type, :aliases => :F, desc: 'Set fetch type. Default: http'
+      option :body, :aliases => :b, desc: 'Set request body'
+      option :force_fetch, :aliases => :f, type: :boolean, desc: 'Set true to force fetch page that is not within freshness criteria. Default: false'
+      option :freshness, :aliases => :s, desc: 'Set how fresh the page cache is. Accepts timestap format.'
+      option :ua_type, :aliases => :u, desc: 'Set user agent type. Default: desktop'
+      option :no_redirect, :aliases => :n, type: :boolean, desc: 'Set true to not follow redirect. Default: false'
+      def add(scraper_name, url)
+        begin
+          options[:headers] = JSON.parse(options[:headers]) if options[:headers]
+          options[:vars] = JSON.parse(options[:vars]) if options[:vars]
+          method = options[:method]
+
+          if options[:job]
+            client = Client::JobPage.new(options)
+            puts "#{client.enqueue(options[:job], method, url, options)}"
+          else
+            client = Client::ScraperJobPage.new(options)
+            puts "#{client.enqueue(scraper_name, method, url, options)}"
+          end
+
+        rescue JSON::ParserError
+          if options[:headers]
+            puts "Error: #{options[:headers]} on headers is not a valid JSON"
+          end
+          if options[:vars]
+            puts "Error: #{options[:vars]} on vars is not a valid JSON"
+          end
+        end
+      end
+
+
+      desc "update <scraper_name> <gid>", "Update a page in a scraper's current job"
+      long_desc <<-LONGDESC
+          Updates a page in a scraper's current job. Only page_type or page vars is updateable.\x5
+          LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :page_type, :aliases => :t, desc: 'Set page type'
+      option :priority, type: :numeric, desc: 'Set fetch priority. The higher the value, the sooner the page gets fetched. Default: 0'
+      option :vars, :aliases => :v, type: :string, desc: 'Set user-defined page variables. Must be in json format. i.e: {"Foo":"bar"}'
+      def update(scraper_name, gid)
+        begin
+          options[:vars] = JSON.parse(options[:vars]) if options[:vars]
+
+          if options[:job]
+            client = Client::JobPage.new(options)
+            puts "#{client.update(options[:job], gid, options)}"
+          else
+            client = Client::ScraperJobPage.new(options)
+            puts "#{client.update(scraper_name, gid, options)}"
+          end
+
+        rescue JSON::ParserError
+          if options[:vars]
+            puts "Error: #{options[:vars]} on vars is not a valid JSON"
+          end
+        end
+      end 
+
+      desc "refetch <scraper_name>", "Refetch Pages on a scraper's current job"
+      long_desc <<-LONGDESC
+        Refetch pages in a scraper's current job. You need to specify either a --gid or --fetch-fail or --parse-fail.\x5
+      LONGDESC
+      option :gid, :aliases => :g, type: :string, desc: 'Refetch a specific GID'
+      option :fetch_fail, type: :boolean, desc: 'Refetches only pages that fails fetching.'
+      option :parse_fail, type: :boolean, desc: 'Refetches only pages that fails parsing.'
+      def refetch(scraper_name)
+        if !options.key?(:gid) && !options.key?(:fetch_fail) && !options.key?(:parse_fail)
+          puts "Must specify either a --gid or --fetch-fail or --parse-fail"
+        else
+          client = Client::ScraperJobPage.new(options)
+          puts "#{client.refetch(scraper_name)}"
+        end
+      end
+
+      desc "reset <scraper_name> <gid>", "Reset fetching and parsing of a page in a scraper's current job"
+      long_desc <<-LONGDESC
+          Reset fetching and parsing of a page in a scraper's current job.\x5
+          LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      def reset(scraper_name, gid)
+        begin
+          options[:vars] = JSON.parse(options[:vars]) if options[:vars]
+
+          if options[:job]
+            client = Client::JobPage.new(options)
+            puts "#{client.reset(options[:job], gid, options)}"
+          else
+            client = Client::ScraperJobPage.new(options)
+            puts "#{client.reset(scraper_name, gid, options)}"
+          end
+
+        rescue JSON::ParserError
+          if options[:vars]
+            puts "Error: #{options[:vars]} on vars is not a valid JSON"
+          end
+        end
+      end 
+
+      desc "show <scraper_name> <gid>", "Show a page in scraper's current job"
+      long_desc <<-LONGDESC
+          Shows a page in a scraper's current job.\x5
+          LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      def show(scraper_name, gid)
+        if options[:job]
+          client = Client::JobPage.new(options)
+          puts "#{client.find(options[:job], gid)}"
+        else
+          client = Client::ScraperJobPage.new(options)
+          puts "#{client.find(scraper_name, gid)}"
+        end
+      end
+
+      desc "log <scraper_name> <gid>", "List log entries related to a job page"
+      long_desc <<-LONGDESC
+          Shows log related to a page in the job. Defaults to showing the most recent entries\x5
+          LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :head, :aliases => :H, desc: 'Show the oldest log entries. If not set, newest entries is shown'
+      option :parsing, :aliases => :p, type: :boolean, desc: 'Show only log entries related to parsing'
+      option :more, :aliases => :m, desc: 'Show next set of log entries. Enter the `More token`'
+      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 5000 per page.'
+      def log(scraper_name, gid)
+        client = Client::JobLog.new(options)
+
+        query = {}
+        query["order"] = options.delete(:head) if options[:head]
+        query["job_type"] = "parsing" if options[:parsing]
+        
+        query["page_token"] = options.delete(:more) if options[:more]
+        query["per_page"] = options.delete(:per_page) if options[:per_page]
+
+        puts "Fetching page logs..."
+
+        if options[:job]
+          result = client.all_job_page_log(options[:job], gid, {query: query})
+        else
+          result = client.scraper_all_job_page_log(scraper_name, gid, {query: query})
+        end
+        
+        if result['entries'].nil? || result["entries"].length == 0
+          puts "No logs yet, please try again later."
+        else
+
+          more_token = result["more_token"]
+
+          result["entries"].each do |entry|
+            puts "#{entry["timestamp"]} #{entry["severity"]}: #{entry["payload"]}" if entry.is_a?(Hash)
+          end
+
+          unless more_token.nil?
+            puts "to see more entries, add: \"--more #{more_token}\""
+          end
+        end
+      end
+
+    end
+  end
+
+end

data/lib/answersengine/cli/seeder.rb

@@ -0,0 +1,40 @@
+module AnswersEngine
+  class CLI < Thor
+    class Seeder < Thor
+      desc "try <scraper_name> <seeder_file>", "Tries a seeder file"
+      long_desc <<-LONGDESC
+            Takes a seeder script and tries to execute it without saving anything.\x5
+            <seeder_file>: Seeder script file will be executed.\x5
+          LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      def try_seed(scraper_name, seeder_file)
+        if options[:job]
+          job_id = options[:job]
+        else
+          job = Client::ScraperJob.new(options).find(scraper_name)
+          job_id = job['id']
+        end
+        
+        puts AnswersEngine::Scraper::Seeder.exec_seeder(seeder_file, job_id, false)
+      end
+
+      desc "exec <scraper_name> <seeder_file>", "Executes a seeder script onto a scraper's current job."
+      long_desc <<-LONGDESC
+            Takes a seeder script and execute it against a job and enqueues the pages into the scraper's current job\x5
+            <seeder_file>: Seeder script file that will be executed on the scraper's current job.\x5
+          LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      def exec_parse(scraper_name, seeder_file)
+        if options[:job]
+          job_id = options[:job]
+        else
+          job = Client::ScraperJob.new(options).find(scraper_name)
+          job_id = job['id']
+        end
+
+        puts AnswersEngine::Scraper::Seeder.exec_seeder(seeder_file, job_id, true)
+      end
+    end
+  end
+
+end

data/lib/answersengine/client.rb

@@ -0,0 +1,23 @@
+require "answersengine/client/base"
+require 'answersengine/client/export'
+require "answersengine/client/scraper"
+require "answersengine/client/scraper_deployment"
+require "answersengine/client/scraper_job_output"
+require "answersengine/client/scraper_job_page"
+require "answersengine/client/scraper_exporter"
+require "answersengine/client/scraper_export"
+require "answersengine/client/scraper_job"
+require 'answersengine/client/job_export'
+require "answersengine/client/job"
+require "answersengine/client/job_log"
+require "answersengine/client/global_page"
+require "answersengine/client/job_page"
+require "answersengine/client/job_output"
+require "answersengine/client/job_stat"
+require "answersengine/client/backblaze_content"
+
+
+module AnswersEngine
+  module Client
+  end
+end

data/lib/answersengine/client/backblaze_content.rb

@@ -0,0 +1,45 @@
+require 'zlib'
+require 'httparty'
+
+module AnswersEngine
+  module Client
+    class BackblazeContent
+      include HTTParty    
+      
+      def get_content(url)
+        self.class.get(url, format: :plain)
+      end
+
+      def get_gunzipped_content(url)
+        # Zlib.gunzip(get_content(url))
+        gunzip(get_content(url))
+      end
+
+      def gunzip(string)
+        sio = StringIO.new(string)
+        gz = Zlib::GzipReader.new(sio, encoding: Encoding::ASCII_8BIT)
+        _content = ""
+        begin 
+          _content = gz.read
+        rescue => e
+          # if unexpected eof error, then readchar until error, and ignore it
+          if e.to_s == 'unexpected end of file'
+            begin 
+              while !gz.eof?
+                _content += gz.readchar
+              end
+            rescue => e
+              puts "Ignored Zlib error: #{e.to_s}"
+            end
+          else 
+            raise e
+          end
+        end
+
+        return _content
+      ensure
+        gz.close if gz.respond_to?(:close)
+      end
+    end
+  end
+end