answersengine 0.10.1 → 0.10.2

data/lib/answersengine/cli/scraper_deployment.rb

@@ -1,24 +0,0 @@
-module AnswersEngine
-  class CLI < Thor
-    class ScraperDeployment < Thor
-
-      package_name "scraper deployment"
-      def self.banner(command, namespace = nil, subcommand = false)
-        "#{basename} #{@package_name} #{command.usage}"
-      end
-
-      
-      desc "list <scraper_name>", "List deployments on a scraper"
-      long_desc <<-LONGDESC
-        List deployments on a scraper.
-      LONGDESC
-      option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
-      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
-      def list(scraper_name)
-        client = Client::ScraperDeployment.new(options)
-        puts "#{client.all(scraper_name)}"
-      end
-    end
-  end
-
-end

data/lib/answersengine/cli/scraper_export.rb

@@ -1,51 +0,0 @@
-module AnswersEngine
-  class CLI < Thor
-    class ScraperExport < Thor
-      package_name "scraper export"
-      def self.banner(command, namespace = nil, subcommand = false)
-        "#{basename} #{@package_name} #{command.usage}"
-      end
-
-      desc "show <export_id>", "Show a scraper's export"
-      def show(export_id)
-        client = Client::ScraperExport.new(options)
-        puts "#{client.find(export_id)}"
-      end
-
-
-      desc "list", "Gets a list of exports"
-      long_desc <<-LONGDESC
-        List exports.
-      LONGDESC
-      option :scraper_name, :aliases => :s, type: :string, desc: 'Filter by a specific scraper_name'
-      option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
-      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
-      def list()
-        if options[:scraper_name]
-          client = Client::ScraperExport.new(options)
-          puts "#{client.all(options[:scraper_name])}"
-        else
-          client = Client::Export.new(options)
-          puts "#{client.all}"
-        end
-      end
-
-      desc "download <export_id>", "Download the exported file"
-      def download(export_id)
-        client = Client::ScraperExport.new(options)
-        result = JSON.parse(client.download(export_id).to_s)
-        
-        if result['signed_url']
-          puts "Download url: \"#{result['signed_url']}\""
-          `open "#{result['signed_url']}"`
-        else
-          puts "Exported file does not exist"
-        end        
-      end
-
-
-
-    end
-  end
-
-end

data/lib/answersengine/cli/scraper_exporter.rb

@@ -1,40 +0,0 @@
-module AnswersEngine
-  class CLI < Thor
-    class ScraperExporter < Thor
-      package_name "scraper exporter"
-      def self.banner(command, namespace = nil, subcommand = false)
-        "#{basename} #{@package_name} #{command.usage}"
-      end
-
-      desc "show <scraper_name> <exporter_name>", "Show a scraper's exporter"
-      def show(scraper_name, exporter_name)
-        client = Client::ScraperExporter.new(options)
-        puts "#{client.find(scraper_name, exporter_name)}"
-      end
-
-      desc "start <scraper_name> <exporter_name>", "Starts an export"
-      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
-      def start(scraper_name, exporter_name)
-        if options[:job]
-          client = Client::JobExport.new(options)
-          puts "#{client.create(options[:job], exporter_name)}"
-        else
-          client = Client::ScraperExport.new(options)
-          puts "#{client.create(scraper_name, exporter_name)}"
-        end
-      end
-
-      desc "list <scraper_name>", "gets a list of exporters on a scraper"
-      long_desc <<-LONGDESC
-        List exporters on a scraper.
-      LONGDESC
-      option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
-      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
-      def list(scraper_name)
-        client = Client::ScraperExporter.new(options)
-        puts "#{client.all(scraper_name)}"
-      end
-    end
-  end
-
-end

data/lib/answersengine/cli/scraper_finisher.rb

@@ -1,20 +0,0 @@
-module AnswersEngine
-  class CLI < Thor
-    class ScraperFinisher < Thor
-
-      package_name "scraper finisher"
-      def self.banner(command, namespace = nil, subcommand = false)
-        "#{basename} #{@package_name} #{command.usage}"
-      end
-
-      desc "reset <scraper_name>", "Reset finisher on a scraper's current job"
-      long_desc <<-LONGDESC
-        Reset finisher on a scraper's current job.\x5
-      LONGDESC
-      def reset(scraper_name)
-        client = Client::ScraperFinisher.new(options)
-        puts "#{client.reset(scraper_name)}"
-      end
-    end
-  end
-end

data/lib/answersengine/cli/scraper_job.rb

@@ -1,75 +0,0 @@
-module AnswersEngine
-  class CLI < Thor
-    class ScraperJob < Thor
-      package_name "scraper job"
-      def self.banner(command, namespace = nil, subcommand = false)
-        "#{basename} #{@package_name} #{command.usage}"
-      end
-
-      desc "show <scraper_name>", "Show a scraper's current job"
-      def show(scraper_name)
-        client = Client::ScraperJob.new(options)
-        puts "#{client.find(scraper_name)}"
-      end
-
-
-      desc "list <scraper_name>", "gets a list of jobs on a scraper"
-      long_desc <<-LONGDESC
-        List jobs on a scraper.
-      LONGDESC
-      option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
-      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
-      def list(scraper_name)
-        client = Client::ScraperJob.new(options)
-        puts "#{client.all(scraper_name)}"
-      end
-
-
-      desc "cancel <scraper_name>", "cancels a scraper's current job"
-      long_desc <<-LONGDESC
-        Cancels a scraper's current job
-      LONGDESC
-      def cancel(scraper_name)
-        client = Client::ScraperJob.new(options)
-        puts "#{client.cancel(scraper_name)}"
-      end
-
-      desc "resume <scraper_name>", "resumes a scraper's current job"
-      long_desc <<-LONGDESC
-        Resumes a scraper's current job
-      LONGDESC
-      def resume(scraper_name)
-        client = Client::ScraperJob.new(options)
-        puts "#{client.resume(scraper_name)}"
-      end
-
-      desc "pause <scraper_name>", "pauses a scraper's current job"
-      long_desc <<-LONGDESC
-        pauses a scraper's current job
-      LONGDESC
-      def pause(scraper_name)
-        client = Client::ScraperJob.new(options)
-        puts "#{client.pause(scraper_name)}"
-      end
-
-
-      desc "update <scraper_name>", "updates a scraper's current job"
-      long_desc <<-LONGDESC
-        Updates a scraper's current job.
-      LONGDESC
-      option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Scraper job must be restarted(paused then resumed, or cancelled then resumed) for it to take effect. Default: 1. '
-      option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Scraper job must be restarted(paused then resumed, or cancelled then resumed) for it to take effect. Default: 0. '
-      option :proxy_type, desc: 'Set the Proxy type. Default: standard'
-      def update(scraper_name)
-        client = Client::ScraperJob.new(options)
-        puts "#{client.update(scraper_name, options)}"
-      end
-
-      desc "var SUBCOMMAND ...ARGS", "for managing scraper's job variables"
-      subcommand "var", ScraperJobVar
-
-
-    end
-  end
-
-end

data/lib/answersengine/cli/scraper_job_var.rb

@@ -1,48 +0,0 @@
-module AnswersEngine
-  class CLI < Thor
-    class ScraperJobVar < Thor
-
-      package_name "job var"
-      def self.banner(command, namespace = nil, subcommand = false)
-        "#{basename} scraper #{@package_name} #{command.usage}"
-      end
-
-      desc "list <scraper_name>", "List environment variables on the scrape job"
-      long_desc <<-LONGDESC
-        List all environment variables on the scrape job.
-      LONGDESC
-      option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
-      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
-      def list(scraper_name)
-        client = Client::ScraperJobVar.new(options)
-        puts "#{client.all(scraper_name)}"
-      end
-
-      desc "set <scraper_name> <var_name> <value>", "Set an environment var on the scrape job"
-      long_desc <<-LONGDESC
-          Creates an environment variable\x5
-          <var_name>: Var name can only consist of alphabets, numbers, underscores. Name must be unique to your scrape job, otherwise it will be overwritten.\x5
-          <value>: Value of variable.\x5
-          LONGDESC
-      option :secret, type: :boolean, desc: 'Set true to make it decrypt the value. Default: false' 
-      def set(scraper_name, var_name, value)
-        # puts "options #{options}"
-        client = Client::ScraperJobVar.new(options)
-        puts "#{client.set(scraper_name, var_name, value, options)}"
-      end
-
-      desc "show <scraper_name> <var_name>", "Show an environment variable on the scrape job"
-      def show(scraper_name, var_name)
-        client = Client::ScraperJobVar.new(options)
-        puts "#{client.find(scraper_name, var_name)}"
-      end
-
-      desc "unset <scraper_name> <var_name>", "Deletes an environment variable on the scrape job"
-      def unset(scraper_name, var_name)
-        client = Client::ScraperJobVar.new(options)
-        puts "#{client.unset(scraper_name, var_name)}"
-      end
-    end
-  end
-
-end

data/lib/answersengine/cli/scraper_page.rb

@@ -1,203 +0,0 @@
-module AnswersEngine
-  class CLI < Thor
-    class ScraperPage < Thor
-
-      package_name "scraper page"
-      def self.banner(command, namespace = nil, subcommand = false)
-        "#{basename} #{@package_name} #{command.usage}"
-      end
-
-      desc "list <scraper_name>", "List Pages on a scraper's current job"
-      long_desc <<-LONGDESC
-        List all pages in a scraper's current job.\x5
-      LONGDESC
-      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
-      option :page_type, :aliases => :t, type: :string, desc: 'Filter by page_type'
-      option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
-      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
-      option :fetch_fail, type: :boolean, desc: 'Returns only pages that fails fetching.'
-      option :parse_fail, type: :boolean, desc: 'Returns only pages that fails parsing.'
-      def list(scraper_name)
-        if options[:job]
-          client = Client::JobPage.new(options)
-          puts "#{client.all(options[:job])}"
-        else
-          client = Client::ScraperJobPage.new(options)
-          puts "#{client.all(scraper_name)}"
-        end
-      end
-
-      desc "add <scraper_name> <url>", "Enqueues a page to a scraper's current job"
-      long_desc <<-LONGDESC
-          Enqueues a page to a scraper's current job\x5
-          LONGDESC
-      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
-      option :method, :aliases => :m, type: :string, desc: 'Set request method. Default: GET'
-      option :headers, :aliases => :H, type: :string, banner: :JSON, desc: 'Set request headers. Must be in json format. i.e: {"Foo":"bar"} '
-      option :cookie, :aliases => :c, type: :string, desc: 'Set request cookie.'
-      option :vars, :aliases => :v, type: :string, banner: :JSON, desc: 'Set user-defined page variables. Must be in json format. i.e: {"Foo":"bar"}'
-      option :page_type, :aliases => :t, desc: 'Set page type'
-      option :priority, type: :numeric, desc: 'Set fetch priority. The higher the value, the sooner the page gets fetched. Default: 0'
-      option :fetch_type, :aliases => :F, desc: 'Set fetch type. Default: http'
-      option :body, :aliases => :b, desc: 'Set request body'
-      option :force_fetch, :aliases => :f, type: :boolean, desc: 'Set true to force fetch page that is not within freshness criteria. Default: false'
-      option :freshness, :aliases => :s, desc: 'Set how fresh the page cache is. Accepts timestap format.'
-      option :ua_type, :aliases => :u, desc: 'Set user agent type. Default: desktop'
-      option :no_redirect, :aliases => :n, type: :boolean, desc: 'Set true to not follow redirect. Default: false'
-      def add(scraper_name, url)
-        begin
-          options[:headers] = JSON.parse(options[:headers]) if options[:headers]
-          options[:vars] = JSON.parse(options[:vars]) if options[:vars]
-          method = options[:method]
-
-          if options[:job]
-            client = Client::JobPage.new(options)
-            puts "#{client.enqueue(options[:job], method, url, options)}"
-          else
-            client = Client::ScraperJobPage.new(options)
-            puts "#{client.enqueue(scraper_name, method, url, options)}"
-          end
-
-        rescue JSON::ParserError
-          if options[:headers]
-            puts "Error: #{options[:headers]} on headers is not a valid JSON"
-          end
-          if options[:vars]
-            puts "Error: #{options[:vars]} on vars is not a valid JSON"
-          end
-        end
-      end
-
-
-      desc "update <scraper_name> <gid>", "Update a page in a scraper's current job"
-      long_desc <<-LONGDESC
-          Updates a page in a scraper's current job. Only page_type or page vars is updateable.\x5
-          LONGDESC
-      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
-      option :page_type, :aliases => :t, desc: 'Set page type'
-      option :priority, type: :numeric, desc: 'Set fetch priority. The higher the value, the sooner the page gets fetched. Default: 0'
-      option :vars, :aliases => :v, type: :string, desc: 'Set user-defined page variables. Must be in json format. i.e: {"Foo":"bar"}'
-      def update(scraper_name, gid)
-        begin
-          options[:vars] = JSON.parse(options[:vars]) if options[:vars]
-
-          if options[:job]
-            client = Client::JobPage.new(options)
-            puts "#{client.update(options[:job], gid, options)}"
-          else
-            client = Client::ScraperJobPage.new(options)
-            puts "#{client.update(scraper_name, gid, options)}"
-          end
-
-        rescue JSON::ParserError
-          if options[:vars]
-            puts "Error: #{options[:vars]} on vars is not a valid JSON"
-          end
-        end
-      end
-
-      desc "refetch <scraper_name>", "Refetch Pages on a scraper's current job"
-      long_desc <<-LONGDESC
-        Refetch pages in a scraper's current job. You need to specify either a --gid or --fetch-fail or --parse-fail or --status.\x5
-      LONGDESC
-      option :gid, :aliases => :g, type: :string, desc: 'Refetch a specific GID'
-      option :fetch_fail, type: :boolean, desc: 'Refetches only pages that fails fetching.'
-      option :parse_fail, type: :boolean, desc: 'Refetches only pages that fails parsing.'
-      option :status, type: :string, desc: 'Refetches only pages with a specific status.'
-      def refetch(scraper_name)
-        if !options.key?(:gid) && !options.key?(:fetch_fail) && !options.key?(:parse_fail) && !options.key?(:status)
-          puts "Must specify either a --gid, --fetch-fail, --parse-fail or --status"
-          return
-        end
-        client = Client::ScraperJobPage.new(options)
-        puts "#{client.refetch(scraper_name)}"
-      end
-
-      desc "reparse <scraper_name>", "Reparse Pages on a scraper's current job"
-      long_desc <<-LONGDESC
-        Reparse pages in a scraper's current job. You need to specify either a --gid or --parse-fail or --status.\x5
-      LONGDESC
-      option :gid, :aliases => :g, type: :string, desc: 'Reparse a specific GID'
-      option :parse_fail, type: :boolean, desc: 'Reparse only pages that fails parsing.'
-      option :status, type: :string, desc: 'Reparse only pages with a specific status.'
-      def reparse(scraper_name)
-        begin
-          options[:vars] = JSON.parse(options[:vars]) if options[:vars]
-
-          if !options.key?(:gid) && !options.key?(:parse_fail) && !options.key?(:status)
-            puts "Must specify either a --gid, --parse-fail or --status"
-            return
-          end
-
-          client = Client::ScraperJobPage.new(options)
-          puts "#{client.reparse(scraper_name)}"
-
-        rescue JSON::ParserError
-          if options[:vars]
-            puts "Error: #{options[:vars]} on vars is not a valid JSON"
-          end
-        end
-      end
-
-      desc "show <scraper_name> <gid>", "Show a page in scraper's current job"
-      long_desc <<-LONGDESC
-          Shows a page in a scraper's current job.\x5
-          LONGDESC
-      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
-      def show(scraper_name, gid)
-        if options[:job]
-          client = Client::JobPage.new(options)
-          puts "#{client.find(options[:job], gid)}"
-        else
-          client = Client::ScraperJobPage.new(options)
-          puts "#{client.find(scraper_name, gid)}"
-        end
-      end
-
-      desc "log <scraper_name> <gid>", "List log entries related to a job page"
-      long_desc <<-LONGDESC
-          Shows log related to a page in the job. Defaults to showing the most recent entries\x5
-          LONGDESC
-      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
-      option :head, :aliases => :H, desc: 'Show the oldest log entries. If not set, newest entries is shown'
-      option :parsing, :aliases => :p, type: :boolean, desc: 'Show only log entries related to parsing'
-      option :more, :aliases => :m, desc: 'Show next set of log entries. Enter the `More token`'
-      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 5000 per page.'
-      def log(scraper_name, gid)
-        client = Client::JobLog.new(options)
-
-        query = {}
-        query["order"] = options.delete(:head) if options[:head]
-        query["job_type"] = "parsing" if options[:parsing]
-
-        query["page_token"] = options.delete(:more) if options[:more]
-        query["per_page"] = options.delete(:per_page) if options[:per_page]
-
-        puts "Fetching page logs..."
-
-        if options[:job]
-          result = client.all_job_page_log(options[:job], gid, {query: query})
-        else
-          result = client.scraper_all_job_page_log(scraper_name, gid, {query: query})
-        end
-
-        if result['entries'].nil? || result["entries"].length == 0
-          puts "No logs yet, please try again later."
-        else
-
-          more_token = result["more_token"]
-
-          result["entries"].each do |entry|
-            puts "#{entry["timestamp"]} #{entry["severity"]}: #{entry["payload"]}" if entry.is_a?(Hash)
-          end
-
-          unless more_token.nil?
-            puts "to see more entries, add: \"--more #{more_token}\""
-          end
-        end
-      end
-
-    end
-  end
-
-end