answersengine 0.10.1 → 0.10.2

data/examples/fetchtest/seeders/csv_seeder.rb

@@ -1,12 +0,0 @@
-CSV.foreach("./seeders/list_of_urls.csv",:headers => true) do |row|
-  pages << {
-    url: row['url'],
-    page_type: row['page_type'],
-    vars: {"abc":[1,2,3], "def": "defcontent"}  
-  }
-
-  # Save pages to the job partially if record counts will be too large
-  max_records = 100
-  save_pages(pages) if $. % max_records == 0
-end
-

data/examples/fetchtest/seeders/failed.rb

@@ -1 +0,0 @@
-raise "fail from seeder"

data/examples/fetchtest/seeders/list_of_urls.csv

@@ -1,5 +0,0 @@
-url,page_type
-http://fetchtest.datahen.com,home
-http://fetchtest.datahen.com/statuses/200,statuses
-http://fetchtest.datahen.com/statuses/200?q=1,statuses
-http://fetchtest.datahen.com/statuses/200?q=2,statuses

data/examples/fetchtest/seeders/seed.rb

@@ -1,28 +0,0 @@
-puts "hello from seeder"
-
-pages << {
-  url: "http://fetchtest.datahen.com/statuses/200?q=queuedFromParser",
-  vars: {"abc":[1], "def": "defcontent"}
-}
-
-pages << {
-  url: "http://fetchtest.datahen.com/statuses/200?q=queuedFromParser2",
-  vars: {"abc":[2], "def": "defcontent"}
-}
-
-save_pages(pages)
-
-pages << {
-  url: "http://fetchtest.datahen.com/statuses/200?q=queuedFromParser3",
-  vars: {"abc":[3], "def": "defcontent"}
-}
-
-pages << {
-  url: "http://fetchtest.datahen.com/statuses/200?q=queuedFromParser4",
-  vars: {"abc":[3], "def": "defcontent"}
-}
-
-pages << {
-  url: "http://fetchtest.datahen.com/statuses/200?q=queuedFromParser5",
-  vars: {"abc":[3], "def": "defcontent"}
-}

data/examples/fetchtest/seeders/test_reset_page.rb

@@ -1,4 +0,0 @@
-pages << {
-  url: "http://fetchtest.datahen.com/statuses/200?q=queuedFromParser",
-  reset: true
-}

data/lib/answersengine/cli.rb

@@ -1,45 +0,0 @@
-require 'thor'
-require 'answersengine/scraper'
-require 'answersengine/cli/scraper_var'
-require 'answersengine/cli/scraper_exporter'
-require 'answersengine/cli/scraper_export'
-require 'answersengine/cli/scraper_job_var'
-require 'answersengine/cli/scraper_job'
-require 'answersengine/cli/scraper_finisher'
-require 'answersengine/cli/global_page'
-require 'answersengine/cli/scraper_page'
-require 'answersengine/cli/job_output'
-require 'answersengine/cli/job'
-require 'answersengine/cli/scraper_deployment'
-require 'answersengine/cli/scraper'
-require 'answersengine/cli/parser'
-require 'answersengine/cli/seeder'
-require 'answersengine/cli/finisher'
-require 'answersengine/cli/env_var'
-
-
-
-module AnswersEngine
-  class CLI < Thor
-    desc "scraper SUBCOMMAND ...ARGS", "manage scrapers"
-    subcommand "scraper", Scraper
-
-    desc "job SUBCOMMAND ...ARGS", "manage scrapers jobs"
-    subcommand "job", Job
-
-    desc "globalpage SUBCOMMAND ...ARGS", "interacts with global page"
-    subcommand "globalpage", GlobalPage
-
-    desc "parser SUBCOMMAND ...ARGS", "for parsing related activities"
-    subcommand "parser", Parser
-
-    desc "seeder SUBCOMMAND ...ARGS", "for seeding related activities"
-    subcommand "seeder", Seeder
-
-    desc "seeder SUBCOMMAND ...ARGS", "for seeding related activities"
-    subcommand "finisher", Finisher
-
-    desc "var SUBCOMMAND ...ARGS", "for environment variable related activities"
-    subcommand "var", EnvVar
-  end
-end

data/lib/answersengine/cli/env_var.rb

@@ -1,48 +0,0 @@
-module AnswersEngine
-  class CLI < Thor
-    class EnvVar < Thor
-      desc "list", "List environment variables on the account"
-      
-      long_desc <<-LONGDESC
-        List all environment variables on the account.
-      LONGDESC
-      option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
-      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
-      def list
-        client = Client::EnvVar.new(options)
-        puts "#{client.all}"
-      end
-
-      desc "set <name> <value>", "Set an environment var on the account"
-      long_desc <<-LONGDESC
-          Creates an environment variable\x5
-          <name>: Var name can only consist of alphabets, numbers, underscores. Name must be unique to your account, otherwise it will be overwritten.\x5
-          <value>: Value of variable.\x5
-          LONGDESC
-      option :secret, type: :boolean, desc: 'Set true to make it decrypt the value. Default: false' 
-      def set(name, value)
-        # puts "options #{options}"
-        client = Client::EnvVar.new(options)
-        puts "#{client.set(name, value, options)}"
-      end
-
-      desc "show <name>", "Show an environment variable on the account"
-      def show(name)
-        client = Client::EnvVar.new(options)
-        puts "#{client.find(name)}"
-      end
-
-      desc "unset <name>", "Deletes an environment variable on the account"
-      def unset(name)
-        client = Client::EnvVar.new(options)
-        puts "#{client.unset(name)}"
-      end
-
-
-      
-
-
-    end
-  end
-
-end

data/lib/answersengine/cli/finisher.rb

@@ -1,40 +0,0 @@
-module AnswersEngine
-  class CLI < Thor
-    class Finisher < Thor
-      desc "try <scraper_name> <finisher_file>", "Tries a finisher file"
-      long_desc <<-LONGDESC
-            Takes a finisher script and tries to execute it without saving anything.\x5
-            <seeder_file>: Finisher script file will be executed.\x5
-          LONGDESC
-      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
-      def try_finisher(scraper_name, finisher_file)
-        if options[:job]
-          job_id = options[:job]
-        else
-          job = Client::ScraperJob.new(options).find(scraper_name)
-          job_id = job['id']
-        end
-
-        puts AnswersEngine::Scraper::Finisher.exec_finisher(finisher_file, job_id, false)
-      end
-
-      desc "exec <scraper_name> <finisher_file>", "Executes a finisher script onto a scraper's current job."
-      long_desc <<-LONGDESC
-            Takes a finisher script and execute it against a job and save outputs into the scraper's current job\x5
-            <finisher_file>: Finisher script file that will be executed on the scraper's current job.\x5
-          LONGDESC
-      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
-      def exec_parse(scraper_name, finisher_file)
-        if options[:job]
-          job_id = options[:job]
-        else
-          job = Client::ScraperJob.new(options).find(scraper_name)
-          job_id = job['id']
-        end
-
-        puts AnswersEngine::Scraper::Finisher.exec_finisher(finisher_file, job_id, true)
-      end
-    end
-  end
-
-end

data/lib/answersengine/cli/global_page.rb

@@ -1,39 +0,0 @@
-module AnswersEngine
-  class CLI < Thor
-    class GlobalPage < Thor
-
-      desc "show <gid>", "Show a global page"
-      def show(gid)
-        client = Client::GlobalPage.new(options)
-        puts "#{client.find(gid)}"
-      end
-
-      desc "content <gid>", "Show content of a globalpage"
-      def content(gid)
-        client = Client::GlobalPage.new(options)
-        result = JSON.parse(client.find_content(gid).to_s)
-        
-        if result['available'] == true
-          puts "Preview content url: \"#{result['preview_url']}\""
-          `open "#{result['preview_url']}"`
-        else
-          puts "Content does not exist"
-        end        
-      end
-
-      desc "failedcontent <gid>", "Show failed content of a globalpage"
-      def failedcontent(gid)
-        client = Client::GlobalPage.new(options)
-        result = JSON.parse(client.find_failed_content(gid).to_s)
-        
-        if result['available'] == true
-          puts "Preview failed content url: \"#{result['preview_url']}\""
-          `open "#{result['preview_url']}"`
-        else
-          puts "Failed Content does not exist"
-        end        
-      end
-      
-    end
-  end
-end

data/lib/answersengine/cli/job.rb

@@ -1,30 +0,0 @@
-module AnswersEngine
-  class CLI < Thor
-    class Job < Thor
-      package_name "job"
-      def self.banner(command, namespace = nil, subcommand = false)
-        "#{basename} #{@package_name} #{command.usage}"
-      end
-
-
-      desc "list", "gets a list of jobs"
-      option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
-      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
-      long_desc <<-LONGDESC
-        List scrape jobs.
-      LONGDESC
-      def list()
-        client = Client::Job.new(options)
-        puts "#{client.all()}"
-      end
-
-      desc "show <job_id>", "Show a job"
-      def show(job_id)
-        client = Client::Job.new(options)
-        puts "#{client.find(job_id)}"
-      end
-
-    end
-  end
-
-end

data/lib/answersengine/cli/job_output.rb

@@ -1,69 +0,0 @@
-module AnswersEngine
-  class CLI < Thor
-    class JobOutput < Thor
-
-      package_name "scraper output"
-      def self.banner(command, namespace = nil, subcommand = false)
-        "#{basename} #{@package_name} #{command.usage}"
-      end
-
-      desc "list <scraper_name>", "List output records in a collection that is in the current job"
-      long_desc <<-LONGDESC
-        List all output records in a collection that is in the current job of a scraper\n
-      LONGDESC
-      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
-      option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
-      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
-      option :collection, :aliases => :c, desc: "Shows outputs from a specific collection.(defaults to 'default' collection)"
-      option :query, :aliases => :q, type: :string, banner: :JSON, desc: 'Set search query. Must be in json format. i.e: {"Foo":"bar"} '
-      def list(scraper_name)
-        collection = options.fetch(:collection) { 'default' }
-        if options[:job]
-          client = Client::JobOutput.new(options)
-          puts "#{client.all(options[:job], collection)}"
-        else
-          client = Client::ScraperJobOutput.new(options)
-          puts "#{client.all(scraper_name, collection)}"
-        end
-      end
-
-      desc "show <scraper_name> <record_id>", "Show one output record in a collection that is in the current job of a scraper"
-      long_desc <<-LONGDESC
-        Shows an output record in a collection that is in the current job of a scraper\n
-        <record_id>: ID of the output record.\x5
-      LONGDESC
-      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
-      option :collection, :aliases => :c, desc: "Shows output from a specific collection.(defaults to 'default' collection)"
-      def show(scraper_name, id)
-        collection = options.fetch(:collection) { 'default' }
-        if options[:job]
-          client = Client::JobOutput.new(options)
-          puts "#{client.find(options[:job], collection, id)}"
-        else
-          client = Client::ScraperJobOutput.new(options)
-          puts "#{client.find(scraper_name, collection, id)}"
-        end
-      end
-
-      desc "collections <scraper_name>", "list job output collections that are inside a current job of a scraper."
-      long_desc <<-LONGDESC
-        List job output collections that are inside a current job of a scraper.\x5
-      LONGDESC
-      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
-      option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
-      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
-      def collections(scraper_name)
-
-        if options[:job]
-          client = Client::JobOutput.new(options)
-          puts "#{client.collections(options[:job])}"
-        else
-          client = Client::ScraperJobOutput.new(options)
-          puts "#{client.collections(scraper_name)}"
-        end
-      end
-
-    end
-  end
-
-end

data/lib/answersengine/cli/parser.rb

@@ -1,64 +0,0 @@
-module AnswersEngine
-  class CLI < Thor
-    class Parser < Thor
-      desc "try <scraper_name> <parser_file> <GID>", "Tries a parser on a Job Page"
-      long_desc <<-LONGDESC
-            Takes a parser script and runs it against a job page\x5
-            <parser_file>: Parser script file that will be executed on the page.\x5
-            <GID>: Global ID of the page.\x5
-          LONGDESC
-      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
-      option :global, :aliases => :g, type: :boolean, default: false, desc: 'Use globalpage instead of a job page'
-      option :vars, :aliases => :v, type: :string, desc: 'Set user-defined page variables. Must be in json format. i.e: {"Foo":"bar"}'
-      def try_parse(scraper_name, parser_file, gid)
-        begin 
-          
-            if options[:job]
-              job_id = options[:job]
-            elsif options[:global] 
-              job_id = nil
-            else
-              job = Client::ScraperJob.new(options).find(scraper_name)
-              job_id = job['id']
-            end
-
-
-          vars = JSON.parse(options[:vars]) if options[:vars]
-          puts AnswersEngine::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, false, vars)
-
-          rescue JSON::ParserError
-          if options[:vars]
-            puts "Error: #{options[:vars]} on vars is not a valid JSON"
-          end
-        end
-      end
-
-      desc "exec <scraper_name> <parser_file> <GID>...<GID>", "Executes a parser script on one or more Job Pages within a scraper's current job"
-      long_desc <<-LONGDESC
-            Takes a parser script executes it against a job page(s) and save the output to the scraper's current job\x5
-            <parser_file>: Parser script file will be executed on the page.\x5
-            <GID>: Global ID of the page.\x5
-          LONGDESC
-      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
-      def exec_parse(scraper_name, parser_file, *gids)
-        gids.each do |gid|
-          begin
-            puts "Parsing #{gid}"
-
-            if options[:job]
-              job_id = options[:job]
-            else
-              job = Client::ScraperJob.new(options).find(scraper_name)
-              job_id = job['id']
-            end
-
-            puts AnswersEngine::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, true)
-          rescue => e
-            puts e
-          end
-        end
-      end
-    end
-  end
-
-end

data/lib/answersengine/cli/scraper.rb

@@ -1,185 +0,0 @@
-module AnswersEngine
-  class CLI < Thor
-    class Scraper < Thor
-      desc "list", "List scrapers"
-
-      long_desc <<-LONGDESC
-        List all scrapers.
-      LONGDESC
-      option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
-      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
-      def list
-        client = Client::Scraper.new(options)
-        puts "#{client.all}"
-      end
-
-      desc "create <scraper_name> <git_repository>", "Create a scraper"
-      long_desc <<-LONGDESC
-          Creates a scraper\x5
-          <scraper_name>: Scraper name can only consist of alphabets, numbers, underscores and dashes. Name must be unique to your account.\x5
-          <git_repository>: URL to a valid Git repository.\x5
-          LONGDESC
-      option :branch, :aliases => :b, desc: 'Set the Git branch to use. Default: master'
-      option :freshness_type, :aliases => :t, desc: 'Set how fresh the page cache is. Possible values: day, week, month, year. Default: any'
-      option :proxy_type, desc: 'Set the Proxy type. Default: standard'
-      option :force_fetch, :aliases => :f, type: :boolean, desc: 'Set true to force fetch page that is not within freshness criteria. Default: false'
-      option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Default: 1'
-      option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
-      option :disable_scheduler, type: :boolean, desc: 'Set true to disable scheduler. Default: false'
-      option :cancel_current_job, type: :boolean, desc: 'Set true to cancel currently active job if scheduler starts. Default: false'
-      option :schedule, type: :string, desc: 'Set the schedule of the scraper to run. Must be in CRON format.'
-      option :timezone, type: :string, desc: "Set the scheduler's timezone. Must be in IANA Timezone format. Defaults to \"America/Toronto\""
-      def create(scraper_name, git_repository)
-        # puts "options #{options}"
-        client = Client::Scraper.new(options)
-        puts "#{client.create(scraper_name, git_repository, options)}"
-      end
-
-      desc "update <scraper_name>", "Update a scraper"
-      long_desc <<-LONGDESC
-          Updates a scraper\x5
-          LONGDESC
-      option :branch, :aliases => :b, desc: 'Set the Git branch to use. Default: master'
-      option :name, :aliases => :n, desc: 'Set the scraper name. Name can only consist of alphabets, numbers, underscores and dashes. Name must be unique to your account'
-      option :repo, :aliases => :r, desc: 'Set the URL to a valid Git repository'
-      option :freshness_type, :aliases => :t, desc: 'Set how fresh the page cache is. Possible values: day, week, month, year. Default: any'
-      option :proxy_type, desc: 'Set the Proxy type. Default: standard'
-      option :force_fetch, :aliases => :f, type: :boolean, desc: 'Set true to force fetch page that is not within freshness criteria. Default: false'
-      option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Default: 1'
-      option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
-      option :disable_scheduler, type: :boolean, desc: 'Set true to disable scheduler. Default: false'
-      option :cancel_current_job, type: :boolean, desc: 'Set true to cancel currently active job if scheduler starts. Default: false'
-      option :schedule, type: :string, desc: 'Set the schedule of the scraper to run. Must be in CRON format.'
-      option :timezone, type: :string, desc: "Set the scheduler's timezone. Must be in IANA Timezone format. Defaults to \"America/Toronto\""
-      def update(scraper_name)
-        client = Client::Scraper.new(options)
-        puts "#{client.update(scraper_name, options)}"
-      end
-
-
-      desc "show <scraper_name>", "Show a scraper"
-      def show(scraper_name)
-        client = Client::Scraper.new(options)
-        puts "#{client.find(scraper_name)}"
-      end
-
-      desc "delete <scraper_name>", "Delete a scraper and related records"
-      def delete(scraper_name)
-        client = Client::Scraper.new(options)
-        puts "#{client.delete(scraper_name)}"
-      end
-
-
-      desc "deploy <scraper_name>", "Deploy a scraper"
-      long_desc <<-LONGDESC
-          Deploys a scraper
-          LONGDESC
-      def deploy(scraper_name)
-        client = Client::ScraperDeployment.new()
-        puts "Deploying scraper. This may take a while..."
-        puts "#{client.deploy(scraper_name)}"
-      end
-
-      desc "start <scraper_name>", "Creates a scraping job and runs it"
-      long_desc <<-LONGDESC
-          Starts a scraper by creating an active scrape job\x5
-          LONGDESC
-      option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Default: 1'
-      option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
-      option :proxy_type, desc: 'Set the Proxy type. Default: standard'
-      def start(scraper_name)
-        client = Client::ScraperJob.new(options)
-        puts "Starting a scrape job..."
-        puts "#{client.create(scraper_name, options)}"
-      end
-
-
-      desc "log <scraper_name>", "List log entries related to a scraper's current job"
-      long_desc <<-LONGDESC
-          Shows log related to a scraper's current job. Defaults to showing the most recent entries\x5
-          LONGDESC
-      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
-      option :head, :aliases => :H, desc: 'Show the oldest log entries. If not set, newest entries is shown'
-      option :parsing, :aliases => :p, type: :boolean, desc: 'Show only log entries related to parsing errors'
-      option :seeding, :aliases => :s, type: :boolean, desc: 'Show only log entries related to seeding errors'
-      option :more, :aliases => :m, desc: 'Show next set of log entries. Enter the `More token`'
-      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 5000 per page.'
-      def log(scraper_name)
-        client = Client::JobLog.new(options)
-
-        query = {}
-        query["order"] = options.delete(:head) if options[:head]
-        query["job_type"] = "parsing" if options[:parsing]
-        query["job_type"] = "seeding" if options[:seeding]
-        query["page_token"] = options.delete(:more) if options[:more]
-        query["per_page"] = options.delete(:per_page) if options[:per_page]
-
-        puts "Fetching logs..."
-
-        if options[:job]
-          result = client.all_job_log(options[:job], {query: query})
-        else
-          result = client.scraper_all_job_log(scraper_name, {query: query})
-        end
-
-        if result['entries'].nil? || result["entries"].length == 0
-          puts "No logs yet, please try again later."
-        else
-          more_token = result["more_token"]
-
-          result["entries"].each do |entry|
-            puts "#{entry["timestamp"]} #{entry["severity"]}: #{entry["payload"]}" if entry.is_a?(Hash)
-          end
-
-          unless more_token.nil?
-            puts "-----------"
-            puts "To see more entries, add: \"--more #{more_token}\""
-          end
-        end
-      end
-
-      desc "stats <scraper_name>", "Get the current stat for a job"
-      long_desc <<-LONGDESC
-        Get stats for a scraper's current job\n
-      LONGDESC
-      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
-      def stats(scraper_name)
-        client = Client::JobStat.new(options)
-        if options[:job]
-          puts "#{client.job_current_stats(options[:job])}"
-        else
-          puts "#{client.scraper_job_current_stats(scraper_name)}"
-        end
-
-      end
-
-
-      desc "job SUBCOMMAND ...ARGS", "manage scrapers jobs"
-      subcommand "job", ScraperJob
-
-      desc "deployment SUBCOMMAND ...ARGS", "manage scrapers deployments"
-      subcommand "deployment", ScraperDeployment
-
-      desc "finisher SUBCOMMAND ...ARGS", "manage scrapers finishers"
-      subcommand "finisher", ScraperFinisher
-
-      desc "output SUBCOMMAND ...ARGS", "view scraper outputs"
-      subcommand "output", JobOutput
-
-      desc "page SUBCOMMAND ...ARGS", "manage pages on a job"
-      subcommand "page", ScraperPage
-
-      desc "export SUBCOMMAND ...ARGS", "manage scraper's exports"
-      subcommand "export", ScraperExport
-
-      desc "exporter SUBCOMMAND ...ARGS", "manage scraper's exporters"
-      subcommand "exporter", ScraperExporter
-
-      desc "var SUBCOMMAND ...ARGS", "for managing scraper's variables"
-      subcommand "var", ScraperVar
-
-
-    end
-  end
-
-end