datahen 0.10.4

data/lib/datahen.rb

@@ -0,0 +1,5 @@
+require "datahen/version"
+require "datahen/scraper"
+
+module Datahen
+end

data/lib/datahen/cli.rb

@@ -0,0 +1,45 @@
+require 'thor'
+require 'datahen/scraper'
+require 'datahen/cli/scraper_var'
+require 'datahen/cli/scraper_exporter'
+require 'datahen/cli/scraper_export'
+require 'datahen/cli/scraper_job_var'
+require 'datahen/cli/scraper_job'
+require 'datahen/cli/scraper_finisher'
+require 'datahen/cli/global_page'
+require 'datahen/cli/scraper_page'
+require 'datahen/cli/job_output'
+require 'datahen/cli/job'
+require 'datahen/cli/scraper_deployment'
+require 'datahen/cli/scraper'
+require 'datahen/cli/parser'
+require 'datahen/cli/seeder'
+require 'datahen/cli/finisher'
+require 'datahen/cli/env_var'
+
+
+
+module Datahen
+  class CLI < Thor
+    desc "scraper SUBCOMMAND ...ARGS", "manage scrapers"
+    subcommand "scraper", Scraper
+
+    desc "job SUBCOMMAND ...ARGS", "manage scrapers jobs"
+    subcommand "job", Job
+
+    desc "globalpage SUBCOMMAND ...ARGS", "interacts with global page"
+    subcommand "globalpage", GlobalPage
+
+    desc "parser SUBCOMMAND ...ARGS", "for parsing related activities"
+    subcommand "parser", Parser
+
+    desc "seeder SUBCOMMAND ...ARGS", "for seeding related activities"
+    subcommand "seeder", Seeder
+
+    desc "seeder SUBCOMMAND ...ARGS", "for seeding related activities"
+    subcommand "finisher", Finisher
+
+    desc "var SUBCOMMAND ...ARGS", "for environment variable related activities"
+    subcommand "var", EnvVar
+  end
+end

data/lib/datahen/cli/env_var.rb

@@ -0,0 +1,48 @@
+module Datahen
+  class CLI < Thor
+    class EnvVar < Thor
+      desc "list", "List environment variables on the account"
+      
+      long_desc <<-LONGDESC
+        List all environment variables on the account.
+      LONGDESC
+      option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
+      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
+      def list
+        client = Client::EnvVar.new(options)
+        puts "#{client.all}"
+      end
+
+      desc "set <name> <value>", "Set an environment var on the account"
+      long_desc <<-LONGDESC
+          Creates an environment variable\x5
+          <name>: Var name can only consist of alphabets, numbers, underscores. Name must be unique to your account, otherwise it will be overwritten.\x5
+          <value>: Value of variable.\x5
+          LONGDESC
+      option :secret, type: :boolean, desc: 'Set true to make it decrypt the value. Default: false' 
+      def set(name, value)
+        # puts "options #{options}"
+        client = Client::EnvVar.new(options)
+        puts "#{client.set(name, value, options)}"
+      end
+
+      desc "show <name>", "Show an environment variable on the account"
+      def show(name)
+        client = Client::EnvVar.new(options)
+        puts "#{client.find(name)}"
+      end
+
+      desc "unset <name>", "Deletes an environment variable on the account"
+      def unset(name)
+        client = Client::EnvVar.new(options)
+        puts "#{client.unset(name)}"
+      end
+
+
+      
+
+
+    end
+  end
+
+end

data/lib/datahen/cli/finisher.rb

@@ -0,0 +1,40 @@
+module Datahen
+  class CLI < Thor
+    class Finisher < Thor
+      desc "try <scraper_name> <finisher_file>", "Tries a finisher file"
+      long_desc <<-LONGDESC
+            Takes a finisher script and tries to execute it without saving anything.\x5
+            <seeder_file>: Finisher script file will be executed.\x5
+          LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      def try_finisher(scraper_name, finisher_file)
+        if options[:job]
+          job_id = options[:job]
+        else
+          job = Client::ScraperJob.new(options).find(scraper_name)
+          job_id = job['id']
+        end
+
+        puts Datahen::Scraper::Finisher.exec_finisher(finisher_file, job_id, false)
+      end
+
+      desc "exec <scraper_name> <finisher_file>", "Executes a finisher script onto a scraper's current job."
+      long_desc <<-LONGDESC
+            Takes a finisher script and execute it against a job and save outputs into the scraper's current job\x5
+            <finisher_file>: Finisher script file that will be executed on the scraper's current job.\x5
+          LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      def exec_parse(scraper_name, finisher_file)
+        if options[:job]
+          job_id = options[:job]
+        else
+          job = Client::ScraperJob.new(options).find(scraper_name)
+          job_id = job['id']
+        end
+
+        puts Datahen::Scraper::Finisher.exec_finisher(finisher_file, job_id, true)
+      end
+    end
+  end
+
+end

data/lib/datahen/cli/global_page.rb

@@ -0,0 +1,39 @@
+module Datahen
+  class CLI < Thor
+    class GlobalPage < Thor
+
+      desc "show <gid>", "Show a global page"
+      def show(gid)
+        client = Client::GlobalPage.new(options)
+        puts "#{client.find(gid)}"
+      end
+
+      desc "content <gid>", "Show content of a globalpage"
+      def content(gid)
+        client = Client::GlobalPage.new(options)
+        result = JSON.parse(client.find_content(gid).to_s)
+        
+        if result['available'] == true
+          puts "Preview content url: \"#{result['preview_url']}\""
+          `open "#{result['preview_url']}"`
+        else
+          puts "Content does not exist"
+        end        
+      end
+
+      desc "failedcontent <gid>", "Show failed content of a globalpage"
+      def failedcontent(gid)
+        client = Client::GlobalPage.new(options)
+        result = JSON.parse(client.find_failed_content(gid).to_s)
+        
+        if result['available'] == true
+          puts "Preview failed content url: \"#{result['preview_url']}\""
+          `open "#{result['preview_url']}"`
+        else
+          puts "Failed Content does not exist"
+        end        
+      end
+      
+    end
+  end
+end

data/lib/datahen/cli/job.rb

@@ -0,0 +1,30 @@
+module Datahen
+  class CLI < Thor
+    class Job < Thor
+      package_name "job"
+      def self.banner(command, namespace = nil, subcommand = false)
+        "#{basename} #{@package_name} #{command.usage}"
+      end
+
+
+      desc "list", "gets a list of jobs"
+      option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
+      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
+      long_desc <<-LONGDESC
+        List scrape jobs.
+      LONGDESC
+      def list()
+        client = Client::Job.new(options)
+        puts "#{client.all()}"
+      end
+
+      desc "show <job_id>", "Show a job"
+      def show(job_id)
+        client = Client::Job.new(options)
+        puts "#{client.find(job_id)}"
+      end
+
+    end
+  end
+
+end

data/lib/datahen/cli/job_output.rb

@@ -0,0 +1,69 @@
+module Datahen
+  class CLI < Thor
+    class JobOutput < Thor
+
+      package_name "scraper output"
+      def self.banner(command, namespace = nil, subcommand = false)
+        "#{basename} #{@package_name} #{command.usage}"
+      end
+
+      desc "list <scraper_name>", "List output records in a collection that is in the current job"
+      long_desc <<-LONGDESC
+        List all output records in a collection that is in the current job of a scraper\n
+      LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
+      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
+      option :collection, :aliases => :c, desc: "Shows outputs from a specific collection.(defaults to 'default' collection)"
+      option :query, :aliases => :q, type: :string, banner: :JSON, desc: 'Set search query. Must be in json format. i.e: {"Foo":"bar"} '
+      def list(scraper_name)
+        collection = options.fetch(:collection) { 'default' }
+        if options[:job]
+          client = Client::JobOutput.new(options)
+          puts "#{client.all(options[:job], collection)}"
+        else
+          client = Client::ScraperJobOutput.new(options)
+          puts "#{client.all(scraper_name, collection)}"
+        end
+      end
+
+      desc "show <scraper_name> <record_id>", "Show one output record in a collection that is in the current job of a scraper"
+      long_desc <<-LONGDESC
+        Shows an output record in a collection that is in the current job of a scraper\n
+        <record_id>: ID of the output record.\x5
+      LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :collection, :aliases => :c, desc: "Shows output from a specific collection.(defaults to 'default' collection)"
+      def show(scraper_name, id)
+        collection = options.fetch(:collection) { 'default' }
+        if options[:job]
+          client = Client::JobOutput.new(options)
+          puts "#{client.find(options[:job], collection, id)}"
+        else
+          client = Client::ScraperJobOutput.new(options)
+          puts "#{client.find(scraper_name, collection, id)}"
+        end
+      end
+
+      desc "collections <scraper_name>", "list job output collections that are inside a current job of a scraper."
+      long_desc <<-LONGDESC
+        List job output collections that are inside a current job of a scraper.\x5
+      LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
+      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
+      def collections(scraper_name)
+
+        if options[:job]
+          client = Client::JobOutput.new(options)
+          puts "#{client.collections(options[:job])}"
+        else
+          client = Client::ScraperJobOutput.new(options)
+          puts "#{client.collections(scraper_name)}"
+        end
+      end
+
+    end
+  end
+
+end

data/lib/datahen/cli/parser.rb

@@ -0,0 +1,64 @@
+module Datahen
+  class CLI < Thor
+    class Parser < Thor
+      desc "try <scraper_name> <parser_file> <GID>", "Tries a parser on a Job Page"
+      long_desc <<-LONGDESC
+            Takes a parser script and runs it against a job page\x5
+            <parser_file>: Parser script file that will be executed on the page.\x5
+            <GID>: Global ID of the page.\x5
+          LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :global, :aliases => :g, type: :boolean, default: false, desc: 'Use globalpage instead of a job page'
+      option :vars, :aliases => :v, type: :string, desc: 'Set user-defined page variables. Must be in json format. i.e: {"Foo":"bar"}'
+      def try_parse(scraper_name, parser_file, gid)
+        begin 
+          
+            if options[:job]
+              job_id = options[:job]
+            elsif options[:global] 
+              job_id = nil
+            else
+              job = Client::ScraperJob.new(options).find(scraper_name)
+              job_id = job['id']
+            end
+
+
+          vars = JSON.parse(options[:vars]) if options[:vars]
+          puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, false, vars)
+
+          rescue JSON::ParserError
+          if options[:vars]
+            puts "Error: #{options[:vars]} on vars is not a valid JSON"
+          end
+        end
+      end
+
+      desc "exec <scraper_name> <parser_file> <GID>...<GID>", "Executes a parser script on one or more Job Pages within a scraper's current job"
+      long_desc <<-LONGDESC
+            Takes a parser script executes it against a job page(s) and save the output to the scraper's current job\x5
+            <parser_file>: Parser script file will be executed on the page.\x5
+            <GID>: Global ID of the page.\x5
+          LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      def exec_parse(scraper_name, parser_file, *gids)
+        gids.each do |gid|
+          begin
+            puts "Parsing #{gid}"
+
+            if options[:job]
+              job_id = options[:job]
+            else
+              job = Client::ScraperJob.new(options).find(scraper_name)
+              job_id = job['id']
+            end
+
+            puts Datahen::Scraper::Parser.exec_parser_page(parser_file, gid, job_id, true)
+          rescue => e
+            puts e
+          end
+        end
+      end
+    end
+  end
+
+end

data/lib/datahen/cli/scraper.rb

@@ -0,0 +1,185 @@
+module Datahen
+  class CLI < Thor
+    class Scraper < Thor
+      desc "list", "List scrapers"
+
+      long_desc <<-LONGDESC
+        List all scrapers.
+      LONGDESC
+      option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
+      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
+      def list
+        client = Client::Scraper.new(options)
+        puts "#{client.all}"
+      end
+
+      desc "create <scraper_name> <git_repository>", "Create a scraper"
+      long_desc <<-LONGDESC
+          Creates a scraper\x5
+          <scraper_name>: Scraper name can only consist of alphabets, numbers, underscores and dashes. Name must be unique to your account.\x5
+          <git_repository>: URL to a valid Git repository.\x5
+          LONGDESC
+      option :branch, :aliases => :b, desc: 'Set the Git branch to use. Default: master'
+      option :freshness_type, :aliases => :t, desc: 'Set how fresh the page cache is. Possible values: day, week, month, year. Default: any'
+      option :proxy_type, desc: 'Set the Proxy type. Default: standard'
+      option :force_fetch, :aliases => :f, type: :boolean, desc: 'Set true to force fetch page that is not within freshness criteria. Default: false'
+      option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Default: 1'
+      option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
+      option :disable_scheduler, type: :boolean, desc: 'Set true to disable scheduler. Default: false'
+      option :cancel_current_job, type: :boolean, desc: 'Set true to cancel currently active job if scheduler starts. Default: false'
+      option :schedule, type: :string, desc: 'Set the schedule of the scraper to run. Must be in CRON format.'
+      option :timezone, type: :string, desc: "Set the scheduler's timezone. Must be in IANA Timezone format. Defaults to \"America/Toronto\""
+      def create(scraper_name, git_repository)
+        # puts "options #{options}"
+        client = Client::Scraper.new(options)
+        puts "#{client.create(scraper_name, git_repository, options)}"
+      end
+
+      desc "update <scraper_name>", "Update a scraper"
+      long_desc <<-LONGDESC
+          Updates a scraper\x5
+          LONGDESC
+      option :branch, :aliases => :b, desc: 'Set the Git branch to use. Default: master'
+      option :name, :aliases => :n, desc: 'Set the scraper name. Name can only consist of alphabets, numbers, underscores and dashes. Name must be unique to your account'
+      option :repo, :aliases => :r, desc: 'Set the URL to a valid Git repository'
+      option :freshness_type, :aliases => :t, desc: 'Set how fresh the page cache is. Possible values: day, week, month, year. Default: any'
+      option :proxy_type, desc: 'Set the Proxy type. Default: standard'
+      option :force_fetch, :aliases => :f, type: :boolean, desc: 'Set true to force fetch page that is not within freshness criteria. Default: false'
+      option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Default: 1'
+      option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
+      option :disable_scheduler, type: :boolean, desc: 'Set true to disable scheduler. Default: false'
+      option :cancel_current_job, type: :boolean, desc: 'Set true to cancel currently active job if scheduler starts. Default: false'
+      option :schedule, type: :string, desc: 'Set the schedule of the scraper to run. Must be in CRON format.'
+      option :timezone, type: :string, desc: "Set the scheduler's timezone. Must be in IANA Timezone format. Defaults to \"America/Toronto\""
+      def update(scraper_name)
+        client = Client::Scraper.new(options)
+        puts "#{client.update(scraper_name, options)}"
+      end
+
+
+      desc "show <scraper_name>", "Show a scraper"
+      def show(scraper_name)
+        client = Client::Scraper.new(options)
+        puts "#{client.find(scraper_name)}"
+      end
+
+      desc "delete <scraper_name>", "Delete a scraper and related records"
+      def delete(scraper_name)
+        client = Client::Scraper.new(options)
+        puts "#{client.delete(scraper_name)}"
+      end
+
+
+      desc "deploy <scraper_name>", "Deploy a scraper"
+      long_desc <<-LONGDESC
+          Deploys a scraper
+          LONGDESC
+      def deploy(scraper_name)
+        client = Client::ScraperDeployment.new()
+        puts "Deploying scraper. This may take a while..."
+        puts "#{client.deploy(scraper_name)}"
+      end
+
+      desc "start <scraper_name>", "Creates a scraping job and runs it"
+      long_desc <<-LONGDESC
+          Starts a scraper by creating an active scrape job\x5
+          LONGDESC
+      option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Default: 1'
+      option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
+      option :proxy_type, desc: 'Set the Proxy type. Default: standard'
+      def start(scraper_name)
+        client = Client::ScraperJob.new(options)
+        puts "Starting a scrape job..."
+        puts "#{client.create(scraper_name, options)}"
+      end
+
+
+      desc "log <scraper_name>", "List log entries related to a scraper's current job"
+      long_desc <<-LONGDESC
+          Shows log related to a scraper's current job. Defaults to showing the most recent entries\x5
+          LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :head, :aliases => :H, desc: 'Show the oldest log entries. If not set, newest entries is shown'
+      option :parsing, :aliases => :p, type: :boolean, desc: 'Show only log entries related to parsing errors'
+      option :seeding, :aliases => :s, type: :boolean, desc: 'Show only log entries related to seeding errors'
+      option :more, :aliases => :m, desc: 'Show next set of log entries. Enter the `More token`'
+      option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 5000 per page.'
+      def log(scraper_name)
+        client = Client::JobLog.new(options)
+
+        query = {}
+        query["order"] = options.delete(:head) if options[:head]
+        query["job_type"] = "parsing" if options[:parsing]
+        query["job_type"] = "seeding" if options[:seeding]
+        query["page_token"] = options.delete(:more) if options[:more]
+        query["per_page"] = options.delete(:per_page) if options[:per_page]
+
+        puts "Fetching logs..."
+
+        if options[:job]
+          result = client.all_job_log(options[:job], {query: query})
+        else
+          result = client.scraper_all_job_log(scraper_name, {query: query})
+        end
+
+        if result['entries'].nil? || result["entries"].length == 0
+          puts "No logs yet, please try again later."
+        else
+          more_token = result["more_token"]
+
+          result["entries"].each do |entry|
+            puts "#{entry["timestamp"]} #{entry["severity"]}: #{entry["payload"]}" if entry.is_a?(Hash)
+          end
+
+          unless more_token.nil?
+            puts "-----------"
+            puts "To see more entries, add: \"--more #{more_token}\""
+          end
+        end
+      end
+
+      desc "stats <scraper_name>", "Get the current stat for a job"
+      long_desc <<-LONGDESC
+        Get stats for a scraper's current job\n
+      LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      def stats(scraper_name)
+        client = Client::JobStat.new(options)
+        if options[:job]
+          puts "#{client.job_current_stats(options[:job])}"
+        else
+          puts "#{client.scraper_job_current_stats(scraper_name)}"
+        end
+
+      end
+
+
+      desc "job SUBCOMMAND ...ARGS", "manage scrapers jobs"
+      subcommand "job", ScraperJob
+
+      desc "deployment SUBCOMMAND ...ARGS", "manage scrapers deployments"
+      subcommand "deployment", ScraperDeployment
+
+      desc "finisher SUBCOMMAND ...ARGS", "manage scrapers finishers"
+      subcommand "finisher", ScraperFinisher
+
+      desc "output SUBCOMMAND ...ARGS", "view scraper outputs"
+      subcommand "output", JobOutput
+
+      desc "page SUBCOMMAND ...ARGS", "manage pages on a job"
+      subcommand "page", ScraperPage
+
+      desc "export SUBCOMMAND ...ARGS", "manage scraper's exports"
+      subcommand "export", ScraperExport
+
+      desc "exporter SUBCOMMAND ...ARGS", "manage scraper's exporters"
+      subcommand "exporter", ScraperExporter
+
+      desc "var SUBCOMMAND ...ARGS", "for managing scraper's variables"
+      subcommand "var", ScraperVar
+
+
+    end
+  end
+
+end