answersengine 0.2.33

data/lib/answersengine/scraper/parser.rb

@@ -0,0 +1,18 @@
+module AnswersEngine
+  module Scraper
+    class Parser
+      def self.exec_parser_page(filename, gid, job_id=nil, save=false, vars = {})
+        extname = File.extname(filename)
+        case extname
+        when '.rb'
+          executor = RubyParserExecutor.new(filename: filename, gid: gid, job_id: job_id, vars: vars)
+          executor.exec_parser(save)
+        else
+          puts "Unable to find a parser executor for file type \"#{extname}\""
+        end
+      end
+
+    
+    end
+  end
+end

data/lib/answersengine/scraper/ruby_parser_executor.rb

@@ -0,0 +1,141 @@
+module AnswersEngine
+  module Scraper
+    class RubyParserExecutor < Executor
+      attr_accessor :save
+
+      def initialize(options={})
+        @filename = options.fetch(:filename) { raise "Filename is required"}
+        @gid = options.fetch(:gid) { raise "GID is required"}
+        @job_id = options.fetch(:job_id)
+        @page_vars = options.fetch(:vars) { {} }
+      end
+
+      def self.exposed_methods
+        [
+          :content,
+          :failed_content,
+          :outputs,
+          :pages,
+          :page,
+          :save_pages,
+          :save_outputs,
+          :find_output,
+          :find_outputs
+        ].freeze
+      end
+
+      def exec_parser(save=false)
+        @save = save
+        if save
+          puts "Executing parser script"
+        else
+          puts "Trying parser script"
+        end
+
+        eval_parser_script(save)
+      end
+
+      def init_page_vars(page)
+        if !@page_vars.nil? && !@page_vars.empty?
+          page['vars'] = @page_vars
+        end
+        page
+      end
+
+      def update_to_server(opts = {})
+        parsing_update(
+          job_id: opts[:job_id],
+          gid: opts[:gid],
+          pages: opts[:pages],
+          outputs: opts[:outputs],
+          parsing_status: opts[:status])
+      end
+
+      def update_parsing_starting_status
+        return unless save
+
+        response = parsing_update(
+          job_id: job_id,
+          gid: gid,
+          parsing_status: :starting)
+
+        if response.code == 200
+          puts "Page Parsing Status Updated."
+        else
+          puts "Error: Unable to save Page Parsing Status to server: #{response.body}"
+          raise "Unable to save Page Parsing Status to server: #{response.body}"
+        end
+      end
+
+      def update_parsing_done_status
+        return unless save
+
+        response = parsing_update(
+          job_id: job_id,
+          gid: gid,
+          parsing_status: :done)
+
+        if response.code == 200
+          puts "Page Parsing Done."
+        else
+          puts "Error: Unable to save Page Parsing Done Status to server: #{response.body}"
+          raise "Unable to save Page Parsing Done Status to server: #{response.body}"
+        end
+      end
+
+      def save_type
+        :parsing
+      end
+
+      def eval_parser_script(save=false)
+        update_parsing_starting_status
+
+        proc = Proc.new do
+          page = init_page
+          outputs = []
+          pages = []
+          page = init_page_vars(page)
+
+          begin
+            context = isolated_binding({
+              outputs: outputs,
+              pages: pages,
+              page: page
+            })
+            eval_with_context filename, context
+          rescue SyntaxError => e
+            handle_error(e) if save
+            raise e
+          rescue => e
+            handle_error(e) if save
+            raise e
+          end
+
+          puts "=========== Parsing Executed ==========="
+          save_pages_and_outputs(pages, outputs, :parsing)
+          update_parsing_done_status
+        end
+        proc.call
+      end
+
+      def content
+        @content ||= get_content(gid)
+      end
+
+      def failed_content
+        @failed_content ||= get_failed_content(gid)
+      end
+
+      def handle_error(e)
+        error = ["Parsing #{e.class}: #{e.to_s} (Job:#{job_id} GID:#{gid})",clean_backtrace(e.backtrace)].join("\n")
+
+        parsing_update(
+          job_id: job_id,
+          gid: gid,
+          parsing_status: :failed,
+          log_error: error)
+      end
+
+    end
+  end
+end

data/lib/answersengine/scraper/ruby_seeder_executor.rb

@@ -0,0 +1,114 @@
+module AnswersEngine
+  module Scraper
+    class RubySeederExecutor < Executor
+      attr_accessor :save
+
+      def initialize(options={})
+        @filename = options.fetch(:filename) { raise "Filename is required"}
+        @job_id = options[:job_id]
+      end
+
+      def self.exposed_methods
+        [
+          :outputs,
+          :pages,
+          :save_pages,
+          :save_outputs,
+          :find_output,
+          :find_outputs
+        ].freeze
+      end
+
+      def exec_seeder(save=false)
+        @save = save
+        if save
+          puts "Executing seeder script"
+        else
+          puts "Trying seeder script"
+        end
+
+        eval_seeder_script(save)
+      end
+
+      def eval_seeder_script(save=false)
+        update_seeding_starting_status
+
+        proc = Proc.new do
+          outputs = []
+          pages = []
+
+          begin
+            context = isolated_binding({
+              outputs: outputs,
+              pages: pages
+            })
+            eval_with_context filename, context
+          rescue SyntaxError => e
+            handle_error(e) if save
+            raise e
+          rescue => e
+            handle_error(e) if save
+            raise e
+          end
+
+          puts "=========== Seeding Executed ==========="
+          save_pages_and_outputs(pages, outputs, :seeding)
+          update_seeding_done_status
+        end
+        proc.call
+      end
+
+      def save_type
+        :seeding
+      end
+
+      def update_to_server(opts = {})
+        seeding_update(
+          job_id: opts[:job_id],
+          pages: opts[:pages],
+          outputs: opts[:outputs],
+          seeding_status: opts[:status])
+      end
+
+      def update_seeding_starting_status
+        return unless save
+
+        response = seeding_update(
+          job_id: job_id,
+          seeding_status: :starting)
+
+        if response.code == 200
+          puts "Seeding Status Updated."
+        else
+          puts "Error: Unable to save Seeding Status to server: #{response.body}"
+          raise "Unable to save Seeding Status to server: #{response.body}"
+        end
+      end
+
+      def update_seeding_done_status
+        return unless save
+
+        response = seeding_update(
+          job_id: job_id,
+          seeding_status: :done)
+
+        if response.code == 200
+          puts "Seeding Done."
+        else
+          puts "Error: Unable to save Seeding Done Status to server: #{response.body}"
+          raise "Unable to save Seeding Done Status to server: #{response.body}"
+        end
+      end
+
+      def handle_error(e)
+        error = ["Seeding #{e.class}: #{e.to_s} (Job:#{job_id}",clean_backtrace(e.backtrace)].join("\n")
+
+        seeding_update(
+          job_id: job_id,
+          seeding_status: :failed,
+          log_error: error)
+      end
+
+    end
+  end
+end

data/lib/answersengine/scraper/seeder.rb

@@ -0,0 +1,18 @@
+module AnswersEngine
+  module Scraper
+    class Seeder
+
+      def self.exec_seeder(filename, job_id=nil, save=false)
+        extname = File.extname(filename)
+        case extname
+        when '.rb'
+          executor = RubySeederExecutor.new(filename: filename, job_id: job_id)
+          executor.exec_seeder(save)
+        else
+          puts "Unable to find a seeder executor for file type \"#{extname}\""
+        end
+      end
+
+    end
+  end
+end

data/lib/answersengine/version.rb

@@ -0,0 +1,3 @@
+module AnswersEngine
+  VERSION = "0.2.33"
+end

metadata

@@ -0,0 +1,255 @@
+--- !ruby/object:Gem::Specification
+name: answersengine
+version: !ruby/object:Gem::Version
+  version: 0.2.33
+platform: ruby
+authors:
+- Parama Danoesubroto
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-03-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.20.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.20.3
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.16.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.16.2
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.11'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.16.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.16.1
+- !ruby/object:Gem::Dependency
+  name: simplecov-console
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.2
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: AnswersEngine toolbelt to develop scrapers and other scripts
+email:
+- parama@answersengine.com
+executables:
+- answersengine
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- answersengine.gemspec
+- bin/console
+- bin/setup
+- examples/fetchtest/libraries/hello.rb
+- examples/fetchtest/libraries/hello_fail.rb
+- examples/fetchtest/parsers/failed.rb
+- examples/fetchtest/parsers/find_outputs.rb
+- examples/fetchtest/parsers/home.rb
+- examples/fetchtest/parsers/nested_fail.rb
+- examples/fetchtest/parsers/simple.rb
+- examples/fetchtest/seeders/csv_seeder.rb
+- examples/fetchtest/seeders/failed.rb
+- examples/fetchtest/seeders/list_of_urls.csv
+- examples/fetchtest/seeders/seed.rb
+- examples/fetchtest/seeders/test_reset_page.rb
+- exe/answersengine
+- lib/answersengine.rb
+- lib/answersengine/cli.rb
+- lib/answersengine/cli/global_page.rb
+- lib/answersengine/cli/job.rb
+- lib/answersengine/cli/job_output.rb
+- lib/answersengine/cli/parser.rb
+- lib/answersengine/cli/scraper.rb
+- lib/answersengine/cli/scraper_deployment.rb
+- lib/answersengine/cli/scraper_export.rb
+- lib/answersengine/cli/scraper_exporter.rb
+- lib/answersengine/cli/scraper_job.rb
+- lib/answersengine/cli/scraper_page.rb
+- lib/answersengine/cli/seeder.rb
+- lib/answersengine/client.rb
+- lib/answersengine/client/backblaze_content.rb
+- lib/answersengine/client/base.rb
+- lib/answersengine/client/export.rb
+- lib/answersengine/client/global_page.rb
+- lib/answersengine/client/job.rb
+- lib/answersengine/client/job_export.rb
+- lib/answersengine/client/job_log.rb
+- lib/answersengine/client/job_output.rb
+- lib/answersengine/client/job_page.rb
+- lib/answersengine/client/job_stat.rb
+- lib/answersengine/client/scraper.rb
+- lib/answersengine/client/scraper_deployment.rb
+- lib/answersengine/client/scraper_export.rb
+- lib/answersengine/client/scraper_exporter.rb
+- lib/answersengine/client/scraper_job.rb
+- lib/answersengine/client/scraper_job_output.rb
+- lib/answersengine/client/scraper_job_page.rb
+- lib/answersengine/plugin.rb
+- lib/answersengine/plugin/context_exposer.rb
+- lib/answersengine/scraper.rb
+- lib/answersengine/scraper/executor.rb
+- lib/answersengine/scraper/parser.rb
+- lib/answersengine/scraper/ruby_parser_executor.rb
+- lib/answersengine/scraper/ruby_seeder_executor.rb
+- lib/answersengine/scraper/seeder.rb
+- lib/answersengine/version.rb
+homepage: https://answersengine.com
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.2.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14.1
+signing_key: 
+specification_version: 4
+summary: AnswersEngine toolbelt for developers
+test_files: []