datahen 0.10.4

data/lib/datahen/scraper/finisher.rb

@@ -0,0 +1,18 @@
+module Datahen
+  module Scraper
+    class Finisher
+
+      def self.exec_finisher(filename, job_id=nil, save=false)
+        extname = File.extname(filename)
+        case extname
+        when '.rb'
+          executor = RubyFinisherExecutor.new(filename: filename, job_id: job_id)
+          executor.exec_finisher(save)
+        else
+          puts "Unable to find a finisher executor for file type \"#{extname}\""
+        end
+      end
+
+    end
+  end
+end

data/lib/datahen/scraper/parser.rb

@@ -0,0 +1,18 @@
+module Datahen
+  module Scraper
+    class Parser
+      def self.exec_parser_page(filename, gid, job_id=nil, save=false, vars = {})
+        extname = File.extname(filename)
+        case extname
+        when '.rb'
+          executor = RubyParserExecutor.new(filename: filename, gid: gid, job_id: job_id, vars: vars)
+          executor.exec_parser(save)
+        else
+          puts "Unable to find a parser executor for file type \"#{extname}\""
+        end
+      end
+
+    
+    end
+  end
+end

data/lib/datahen/scraper/ruby_finisher_executor.rb

@@ -0,0 +1,116 @@
+module Datahen
+  module Scraper
+    class RubyFinisherExecutor < Executor
+      attr_accessor :save
+
+      def initialize(options={})
+        @filename = options.fetch(:filename) { raise "Filename is required"}
+        @job_id = options[:job_id]
+      end
+
+      def self.exposed_methods
+        [
+          :outputs,
+          :save_outputs,
+          :find_output,
+          :find_outputs
+        ].freeze
+      end
+
+      def exec_finisher(save=false)
+        @save = save
+        if save
+          puts "Executing finisher script"
+        else
+          puts "Trying finisher script"
+        end
+
+        eval_finisher_script(save)
+      end
+
+      def eval_finisher_script(save=false)
+        update_finisher_starting_status
+
+        proc = Proc.new do
+          outputs = []
+
+          begin
+            context = isolated_binding({
+              outputs: outputs,
+              job_id: job_id
+            })
+            eval_with_context filename, context
+          rescue SyntaxError => e
+            handle_error(e) if save
+            raise e
+          rescue => e
+            handle_error(e) if save
+            raise e
+          end
+
+          puts "=========== Finisher Executed ==========="
+          begin
+            save_outputs(outputs)
+          rescue => e
+            handle_error(e) if save
+            raise e
+          end
+          
+          update_finisher_done_status
+        end
+        proc.call
+      end
+
+      def save_type
+        :executing
+      end
+
+      def update_to_server(opts = {})
+        finisher_update(
+          job_id: opts[:job_id],
+          outputs: opts[:outputs],
+          finisher_status: opts[:status])
+      end
+
+      def update_finisher_starting_status
+        return unless save
+
+        response = finisher_update(
+          job_id: job_id,
+          finisher_status: :starting)
+
+        if response.code == 200
+          puts "Finisher Status Updated."
+        else
+          puts "Error: Unable to save Finisher Status to server: #{response.body}"
+          raise "Unable to save Finisher Status to server: #{response.body}"
+        end
+      end
+
+      def update_finisher_done_status
+        return unless save
+
+        response = finisher_update(
+          job_id: job_id,
+          finisher_status: :done)
+
+        if response.code == 200
+          puts "Finisher Done."
+        else
+          puts "Error: Unable to save Finisher Done Status to server: #{response.body}"
+          raise "Unable to save Finisher Done Status to server: #{response.body}"
+        end
+      end
+
+      def handle_error(e)
+        error = ["Finisher #{e.class}: #{e.to_s} (Job:#{job_id}",clean_backtrace(e.backtrace)].join("\n")
+
+        finisher_update(
+          job_id: job_id,
+          finisher_status: :failed,
+          log_error: error)
+      end
+
+    end
+  end
+end

data/lib/datahen/scraper/ruby_parser_executor.rb

@@ -0,0 +1,200 @@
+module Datahen
+  module Scraper
+    class RubyParserExecutor < Executor
+      attr_accessor :save
+      # Refetch self page flag.
+      # @return [Boollean]
+      # @note It is stronger than #reparse_self flag.
+      attr_accessor :refetch_self
+      # Reparse self page flag.
+      # @return [Boollean]
+      attr_accessor :reparse_self
+
+      def initialize(options={})
+        @filename = options.fetch(:filename) { raise "Filename is required"}
+        @gid = options.fetch(:gid) { raise "GID is required"}
+        @job_id = options.fetch(:job_id)
+        @page_vars = options.fetch(:vars) { {} }
+      end
+
+      def self.exposed_methods
+        [
+          :content,
+          :failed_content,
+          :outputs,
+          :pages,
+          :page,
+          :save_pages,
+          :save_outputs,
+          :find_output,
+          :find_outputs,
+          :refetch,
+          :reparse
+        ].freeze
+      end
+
+      def exec_parser(save=false)
+        @save = save
+        if save
+          puts "Executing parser script"
+        else
+          puts "Trying parser script"
+        end
+
+        eval_parser_script(save)
+      end
+
+      def init_page_vars(page)
+        if !@page_vars.nil? && !@page_vars.empty?
+          page['vars'] = @page_vars
+        end
+        page
+      end
+
+      def update_to_server(opts = {})
+        parsing_update(
+          job_id: opts[:job_id],
+          gid: opts[:gid],
+          pages: opts[:pages],
+          outputs: opts[:outputs],
+          parsing_status: opts[:status])
+      end
+
+      def update_parsing_starting_status
+        return unless save
+
+        response = parsing_update(
+          job_id: job_id,
+          gid: gid,
+          parsing_status: :starting)
+
+        if response.code == 200
+          puts "Page Parsing Status Updated."
+        else
+          puts "Error: Unable to save Page Parsing Status to server: #{response.body}"
+          raise "Unable to save Page Parsing Status to server: #{response.body}"
+        end
+      end
+
+      def update_parsing_done_status
+        return unless save
+
+        response = parsing_update(
+          job_id: job_id,
+          gid: gid,
+          parsing_status: :done)
+
+        if response.code == 200
+          puts "Page Parsing Done."
+        else
+          puts "Error: Unable to save Page Parsing Done Status to server: #{response.body}"
+          raise "Unable to save Page Parsing Done Status to server: #{response.body}"
+        end
+      end
+
+      def save_type
+        :parsing
+      end
+
+      def refetch_page gid
+        if save
+          Client::ScraperJobPage.new({gid: gid}).refetch_by_job(self.job_id)
+          puts "Refetch page #{gid}"
+        else
+          puts "Would have refetch page #{gid}"
+        end
+      end
+
+      def refetch page_gid
+        raise ArgumentError.new("page_gid needs to be a String.") unless page_gid.is_a?(String)
+        if page_gid == gid
+          self.refetch_self = true
+          return
+        end
+        refetch_page page_gid
+      end
+
+      def reparse_page gid
+        if save
+          Client::ScraperJobPage.new({gid: gid}).reparse_by_job(self.job_id)
+          puts "Reparse page #{gid}"
+        else
+          puts "Would have reparse page #{gid}"
+        end
+      end
+
+      def reparse page_gid
+        raise ArgumentError.new("page_gid needs to be a String.") unless page_gid.is_a?(String)
+        if page_gid == gid
+          self.reparse_self = true
+          return
+        end
+        reparse_page page_gid
+      end
+
+      def eval_parser_script(save=false)
+        update_parsing_starting_status
+
+        proc = Proc.new do
+          page = init_page
+          outputs = []
+          pages = []
+          page = init_page_vars(page)
+          self.refetch_self = false
+          self.reparse_self = false
+
+          begin
+            context = isolated_binding({
+              outputs: outputs,
+              pages: pages,
+              page: page
+            })
+            eval_with_context filename, context
+          rescue SyntaxError => e
+            handle_error(e) if save
+            raise e
+          rescue => e
+            handle_error(e) if save
+            raise e
+          end
+
+          puts "=========== Parsing Executed ==========="
+          begin
+            save_pages_and_outputs(pages, outputs, :parsing)
+          rescue => e
+            handle_error(e) if save
+            raise e
+          end
+          
+          if refetch_self
+            refetch_page gid
+          elsif reparse_self
+            reparse_page gid
+          else
+            update_parsing_done_status
+          end
+        end
+        proc.call
+      end
+
+      def content
+        @content ||= get_content(gid)
+      end
+
+      def failed_content
+        @failed_content ||= get_failed_content(gid)
+      end
+
+      def handle_error(e)
+        error = ["Parsing #{e.class}: #{e.to_s} (Job:#{job_id} GID:#{gid})",clean_backtrace(e.backtrace)].join("\n")
+
+        parsing_update(
+          job_id: job_id,
+          gid: gid,
+          parsing_status: :failed,
+          log_error: error)
+      end
+
+    end
+  end
+end

data/lib/datahen/scraper/ruby_seeder_executor.rb

@@ -0,0 +1,120 @@
+module Datahen
+  module Scraper
+    class RubySeederExecutor < Executor
+      attr_accessor :save
+
+      def initialize(options={})
+        @filename = options.fetch(:filename) { raise "Filename is required"}
+        @job_id = options[:job_id]
+      end
+
+      def self.exposed_methods
+        [
+          :outputs,
+          :pages,
+          :save_pages,
+          :save_outputs,
+          :find_output,
+          :find_outputs
+        ].freeze
+      end
+
+      def exec_seeder(save=false)
+        @save = save
+        if save
+          puts "Executing seeder script"
+        else
+          puts "Trying seeder script"
+        end
+
+        eval_seeder_script(save)
+      end
+
+      def eval_seeder_script(save=false)
+        update_seeding_starting_status
+
+        proc = Proc.new do
+          outputs = []
+          pages = []
+
+          begin
+            context = isolated_binding({
+              outputs: outputs,
+              pages: pages
+            })
+            eval_with_context filename, context
+          rescue SyntaxError => e
+            handle_error(e) if save
+            raise e
+          rescue => e
+            handle_error(e) if save
+            raise e
+          end
+
+          puts "=========== Seeding Executed ==========="
+          begin
+            save_pages_and_outputs(pages, outputs, :seeding)
+          rescue => e
+            handle_error(e) if save
+            raise e
+          end
+
+          update_seeding_done_status
+        end
+        proc.call
+      end
+
+      def save_type
+        :seeding
+      end
+
+      def update_to_server(opts = {})
+        seeding_update(
+          job_id: opts[:job_id],
+          pages: opts[:pages],
+          outputs: opts[:outputs],
+          seeding_status: opts[:status])
+      end
+
+      def update_seeding_starting_status
+        return unless save
+
+        response = seeding_update(
+          job_id: job_id,
+          seeding_status: :starting)
+
+        if response.code == 200
+          puts "Seeding Status Updated."
+        else
+          puts "Error: Unable to save Seeding Status to server: #{response.body}"
+          raise "Unable to save Seeding Status to server: #{response.body}"
+        end
+      end
+
+      def update_seeding_done_status
+        return unless save
+
+        response = seeding_update(
+          job_id: job_id,
+          seeding_status: :done)
+
+        if response.code == 200
+          puts "Seeding Done."
+        else
+          puts "Error: Unable to save Seeding Done Status to server: #{response.body}"
+          raise "Unable to save Seeding Done Status to server: #{response.body}"
+        end
+      end
+
+      def handle_error(e)
+        error = ["Seeding #{e.class}: #{e.to_s} (Job:#{job_id}",clean_backtrace(e.backtrace)].join("\n")
+
+        seeding_update(
+          job_id: job_id,
+          seeding_status: :failed,
+          log_error: error)
+      end
+
+    end
+  end
+end