answersengine 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be8e1b2d4a1ca60a2b0f1ccc1fd5c00f33aa2f412214d9294e357ec2bfd353f1
-  data.tar.gz: ec3e082f0b8905a313f43a1d7d6018640c06baa6d06501f62126cab2dbe0af32
+  metadata.gz: e1025b7622002c5eb5fef3328f51171e574404f8c54d2c334a88a9a432a02e34
+  data.tar.gz: ce8c68dbeecd1c6151fc294395b59051cac3584419012991d31d184204160732
 SHA512:
-  metadata.gz: 5dec73a502a3431a5471f02a2b4093d57ed0cd23528a5ea6ec3915ea67031e724471df15d36499e5217a37ad0f32a51a87e7329d095670baedde6f5186ee2f42
-  data.tar.gz: a2864beba98cdc6660e091649e89fb0756f0edd3528443ba8231ae17d079cd1efa8adab80329271f32b7dc80ab68b0f1ca9d12897dea82ab0284d66bda2566f3
+  metadata.gz: a0708e1189d35a1b3b3b8a19ad98340dc461a88079b7e5587f3622183e6235531299ceef7e42931e958d6ac62c17778183f7d26e730c01df5c83b6eab364b18b
+  data.tar.gz: 97085efab1572619ef43d7014da56d768686b72a4c60ae00bdea516a619832444370d0a8024678f9896111d9eedd17d5ef4679e4c2a524c7b9918664f4557b4f

data/lib/answersengine/client/scraper_job_page.rb

@@ -26,11 +26,21 @@ module AnswersEngine
         self.class.put("/scrapers/#{scraper_name}/current_job/pages/refetch", params)
       end
 
+      def refetch_by_job(job_id, opts={})
+        params = @options.merge(opts)
+        self.class.put("/jobs/#{job_id}/pages/refetch", params)
+      end
+
       def reparse(scraper_name, opts={})
         params = @options.merge(opts)
         self.class.put("/scrapers/#{scraper_name}/current_job/pages/reparse", params)
       end
 
+      def reparse_by_job(job_id, opts={})
+        params = @options.merge(opts)
+        self.class.put("/jobs/#{job_id}/pages/reparse", params)
+      end
+
       def enqueue(scraper_name, method, url, opts={})
         body = {}
         body[:method] =  method != "" ? method : "GET"

data/lib/answersengine/scraper/ruby_parser_executor.rb

@@ -2,6 +2,13 @@ module AnswersEngine
   module Scraper
     class RubyParserExecutor < Executor
       attr_accessor :save
+      # Refetch self page flag.
+      # @return [Boollean]
+      # @note It is stronger than #reparse_self flag.
+      attr_accessor :refetch_self
+      # Reparse self page flag.
+      # @return [Boollean]
+      attr_accessor :reparse_self
 
       def initialize(options={})
         @filename = options.fetch(:filename) { raise "Filename is required"}
@@ -20,7 +27,9 @@ module AnswersEngine
           :save_pages,
           :save_outputs,
           :find_output,
-          :find_outputs
+          :find_outputs,
+          :refetch,
+          :reparse
         ].freeze
       end
 
@@ -87,6 +96,42 @@ module AnswersEngine
         :parsing
       end
 
+      def refetch_page gid
+        if save
+          Client::ScraperJobPage.new({gid: gid}).refetch_by_job(self.job_id)
+          puts "Refetch page #{gid}"
+        else
+          puts "Would have refetch page #{gid}"
+        end
+      end
+
+      def refetch page_gid
+        raise ArgumentError.new("page_gid needs to be a String.") unless page_gid.is_a?(String)
+        if page_gid == gid
+          self.refetch_self = true
+          return
+        end
+        refetch_page page_gid
+      end
+
+      def reparse_page gid
+        if save
+          Client::ScraperJobPage.new({gid: gid}).reparse_by_job(self.job_id)
+          puts "Reparse page #{gid}"
+        else
+          puts "Would have reparse page #{gid}"
+        end
+      end
+
+      def reparse page_gid
+        raise ArgumentError.new("page_gid needs to be a String.") unless page_gid.is_a?(String)
+        if page_gid == gid
+          self.reparse_self = true
+          return
+        end
+        reparse_page page_gid
+      end
+
       def eval_parser_script(save=false)
         update_parsing_starting_status
 
@@ -95,6 +140,8 @@ module AnswersEngine
           outputs = []
           pages = []
           page = init_page_vars(page)
+          self.refetch_self = false
+          self.reparse_self = false
 
           begin
             context = isolated_binding({
@@ -113,7 +160,13 @@ module AnswersEngine
 
           puts "=========== Parsing Executed ==========="
           save_pages_and_outputs(pages, outputs, :parsing)
-          update_parsing_done_status
+          if refetch_self
+            refetch_page gid
+          elsif reparse_self
+            reparse_page gid
+          else
+            update_parsing_done_status
+          end
         end
         proc.call
       end

data/lib/answersengine/version.rb

@@ -1,3 +1,3 @@
 module AnswersEngine
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: answersengine
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-08-14 00:00:00.000000000 Z
+date: 2019-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor