answersengine 0.3.3 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9ee80d67995d7fd6b3fac3b64b4d20abdc4c2dee40919863bac3a29b7aad0553
-  data.tar.gz: 6dc0454f1bb29ec09f59e69c2085f00c1eb2e2eea5bfc397669be424148964a9
+  metadata.gz: f2515f220abb2835bf815a55a62382f6929e32a2629fda0cc7504f49704ef984
+  data.tar.gz: f898220e6b2dc947316915531c6ffc6faa7011de27f1a574d92f832984d85d8c
 SHA512:
-  metadata.gz: cb52e957ac46ea26ce2c43c1daad6568e00e40aa617ec764a450837372d737abf1bd4fde706cad543972535d3c8b041826c4b2cc1ef7fb3adc3f932ce4936320
-  data.tar.gz: 2f9db966d3f56efbbcfa54e79fa57798ea36baf937a2df6872905027cfb324a110958e31a11291861be39e90d9c472bafdc8579f68a44321fb29728229acdc97
+  metadata.gz: d8a750d0d2c7099b3e8d46a39d0a5f0b874296491a5ade11d6a56c8ca7611011f7d156e39e785b19bf0c735d109bf494c20738d14db2f6aa114668422485cf32
+  data.tar.gz: 0bd4e8cf8fbdec06a034769635272ff4f295d3741704930bf0f0209f7cf055ba267080cc61f65f139e760484dc4dda5984d3d6ad8066b16f42d46b1deb1df00b

data/lib/answersengine/cli/scraper_page.rb

@@ -94,7 +94,7 @@ module AnswersEngine
             puts "Error: #{options[:vars]} on vars is not a valid JSON"
           end
         end
-      end 
+      end
 
       desc "refetch <scraper_name>", "Refetch Pages on a scraper's current job"
       long_desc <<-LONGDESC
@@ -106,35 +106,36 @@ module AnswersEngine
       def refetch(scraper_name)
         if !options.key?(:gid) && !options.key?(:fetch_fail) && !options.key?(:parse_fail)
           puts "Must specify either a --gid or --fetch-fail or --parse-fail"
-        else
-          client = Client::ScraperJobPage.new(options)
-          puts "#{client.refetch(scraper_name)}"
+          return
         end
+        client = Client::ScraperJobPage.new(options)
+        puts "#{client.refetch(scraper_name)}"
       end
 
-      desc "reset <scraper_name> <gid>", "Reset fetching and parsing of a page in a scraper's current job"
+      desc "reparse <scraper_name>", "Reparse Pages on a scraper's current job"
       long_desc <<-LONGDESC
-          Reset fetching and parsing of a page in a scraper's current job.\x5
-          LONGDESC
-      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
-      def reset(scraper_name, gid)
+        Reparse pages in a scraper's current job. You need to specify either a --gid or --parse-fail.\x5
+      LONGDESC
+      option :gid, :aliases => :g, type: :string, desc: 'Reparse a specific GID'
+      option :parse_fail, type: :boolean, desc: 'Reparse only pages that fails parsing.'
+      def reparse(scraper_name)
         begin
           options[:vars] = JSON.parse(options[:vars]) if options[:vars]
 
-          if options[:job]
-            client = Client::JobPage.new(options)
-            puts "#{client.reset(options[:job], gid, options)}"
-          else
-            client = Client::ScraperJobPage.new(options)
-            puts "#{client.reset(scraper_name, gid, options)}"
+          if !options.key?(:gid) && !options.key?(:parse_fail)
+            puts "Must specify either a --gid or --parse-fail"
+            return
           end
 
+          client = Client::ScraperJobPage.new(options)
+          puts "#{client.reparse(scraper_name)}"
+
         rescue JSON::ParserError
           if options[:vars]
             puts "Error: #{options[:vars]} on vars is not a valid JSON"
           end
         end
-      end 
+      end
 
       desc "show <scraper_name> <gid>", "Show a page in scraper's current job"
       long_desc <<-LONGDESC
@@ -166,7 +167,7 @@ module AnswersEngine
         query = {}
         query["order"] = options.delete(:head) if options[:head]
         query["job_type"] = "parsing" if options[:parsing]
-        
+
         query["page_token"] = options.delete(:more) if options[:more]
         query["per_page"] = options.delete(:per_page) if options[:per_page]
 
@@ -177,7 +178,7 @@ module AnswersEngine
         else
           result = client.scraper_all_job_page_log(scraper_name, gid, {query: query})
         end
-        
+
         if result['entries'].nil? || result["entries"].length == 0
           puts "No logs yet, please try again later."
         else

data/lib/answersengine/client/job.rb

@@ -40,7 +40,7 @@ module AnswersEngine
         body[:pages] = opts.fetch(:pages) {[]}
         body[:seeding_status] = opts.fetch(:seeding_status){ nil }
         body[:log_error] = opts[:log_error] if opts[:log_error]
-        
+
         @options.merge!({body: body.to_json})
 
         self.class.put("/jobs/#{job_id}/seeding_update", @options)
@@ -50,4 +50,3 @@ module AnswersEngine
 
   end
 end
-

data/lib/answersengine/client/job_page.rb

@@ -10,22 +10,18 @@ module AnswersEngine
       end
 
       def update(job_id, gid, opts={})
-        body = {}        
+        body = {}
         body[:page_type] = opts[:page_type] if opts[:page_type]
         body[:priority] = opts[:priority] if opts[:priority]
         body[:vars] = opts[:vars] if opts[:vars]
-        
+
         @options.merge!({body: body.to_json})
 
         self.class.put("/jobs/#{job_id}/pages/#{gid}", @options)
       end
 
-      def reset(job_id, gid, opts={})
-        self.class.put("/jobs/#{job_id}/pages/#{gid}/reset", @options)
-      end
-
       def enqueue(job_id, method, url, opts={})
-        body = {}        
+        body = {}
         body[:method] =  method != "" ? method : "GET"
         body[:url] =  url
         body[:page_type] = opts[:page_type] if opts[:page_type]
@@ -39,7 +35,7 @@ module AnswersEngine
         body[:ua_type] = opts[:ua_type] if opts[:ua_type]
         body[:no_redirect] = opts[:no_redirect] if opts[:no_redirect]
         body[:cookie] = opts[:cookie] if opts[:cookie]
-        
+
         @options.merge!({body: body.to_json})
 
         self.class.post("/jobs/#{job_id}/pages", @options)
@@ -51,7 +47,7 @@ module AnswersEngine
         body[:pages] = opts.fetch(:pages) {[]}
         body[:parsing_status] = opts.fetch(:parsing_status){ nil }
         body[:log_error] = opts[:log_error] if opts[:log_error]
-        
+
         @options.merge!({body: body.to_json})
 
         self.class.put("/jobs/#{job_id}/pages/#{gid}/parsing_update", @options)
@@ -59,4 +55,3 @@ module AnswersEngine
     end
   end
 end
-

data/lib/answersengine/client/scraper_job_page.rb

@@ -10,26 +10,28 @@ module AnswersEngine
       end
 
       def update(scraper_name, gid, opts={})
-        body = {}        
+        body = {}
         body[:page_type] = opts[:page_type] if opts[:page_type]
         body[:priority] = opts[:priority] if opts[:priority]
         body[:vars] = opts[:vars] if opts[:vars]
-        
+
         @options.merge!({body: body.to_json})
 
         self.class.put("/scrapers/#{scraper_name}/current_job/pages/#{gid}", @options)
       end
 
-      def refetch(scraper_name, opts={})
-        self.class.put("/scrapers/#{scraper_name}/current_job/pages/refetch", @options)
+      def refetch(scraper_name, opts = nil)
+        opts ||= @options
+        self.class.put("/scrapers/#{scraper_name}/current_job/pages/refetch", opts)
       end
 
-      def reset(scraper_name, gid, opts={})
-        self.class.put("/scrapers/#{scraper_name}/current_job/pages/#{gid}/reset", @options)
+      def reparse(scraper_name, opts = nil)
+        opts ||= @options
+        self.class.put("/scrapers/#{scraper_name}/current_job/pages/reparse", opts)
       end
 
       def enqueue(scraper_name, method, url, opts={})
-        body = {}        
+        body = {}
         body[:method] =  method != "" ? method : "GET"
         body[:url] =  url
         body[:page_type] = opts[:page_type] if opts[:page_type]
@@ -43,7 +45,7 @@ module AnswersEngine
         body[:ua_type] = opts[:ua_type] if opts[:ua_type]
         body[:no_redirect] = opts[:no_redirect] if opts[:no_redirect]
         body[:cookie] = opts[:cookie] if opts[:cookie]
-        
+
         @options.merge!({body: body.to_json})
 
         self.class.post("/scrapers/#{scraper_name}/current_job/pages", @options)
@@ -52,4 +54,3 @@ module AnswersEngine
     end
   end
 end
-

data/lib/answersengine/version.rb

@@ -1,3 +1,3 @@
 module AnswersEngine
-  VERSION = "0.3.3"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: answersengine
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.4.0
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-03-24 00:00:00.000000000 Z
+date: 2019-07-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor