datahen 1.3.1 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: af6aefb0106af924ba636a91de29b27953fbf403ede51a779b9787a3f135d1b9
-  data.tar.gz: ce6f5dedbdff08f2034cb631e103c905ee82642b86349fdbbecc878281764c6a
+  metadata.gz: ae63999d11bc052d81e3b1de67a0741702dd980e719dd544b5e689f0383e7a34
+  data.tar.gz: faf53b662afa26409bff83c3007127211863ce33ff45b8c60aab56491fdcafe7
 SHA512:
-  metadata.gz: ccaa0107c92694865efbbf8c1b8e8d5d100d60145c3f33439abe5561ea6f220efa8f499764b152fdc90f0173433e380b697b4bd3eaee68d7b0681815c789bcc5
-  data.tar.gz: c4859147460d3600e76d903755eef91cb114c65b7abb17c0ab093fa9f23e7e298dd02a42fc1da8a25c6d50edee06c17b8175dca4d61b62ec012c947f1904d2f4
+  metadata.gz: b0e7a0ddc975202df66785211cee796e1aef61de921f99ba0481645f59fb65963c03517e1d4f5b471d2ed108087011f20d6b693d6d199e2e96c860412b675415
+  data.tar.gz: 70a13268ba6f3df8f560a1b4d65b261ec4744bbd71eb22b04c25dc5809e70af3541cc30d9dad0e4a4cc22c3a353b90bc24c4a43cfc70e7f82fc2080001596d38

data/lib/datahen/cli/scraper_page.rb

@@ -28,46 +28,48 @@ module Datahen
         end
       end
 
-      desc "add <scraper_name> <url>", "Enqueues a page to a scraper's current job"
+      desc "add <scraper_name> <page_json>", "Enqueues a page to a scraper's current job"
       long_desc <<-LONGDESC
           Enqueues a page to a scraper's current job\x5
           LONGDESC
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
-      option :method, :aliases => :m, type: :string, desc: 'Set request method. Default: GET'
-      option :headers, :aliases => :H, type: :string, banner: :JSON, desc: 'Set request headers. Must be in json format. i.e: {"Foo":"bar"} '
-      option :cookie, :aliases => :c, type: :string, desc: 'Set request cookie.'
-      option :vars, :aliases => :v, type: :string, banner: :JSON, desc: 'Set user-defined page variables. Must be in json format. i.e: {"Foo":"bar"}'
-      option :page_type, :aliases => :t, desc: 'Set page type'
-      option :priority, type: :numeric, desc: 'Set fetch priority. The higher the value, the sooner the page gets fetched. Default: 0'
-      option :fetch_type, :aliases => :F, desc: 'Set fetch type. Default: http'
-      option :body, :aliases => :b, desc: 'Set request body'
-      option :force_fetch, :aliases => :f, type: :boolean, desc: 'Set true to force fetch page that is not within freshness criteria. Default: false'
-      option :freshness, :aliases => :s, desc: 'Set how fresh the page cache is. Accepts timestap format.'
-      option :ua_type, :aliases => :u, desc: 'Set user agent type. Default: desktop'
-      option :no_redirect, :aliases => :n, type: :boolean, desc: 'Set true to not follow redirect. Default: false'
-      option :max_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
-      option :retry_interval, type: :numeric, desc: 'Set a value to set retry time interval on seconds when refetching a page. Set a value grather than 0 to set it as new time to refetch, 0 means default time. Default: 0'
-      def add(scraper_name, url)
+      def add(scraper_name, page_json)
         begin
-          options[:headers] = JSON.parse(options[:headers]) if options[:headers]
-          options[:vars] = JSON.parse(options[:vars]) if options[:vars]
-          method = options[:method]
+          page = JSON.parse(page_json)
 
           if options[:job]
             client = Client::JobPage.new(options)
-            puts "#{client.enqueue(options[:job], method, url, options)}"
+            puts "#{client.enqueue(options[:job], page, options)}"
           else
             client = Client::ScraperJobPage.new(options)
-            puts "#{client.enqueue(scraper_name, method, url, options)}"
+            puts "#{client.enqueue(scraper_name, page, options)}"
           end
 
         rescue JSON::ParserError
-          if options[:headers]
-            puts "Error: #{options[:headers]} on headers is not a valid JSON"
-          end
-          if options[:vars]
-            puts "Error: #{options[:vars]} on vars is not a valid JSON"
+            puts "Error: Invalid JSON"
+        end
+      end
+
+
+      desc "getgid <scraper_name> <page_json>", "Get the generated GID for a scraper's current job"
+      long_desc <<-LONGDESC
+          Get the generated GID for a scraper's current job.\x5
+          LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      def getgid(scraper_name, page_json)
+        begin
+          page = JSON.parse(page_json)
+          
+          if options[:job]
+            client = Client::JobPage.new(options)
+            puts "#{client.get_gid(options[:job], page,  options)}"
+          else
+            client = Client::ScraperJobPage.new(options)
+            puts "#{client.get_gid(scraper_name, page, options)}"
           end
+
+        rescue JSON::ParserError
+          puts "Error: Invalid JSON"
         end
       end
 

data/lib/datahen/client/job_page.rb

@@ -24,28 +24,18 @@ module Datahen
         self.class.put("/jobs/#{job_id}/pages/#{gid}", params)
       end
 
-      def enqueue(job_id, method, url, opts={})
-        body = {}
-        body[:method] =  method != "" ? method : "GET"
-        body[:url] =  url
-        body[:page_type] = opts[:page_type] if opts[:page_type]
-        body[:priority] = opts[:priority] if opts[:priority]
-        body[:fetch_type] = opts[:fetch_type] if opts[:fetch_type]
-        body[:body] = opts[:body] if opts[:body]
-        body[:headers] = opts[:headers] if opts[:headers]
-        body[:vars] = opts[:vars] if opts[:vars]
-        body[:force_fetch] = opts[:force_fetch] if opts[:force_fetch]
-        body[:freshness] = opts[:freshness] if opts[:freshness]
-        body[:ua_type] = opts[:ua_type] if opts[:ua_type]
-        body[:no_redirect] = opts[:no_redirect] if opts[:no_redirect]
-        body[:cookie] = opts[:cookie] if opts[:cookie]
-        body[:max_size] = opts[:max_size] if opts[:max_size]
-        body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
-        body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
-
-        params = @options.merge({body: body.to_json})
+      def enqueue(job_id, page, opts={})
+        params = @options.merge(opts).merge({body: page.to_json})
 
         self.class.post("/jobs/#{job_id}/pages", params)
+        
+      end
+
+      def get_gid(job_id, page, opts={})
+      
+        params = @options.merge(opts).merge({body: page.to_json})
+
+        self.class.post("/jobs/#{job_id}/generate_gid", params)
       end
 
       def dequeue(job_id, limit, page_types, parse_fetching_failed, opts = {})

data/lib/datahen/client/scraper_job_page.rb

@@ -47,30 +47,19 @@ module Datahen
         self.class.put("/scrapers/#{scraper_name}/current_job/pages/limbo", params)
       end
 
-      def enqueue(scraper_name, method, url, opts={})
-        body = {}
-        body[:method] =  method != "" ? method : "GET"
-        body[:url] =  url
-        body[:page_type] = opts[:page_type] if opts[:page_type]
-        body[:priority] = opts[:priority] if opts[:priority]
-        body[:fetch_type] = opts[:fetch_type] if opts[:fetch_type]
-        body[:body] = opts[:body] if opts[:body]
-        body[:headers] = opts[:headers] if opts[:headers]
-        body[:vars] = opts[:vars] if opts[:vars]
-        body[:force_fetch] = opts[:force_fetch] if opts[:force_fetch]
-        body[:freshness] = opts[:freshness] if opts[:freshness]
-        body[:ua_type] = opts[:ua_type] if opts[:ua_type]
-        body[:no_redirect] = opts[:no_redirect] if opts[:no_redirect]
-        body[:cookie] = opts[:cookie] if opts[:cookie]
-        body[:max_size] = opts[:max_size] if opts[:max_size]
-        body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
-        body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
-
-        params = @options.merge({body: body.to_json})
+      def enqueue(scraper_name, page, opts={})
+      params = @options.merge(opts).merge({body: page.to_json})
 
         self.class.post("/scrapers/#{scraper_name}/current_job/pages", params)
       end
 
+      def get_gid(scraper_name, page, opts={})
+      
+        params = @options.merge(opts).merge({body: page.to_json})
+
+        self.class.post("/scrapers/#{scraper_name}/current_job/generate_gid", params)
+      end
+
       def find_content(scraper_name, gid)
         self.class.get("/scrapers/#{scraper_name}/current_job/pages/#{gid}/content", @options)
       end

data/lib/datahen/scraper/batch_parser.rb

@@ -219,7 +219,7 @@ module Datahen
         self.dequeuer_is_alive!
 
         # ensure a valid response or try again
-        if response.nil? || response.response.code.to_i != 200
+        if response.body.nil? || response.body.empty? || response.response.code.to_i != 200
           self.repeat_puts(response.nil? ? 'null' : response.body)
           self.recollect_garbage
           return 0

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "1.3.1"
+  VERSION = "1.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.4.0
 platform: ruby
 authors:
 - Parama Danoesubroto
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-06-05 00:00:00.000000000 Z
+date: 2023-11-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -278,7 +278,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://datahen.com
   source_code_uri: https://github.com/DataHenOfficial/datahen-ruby
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -293,8 +293,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: DataHen toolbelt for developers
 test_files: []