datahen 1.2.2 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bcc40ebf4e65038d925d3b234a42dce05d66e38ac98c254bc1bcf0800475a266
-  data.tar.gz: d9d021b4f17b6f409fed11de2f64092396df4adf3b382a910be3336d8f74e661
+  metadata.gz: 8acff6d62851c24f1bd86d993255ca3c2233ec9b45f933964bf804b49e637611
+  data.tar.gz: 978f1ae63011cd20ad978640b5fa437416cc65930b29f143f6431c14bd059840
 SHA512:
-  metadata.gz: 05b969a63bdd90b1cfe944a303fe203e6def8c5307f072c0eb152e050c5717bec8ac464a25916fbe07ccf890abc7af6befebef831d39c968c591a630915b47b6
-  data.tar.gz: 632c4ad6a55b201bd6fae08830449958558f2d220b570b5e14aee7285b433b2e5439b46e489bfeb696417af6537b3eb636216e1eb8ecc400dbaa0a85f5f8dd8d
+  metadata.gz: 4531cadcddab26fd218bd1c7462114985d72460dd1d68aaca2338bea2f7d9e16278b639795d379feaab62f68924376fd17fd93df2916069cb47797210ad6b370
+  data.tar.gz: dcd734cb8ed3876bbf7f62ed6a66004c8d21b7f8009f7b7591144887ddf5a1d12c07b6768a54e84daeaf09d20db6ea7702adaf9c526c83043351a7cf04612388

data/lib/datahen/cli/scraper_job.rb

@@ -133,6 +133,21 @@ module Datahen
         end
       end
 
+      desc "sync_schema <scraper_name>", "deploy schema config"
+      long_desc <<-LONGDESC
+        Deploy a scraper's schema config
+      LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      def sync_schema(scraper_name)
+        if options[:job]
+          client = Client::Job.new(options)
+          puts "#{client.sync_schema(options[:job])}"
+        else
+          client = Client::ScraperJob.new(options)
+          puts "#{client.sync_schema(scraper_name)}"
+        end
+      end
+
       desc "var SUBCOMMAND ...ARGS", "for managing scraper's job variables"
       subcommand "var", ScraperJobVar
 

data/lib/datahen/client/job.rb

@@ -94,6 +94,12 @@ module Datahen
         self.class.delete("/jobs/#{job_id}", params)
       end
 
+      def sync_schema(job_id, opts={})
+        params = @options.merge(opts)
+
+        self.class.put("/sync/jobs/#{job_id}/schema", params)
+      end
+
     end
 
   end

data/lib/datahen/client/scraper.rb

@@ -72,6 +72,7 @@ module Datahen
 
         self.class.get("/scrapers/#{scraper_name}/profile", params)
       end
+
     end
   end
 end

data/lib/datahen/client/scraper_job.rb

@@ -75,6 +75,11 @@ module Datahen
         params = @options.merge(opts)
         self.class.delete("/scrapers/#{scraper_name}/current_job", params)
       end
+
+      def sync_schema(scraper_name, opts={})
+        params = @options.merge(opts)
+        self.class.put("/scrapers/#{scraper_name}/current_job/sync/schema", params)
+      end
     end
   end
 end

data/lib/datahen/scraper/ruby_parser_executor.rb

@@ -108,16 +108,32 @@ module Datahen
         end
       end
 
+      def update_parsing_status page_gid, status
+        return unless save
+
+        response = parsing_update(
+          job_id: job_id,
+          gid: page_gid,
+          parsing_status: status)
+
+        if response.code == 200
+          puts "Page #{page_gid} status changed to #{status}."
+        else
+          puts "Error: Unable to change page  #{page_gid} status: #{response.body} to #{status}"
+          raise "Unable to change page  #{page_gid} status: #{response.body} to #{status}"
+        end
+      end
+
       def save_type
         :parsing
       end
 
-      def refetch_page gid
+      def refetch_page page_gid
         if save
-          Client::JobPage.new({gid: gid}).refetch(self.job_id)
-          puts "Refetch page #{gid}"
+          update_parsing_status page_gid, :to_refetch
+          puts "Refetch page #{page_gid}"
         else
-          puts "Would have refetch page #{gid}"
+          puts "Would have refetch page #{page_gid}"
         end
       end
 
@@ -130,12 +146,12 @@ module Datahen
         refetch_page page_gid
       end
 
-      def reparse_page gid
+      def reparse_page page_gid
         if save
-          Client::JobPage.new({gid: gid}).reparse(self.job_id)
-          puts "Reparse page #{gid}"
+          update_parsing_status page_gid, :to_reparse
+          puts "Reparse page #{page_gid}"
         else
-          puts "Would have reparse page #{gid}"
+          puts "Would have reparse page #{page_gid}"
         end
       end
 
@@ -148,12 +164,12 @@ module Datahen
         reparse_page page_gid
       end
 
-      def limbo_page gid
+      def limbo_page page_gid
         if save
-          Client::JobPage.new({gid: gid}).limbo(self.job_id)
-          puts "Limbo page #{gid}"
+          update_parsing_status page_gid, :limbo
+          puts "Limbo page #{page_gid}"
         else
-          puts "Would have limbo page #{gid}"
+          puts "Would have limbo page #{page_gid}"
         end
       end
 
@@ -204,13 +220,13 @@ module Datahen
           end
 
           if refetch_self
-            refetch_page gid
+            update_parsing_status gid, :to_refetch
           elsif reparse_self
-            reparse_page gid
+            update_parsing_status gid, :to_reparse
           elsif limbo_self
-            limbo_page gid
+            update_parsing_status gid, :limbo
           else
-            update_parsing_done_status
+            update_parsing_status gid, :done
           end
         end
         proc.call

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "1.2.2"
+  VERSION = "1.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 1.2.2
+  version: 1.3.0
 platform: ruby
 authors:
 - Parama Danoesubroto
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-12-14 00:00:00.000000000 Z
+date: 2023-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -278,7 +278,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://datahen.com
   source_code_uri: https://github.com/DataHenOfficial/datahen-ruby
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -293,8 +293,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: DataHen toolbelt for developers
 test_files: []