datahen 0.14.9 → 0.14.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 63ae3d8aba6bb70a89033d592db3242905d70d7e0ea6a90455c370cf3b8a2dde
-  data.tar.gz: 3f4644be8702eb789f8d4c3e099750dade2e9d9f83ab6a359f222972d11938bc
+  metadata.gz: 206ac0cf7cc5dc64deecff24f1d731f353951390c4c96d3cc6721a4dd0ecb258
+  data.tar.gz: 00d8e528649b2296d585d5008ebbf2de62d100449e5841f4f220147fcebc2484
 SHA512:
-  metadata.gz: 20f50d8b4a52d360fe07f32aaa0350a190b80c30157d0d2c1e33ca745013ed8ab1a9a97d81acfc400fa6ff3250394ea44e69bfe894318d4798ec8d99567ab736
-  data.tar.gz: 2724495723be6e2d249e8697e28102674c3685ce4cb384bcefaeca03d606361bf2e0e24c098954ad5b872b7e067a0cc5d0c1ed84a10b7b703144ad3a2ba16b4f
+  metadata.gz: 2c66c9527f38db20a7f0ed589c560a2dbafdd038e1834b8552305cd665fec8063fab3c6dcd31eab9df07f2bc04ce5682272bea61772f0dfac20cf84f05b4be35
+  data.tar.gz: 6c327375e3e1eba2235194f0b58ab280b5ecbbb36fd0cf4808d9cf89b539240d0fead54ee8a4019bd0f196ddd2c7621b2ace8c1dd2da80ba6b665b1ca41a015f

data/lib/datahen/cli/global_page.rb

@@ -15,7 +15,10 @@ module Datahen
 
         if result['available'] == true
           puts "Preview content url: \"#{result['preview_url']}\""
-          `open "#{result['preview_url']}"`
+          begin
+            `open "#{result['preview_url']}"`
+          rescue
+          end
         else
           puts "Content does not exist"
         end

data/lib/datahen/cli/job.rb

@@ -6,10 +6,10 @@ module Datahen
         "#{basename} #{@package_name} #{command.usage}"
       end
 
-
       desc "list", "gets a list of jobs"
       option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
       option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
+      option :status, type: :string, desc: 'Returns jobs with a certain status'
       long_desc <<-LONGDESC
         List scrape jobs.
       LONGDESC
@@ -34,8 +34,7 @@ module Datahen
         client = Client::JobStat.new(options)
         puts "#{client.job_current_stats(job_id, options)}"
       end
-
-
+      
     end
   end
 

data/lib/datahen/cli/scraper.rb

@@ -30,6 +30,8 @@ module Datahen
       option :schedule, type: :string, desc: 'Set the schedule of the scraper to run. Must be in CRON format.'
       option :timezone, type: :string, desc: "Set the scheduler's timezone. Must be in IANA Timezone format. Defaults to \"America/Toronto\""
       option :profile, type: :string, desc: 'Set the profiles (comma separated) to apply to the job. Default: default'
+      option :multiple_jobs, type: :boolean, desc: 'Set true to enable multiple jobs. Default: false'
+      option :max_job_count, type: :numeric, desc: 'Set a value to set max number of jobs available. Set -1 for unlimited. Default: 3'
       def create(scraper_name, git_repository)
         # puts "options #{options}"
         client = Client::Scraper.new(options)
@@ -53,6 +55,8 @@ module Datahen
       option :schedule, type: :string, desc: 'Set the schedule of the scraper to run. Must be in CRON format.'
       option :timezone, type: :string, desc: "Set the scheduler's timezone. Must be in IANA Timezone format. Defaults to \"America/Toronto\""
       option :profile, type: :string, desc: 'Set the profiles (comma separated) to apply to the job. Default: default'
+      option :multiple_jobs, type: :boolean, desc: 'Set true to enable multiple jobs. Default: false'
+      option :max_job_count, type: :numeric, desc: 'Set a value to set max number of jobs available. Set -1 for unlimited. Default: 3'
       def update(scraper_name)
         client = Client::Scraper.new(options)
         puts "#{client.update(scraper_name, options)}"

data/lib/datahen/cli/scraper_export.rb

@@ -36,7 +36,10 @@ module Datahen
 
         if result['signed_url']
           puts "Download url: \"#{result['signed_url']}\""
-          `open "#{result['signed_url']}"`
+          begin
+            `open "#{result['signed_url']}"`
+          rescue
+          end
         else
           puts "Exported file does not exist"
         end

data/lib/datahen/cli/scraper_job.rb

@@ -26,6 +26,7 @@ module Datahen
       LONGDESC
       option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
       option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
+      option :status, type: :string, desc: 'Returns jobs with a certain status'
       def list(scraper_name)
         client = Client::ScraperJob.new(options)
         puts "#{client.all(scraper_name)}"
@@ -47,6 +48,21 @@ module Datahen
         end
       end
 
+      desc "delete <scraper_name>", "delete a scraper's current job"
+      long_desc <<-LONGDESC
+        Delete a scraper's current job
+      LONGDESC
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      def delete(scraper_name)
+        if options[:job]
+          client = Client::Job.new(options)
+          puts "#{client.delete(options[:job])}"
+        else
+          client = Client::ScraperJob.new(options)
+          puts "#{client.delete(scraper_name)}"
+        end
+      end
+
       desc "resume <scraper_name>", "resumes a scraper's current job"
       long_desc <<-LONGDESC
         Resumes a scraper's current job
@@ -67,13 +83,14 @@ module Datahen
         Pauses a scraper's current job
       LONGDESC
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      option :force, :aliases => :f, type: :boolean, desc: 'Force a job to be paused from a done or cancelled status'
       def pause(scraper_name)
         if options[:job]
           client = Client::Job.new(options)
-          puts "#{client.pause(options[:job])}"
+          puts "#{client.pause(options[:job], options)}"
         else
           client = Client::ScraperJob.new(options)
-          puts "#{client.pause(scraper_name)}"
+          puts "#{client.pause(scraper_name, options)}"
         end
       end
 

data/lib/datahen/cli/scraper_page.rb

@@ -224,7 +224,10 @@ module Datahen
 
         if result['available'] == true
           puts "Preview content url: \"#{result['preview_url']}\""
-          `open "#{result['preview_url']}"`
+          begin
+            `open "#{result['preview_url']}"`
+          rescue
+          end
         else
           puts "Content does not exist"
         end
@@ -244,7 +247,10 @@ module Datahen
 
         if result['available'] == true
           puts "Preview failed content url: \"#{result['preview_url']}\""
-          `open "#{result['preview_url']}"`
+          begin
+            `open "#{result['preview_url']}"`
+          rescue
+          end
         else
           puts "Failed Content does not exist"
         end

data/lib/datahen/client/auth_token.rb

@@ -16,7 +16,7 @@ module Datahen
             role: role,
             description: description}
 
-        params = @options.merge({body: body.to_json})
+        params = @options.merge({body: body.to_json}).merge(opts)
         self.class.post("/auth_tokens", params)
       end
 

data/lib/datahen/client/base.rb

@@ -58,6 +58,7 @@ module Datahen
         query[:limit] = opts[:limit] if opts[:limit]
         query[:order] = opts[:order] if opts[:order]
         query[:filter] = opts[:filter] if opts[:filter]
+        query[:force] = opts[:force] if opts[:force]
 
         if opts[:query]
           if opts[:query].is_a?(Hash)

data/lib/datahen/client/job.rb

@@ -71,6 +71,11 @@ module Datahen
         self.class.get("/jobs/#{job_id}/profile", params)
       end
 
+      def delete(job_id, opts={})
+        params = @options.merge(opts)
+        self.class.delete("/jobs/#{job_id}", params)
+      end
+
     end
 
   end

data/lib/datahen/client/scraper.rb

@@ -26,6 +26,8 @@ module Datahen
         body[:schedule] = opts[:schedule] if opts[:schedule]
         body[:timezone] = opts[:timezone] if opts[:timezone]
         body[:profile] = opts[:profile] if opts[:profile]
+        body[:multiple_jobs] = opts[:multiple_jobs] if opts[:multiple_jobs]
+        body[:max_job_count] = opts[:max_job_count] if opts[:max_job_count]
         params = @options.merge({body: body.to_json})
         self.class.post("/scrapers", params)
       end
@@ -45,6 +47,8 @@ module Datahen
         body[:schedule] = opts[:schedule] if opts[:schedule]
         body[:timezone] = opts[:timezone] if opts[:timezone]
         body[:profile] = opts[:profile] if opts[:profile]
+        body[:multiple_jobs] = opts[:multiple_jobs] if opts.has_key?("multiple_jobs") || opts.has_key?(:multiple_jobs)
+        body[:max_job_count] = opts[:max_job_count] if opts.has_key?("max_job_count") || opts.has_key?(:max_job_count)
         params = @options.merge({body: body.to_json})
 
         self.class.put("/scrapers/#{scraper_name}", params)

data/lib/datahen/client/scraper_job.rb

@@ -55,6 +55,11 @@ module Datahen
 
         self.class.get("/scrapers/#{scraper_name}/current_job/profile", params)
       end
+
+      def delete(scraper_name, opts={})
+        params = @options.merge(opts)
+        self.class.delete("/scrapers/#{scraper_name}/current_job", params)
+      end
     end
   end
 end

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "0.14.9"
+  VERSION = "0.14.16"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 0.14.9
+  version: 0.14.16
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-08-08 00:00:00.000000000 Z
+date: 2020-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -266,7 +266,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: DataHen toolbelt for developers