caseblocks_bucket_extractor 0.0.11 → 0.0.12

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: 19cdabd20e7d7276f24c7ae042c9480f59ec6196
-  data.tar.gz: 0257f0d5f4105e86449537ec2c73a20ad32b1d04
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    NDk0ZTE5Y2MzYjkzNzdkOWI4MjgwZjRkYmNkNDVhYjMwNGY4NmY4MQ==
+  data.tar.gz: !binary |-
+    OTk5YWRhZTNmMDA3ZWE5ZDc5NzUzMzdhOGE4MTE3ZWFkYjE1YzQ3MA==
 SHA512:
-  metadata.gz: acc06ae75580605a3b97200e6700bb6e89791ebb514a5d81a5f2a1b40bd2c25d3183dbf7f7d32b9a693cbe652967860cd785e3e776451e710bc77cce12875879
-  data.tar.gz: c6c602ead6f7d518b447e2627791c9ee08755fd13eb6e79be38f1c612cc3aa8f933ea962494e3b9b30d50fbb44d148c2a3b5add4c18be7e9c31eb2ccc4978db0
+  metadata.gz: !binary |-
+    OTRlYjE0NzI5YjVlYWUxNDI4ZjYzYjM3NjEyMDhjNmU1NGJiZDMzNzNkNGY2
+    NjY5MzkxMzYxOTliZWRkNmIyNDFjODMxYmUwMGM3OWE1YjI4OGM0MGE5YmI2
+    NjgzMjc1NzQ5MzBlOWY5NDcwMzA5MDQzYmNjMjM5ZGMzZTUyM2E=
+  data.tar.gz: !binary |-
+    ZTkxZmUxMDAwOGE5ZGI0ZWNhZjI0ODAzMDVkYjg1OTBlYjg3ZmE5ZDMzYmU1
+    MDA1MTVmOGQ1YzJhY2UyMTgxNzFmY2FhNWFkZTIzOTBiY2M0Zjk5MWI1ZTI4
+    YzU2NDZkMmQwYWZiNDQwMjUyNWE1MTZjZmI4NzU3MWJlYmNjZmE=

data/lib/caseblocks_bucket_extractor.rb

@@ -1,6 +1,7 @@
 require 'httparty'
 require 'json'
 require 'open-uri'
+require 'zip'
 
 class CaseblocksBucketExtractor
   include HTTParty
@@ -12,7 +13,7 @@ class CaseblocksBucketExtractor
   headers "Accept" => "application/json"
   headers "Content-Type" => "application/json"
 
-  def initialize(url, auth_token, download_options={:retry_limit => 60,:time_between_retries => 60})
+  def initialize(url, auth_token, extract_path, download_options={:retry_limit => 60,:time_between_retries => 60})
     self.class.base_uri url
 
     puts url
@@ -21,9 +22,11 @@ class CaseblocksBucketExtractor
 
     @retry_limit = download_options[:retry_limit]
     @time_between_retries = download_options[:time_between_retries]
+    @extract_path = extract_path
   end
 
   def download(case_type_name, bucket_slug, file_options)
+
     response = self.class.get("/case_blocks/#{URI::encode(case_type_name)}/buckets/#{bucket_slug}/schedule_download_job")
     job_url = get_job_url(response.body)
 
@@ -50,11 +53,14 @@ class CaseblocksBucketExtractor
       hol_up
       count_of_retries -= 1
 
-      response = HTTParty.get("#{job_url}?auth_token=#{self.auth_token}")
+      url = "#{job_url}?auth_token=#{self.auth_token}"
+      puts url
+      response = HTTParty.get(url)
 
       begin
         result = JSON.parse(response.body)
       rescue
+        puts response.body
         puts "STATUS UPDATE: cannot parse JSON received response code with body of #{response.body}"
         next
       end
@@ -73,11 +79,27 @@ class CaseblocksBucketExtractor
     files = json["job"]["job_log"].find{|f| f["files"] }
     file_options.each do |file_option|
       json_file = files["files"].find{|f| f["file"].include?(file_option[:name])}
-      download_file(file_option[:path], json_file["link"])
+      if json_file
+        download_file(file_option[:path], json_file["link"]) 
+        puts "Reading zip file #{file_option[:path]}"
+        Zip::File.open(file_option[:path]) do |zip_file|
+          # Handle entries one by one
+          zip_file.each do |entry|
+            # Extract to file/directory/symlink
+            extract_filepath = "#{@extract_path}/#{entry.name}"
+            puts "Extracting #{extract_filepath}"
+            entry.extract(extract_filepath) { true }
+          end
+        end
+
+      else
+        puts "Unable to find link for #{file_option[:name]}"
+      end
     end
   end
 
   def download_file(file_location,s3_link)
+    puts "Downloading #{s3_link} to #{file_location}"
     File.open(file_location, "wb") do |saved_file|
       open(s3_link, 'rb') do |read_file|
         IO.copy_stream(read_file, saved_file)
@@ -88,6 +110,8 @@ class CaseblocksBucketExtractor
   def get_job_url(response_body)
     begin
       job_url = JSON.parse(response_body)["job_url"]
+      job_url = job_url.gsub("http:", "https:") if job_url =~ /http:/
+      job_url = "#{job_url}.json" unless job_url =~ /\.json\z/
     rescue
       puts "SCHEDULE_DOWNLOAD: cannot get job_url with body of #{response_body}"
       return nil

metadata

@@ -1,10 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: caseblocks_bucket_extractor
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 0.0.12
 platform: ruby
 authors:
 - Charlie Quinn, Stewart McKee, Colin Gemmell
+- Ijonas Kisselbach
 autorequire: 
 bindir: bin
 cert_chain: []
@@ -28,25 +29,39 @@ dependencies:
   name: slop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.6.0
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
-description: A simple hello world gem
+        version: 1.1.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.7
+description: A gem to download bucket data from CaseBlocks
 email: development@emergeadapt.com
 executables:
 - bucket_downloader
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/caseblocks_bucket_extractor.rb
 - bin/bucket_downloader
+- lib/caseblocks_bucket_extractor.rb
 homepage: http://rubygems.org/gems/caseblocks_bucket_extractor
 licenses: []
 metadata: {}
@@ -56,17 +71,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.2
+rubygems_version: 2.4.6
 signing_key: 
 specification_version: 4
 summary: Gem to kick off, poll and save bucket downloads from CaseBlocks