pure-extractor 0.6.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 37ee6bb92ce3fc3dc87712ab0d570d502a3808da
-  data.tar.gz: 3797b31f40e188eed3f64e21a9c3ced09a6e8282
+  metadata.gz: 0219796a63f3282dbcd658e866487dfafaf5e6e5
+  data.tar.gz: 690d3103961bf267a92faeb1d7587597997d32da
 SHA512:
-  metadata.gz: 354bb45f1f841b65e8ef2bf42567a529bddd711e49da98c8cee24d32be9335d663838b11473e2f63abc87752e63145e694405f7fc1ac621edcb59297e8df0210
-  data.tar.gz: 7574ccb6aade9d1a09da68c279dae15050faecb8cc5e33c626f09811f60eb6d2fc88f2a7cf3425c0c8085bb2680abedf5e307573a6a0738af535b01fb5a7f495
+  metadata.gz: cb3681cb35c6649b68865604a565a6bebaac1451fc6d42302078ba94c71c1654d50022d2d19b658bf1f70ed44d8fa3e53f8743df5b0c73c3932728f24beee314
+  data.tar.gz: a2604591ba5435cebe5900a05f63fee26dc57e1a6964f0958ee9a527133220fb61ef76582100a259d672b19ae35d93cadbf52f3565890a1c6c915103e80a8d59

data/.gitignore

@@ -8,3 +8,4 @@
 /spec/reports/
 /tmp/
 *.gem
+.rbenv-gemsets

data/lib/pure/extractor/commands/pure_command.rb

@@ -5,10 +5,11 @@ module Pure
     module Commands
       class PureCommand < Clamp::Command
         
-        option ["-o", "--output-file"], "file", "file to output to, when extracting all this is the folder to place output files", required: true
+        option ["-o", "--output-dir"], "output-dir", "Directory to store generated files in", required: true
         option ["-s", "--server"], "server", "Full url to Pure WS rest server", required: true
         option ["-u", "--username"], "username", "Username to connect to Pure WS"
         option ["-p", "--password"], "password", "Password to connect to Pure WS"
+        option ["-c", "--chunk-size"], "chunk-size", "Number of entities to extract per file, defaults to 200"
         
       end
     end

data/lib/pure/extractor/commands/pure_extractor.rb

@@ -8,7 +8,7 @@ module Pure
         
         include Pure::Extractor::ConfigurePuree
         
-        valid_extracts = [:organisation, :people, :projects, :publications, :datasets, :all]
+        valid_extracts = [:organisation, :people, :projects, :publications, :datasets]
         
         parameter "EXTRACT", "what to extract from pure, valid options are #{valid_extracts.map{|v| v.to_s}}" do |s|
           
@@ -27,26 +27,8 @@ module Pure
         def execute
           
           configure_puree server, username, password
-        
-          case extract
-            
-          when :all
-            
-            valid_extracts.each do |extract|
-              
-              next unless extract != :all
-              
-              filename = output_file + "/" + extract.to_s + ".json"
-              
-              Pure::Extractor.extract pure_collections[extract], filename
-              
-            end
-            
-          else
-            
-            Pure::Extractor.extract pure_collections[extract], output_file
             
-          end
+          Pure::Extractor.extract pure_collections[extract], chunk_size, output_dir
           
         end
         

data/lib/pure/extractor/version.rb

@@ -1,5 +1,5 @@
 module Pure
   module Extractor
-    VERSION = "0.6.0"
+    VERSION = "1.0.0"
   end
 end

data/lib/pure/extractor.rb

@@ -6,7 +6,7 @@ require 'ruby-progressbar'
 module Pure
   module Extractor
     
-    def self.extract type, output_file
+    def self.extract type, chunk_size, output_directory
       
       collection = Puree::Collection.new resource: type
       
@@ -17,13 +17,23 @@ module Pure
       progress_bar = ProgressBar.create(format: "%a %e %b\u{15E7}%i %p%% %t", progress_mark: ' ', remainder_mark: "\u{FF65}", total: collection_count)
       
       offset = 0
-      limit = 20
-      
-      results = []
+      file_id = 0
+
+      if chunk_size.nil? || chunk_size.empty?
+        chunk_size = 200
+      end
+
+      chunk_size = chunk_size.to_i
       
       while offset < collection_count do
+
+        file_id += 1
+
+        filename = type.to_s + "_#{file_id.to_s.rjust(6, '0')}"
+
+        output_file = output_directory + "/#{filename}.json"
         
-        returned_collection = collection.find limit: limit, offset: offset
+        returned_collection = collection.find limit: chunk_size, offset: offset
         
         returned_collection.each do |item|
           
@@ -31,17 +41,15 @@ module Pure
         
         end
 
-        results.concat(returned_collection)
+        formatted_results = format_results_for_type type, returned_collection
+
+        write_results_to_file formatted_results, output_file
         
-        update_progress_bar progress_bar, limit, collection_count
+        update_progress_bar progress_bar, chunk_size, collection_count
 
-        offset += limit
+        offset += chunk_size
 
       end
-
-      formatted_results = format_results_for_type type, results
-      
-      write_results_to_file formatted_results, output_file, type.to_s
       
     end
 
@@ -121,9 +129,7 @@ module Pure
       
     end
     
-    def self.write_results_to_file results, file, collection_name
-      
-      puts "Writing #{collection_name} to #{file}"
+    def self.write_results_to_file results, file
       
       File.open(file, "w") do |f|
         f.write(JSON.pretty_generate(results))

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pure-extractor
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Stephen Robinson
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-12-09 00:00:00.000000000 Z
+date: 2017-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: clamp