pure-extractor 1.1.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b238e51c6777d6dd560f62b11c44e18251042918
-  data.tar.gz: 70feca57d15b4d1906010215247bd305a43d56b2
+  metadata.gz: c13f77c8908044946b4d7939c45e1b925e43a43d
+  data.tar.gz: bb730f66444a12fe629e395f2fffe970447d910d
 SHA512:
-  metadata.gz: e0351116e8b178eee808f470751460f59056e6d22f1fc3d7ecc13a85a399460102267e1f685d3d3e348ab72d3dfb1fc7a1aa41b5544974f024647a11651d4669
-  data.tar.gz: 2b63cd36a9e500d843382a14520e48eeaeb4b2fa18ae7e1a46fd268a63d3190423522ffd8986c47ce0c3337f3e15c5377ab2e1fb9cc6792e6fb5f4dcbd1ac1d7
+  metadata.gz: d80c1ae5553e9c044a5aa88b9ae61bbb68e8d7c4fba72c46ff08823a86bdeee6e1c45c0ebeb28096ad1d4b5cc91dfaeb3f3d9c112ec915ee9b5db850b8d34625
+  data.tar.gz: 69333eb907dffb2e51633907dc350646e1417ced8b2ddf93ef2425d592b01229ee16308ff36cdfac194baeba6683523608b5699ae4750324b933d01de0ddf042

data/.gitignore

@@ -9,3 +9,4 @@
 /tmp/
 *.gem
 .rbenv-gemsets
+.byebug_history

data/README.md

@@ -44,6 +44,8 @@ pure-extractor -s SERVER_URL -u USERNAME -p PASSWORD -o OUTPUT_DIRECTORY -c CHUN
 | -u, --username | Username for the Pure WS Rest service, not required if Pure WS requests are unauthenticated |
 | -p, --password | Password for the Pure WS Rest service, not required if Pure WS requests are unauthenticated |
 | -c, --chunk-size | The number of entries to return per chunk and store per file, defaults to 200 if not set |
+| -d, --request-delay | Flag to add random delay between sending API requests so as not to overload server, defaults to **false** |
+| -i, --interactive | Run in interactive mode, used not running in docker, defaults to **false** |
 
 ## Development
 

data/lib/pure/extractor.rb

@@ -1,141 +1,12 @@
 require "pure/extractor/version"
-require "pure/extractor/configure_puree"
 require "pure/extractor/commands/pure_extractor"
 require 'ruby-progressbar'
+require 'pure/extractor/formatters'
+require 'pure/extractor/extractors'
+require 'puree'
 
 module Pure
   module Extractor
     
-    def self.extract type, chunk_size, output_directory
-      
-      collection = Puree::Collection.new resource: type
-      
-      collection_count = collection.count
-      
-      puts collection_count
-      
-      progress_bar = ProgressBar.create(format: "%a %e %b\u{15E7}%i %p%% %t", progress_mark: ' ', remainder_mark: "\u{FF65}", total: collection_count)
-      
-      offset = 0
-      file_id = 0
-
-      if chunk_size.nil? || chunk_size.empty?
-        chunk_size = 200
-      end
-
-      chunk_size = chunk_size.to_i
-      
-      while offset < collection_count do
-
-        file_id += 1
-
-        filename = type.to_s + "_#{file_id.to_s.rjust(6, '0')}"
-
-        output_file = output_directory + "/#{filename}.json"
-        
-        returned_collection = collection.find limit: chunk_size, offset: offset
-        
-        returned_collection.each do |item|
-          
-          delete_keys_for_type type, item
-        
-        end
-
-        formatted_results = format_results_for_type type, returned_collection
-
-        write_results_to_file formatted_results, output_file
-        
-        update_progress_bar progress_bar, chunk_size, collection_count
-
-        offset += chunk_size
-
-      end
-      
-    end
-
-    def self.format_results_for_type type, results
-
-      formatted_results = []
-
-      case type
-
-        when :organisation
-
-          results.each do |result|
-
-            formatted_result = {
-                system: {
-                    uuid: result["uuid"],
-                    modified_at: result["modified"]
-                },
-                details: {
-                    name: result["name"],
-                    description: nil,
-                    url: result["url"][0],
-                    isni: nil,
-                    type: result["type"]
-                },
-                parent: {
-                    uuid: result["parent"]["uuid"]
-                }
-            }
-
-            formatted_results.push formatted_result
-
-          end
-
-        else
-          formatted_results = results
-
-      end
-
-      formatted_results
-
-    end
-    
-    def self.delete_keys_for_type type, item
-      
-      keys = []
-      nested_keys = {}
-      
-      case type
-        
-      when :dataset
-        
-        keys = ["keyword", "file", "associated", "link", "spatial"]
-        nested_keys = { "person" => ["external", "other"] }
-        
-      end
-      
-      keys.each do |key|
-        item.delete(key)
-      end
-      
-      nested_keys.each do |key, attribute|
-        item[key].delete(attribute)
-      end
-      
-      item
-      
-    end
-    
-    def self.update_progress_bar progress_bar, limit, collection_count
-      
-      if (progress_bar.progress + limit) < collection_count
-        progress_bar.progress += limit 
-      else
-        progress_bar.progress = collection_count
-      end
-      
-    end
-    
-    def self.write_results_to_file results, file
-      
-      File.open(file, "w") do |f|
-        f.write(JSON.pretty_generate(results))
-      end
-      
-    end
-    
   end
 end

data/lib/pure/extractor/commands/pure_command.rb

@@ -10,7 +10,9 @@ module Pure
         option ["-u", "--username"], "username", "Username to connect to Pure WS"
         option ["-p", "--password"], "password", "Password to connect to Pure WS"
         option ["-c", "--chunk-size"], "chunk-size", "Number of entities to extract per file, defaults to 200"
-        
+        option ["-d", "--request-delay"], :flag, "Add random delay between sending API Requests, default to false", default: false
+        option ["-i", "--interactive"], :flag, "Run in interactive mode, displaying progress bar. This is the default mode", default: false
+
       end
     end
   end

data/lib/pure/extractor/commands/pure_extractor.rb

@@ -5,9 +5,7 @@ module Pure
   module Extractor
     module Commands
       class PureExtractorCommand < PureCommand
-        
-        include Pure::Extractor::ConfigurePuree
-        
+
         valid_extracts = [:organisation, :people, :projects, :publications, :datasets]
         
         parameter "EXTRACT", "what to extract from pure, valid options are #{valid_extracts.map{|v| v.to_s}}" do |s|
@@ -26,9 +24,29 @@ module Pure
         
         def execute
           
-          configure_puree server, username, password
-            
-          Pure::Extractor.extract pure_collections[extract], chunk_size, output_dir
+          puree_config = {
+              url: server,
+              username: username,
+              password: password,
+              collection: pure_collections[extract],
+              chunk_size: chunk_size,
+              output_directory: output_dir,
+              delay: request_delay?
+          }
+
+          if interactive?
+
+            Pure::Extractor::Extractors::InteractiveExtractor.set_config puree_config
+            Pure::Extractor::Extractors::InteractiveExtractor.extract
+
+          else
+
+            Pure::Extractor::Extractors::LoggingExtractor.set_config puree_config
+            Pure::Extractor::Extractors::LoggingExtractor.extract
+
+          end
+
+
           
         end
         

data/lib/pure/extractor/extractors.rb

@@ -0,0 +1,11 @@
+require 'pure/extractor/extractors/extractor'
+require 'pure/extractor/extractors/interactive'
+require 'pure/extractor/extractors/logging'
+
+module Pure
+  module Extractor
+    module Extractors
+
+    end
+  end
+end

data/lib/pure/extractor/extractors/extractor.rb

@@ -0,0 +1,74 @@
+module Pure
+  module Extractor
+    module Extractors
+
+      class Extractor
+
+        @config = {}
+
+        def self.set_config config
+          @config = config
+        end
+
+        def self.extract_collection_to_disk collection, chunk_size, offset, output_file
+
+          returned_collection = collection.find limit: chunk_size, offset: offset
+
+          formatted_results = format_results_for_type @config[:collection], returned_collection
+
+          write_results_to_file formatted_results, output_file
+
+        end
+
+        def self.random_delay
+
+          random_timeout = 60 + Random.rand(120)
+
+          sleep(random_timeout)
+
+        end
+
+        def self.format_results_for_type type, results
+
+          begin
+            formatter = Module.const_get('Pure::Extractor::Formatters::' + type.to_s.capitalize)
+            formatter.format_array results
+          rescue NameError
+            raise 'No formatter for specified area'
+          end
+
+        end
+
+        def self.write_results_to_file results, file
+
+          File.open(file, "w") do |f|
+            f.write(JSON.pretty_generate(results))
+          end
+
+        end
+
+        def self.filename_for_id id
+          @config[:collection].to_s + "_#{id.to_s.rjust(6, '0')}"
+        end
+
+        def self.output_filepath_for_filename filename
+          @config[:output_directory] + "/#{filename}.json"
+        end
+
+        def self.get_chunk_size
+
+          chunk_size = @config[:chunk_size]
+
+          if chunk_size.nil? || chunk_size.empty?
+            chunk_size = 200
+          end
+
+          chunk_size.to_i
+
+        end
+
+      end
+
+    end
+  end
+end

data/lib/pure/extractor/extractors/interactive.rb

@@ -0,0 +1,60 @@
+require 'pure/extractor/extractors/extractor'
+
+module Pure
+  module Extractor
+    module Extractors
+
+      class InteractiveExtractor < Extractor
+
+        def self.extract
+
+          collection = Puree::Extractor::Collection.new config: @config, resource: @config[:collection]
+
+          collection_count = collection.count
+
+          puts "Extracting #{collection_count} records from #{@config[:collection]} collection"
+
+          progress_bar = ProgressBar.create(format: "%a %e %b\u{15E7}%i %p%% %t", progress_mark: ' ', remainder_mark: "\u{FF65}", total: collection_count)
+
+          offset = 0
+          file_id = 0
+
+          chunk_size = get_chunk_size
+
+          while offset < collection_count do
+
+            random_delay if (offset != 0) && @config[:delay]
+
+            file_id += 1
+
+            filename = filename_for_id file_id
+
+            output_file = output_filepath_for_filename filename
+
+            extract_collection_to_disk collection, chunk_size, offset, output_file
+
+            update_progress_bar progress_bar, chunk_size, collection_count
+
+            offset += chunk_size
+
+          end
+
+          puts "Finished extracting #{collection_count} records from #{@config[:collection]} collection"
+
+        end
+
+        def self.update_progress_bar progress_bar, limit, collection_count
+
+          if (progress_bar.progress + limit) < collection_count
+            progress_bar.progress += limit
+          else
+            progress_bar.progress = collection_count
+          end
+
+        end
+
+      end
+
+    end
+  end
+end

data/lib/pure/extractor/extractors/logging.rb

@@ -0,0 +1,68 @@
+require 'pure/extractor/extractors/extractor'
+
+module Pure
+  module Extractor
+    module Extractors
+
+      class LoggingExtractor < Extractor
+
+        def self.extract
+
+          collection = Puree::Extractor::Collection.new config: @config, resource: @config[:collection]
+
+          collection_count = collection.count
+
+          puts "Extracting #{collection_count} records from #{@config[:collection]} collection"
+
+          offset = 0
+          file_id = 0
+
+          chunk_size = get_chunk_size
+
+          number_of_files = get_number_of_files collection_count
+
+          while offset < collection_count do
+
+            random_delay if (offset != 0) && @config[:delay]
+
+            file_id += 1
+
+            if (offset + chunk_size) > collection_count
+              to_records = collection_count
+            else
+              to_records = offset + chunk_size
+            end
+
+            puts "Extracting records #{offset} - #{to_records} to file #{file_id} of #{number_of_files}"
+
+            filename = filename_for_id file_id
+
+            output_file = output_filepath_for_filename filename
+
+            extract_collection_to_disk collection, chunk_size, offset, output_file
+
+            puts "Extracted records to #{output_file}"
+
+            offset += chunk_size
+
+          end
+
+          puts "Finished extracting #{collection_count} records from #{@config[:collection]} collection"
+
+        end
+
+        def self.get_number_of_files collection_count
+
+          full_files, remaining_records = collection_count.divmod(get_chunk_size)
+
+          return full_files if remaining_records == 0
+
+          full_files + 1
+
+        end
+
+      end
+
+    end
+  end
+end

data/lib/pure/extractor/formatters.rb

@@ -0,0 +1,11 @@
+require 'pure/extractor/formatters/resource'
+require 'pure/extractor/formatters/organisation'
+require 'pure/extractor/formatters/person'
+
+module Pure
+  module Extractor
+    module Formatters
+
+    end
+  end
+end

data/lib/pure/extractor/formatters/organisation.rb

@@ -0,0 +1,47 @@
+require 'pure/extractor/formatters/resource'
+
+module Pure
+  module Extractor
+    module Formatters
+
+      class Organisation < Resource
+
+        def self.format unit
+
+          {
+              system: get_system(unit),
+              details: get_details(unit),
+              parent: {
+                  uuid: get_parent_uuid(unit)
+              }
+          }
+
+        end
+
+        def self.get_details result
+
+          {
+              name: result.name,
+              description: nil,
+              url: result.urls.first,
+              isni: nil,
+              type: result.type
+          }
+
+        end
+
+        def self.get_parent_uuid result
+
+          if result.parent.nil?
+            nil
+          else
+            result.parent.uuid
+          end
+
+        end
+
+      end
+
+    end
+  end
+end

data/lib/pure/extractor/formatters/person.rb

@@ -0,0 +1,28 @@
+require 'pure/extractor/formatters/resource'
+
+module Pure
+  module Extractor
+    module Formatters
+
+      class Person < Resource
+
+        def self.format unit
+
+          {
+              system: get_system(unit),
+              details: {
+                  first_name: unit.name.first,
+                  last_name: unit.name.last,
+                  email: unit.email_addresses.first,
+                  image_url: unit.image_urls.first,
+                  orcid: unit.orcid
+              }
+          }
+
+        end
+
+      end
+
+    end
+  end
+end

data/lib/pure/extractor/formatters/resource.rb

@@ -0,0 +1,31 @@
+module Pure
+  module Extractor
+    module Formatters
+
+      class Resource
+
+        def self.format_array array
+
+          results = []
+
+          array.each do |result|
+            results.push format(result)
+          end
+
+          results
+
+        end
+
+        def self.get_system result
+
+          {
+              uuid: result.uuid,
+              modified_at: result.modified
+          }
+
+        end
+
+      end
+    end
+  end
+end

data/lib/pure/extractor/version.rb

@@ -1,5 +1,5 @@
 module Pure
   module Extractor
-    VERSION = "1.1.0"
+    VERSION = "2.0.0"
   end
 end

data/pure-extractor.gemspec

@@ -27,11 +27,12 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "clamp"
-  spec.add_dependency "puree", "~> 0.19.1"
+  spec.add_dependency "puree", "~> 1.3.0"
   spec.add_dependency "ruby-progressbar"
 
   spec.add_dependency "bundler", "~> 1.12"
   
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "minitest"
+  spec.add_development_dependency "byebug"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pure-extractor
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Stephen Robinson
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-05-23 00:00:00.000000000 Z
+date: 2017-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: clamp
@@ -31,14 +31,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.19.1
+        version: 1.3.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.19.1
+        version: 1.3.0
 - !ruby/object:Gem::Dependency
   name: ruby-progressbar
   requirement: !ruby/object:Gem::Requirement
@@ -95,6 +95,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Command line application to extract data from Pure and write to JSON
   files for DMAO
 email:
@@ -114,7 +128,14 @@ files:
 - lib/pure/extractor.rb
 - lib/pure/extractor/commands/pure_command.rb
 - lib/pure/extractor/commands/pure_extractor.rb
-- lib/pure/extractor/configure_puree.rb
+- lib/pure/extractor/extractors.rb
+- lib/pure/extractor/extractors/extractor.rb
+- lib/pure/extractor/extractors/interactive.rb
+- lib/pure/extractor/extractors/logging.rb
+- lib/pure/extractor/formatters.rb
+- lib/pure/extractor/formatters/organisation.rb
+- lib/pure/extractor/formatters/person.rb
+- lib/pure/extractor/formatters/resource.rb
 - lib/pure/extractor/version.rb
 - pure-extractor.gemspec
 homepage: https://github.com/lulibrary

data/lib/pure/extractor/configure_puree.rb

@@ -1,27 +0,0 @@
-require 'puree'
-
-module Pure
-  module Extractor
-    module ConfigurePuree
-      
-      def configure_puree server, username, password
-        
-        Puree.configure do |config|
-          
-          config.base_url = server
-          
-          if !username.nil? && !password.nil? && !username.empty? && !password.empty?
-            
-            config.username = username
-            config.password = password
-            config.basic_auth = true
-            
-          end
-          
-        end
-        
-      end
-      
-    end
-  end
-end