pure-extractor 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: af0b81223fae580babc4b2c00650139f4598f1b7
-  data.tar.gz: 6a359c0e046ce6b9196aeaab5c4372778f3e1c38
+  metadata.gz: 40f1c2344052bbe0bb2dd38b51ae78e0edca2138
+  data.tar.gz: b5f685c5147b0ae84e6aa7b71b73c1658df6a42b
 SHA512:
-  metadata.gz: d23e4252837367cc9ebc244c24d9b3d201c51cbe218bcc77aa24aff4af889c99df5cdf956890c0eac3f3db752aebf604959ac6f0b24de62448b2705104b82388
-  data.tar.gz: f5353af543b3a3b50e2458da57e44dd262834f815ecbdefbb674de35e4ad3220937b7494314ee7e489c9ea45b97137d1ebbfc9893d031d6f35e99f64d7b2ba36
+  metadata.gz: 72cdca316a23fe256800d219e4e5e168f8fcb0f332967c6c23f6a6f5689e0607fe67c2e9f7b9830ea98bc3a8508c3e4e58b065886f0e2baa554fdd014d4d5554
+  data.tar.gz: 18c0a8c121a20cbb5fafbe0ee9c5a3ce60449e0b15fa600c19bb9d86a6869897b1741532ad5476d35e98d9cbad3cf6b7b6f36cd1217bfdc2a6b9ff20b73e3b87

data/lib/pure/extractor/commands/pure_command.rb

@@ -5,10 +5,10 @@ module Pure
     module Commands
       class PureCommand < Clamp::Command
         
-        option ["-o", "--output-folder"], "folder", "folder to output to", required: true
+        option ["-o", "--output-file"], "file", "file to output to, when extracting all this is the folder to place output files", required: true
         option ["-s", "--server"], "server", "Full url to Pure WS rest server", required: true
-        option ["-u", "--username"], "username", "Username to connect to Pure WS", required: true
-        option ["-p", "--password"], "password", "Password to connect to Pure WS", required: true
+        option ["-u", "--username"], "username", "Username to connect to Pure WS"
+        option ["-p", "--password"], "password", "Password to connect to Pure WS"
         
       end
     end

data/lib/pure/extractor/commands/pure_extractor.rb

@@ -8,9 +8,9 @@ module Pure
         
         include Pure::Extractor::ConfigurePuree
         
-        valid_extracts = [:organisation, :people]
+        valid_extracts = [:organisation, :people, :projects, :publications, :datasets, :all]
         
-        parameter "EXTRACT", "what to extract from pure" do |s|
+        parameter "EXTRACT", "what to extract from pure, valid options are #{valid_extracts.map{|v| v.to_s}}" do |s|
           
           s = s.to_sym
           
@@ -20,15 +20,31 @@ module Pure
           
         end
         
+        def pure_collections
+          {organisation: :organisation, people: :person, projects: :project, publications: :publication, datasets: :dataset}
+        end
+        
         def execute
           
           configure_puree server, username, password
         
           case extract
             
-          when :organisation
-          
-            Pure::Extractor::Organisation.extract output_folder
+          when :all
+            
+            valid_extracts.each do |extract|
+              
+              next unless extract != :all
+              
+              filename = output_file + "/" + extract.to_s + ".json"
+              
+              Pure::Extractor.extract pure_collections[extract], filename
+              
+            end
+            
+          else
+            
+            Pure::Extractor.extract pure_collections[extract], output_file
             
           end
           

data/lib/pure/extractor/configure_puree.rb

@@ -10,7 +10,7 @@ module Pure
           
           config.base_url = server
           
-          if !username.empty? && !password.empty?
+          if !username.nil? && !password.nil? && !username.empty? && !password.empty?
             
             config.username = username
             config.password = password

data/lib/pure/extractor/version.rb

@@ -1,5 +1,5 @@
 module Pure
   module Extractor
-    VERSION = "0.2.0"
+    VERSION = "0.3.0"
   end
 end

data/lib/pure/extractor.rb

@@ -1,11 +1,92 @@
 require "pure/extractor/version"
 require "pure/extractor/configure_puree"
 require "pure/extractor/commands/pure_extractor"
-require "pure/extractor/commands/pure_organisation_extractor"
-require "pure/extractor/organisation"
 
 module Pure
   module Extractor
-    # Your code goes here...
+    
+    def self.extract type, output_file
+      
+      collection = Puree::Collection.new resource: type
+      
+      collection_count = collection.find(limit: 1000000000, full: false).count
+      
+      puts collection_count
+      
+      progress_bar = ProgressBar.create(format: "%a %e %b\u{15E7}%i %p%% %t", progress_mark: ' ', remainder_mark: "\u{FF65}", total: collection_count)
+      
+      offset = 0
+      limit = 20
+      
+      results = []
+      
+      while offset < collection_count do
+        
+        returned_collection = collection.find limit: limit, offset: offset
+        
+        returned_collection.each do |item|
+          
+          delete_keys_for_type type, item
+        
+        end
+
+        results.concat(returned_collection)
+        
+        update_progress_bar progress_bar, limit, collection_count
+
+        offset += limit
+
+      end
+      
+      write_results_to_file results, output_file, type.to_s
+      
+    end
+    
+    def self.delete_keys_for_type type, item
+      
+      keys = []
+      nested_keys = {}
+      
+      case type
+        
+      when :dataset
+        
+        keys = ["keyword", "file", "associated", "link", "spatial"]
+        nested_keys = { "person" => ["external", "other"] }
+        
+      end
+      
+      keys.each do |key|
+        item.delete(key)
+      end
+      
+      nested_keys.each do |key, attribute|
+        item[key].delete(attribute)
+      end
+      
+      item
+      
+    end
+    
+    def self.update_progress_bar progress_bar, limit, collection_count
+      
+      if (progress_bar.progress + limit) < collection_count
+        progress_bar.progress += limit 
+      else
+        progress_bar.progress = collection_count
+      end
+      
+    end
+    
+    def self.write_results_to_file results, file, collection_name
+      
+      puts "Writing #{collection_name} to #{file}"
+      
+      File.open(file, "w") do |f|
+        f.write(results.to_json)
+      end
+      
+    end
+    
   end
 end

data/pure-extractor.gemspec

@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "clamp"
   spec.add_dependency "puree"
+  spec.add_dependency "ruby-progressbar"
 
   spec.add_dependency "bundler", "~> 1.12"
   

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pure-extractor
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Stephen Robinson
@@ -39,6 +39,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -87,7 +101,6 @@ email:
 - library.dit@lancaster.ac.uk
 executables:
 - pure-extractor
-- pure-organisation-extractor
 extensions: []
 extra_rdoc_files: []
 files:
@@ -98,13 +111,10 @@ files:
 - bin/console
 - bin/setup
 - exe/pure-extractor
-- exe/pure-organisation-extractor
 - lib/pure/extractor.rb
 - lib/pure/extractor/commands/pure_command.rb
 - lib/pure/extractor/commands/pure_extractor.rb
-- lib/pure/extractor/commands/pure_organisation_extractor.rb
 - lib/pure/extractor/configure_puree.rb
-- lib/pure/extractor/organisation.rb
 - lib/pure/extractor/version.rb
 - pure-extractor.gemspec
 homepage: https://github.com/lulibrary

data/exe/pure-organisation-extractor

@@ -1,6 +0,0 @@
-#!/usr/bin/env ruby
-
-require "bundler/setup"
-require "pure/extractor"
-
-Pure::Extractor::Commands::PureOrganisationExtractorCommand.run

data/lib/pure/extractor/commands/pure_organisation_extractor.rb

@@ -1,22 +0,0 @@
-require 'pure/extractor/commands/pure_command'
-require 'puree'
-
-module Pure
-  module Extractor
-    module Commands
-      class PureOrganisationExtractorCommand < PureCommand
-        
-        include Pure::Extractor::ConfigurePuree
-        
-        def execute
-          
-          configure_puree server, username, password
-          
-          Pure::Extractor::Organisation.extract output_folder
-          
-        end
-        
-      end
-    end
-  end
-end

data/lib/pure/extractor/organisation.rb

@@ -1,51 +0,0 @@
-require 'puree'
-require 'json'
-
-module Pure
-  module Extractor
-    class Organisation
-      
-      def self.extract output_folder
-        
-        filename = "organisation.json"
-        
-        org = Puree::Collection.new resource: :organisation
-        
-        org_uuids = org.find limit: 1000000000, full: false
-        
-        org_uuids.count
-        
-        offset = 0
-        limit = 20
-        
-        orgs = []
-      
-        while offset < org_uuids.count do
-          
-          returned_orgs = org.find limit: limit, offset: offset
-          
-          returned_orgs.each do |r_org|
-          
-            r_org.delete("address")
-            r_org.delete("email")
-            r_org.delete("organisation")
-            r_org.delete("phone")
-            r_org.delete("url")
-          
-          end
-
-          orgs.concat(returned_orgs)
-
-          offset += limit
-
-        end
-        
-        File.open(output_folder + "/" + filename, "w") do |f|
-          f.write(orgs.to_json)
-        end
-        
-      end
-      
-    end
-  end
-end