heydan 0.1.3.4 → 0.1.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3260ed35f30821ea42bb12967484f2993dcc74c7
-  data.tar.gz: 37b1fd894a085733cd55b925b56e6c1ea908c618
+  metadata.gz: 5eaa35cb9a73592da90cf0b1258f71e6db629fdb
+  data.tar.gz: c2b2d3d5354d8fe3a84dc0aedef4f88fff210f38
 SHA512:
-  metadata.gz: 8df0f65d8988320d296fe4022ef333cd9d546f61f27b94566a18d1d25356871fb3f9c4ae3422b5f67d4b9a4da85ef095dadf4c9d00ba6aefd4ced3b015a479d0
-  data.tar.gz: 372b5c89a7f9a3476ebd7503430496d3bf6002abea2fa7127a464b28a1025964c790b9ff91c86d14de65d90bc33f1d86ce5a1184c428167b69810e15a39147a7
+  metadata.gz: 3ab0c4108d611ea899ae93c8e5ea90be371a5d0970cd9b5ed7aabb0867720f3c4778592434e2aad037b157a8f10b65abb2fcc9962f450485136c6e4208136309
+  data.tar.gz: 8ea5c19c13da2352f32b7cc17befb0f99176ecc830374dbfae970ed39ec40ca1fd3b7a774208a000d1b15c8247ecbf78605bdacf5da3b3ec510b6a716735c432

data/heydan.gemspec

@@ -41,6 +41,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "spreadsheet", "~>1.0"
   spec.add_dependency "dbf", "~> 2.0"
   spec.add_dependency "rubyXL", "~> 3.3"
+  spec.add_dependency "parallel"
 
 
   #development/test depedencies

data/lib/heydan/cli.rb

@@ -52,10 +52,13 @@ module HeyDan
       HeyDan.options = options
       HeyDan::OpenCivicIdentifiers.build
     end
-    desc 'import', "Imports files into elasticsearch"
-    def import()
+
+    desc 'import NUMBER', "Imports files into elasticsearch, set the number to lower than 100 if the files are large"
+    option :parallel, type: :boolean
+    def import(number=100)
       HeyDan::helper_text('import')
-      HeyDan::Import.process
+      options[:parallel] ? HeyDan::Import.process_in_parallel(number) : HeyDan::Import.process(number)
+      
     end
     
     desc "sources SUBCOMMAND ...ARGS", "manage sources"

data/lib/heydan/import.rb

@@ -1,5 +1,6 @@
 require 'ruby-progressbar'
 require 'elasticsearch'
+require 'parallel'
 
 class HeyDan::Import
     attr_accessor :client
@@ -21,16 +22,16 @@ class HeyDan::Import
         client.indices.create index: index
       end
 
-      def process
+      def process(number=100)
         create_index unless check_index
         total = Dir.glob("#{HeyDan.folders[:jurisdictions]}/*").size
         files= Dir.glob("#{HeyDan.folders[:jurisdictions]}/*")
         a=0
-        b=1000
+        b=number
         progress = ProgressBar.create(:title => "Importing #{files.size} jurisdictions into Elastic Search", :starting_at => a, :total => files.size)
         while true do
           @bulk = []
-          b=( files.size - b < 1000 ? -1 : a + 1000)
+          b=( files.size - b < number ? -1 : a + number)
           files[a..b].each do |file|
             jf = HeyDan::JurisdictionFile.new(name: file)
             @bulk << { index:  { _index: 'jurisdictions', _type: jf.type, _id: jf.hash_id, data: jf.get_json } } 
@@ -45,5 +46,17 @@ class HeyDan::Import
           end
         end
       end
+
+      def process_in_parallel(number=100)
+        create_index unless check_index
+        total = Dir.glob("#{HeyDan.folders[:jurisdictions]}/*").size
+        files= Dir.glob("#{HeyDan.folders[:jurisdictions]}/*")
+        results = Parallel.map(files.each_slice(number).to_a) do |chunk|
+          chunk.each do |file|
+            jf = HeyDan::JurisdictionFile.new(name: file)
+            @client.index index: 'jurisdictions', type: jf.type, id: jf.hash_id, body: jf.get_json
+          end
+        end
+      end
     end
 end

data/lib/heydan/version.rb

@@ -1,3 +1,3 @@
 module HeyDan
-  VERSION = "0.1.3.4"
+  VERSION = "0.1.3.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: heydan
 version: !ruby/object:Gem::Version
-  version: 0.1.3.4
+  version: 0.1.3.5
 platform: ruby
 authors:
 - Dan Melton
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-16 00:00:00.000000000 Z
+date: 2015-09-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -192,6 +192,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement