relational_exporter 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9fef27ab7b4fa6f13f35191995f9c8590ac6bd59
-  data.tar.gz: e6bfdaae73b812140fe7ded8747d03384d8a9f4d
+  metadata.gz: 8e0567d62dfad39056f5b3ecd26f6e2392103426
+  data.tar.gz: e5eab301f0312bb97e9d90df2b24a08bb1bda317
 SHA512:
-  metadata.gz: 13c2c195e7c783b734b93d51e6aba5edac1bebf48fad6f3b149e55a28525a2f276eae232ac4aa2d3676702ba04b869d3d70d3fcb09f8a20134cda1204e744419
-  data.tar.gz: 0413a341cfed6cad14169aadfb296a2cc841cf7aadc7909b63fc992efb67bdc48b50d9d607bb676bf40f829ddd8e4e0dd470ff3b9b9c842bf549ac3011dd4fe6
+  metadata.gz: fe7517787f4eb54038f56a6591bcf03e07f8ed13315f621ca45d35b006adee7a3856ae0e28ba33cba6e3161170c667102d2641a218f88e732af89d8ca32e36c1
+  data.tar.gz: ef796345886b4f9aed27fd79df858b0577377ddc67cabf23d2cffdf0c4681f7a251ad83601898111fdc8eb3df5e265db1b4f877b30203cc3a286085b1ec75aa9

data/README.md

@@ -7,7 +7,7 @@ A gem to make it easy to export data from relational databases. RelationalExport
 * Add concurrency for faster exports
 * Support multiple formats (currently only CSV)
 * Improve DSL
-* Clean up the code
+* Clean up the code ([revise worker/job model](https://gist.github.com/schmurfy/3758436))
 
 ## Installation
 

data/lib/relational_exporter/record_worker.rb

@@ -18,6 +18,7 @@ module RelationalExporter
 
       get_rows with_headers
 
+      info "Queueing record #{record_sequence} <#{Actor.current}>…"
       Celluloid::Actor[:csv_builder].queue[record_sequence] = [@header_row, @value_row]
     end
 

data/lib/relational_exporter/version.rb

@@ -1,3 +1,3 @@
 module RelationalExporter
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end

data/lib/relational_exporter.rb

@@ -1,5 +1,6 @@
 require 'byebug'
 require 'csv'
+require 'benchmark'
 require 'hashie'
 require 'relational_exporter/version'
 require 'relational_exporter/active_record_extension'
@@ -27,40 +28,65 @@ module RelationalExporter
       load_models
     end
 
-    def export(output_config, &block)
+    def export(options, &block)
       ActiveRecord::Base.logger = @logger
       Celluloid.logger = @logger
 
-      output_config = Hashie::Mash.new output_config
+      options = Hashie::Mash.new options
 
-      main_klass = output_config.output.model.to_s.classify.constantize
+      main_klass = options.output.model.to_s.classify.constantize
 
-      main_klass.set_scope_from_hash output_config.output.scope.as_json
+      main_klass.set_scope_from_hash options.output.scope.as_json
 
-      csv_builder = RelationalExporter::CsvBuilder.new output_config.file_path
+      total_records = main_klass.find_all_by_scope(options.output.scope.as_json).count
+      remaining_records = total_records
+
+      csv_builder = RelationalExporter::CsvBuilder.new options.file_path
       Celluloid::Actor[:csv_builder] = csv_builder
       result = csv_builder.future.start
-      pool = RelationalExporter::RecordWorker.pool size: 8
+      pool_size = options.workers || 10
+      pool = RelationalExporter::RecordWorker.pool(size: pool_size)
       get_headers = true
 
       record_sequence = -1
-      main_klass.find_all_by_scope(output_config.output.scope.as_json).find_in_batches(batch_size: 100) do |records|
-        records.each do |record|
-          record_sequence += 1
-
-          args = [record_sequence, record, output_config.output.associations, get_headers]
-          if get_headers
-            pool.get_csv_row(*args)
-            get_headers = false
-          else
-            pool.async.get_csv_row(*args)
+      batch_count = 0
+
+      batch_options = Hashie::Mash.new({batch_size: 100}.merge(options.batch_options || {}))
+      limit = options.limit.to_i || nil
+
+      @logger.info "CSV export will process #{limit} of #{total_records} total records."
+
+      all_bm = Benchmark.measure do
+        catch(:hit_limit) do
+          main_klass.find_all_by_scope(options.output.scope.as_json).find_in_batches(batch_options.to_h.symbolize_keys) do |records|
+            batch_count+=1
+            batch_bm = Benchmark.measure do
+              records.each do |record|
+                record_sequence += 1
+                remaining_records -= 1
+
+                args = [record_sequence, record, options.output.associations, get_headers]
+                if get_headers
+                  pool.get_csv_row(*args)
+                  get_headers = false
+                else
+                  pool.async.get_csv_row(*args)
+                end
+
+                throw :hit_limit if record_sequence == limit
+              end
+            end
+
+            @logger.debug "Batch of #{records.size} queued. #{remaining_records} remaining. Benchmark: #{batch_bm}"
           end
         end
-      end
 
-      csv_builder.end_index = record_sequence
+        csv_builder.end_index = record_sequence
+
+        @logger.info "CSV export complete <#{options.file_path}>" if result.value === true
+      end
 
-      @logger.info "CSV export complete" if result.value === true
+      @logger.debug "#{batch_count} batches processed. Benchmark: #{all_bm}"
 
       pool.terminate
       csv_builder.terminate

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relational_exporter
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Andrew Hammond
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-18 00:00:00.000000000 Z
+date: 2014-02-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: hashie