relational_exporter 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8c1b454011ca52b5ec7122722273f8503e1bd55a
-  data.tar.gz: 3c8f0d8b4371f5177d4b04b46debd9ef0df49287
+  metadata.gz: 9fef27ab7b4fa6f13f35191995f9c8590ac6bd59
+  data.tar.gz: e6bfdaae73b812140fe7ded8747d03384d8a9f4d
 SHA512:
-  metadata.gz: 5ce81c194f3d0168cdf9dd88ba22f35e8a1909e269d925a2b560b302f3930054ca0c505d7f975ad50e52988c6cd7ea0880ddd1870e0a5f912a628428500e9890
-  data.tar.gz: 3db2e8d53cfe005eadf99f53034d50a325f0b1c4d477fd7999fae25f3019f77f5225f86854f9190c41398a21736b21af75d8c4ddee571dc36041659298ff0fa6
+  metadata.gz: 13c2c195e7c783b734b93d51e6aba5edac1bebf48fad6f3b149e55a28525a2f276eae232ac4aa2d3676702ba04b869d3d70d3fcb09f8a20134cda1204e744419
+  data.tar.gz: 0413a341cfed6cad14169aadfb296a2cc841cf7aadc7909b63fc992efb67bdc48b50d9d607bb676bf40f829ddd8e4e0dd470ff3b9b9c842bf549ac3011dd4fe6

data/lib/relational_exporter.rb

@@ -3,6 +3,9 @@ require 'csv'
 require 'hashie'
 require 'relational_exporter/version'
 require 'relational_exporter/active_record_extension'
+require 'relational_exporter/csv_builder'
+require 'relational_exporter/record_worker'
+require 'celluloid'
 
 module RelationalExporter
   class Runner
@@ -26,6 +29,7 @@ module RelationalExporter
 
     def export(output_config, &block)
       ActiveRecord::Base.logger = @logger
+      Celluloid.logger = @logger
 
       output_config = Hashie::Mash.new output_config
 
@@ -33,128 +37,36 @@ module RelationalExporter
 
       main_klass.set_scope_from_hash output_config.output.scope.as_json
 
-      header_row = []
-      max_associations = {}
-
-      csv_options = {headers: true}
-      if output_config.file_path.blank?
-        csv_method = :instance
-        csv_args = [STDOUT, csv_options]
-      else
-        csv_method = :open
-        csv_args = [output_config.file_path, 'wb', csv_options]
-      end
-
-      ::CSV.send(csv_method, *csv_args) do |csv|
-        main_klass.find_all_by_scope(output_config.output.scope.as_json).find_in_batches do |batch|
-          batch.each do |single|
-            if block_given?
-              yield single
-
-              return unless single
-            end
-
-            row = []
-
-            # Add main record headers
-            serialized_attributes(single).each do |field, value|
-              header_row << csv_header_prefix_for_key(main_klass, field) if csv.header_row?
-              row << value
-            end
-
-            output_config.output.associations.each do |association_accessor, association_options|
-              association_accessor = association_accessor.to_s.to_sym
-              association_klass = association_accessor.to_s.classify.constantize
-              scope = symbolize_options association_options.scope
-
-              associated = single.send association_accessor
-              # TODO - this might suck for single associations (has_one) because they don't return an ar::associations::collectionproxy
-              associated = associated.find_all_by_scope(scope) unless scope.blank? || !associated.respond_to?(:find_all_by_scope)
-
-              if associated.is_a? Hash
-                associated = [ associated ]
-              elsif associated.blank?
-                associated = []
-              end
-
-              foreign_key = main_klass.reflections[association_accessor].foreign_key rescue nil
-
-              fields = serialized_attributes(association_klass).keys
-
-              fields.reject! {|v| v == foreign_key } if foreign_key
-
-              if csv.header_row?
-                case main_klass.reflections[association_accessor].macro
-                when :has_many
-                  max_associated = association_klass.find_all_by_scope(scope)
-                                                    .joins(main_klass.table_name.to_sym)
-                                                    .order('count_all desc')
-                                                    .group(foreign_key)
-                                                    .limit(1).count.flatten[1]
-                when :has_one
-                  max_associated = 1
-                end
-
-                max_associated = 0 if max_associated.nil?
-
-                max_associations[association_accessor] = max_associated
-
-                max_associated.times do |i|
-                  fields.each do |field|
-                    header_row << csv_header_prefix_for_key(association_klass, field, i+1)
-                  end
-                end
-              end
-
-              get_row_arr(associated, fields, max_associations[association_accessor]) {|field| row << field}
-            end
-
-            csv << header_row if csv.header_row?
-            if row.count != header_row.count
-              @logger.error "Encountered invalid row, skipping."
-            end
-            csv << row
+      csv_builder = RelationalExporter::CsvBuilder.new output_config.file_path
+      Celluloid::Actor[:csv_builder] = csv_builder
+      result = csv_builder.future.start
+      pool = RelationalExporter::RecordWorker.pool size: 8
+      get_headers = true
+
+      record_sequence = -1
+      main_klass.find_all_by_scope(output_config.output.scope.as_json).find_in_batches(batch_size: 100) do |records|
+        records.each do |record|
+          record_sequence += 1
+
+          args = [record_sequence, record, output_config.output.associations, get_headers]
+          if get_headers
+            pool.get_csv_row(*args)
+            get_headers = false
+          else
+            pool.async.get_csv_row(*args)
           end
         end
       end
-    end
-
-    private
-
-    def csv_header_prefix_for_key(klass, key, index=nil)
-      if klass.respond_to?(:active_model_serializer) && !klass.active_model_serializer.nil? && klass.active_model_serializer.respond_to?(:csv_header_prefix_for_key)
-        header_prefix = klass.active_model_serializer.csv_header_prefix_for_key key.to_sym
-      else
-        header_prefix = klass.to_s
-      end
 
-      header_prefix + index.to_s + key.to_s.classify
-    end
-
-    def serialized_attributes(object)
-      return {} if object.nil?
-
-      klass, model = object.is_a?(Class) ? [object, object.first] : [object.class, object]
+      csv_builder.end_index = record_sequence
 
-      return {} if model.nil?
-
-      if model.respond_to?(:active_model_serializer) && !model.active_model_serializer.nil?
-        serialized = model.active_model_serializer.new(model).as_json(root: false)
-      end
+      @logger.info "CSV export complete" if result.value === true
 
-      serialized = model.attributes if serialized.nil?
-      serialized
+      pool.terminate
+      csv_builder.terminate
     end
 
-    def get_row_arr(records, fields, max_count, &block)
-      max_count.times do |i|
-        record = records[i].nil? ? {} : serialized_attributes(records[i])
-        fields.each do |field|
-          val = record[field]
-          yield val
-        end
-      end
-    end
+    private
 
     def symbolize_options(options)
       options = options.as_json

data/lib/relational_exporter/csv_builder.rb

@@ -0,0 +1,71 @@
+require 'celluloid'
+
+module RelationalExporter
+  class CsvBuilder
+    include Celluloid
+    include Celluloid::Logger
+
+    attr_accessor :queue, :end_index
+
+    trap_exit :actor_died
+    def actor_died(actor, reason)
+      warn "Oh no! #{actor.inspect} has died because of a #{reason.class}" unless reason.nil?
+    end
+
+    def initialize(file_path=nil)
+      @header_row = []
+      @index = 0
+      @end_index = nil
+      @queue = {}
+      @file_path = file_path
+    end
+
+    def start
+      csv_args = @file_path.blank? ? STDOUT : @file_path
+
+      csv_options = {headers: true}
+      if @file_path.blank?
+        csv_method = :instance
+        csv_args = [STDOUT, csv_options]
+      else
+        csv_method = :open
+        csv_args = [@file_path, 'wb', csv_options]
+      end
+
+      ::CSV.send(csv_method, *csv_args) do |csv|
+        until @index == @end_index
+          if row = @queue.delete(@index)
+            write_row(row, csv)
+            @index += 1
+          else
+            sleep 1
+          end
+        end
+      end
+
+      true
+    end
+
+    def remaining
+      @end_index - @index if @end_index
+    end
+
+    private
+
+    def write_row(row, csv)
+      headers, values = row.is_a?(Celluloid::Future) ? row.value : row
+      if csv.header_row?
+        @header_row = headers
+        info "Writing headers to file (#{headers.count})"
+        csv << @header_row
+      end
+      if values.count == @header_row.count
+        info "Writing row to file (#{@index})"
+        csv << values
+      else
+        # @logger.error "Encountered invalid row, skipping."
+        error "Bad row! #{values.count} vs #{@header_row.count}", @header_row.join(','), values.join(',')
+      end
+    end
+  end
+end

data/lib/relational_exporter/record_worker.rb

@@ -0,0 +1,130 @@
+require 'celluloid'
+
+module RelationalExporter
+  class RecordWorker
+    include Celluloid
+    include Celluloid::Logger
+
+    trap_exit :actor_died
+    def actor_died(actor, reason)
+      puts "Oh no! #{actor.inspect} has died because of a #{reason.class}" unless reason.nil?
+    end
+
+    @@MAX_ASSOCIATED = {}
+
+    def get_csv_row(record_sequence, record, associations, with_headers=false)
+      @record = record
+      @associations = associations
+
+      get_rows with_headers
+
+      Celluloid::Actor[:csv_builder].queue[record_sequence] = [@header_row, @value_row]
+    end
+
+    def get_rows(get_headers=false)
+      @header_row = []
+      @value_row = []
+      main_klass = @record.class
+
+      RecordWorker.serialized_attributes_for_object_or_class(@record).each do |field, value|
+        @header_row << RecordWorker.csv_header_prefix_for_key(main_klass, field) if get_headers
+        @value_row << value
+      end
+
+      @associations.each do |association_accessor, association_options|
+        association_accessor = association_accessor.to_s.to_sym
+        association_klass = association_accessor.to_s.classify.constantize
+        scope = RecordWorker.symbolize_options association_options.scope
+
+        associated = @record.send association_accessor
+        # TODO - this might suck for single associations (has_one) because they don't return an ar::associations::collectionproxy
+        associated = associated.find_all_by_scope(scope) unless scope.blank? || !associated.respond_to?(:find_all_by_scope)
+
+        if associated.is_a? Hash
+          associated = [ associated ]
+        elsif associated.blank?
+          associated = []
+        end
+
+        foreign_key = main_klass.reflections[association_accessor].foreign_key rescue nil
+
+        fields = RecordWorker.serialized_attributes_for_object_or_class(association_klass).keys
+
+        fields.reject! {|v| v == foreign_key } if foreign_key
+
+        if get_headers
+          @@MAX_ASSOCIATED[association_accessor] ||= begin
+            case main_klass.reflections[association_accessor].macro
+            when :has_many
+              max_associated = association_klass.find_all_by_scope(scope)
+                                                .joins(main_klass.table_name.to_sym)
+                                                .order('count_all desc')
+                                                .group(foreign_key)
+                                                .limit(1).count.flatten[1]
+            when :has_one
+              max_associated = 1
+            end
+
+            max_associated = 0 if max_associated.nil?
+            max_associated
+          end
+
+          @@MAX_ASSOCIATED[association_accessor].times do |i|
+            fields.each do |field|
+              @header_row << RecordWorker.csv_header_prefix_for_key(association_klass, field, i+1)
+            end
+          end
+        end
+
+        RecordWorker.get_maxed_row_arr(associated, fields, @@MAX_ASSOCIATED[association_accessor]) do |field|
+          @value_row << field
+        end
+      end
+    end
+
+    def self.get_maxed_row_arr(records, fields, max_count=0, &block)
+      return if max_count.nil?
+      max_count.times do |i|
+        record = records[i].nil? ? {} : RecordWorker.serialized_attributes_for_object_or_class(records[i])
+        fields.each do |field|
+          val = record[field]
+          yield val
+        end
+      end
+    end
+
+    def self.csv_header_prefix_for_key(klass, key, index=nil)
+      if klass.respond_to?(:active_model_serializer) && !klass.active_model_serializer.nil? && klass.active_model_serializer.respond_to?(:csv_header_prefix_for_key)
+        header_prefix = klass.active_model_serializer.csv_header_prefix_for_key key.to_sym
+      else
+        header_prefix = klass.to_s
+      end
+
+      header_prefix + index.to_s + key.to_s.classify
+    end
+
+    def self.serialized_attributes_for_object_or_class(object)
+      return {} if object.nil?
+
+      klass, model = object.is_a?(Class) ? [object, object.first] : [object.class, object]
+
+      return {} if model.nil?
+
+      if model.respond_to?(:active_model_serializer) && !model.active_model_serializer.nil?
+        serialized = model.active_model_serializer.new(model).as_json(root: false)
+      end
+
+      serialized = model.attributes if serialized.nil?
+      serialized
+    end
+
+    def self.symbolize_options(options)
+      options = options.as_json
+      if options.is_a? Hash
+        options.deep_symbolize_keys!
+      elsif options.is_a? Array
+        options.map { |val| RecordWorker.symbolize_options val }
+      end
+    end
+  end
+end

data/lib/relational_exporter/version.rb

@@ -1,3 +1,3 @@
 module RelationalExporter
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

data/relational_exporter.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'hashie'
   spec.add_dependency 'activesupport'
+  spec.add_dependency 'celluloid'
 
   spec.add_development_dependency 'bundler', '~> 1.3'
   spec.add_development_dependency 'rake'

metadata

@@ -1,83 +1,97 @@
 --- !ruby/object:Gem::Specification
 name: relational_exporter
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Andrew Hammond
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-12 00:00:00.000000000 Z
+date: 2014-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: hashie
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: celluloid
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Export relational databases as flat files
@@ -87,13 +101,15 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - lib/relational_exporter.rb
 - lib/relational_exporter/active_record_extension.rb
+- lib/relational_exporter/csv_builder.rb
+- lib/relational_exporter/record_worker.rb
 - lib/relational_exporter/version.rb
 - relational_exporter.gemspec
 homepage: http://github.com/andrhamm
@@ -106,19 +122,18 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.0
+rubygems_version: 2.2.0
 signing_key: 
 specification_version: 4
 summary: Export relational databases as flat files
 test_files: []
-has_rdoc: