abstract_importer 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3f5a316e4e45d1c7a09af7b4930dda62e690d5c8
-  data.tar.gz: 333effaf5481100cd62f0f7523161747e923d515
+  metadata.gz: 390699648f23aa027bf392f0f9095bf4b6801e3b
+  data.tar.gz: 5f1d0ccb8c4db21649913fa3914b7f6a4f0ba22c
 SHA512:
-  metadata.gz: a49953a3ecee96971314712785578a4b4fb1c09a2461b02448995fe1c3589f471e158c87c6e054aa613386a5ecaa064fd49a03e009d881bc5d29a6702dd23697
-  data.tar.gz: a18398b96c81a7f8f8ecb469c76fa06b7513f534a2c7d6c9b515479c76209161f08e844bc1b7bbf768881dc8b060730aaf2e7d47b44d44a6248d1087c24887fc
+  metadata.gz: 1081c7623c65fcbd0748385d6e30f139dcf0d33703ace0c172e9b3e2d9a07aac3cabff0de5f600dc40dc0de7d28af5de2d0865960252888bb5b101711c8ed252
+  data.tar.gz: b59ab4abc7f8c9634e00c24840a01dfe0974f5b7f57e8f34b57129ad10e8d7f0c226972c3323b67107b3aeec6741b2f9e7aaf072ccc8515dbae308c59ca8072b

data/.travis.yml

@@ -0,0 +1,8 @@
+# .travis.yml
+language: ruby
+rvm:
+  - 2.0.0
+env:
+  - DB=sqlite
+script:
+  - bundle exec rake test

data/README.md

@@ -1,5 +1,9 @@
 # AbstractImporter
 
+[![Build Status](https://travis-ci.org/concordia-publishing-house/abstract_importer.png?branch=master)](https://travis-ci.org/concordia-publishing-house/abstract_importer)
+
+[![Code Climate](https://codeclimate.com/github/concordia-publishing-house/abstract_importer.png)](https://codeclimate.com/github/concordia-publishing-house/abstract_importer)
+
 AbstractImporter provides services for importing complex data from an arbitrary data source. It:
 
  * Preserves relationships between tables that are imported as a set

data/lib/abstract_importer/base.rb

@@ -4,6 +4,7 @@ require 'abstract_importer/reporter'
 require 'abstract_importer/collection'
 require 'abstract_importer/collection_importer'
 require 'abstract_importer/id_map'
+require 'abstract_importer/summary'
 
 
 module AbstractImporter
@@ -28,12 +29,12 @@ module AbstractImporter
       @dry_run      = options.fetch(:dry_run, false)
       
       @id_map       = IdMap.new
-      @summary      = {}
+      @results      = {}
       @import_plan  = self.class.import_plan.to_h
       @collections  = []
     end
     
-    attr_reader :source, :parent, :reporter, :id_map, :summary
+    attr_reader :source, :parent, :reporter, :id_map, :results
     
     def dry_run?
       @dry_run
@@ -57,7 +58,7 @@ module AbstractImporter
       
       teardown
       reporter.finish_all(self, ms)
-      @summary
+      results
     end
     
     def setup
@@ -68,7 +69,7 @@ module AbstractImporter
     end
     
     def import_collection(collection)
-      @summary[collection.name] = CollectionImporter.new(self, collection).perform!
+      results[collection.name] = CollectionImporter.new(self, collection).perform!
     end
     
     def teardown

data/lib/abstract_importer/collection_importer.rb

@@ -30,7 +30,7 @@ module AbstractImporter
       reporter.start_collection(self)
       prepare!
       
-      summary[5] = Benchmark.ms do
+      summary.ms = Benchmark.ms do
         each_new_record &method(:process_record)
       end
       
@@ -42,32 +42,10 @@ module AbstractImporter
     
     
     def prepare!
-      #          [total, existing_records, new_records, already_imported, invalid, milliseconds]
-      @summary = [    0,                0,           0,                0,       0,            0]
-      @already_imported = load_already_imported_records!
+      @summary = Summary.new
       @mappings = prepare_mappings!
     end
     
-    def load_already_imported_records!
-      # We keep a _separate_ list of legacy IDs that
-      # have already been imported. It would optimal to
-      # check @id_map to see if a record has been imported;
-      # but because of a bug with tags, that won't work:
-      #
-      # Tags import from three table: Activity, Skill,
-      # and Training. Those tables yield tags whose
-      # legacy_ids collide. As a result several tags
-      # can share the same ID; and tags that would collide
-      # are [erroneously] not imported.
-      #
-      # Fixing this problem would involve changing the
-      # legacy_id identifier for each tag which would
-      # break the connection between already-imported tags
-      # and new imports.
-      #
-      id_map[table_name]
-    end
-    
     def prepare_mappings!
       mappings = []
       model.reflect_on_all_associations.each do |association|
@@ -117,24 +95,24 @@ module AbstractImporter
     end
     
     def process_record(hash)
-      summary[0] += 1
+      summary.total += 1
       
       if already_imported?(hash)
-        summary[3] += 1
+        summary.already_imported += 1
         return
       end
       
       remap_foreign_keys!(hash)
       
       if redundant_record?(hash)
-        summary[1] += 1
+        summary.redundant += 1
         return
       end
       
       if create_record(hash)
-        summary[2] += 1
+        summary.created += 1
       else
-        summary[4] += 1
+        summary.invalid += 1
       end
     end
     
@@ -143,7 +121,7 @@ module AbstractImporter
     
     
     def already_imported?(hash)
-      @already_imported.key? hash[:id]
+      id_map.contains? table_name, hash[:id]
     end
     
     def remap_foreign_keys!(hash)

data/lib/abstract_importer/id_map.rb

@@ -19,6 +19,10 @@ module AbstractImporter
     end
     alias :[] :get
     
+    def contains?(table_name, id)
+      @id_map[table_name.to_sym].key?(id)
+    end
+    
     def <<(record)
       register(record: record)
     end

data/lib/abstract_importer/reporter.rb

@@ -43,19 +43,9 @@ module AbstractImporter
     end
     
     def finish_collection(collection, summary)
-      ms = summary[5]
-      elapsed = distance_of_time(ms)
-      stat "\n  #{summary[0]} #{collection.name} were found"
-      if summary[0] > 0
-        stat     "#{summary[3]} #{collection.name} were imported previously"
-        stat     "#{summary[1]} #{collection.name} would create duplicates and will not be imported"
-        stat     "#{summary[4]} #{collection.name} were invalid"
-        stat     "#{summary[2]} #{collection.name} were imported"
-      end
-      stat     "#{elapsed} elapsed" << (summary[0] > 0 ? " (#{(ms / summary[0]).to_i}ms each)" : "")
-      
-      print_messages(@notices, "Notices")
-      print_messages(@errors,  "Errors")
+      print_summary summary, collection.name
+      print_messages @notices, "Notices"
+      print_messages @errors,  "Errors"
     end
     
     
@@ -115,6 +105,19 @@ module AbstractImporter
       end
     end
     
+    def print_summary(summary, plural)
+      stat "\n  #{summary.total} #{plural} were found"
+      if summary.total > 0
+        stat "#{summary.already_imported} #{plural} were imported previously"
+        stat "#{summary.redundant} #{plural} would create duplicates and will not be imported"
+        stat "#{summary.invalid} #{plural} were invalid"
+        stat "#{summary.created} #{plural} were imported"
+        stat "#{distance_of_time(summary.ms)} elapsed (#{(summary.ms / summary.total).to_i}ms each)"
+      else
+        stat "#{distance_of_time(summary.ms)} elapsed"
+      end
+    end
+    
     def print_messages(array, caption)
       return if array.empty?
       status "\n--#{caption}#{("-"*(78-caption.length))}\n\n"

data/lib/abstract_importer/summary.rb

@@ -0,0 +1,9 @@
+module AbstractImporter
+  class Summary < Struct.new(:total, :redundant, :created, :already_imported, :invalid, :ms)
+    
+    def initialize
+      super(0,0,0,0,0,0)
+    end
+    
+  end
+end

data/lib/abstract_importer/version.rb

@@ -1,3 +1,3 @@
 module AbstractImporter
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: abstract_importer
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Bob Lail
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-14 00:00:00.000000000 Z
+date: 2013-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -172,6 +172,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -185,6 +186,7 @@ files:
 - lib/abstract_importer/import_options.rb
 - lib/abstract_importer/import_plan.rb
 - lib/abstract_importer/reporter.rb
+- lib/abstract_importer/summary.rb
 - lib/abstract_importer/version.rb
 - test/callback_test.rb
 - test/importer_test.rb
@@ -212,7 +214,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.2
+rubygems_version: 2.0.3
 signing_key: 
 specification_version: 4
 summary: Provides services for the mass-import of complex relational data