data_seeder 0.0.5 → 1.0.0

data/lib/data_seeder/loader.rb

@@ -2,68 +2,42 @@ require 'English'
 
 module DataSeeder
   module Loader
-    attr_accessor :file_config, :key_attribute
-    attr_reader   :path, :path_minus_ext
-
-    def initialize(options={})
-      @only = options[:only]
-      @except = options[:except]
-      if options.has_key?(:purge)
-        @purge = options[:purge]
-      else
-        @purge = true
-      end
-      @old_keys = []
-    end
-
-    def config
-      DataSeeder.config
+    attr_reader :seeder_config, :config, :key_attribute, :klass, :path, :path_minus_ext
+
+    def initialize(config)
+      @seeder_config  = DataSeeder.config
+      @config         = config
+      @key_attribute  = config[:key_attribute] || :id
+      @klass          = config[:klass]
+      @path           = config[:path]
+      @path_minus_ext = config[:path_minus_ext]
+      # Default purge to true if unspecified
+      @config[:purge] = true unless config.has_key?(:purge)
+      @old_ids        = Set.new
     end
 
     def logger
-      DataSeeder.logger
-    end
-
-    def klass!
-      @path_minus_ext.classify.constantize
-    rescue NameError => e
-      raise "#{@path} doesn't match a corresponding model"
-    end
-
-    def klass
-      # This should always translate to a class except for custom loaders
-      @path_minus_ext.classify.constantize rescue nil
+      @seeder_config.logger
     end
 
-    def process(path)
-      @path           = path
-      dot_index       = @path.rindex('.')
-      @path_minus_ext = @path[0, dot_index]
-      @file_config    = {}
-      cfg_file = "#{@path_minus_ext}.cfg"
-      @file_config = eval(File.read(cfg_file)) if File.exist?(cfg_file)
-      File.open(@path, 'r') do |fin|
-        load_file_config(fin) if @file_config.empty?
-        @file_config = ActiveSupport::HashWithIndifferentAccess.new(@file_config)
-        setup
-        load(fin)
-        teardown
-      end
-      call_file_method(:teardown)
+    def process(io)
+      setup
+      load(io)
+      teardown
     end
 
     def setup
-      @key_attribute = self.file_config[:key_attribute] || :id
-      @old_keys = self.klass!.all.pluck(@key_attribute).map(&:to_s) if @purge
-      logger.info { "Loading #{@path}" }
-      call_file_method(:setup)
+      @old_ids = klass.all.pluck(:id).to_set if config[:purge]
     end
 
     def teardown
-      @old_keys.each do |key|
-        if model = self.klass!.find_by(@key_attribute => key)
-          logger.info { "  Destroying #{model_info(model)}"}
-          model.destroy
+      destroy_models(klass, @old_ids)
+    end
+
+    def destroy_models(klass, ids)
+      ids.each do |id|
+        if model = klass.find_by(id: id)
+          destroy_model(model)
         end
       end
     end
@@ -72,68 +46,89 @@ module DataSeeder
     # The changes argument will be the model.changes on an update.
     def model_info(model, changes=nil)
       if changes
-        attr = @file_config[:update_display_method] || @key_attribute
-        "#{model.send(attr)}: #{changes.inspect}"
+        if attr = config[:update_display_method]
+          "#{model.send(attr)}: #{changes.inspect}"
+        elsif @key_attribute.kind_of?(Enumerable)
+          label = @key_attribute.map {|k| "#{k}=#{model.send(k)}"}.join(' ')
+          "#{label}: #{changes.inspect}"
+        else
+          "#{model.send(@key_attribute)}: #{changes.inspect}"
+        end
       else
         model.inspect
       end
     end
 
-    def load_file_config(fin)
-      config_line = fin.readline
-      if match = config_line.match(/^\s*#\s*config:(.*)/)
-        @file_config = eval(match[1])
-      else
-        fin.seek(0)
-        if self.klass && self.klass.respond_to?(:data_seeder_config)
-          @file_config = self.klass.data_seeder_config
-        end
-      end
-    end
-
-    def load(fin)
+    def load(io)
       throw 'Must override load'
     end
 
+    # This doesn't work in some versions of JRuby (version 9.0.3.0?)
     def line_number
       $INPUT_LINE_NUMBER
     end
 
     def save(attr)
-      if @file_config[:use_line_number_as_id]
-        key = self.line_number
+      attr = call_method(:postprocess, attr) || attr
+      if config[:use_line_number_as_id]
+        find_hash = { @key_attribute => self.line_number }
+      elsif @key_attribute.kind_of?(Enumerable)
+        find_hash = {}
+        @key_attribute.each do |k|
+          find_hash[k] = attr[k.to_s] || attr[k.to_sym]
+        end
       else
         key = attr[@key_attribute.to_s] || attr[@key_attribute.to_sym]
         raise "No #{@key_attribute} in #{attr.inspect}" unless key
+        find_hash = { @key_attribute => key }
       end
-      if method = @file_config[:postprocess]
-        method.call(attr)
-      elsif self.klass!.respond_to?(:data_seeder_postprocess)
-        self.klass!.send(:data_seeder_postprocess, attr)
-      end
-      @old_keys.delete(key.to_s)
-      model = self.klass!.find_or_initialize_by(@key_attribute => key)
+      model = self.klass.find_or_initialize_by(find_hash)
       model.attributes = attr
       save_model(model)
     end
 
     def save_model(model)
       if model.new_record?
-        logger.info { "  Saving #{model_info(model)}" }
+        log_save(model)
       else
-        changes = model.changes
-        return if changes.empty?
-        logger.info { "  Updating #{model_info(model, changes)}" }
+        @old_ids.delete(model.id)
+        return unless model.changed?
+        log_update(model)
       end
       model.save!
     end
 
-    def call_file_method(name, *args)
-      if method = @file_config[name]
-        return method.call(*args)
-      else
-        class_method = "data_seeder_#{name}"
-        return self.klass.send(class_method, *args) if @klass.respond_to?(class_method)
+    # Allow override for potential soft-delete
+    def destroy_model(model)
+      log_destroy(model)
+      model.destroy
+    end
+
+    def log_save(model)
+      logger.debug { "Saving #{model_info(model)}" }
+    end
+
+    def log_update(model)
+      logger.debug { "Updating #{model_info(model, model.changes)}" }
+    end
+
+    def log_destroy(model)
+      logger.debug { "Destroying #{model_info(model)}" }
+    end
+
+    def log_indent(&block)
+      @seeder_config.log_indent(&block)
+    end
+
+    def call_method(name, *args)
+      if self.respond_to?(name)
+        return send(name, *args)
+      elsif val = config[name]
+        if val.kind_of?(Proc)
+          return val.call(*args)
+        else
+          return val
+        end
       end
       return nil
     end

data/lib/data_seeder/loader/csv.rb

@@ -9,7 +9,8 @@ module DataSeeder
 
       def load(io)
         @line_number = 0
-        ::CSV.foreach(io, headers: true) do |row|
+        csv = ::CSV.new(io, headers: true)
+        csv.each do |row|
           @line_number += 1
           save(row.to_hash)
         end

data/lib/data_seeder/loader/txt.rb

@@ -4,16 +4,11 @@ module DataSeeder
       include Loader
 
       def load(io)
-        if method = self.file_config[:line]
+        if method = config[:line]
           io.each_line do |line|
             next if line.blank? || line.match(/^\s*#/)
             save(method.call(line))
           end
-        elsif self.klass.respond_to?(:data_seeder_line)
-          io.each_line do |line|
-            next if line.blank? || line.match(/^\s*#/)
-            save(self.klass.send(:data_seeder_line, line))
-          end
         else
           raise "No line method defined for #{self.klass.name}"
         end

data/lib/data_seeder/loader/yaml.rb

@@ -6,7 +6,7 @@ module DataSeeder
       include Loader
 
       def load(io)
-        yaml = ::YAML.load(io.read)
+        yaml = ::YAML.load(ERB.new(io.read).result)
         if yaml.kind_of?(Hash)
           yaml.each do |key, attr|
             attr[self.key_attribute] = key if self.key_attribute

data/lib/data_seeder/version.rb

@@ -1,3 +1,3 @@
 module DataSeeder
-  VERSION = "0.0.5"
+  VERSION = "1.0.0"
 end

data/test/dummy/app/models/app_error_data_seeder.rb

@@ -7,10 +7,10 @@ class AppErrorDataSeeder
     @app = App.find_or_initialize_by(name: self.path_minus_ext)
     @existing_errors = {}
     if @app.new_record?
-      logger.info "Loading errors for new App: #{@app.name}"
+      logger.debug { "Loading errors for new App: #{@app.name}" }
       @app.save!
     else
-      logger.info "Loading errors for existing App: #{@app.name}"
+      logger.debug { "Loading errors for existing App: #{@app.name}" }
       @app.app_errors.each do |app_error|
         @existing_errors[app_error.code] = app_error
       end
@@ -19,10 +19,12 @@ class AppErrorDataSeeder
 
   def teardown
     unless @existing_errors.empty?
-      logger.info { "  The following are begin removed:" }
-      @existing_errors.each do |code, app_error|
-        logger.info "    #{code}: #{app_error.message}"
-        app_error.destroy
+      logger.debug { "The following are begin removed:" }
+      log_indent do
+        @existing_errors.each do |code, app_error|
+          logger.debug { "#{code}: #{app_error.message}" }
+          app_error.destroy
+        end
       end
     end
   end
@@ -40,11 +42,11 @@ class AppErrorDataSeeder
         @existing_errors.delete(code)
         app_error.message = message
         unless app_error.changes.empty?
-          logger.info { "  Changing #{code}: #{app_error.changes}" }
+          logger.debug { "Changing #{code}: #{app_error.changes}" }
           app_error.save!
         end
       else
-        logger.info { "  Creating #{code}: #{message}" }
+        logger.debug { "Creating #{code}: #{message}" }
         @app.app_errors.create!(code: code, message: message)
       end
     end

data/test/dummy/app/models/country.rb

@@ -1,14 +1,2 @@
 class Country < ActiveRecord::Base
-  def self.data_seeder_line(line)
-    {
-      code: line[0,2],
-      name: line[3...-1],
-    }
-  end
-
-  def self.data_seeder_config
-    {
-      key_attribute: :code
-    }
-  end
 end

data/test/dummy/db/seed.test/countries_csv/countries.cfg

@@ -0,0 +1,9 @@
+{ 
+  key_attribute: 'code', 
+  postprocess: ->(attrs) { 
+    { 
+      code: attrs['country_code'],
+      name: attrs['country']
+    }
+  }
+}

data/test/dummy/db/seed.test/countries_csv/countries.csv

@@ -0,0 +1,249 @@
+country_code,country
+AD,"Andorra"
+AE,"United Arab Emirates"
+AF,"Afghanistan"
+AG,"Antigua and Barbuda"
+AI,"Anguilla"
+AL,"Albania"
+AM,"Armenia"
+AO,"Angola"
+AQ,"Antarctica"
+AR,"Argentina"
+AS,"American Samoa"
+AT,"Austria"
+AU,"Australia"
+AW,"Aruba"
+AX,"Åland Islands"
+AZ,"Azerbaijan"
+BA,"Bosnia and Herzegovina"
+BB,"Barbados"
+BD,"Bangladesh"
+BE,"Belgium"
+BF,"Burkina Faso"
+BG,"Bulgaria"
+BH,"Bahrain"
+BI,"Burundi"
+BJ,"Benin"
+BL,"Saint Barthélemy"
+BM,"Bermuda"
+BN,"Brunei Darussalam"
+BO,"Bolivia, Plurinational State of"
+BQ,"Bonaire, Sint Eustatius and Saba"
+BR,"Brazil"
+BS,"Bahamas"
+BT,"Bhutan"
+BV,"Bouvet Island"
+BW,"Botswana"
+BY,"Belarus"
+BZ,"Belize"
+CA,"Canada"
+CC,"Cocos (Keeling) Islands"
+CD,"Congo, the Democratic Republic of the"
+CF,"Central African Republic"
+CG,"Congo"
+CH,"Switzerland"
+CI,"Côte d'Ivoire"
+CK,"Cook Islands"
+CL,"Chile"
+CM,"Cameroon"
+CN,"China"
+CO,"Colombia"
+CR,"Costa Rica"
+CU,"Cuba"
+CV,"Cape Verde"
+CW,"Curaçao"
+CX,"Christmas Island"
+CY,"Cyprus"
+CZ,"Czech Republic"
+DE,"Germany"
+DJ,"Djibouti"
+DK,"Denmark"
+DM,"Dominica"
+DO,"Dominican Republic"
+DZ,"Algeria"
+EC,"Ecuador"
+EE,"Estonia"
+EG,"Egypt"
+EH,"Western Sahara"
+ER,"Eritrea"
+ES,"Spain"
+ET,"Ethiopia"
+FI,"Finland"
+FJ,"Fiji"
+FK,"Falkland Islands (Malvinas)"
+FM,"Micronesia, Federated States of"
+FO,"Faroe Islands"
+FR,"France"
+GA,"Gabon"
+GB,"United Kingdom"
+GD,"Grenada"
+GE,"Georgia"
+GF,"French Guiana"
+GG,"Guernsey"
+GH,"Ghana"
+GI,"Gibraltar"
+GL,"Greenland"
+GM,"Gambia"
+GN,"Guinea"
+GP,"Guadeloupe"
+GQ,"Equatorial Guinea"
+GR,"Greece"
+GS,"South Georgia and the South Sandwich Islands"
+GT,"Guatemala"
+GU,"Guam"
+GW,"Guinea-Bissau"
+GY,"Guyana"
+HK,"Hong Kong"
+HM,"Heard Island and McDonald Islands"
+HN,"Honduras"
+HR,"Croatia"
+HT,"Haiti"
+HU,"Hungary"
+ID,"Indonesia"
+IE,"Ireland"
+IL,"Israel"
+IM,"Isle of Man"
+IN,"India"
+IO,"British Indian Ocean Territory"
+IQ,"Iraq"
+IR,"Iran, Islamic Republic of"
+IS,"Iceland"
+IT,"Italy"
+JE,"Jersey"
+JM,"Jamaica"
+JO,"Jordan"
+JP,"Japan"
+KE,"Kenya"
+KG,"Kyrgyzstan"
+KH,"Cambodia"
+KI,"Kiribati"
+KM,"Comoros"
+KN,"Saint Kitts and Nevis"
+KP,"Korea, Democratic People's Republic of"
+KR,"Korea, Republic of"
+KW,"Kuwait"
+KY,"Cayman Islands"
+KZ,"Kazakhstan"
+LA,"Lao People's Democratic Republic"
+LB,"Lebanon"
+LC,"Saint Lucia"
+LI,"Liechtenstein"
+LK,"Sri Lanka"
+LR,"Liberia"
+LS,"Lesotho"
+LT,"Lithuania"
+LU,"Luxembourg"
+LV,"Latvia"
+LY,"Libya"
+MA,"Morocco"
+MC,"Monaco"
+MD,"Moldova, Republic of"
+ME,"Montenegro"
+MF,"Saint Martin (French part)"
+MG,"Madagascar"
+MH,"Marshall Islands"
+MK,"Macedonia, the former Yugoslav Republic of"
+ML,"Mali"
+MM,"Myanmar"
+MN,"Mongolia"
+MO,"Macao"
+MP,"Northern Mariana Islands"
+MQ,"Martinique"
+MR,"Mauritania"
+MS,"Montserrat"
+MT,"Malta"
+MU,"Mauritius"
+MV,"Maldives"
+MW,"Malawi"
+MX,"Mexico"
+MY,"Malaysia"
+MZ,"Mozambique"
+NA,"Namibia"
+NC,"New Caledonia"
+NE,"Niger"
+NF,"Norfolk Island"
+NG,"Nigeria"
+NI,"Nicaragua"
+NL,"Netherlands"
+NO,"Norway"
+NP,"Nepal"
+NR,"Nauru"
+NU,"Niue"
+NZ,"New Zealand"
+OM,"Oman"
+PA,"Panama"
+PE,"Peru"
+PF,"French Polynesia"
+PG,"Papua New Guinea"
+PH,"Philippines"
+PK,"Pakistan"
+PL,"Poland"
+PM,"Saint Pierre and Miquelon"
+PN,"Pitcairn"
+PR,"Puerto Rico"
+PS,"Palestine, State of"
+PT,"Portugal"
+PW,"Palau"
+PY,"Paraguay"
+QA,"Qatar"
+RE,"Réunion"
+RO,"Romania"
+RS,"Serbia"
+RU,"Russian Federation"
+RW,"Rwanda"
+SA,"Saudi Arabia"
+SB,"Solomon Islands"
+SC,"Seychelles"
+SD,"Sudan"
+SE,"Sweden"
+SG,"Singapore"
+SH,"Saint Helena, Ascension and Tristan da Cunha"
+SI,"Slovenia"
+SJ,"Svalbard and Jan Mayen"
+SK,"Slovakia"
+SL,"Sierra Leone"
+SM,"San Marino"
+SN,"Senegal"
+SO,"Somalia"
+SR,"Suriname"
+SS,"South Sudan"
+ST,"Sao Tome and Principe"
+SV,"El Salvador"
+SX,"Sint Maarten (Dutch part)"
+SY,"Syrian Arab Republic"
+SZ,"Swaziland"
+TC,"Turks and Caicos Islands"
+TD,"Chad"
+TF,"French Southern Territories"
+TG,"Togo"
+TH,"Thailand"
+TJ,"Tajikistan"
+TK,"Tokelau"
+TL,"Timor-Leste"
+TM,"Turkmenistan"
+TN,"Tunisia"
+TO,"Tonga"
+TR,"Turkey"
+TT,"Trinidad and Tobago"
+TV,"Tuvalu"
+TW,"Taiwan, Province of China"
+TZ,"Tanzania, United Republic of"
+UA,"Ukraine"
+UG,"Uganda"
+UM,"United States Minor Outlying Islands"
+US,"United States"
+UY,"Uruguay"
+UZ,"Uzbekistan"
+VA,"Holy See (Vatican City State)"
+VC,"Saint Vincent and the Grenadines"
+VE,"Venezuela, Bolivarian Republic of"
+VG,"Virgin Islands, British"
+VI,"Virgin Islands, U.S."
+VN,"Viet Nam"
+VU,"Vanuatu"
+WF,"Wallis and Futuna"
+WS,"Samoa"
+YE,"Yemen"
+YT,"Mayotte"
+ZA,"South Africa"
+ZM,"Zambia"