data_seeder 0.0.5 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7a93ea7fbedc0caf70fd7e28d3b0c854ab133ecb
-  data.tar.gz: d4111c1efdf4ab10fc59f12d593b1a7159ca6626
+  metadata.gz: e24f096a1fd9c290b8e6046ae8d2a51eedb102da
+  data.tar.gz: 5d96f04e6a4fae47c640330a6a15c4f68a05a8a9
 SHA512:
-  metadata.gz: b050e33cd2128903baafe04efa16974e08d5abe2505088f1db9e659df91d0568f247bf093c4187f798f479b59e09bb9f30f2d3a843877a4e81c27588c5c94ffa
-  data.tar.gz: 63c0b2c9fa7200bdea5fc3e9ce605110542b1c2e850a7ea7f8497000d5a9542d5743bb0e3b63d3566a70c04160574732c68a929f8f2ccb2e5451389e8dcf9527
+  metadata.gz: 3b2de1af354cb2b5a0977b97f47dd8e3d22f127e777c3cdbdcd0a5f9a81ae6b118f1aecd29647f57a21d81ade2ac5ec5d5a83b069508846c5dea28a9d8cd3324
+  data.tar.gz: 72899aa6169debf7eeac35ae8d454c4a0d3befb1297a4ba73ff12935c77c92aa59210db667993a607d2c997708469afa481c7be71c39d9f7cf17827c2c9435ba

data/README.md

@@ -38,23 +38,15 @@ the following table:
 
 And you have a corresponding db/seed/countries.txt file as follows:
 
-    # config: { key_attribute: 'code', line: ->(line) { { code: line[0,2], name: line[3...-1] } } }
     AD Andorra
     AE United Arab Emirates
     AF Arghanistan
 
-The first line in a file can define the config attributes associated with the file.  For this seed file,
-the key_attribute says that it will use the 'code' attribute to lookup existing records (defaults to 'id')
-and the line function
-defines how the line is converted to an attribute hash defining the instance.
-
-Since the first line can get a little busy with config information, you can also store your config in a
-separate .cfg file with the same name.  This contents of this file should eval to a hash.  The above config line would be
-equivalent to a db/seed/countries.cfg file with the following:
+And a db/seed/countries.cfg file as follows:
 
     {
-      key_attribute: :'code',
-      line: ->(attr) {
+      key_attribute: 'code',
+      line: ->(line) {
         {
           code: line[0,2],
           name: line[3...-1]
@@ -62,6 +54,13 @@ equivalent to a db/seed/countries.cfg file with the following:
       }
     }
 
+The cfg file defines the config attributes associated with the file.  This contents of this file
+should eval to a hash.  For this seed file,
+the key_attribute says that it will use the 'code' attribute to lookup existing records (defaults to 'id')
+and the line function
+defines how the line is converted to an attribute hash defining the instance.  
+
+
 Running rake db:seed will result in the following output:
 
     # rake db:seed
@@ -97,11 +96,11 @@ your own custom loaders.
 For instance, suppose you had the following tables:
 
 ```ruby
-create_table "apps", force: :cascade do |t|
+create_table "apps" do |t|
   t.string "name"
 end
 
-create_table "app_errors", force: :cascade do |t|
+create_table "app_errors" do |t|
   t.integer "app_id"
   t.string  "code"
   t.string  "message"
@@ -122,69 +121,14 @@ And you wanted to load up separate error messages for each app such as the follo
     B1 Error message for B1
 
 
-You could create your own custom loader that might look as follows:
-
-```ruby
-require 'data_seeder'
-
-class AppErrorDataSeeder
-  include ::DataSeeder::Loader
-
-  def setup
-    @app = App.find_or_initialize_by(name: self.path_minus_ext)
-    @existing_errors = {}
-    if @app.new_record?
-      logger.info "Loading errors for new App: #{@app.name}"
-      @app.save!
-    else
-      logger.info "Loading errors for existing App: #{@app.name}"
-      @app.app_errors.each do |app_error|
-        @existing_errors[app_error.code] = app_error
-      end
-    end
-  end
-
-  def teardown
-    unless @existing_errors.empty?
-      logger.info { "  The following are begin removed:" }
-      @existing_errors.each do |code, app_error|
-        logger.info "    #{code}: #{app_error.message}"
-        app_error.destroy
-      end
-    end
-  end
-
-  def load(io)
-    io.each_line do |line|
-      line.strip!
-      next if line.blank? || line[0] == ?#
-      space_i   = line.index(' ')
-      raise "Invalid line: #{line}" unless space_i
-      code      = line[0,space_i].strip
-      message   = line[space_i+1..-1].strip
-      app_error = @existing_errors[code]
-      if app_error
-        @existing_errors.delete(code)
-        app_error.message = message
-        unless app_error.changes.empty?
-          logger.info { "  Changing #{code}: #{app_error.changes}" }
-          app_error.save!
-        end
-      else
-        logger.info { "  Creating #{code}: #{message}" }
-        @app.app_errors.create!(code: code, message: message)
-      end
-    end
-  end
-end
-```  
+Look [here](test/dummy/app/models/app_error_data_seeder.rb) for an example of creating your own custom loader.
 
-To add the seeder, you would create the following config/initializers/data_seeder.rb:
+To add this seeder, you would create the following config/initializers/data_seeder.rb:
 
 ```ruby
 MyApp::Application.config.after_initialize do
   DataSeeder.configure do |config|
-    config.add_loader('err', AppErrorDataSeeder.new)
+    config.add_loader('err', AppErrorDataSeeder)
   end
 end
 ```
@@ -200,43 +144,72 @@ Executing DataSeeder.run would result in the following:
       Creating 2: We are seriously foobared
       Creating 3: We are less seriously foobared
 
-TODO
-----
+## Configurable values
+
+#### depends
 
-Ability to specify more than 1 directory for Rails.env overrides.  Could potentially be used if you have that
-x Gigabyte seed file that you don't want to check into source control and only want run on production?
+Value or array that this model depends on such that they must be seeded first.  Examples:
 
-YAML should allow loading as either array or hash. (currently only does hash)
+    {
+      depends: ['countries','states']
+    }
 
-CSV should have options such as only: and except: for using/skipping the specified header columns.
+#### key_attribute
 
-The structure.sql caching within rails uses the file timestamp to determine whether to prepare the test database.  This
-is error prone and forces you to do a 'touch db/structure.sql' to get around the not getting reloaded problem.  Should
-I add a utility to override this rails implementation with a sha-based one like the seed files use?  (or am I the only
-one who has to 'touch db/structure.sql' everytime I switch branches?)
+The attribute used to define uniqueness within the model.  Can be a single attribute or an array. Defaults to 'id'
 
-Add 'sql' loader (with disclaimer that it will temporarily truncate the table)
+#### klass
 
-Ability to stop early when loading up a large seed file for a given environment, i.e., stop after processing the
-first 10 lines when Rails.env.test?
+Defines the ActiveRecord Class if it can't be inferred from the seed file.
 
-I want to allow different seeding for different environments.  For instance development might have a bunch of dummy
-data useful for getting an environment up and running.  I'm thinking either the seed_dir similar to like a PATH
-environment variable where the first one found would override the others, or maybe make it automatic based on the
-directory names and the environment (seed.development/state.yml would override seed/state.yml).
+#### line
 
-The test environment will be the one that will constantly being seeded after migrations or branch changes.  Some of
-the seed files might be large and take a long time to seed.  The above
-strategy using seed.test might be useful but it might also be useful to have a preprocessor type such as .sh so for
-instance you might have seed.test/table_with_lotsa_rows.csv.sh which might consist of the line
-'head -20 ../seed/table_with_lotsa_rows.csv'
+Proc used for converting a line to attributes (txt files only).
 
-Caching of long-running stuff via pg_dump, mysqldump, or other?  This belongs with discussion of the environment-specific
-seeding above.
+#### postprocess
 
-Allow config-driven initialization so that we could require: false in the Gemfile and only load as needed.
+Modify the attributes from the seed file before applying them to the model.
+
+Example:
+
+    {
+      key_attribute: 'code',
+      postprocess: ->(attrs) {
+        {
+          code: attrs['country_code'],
+          name: attrs['country']
+        }
+      }
+    }
+
+#### purge
+
+Destroys rows that no longer exist in the seed file.
+
+#### update_display_method
+
+Model method used for displaying updates to a model.
+
+#### use_line_number_as_id
+
+Use the line number of the seed file as the id
+
+## Incompatibilities from 0.0.x version
+
+Custom seeders should now be specified as a Class and not an instance (MySeeder instead of MySeeder.new)
+
+data_seeder_<config-item> methods within the models are no longer supported.
+
+Using the first line of txt, json, and yaml files as the config is no longer supported.  Move them to
+a separate .cfg file.
+
+
+TODO
+----
+
+Add 'sql' loader (with disclaimer that it will temporarily truncate the table)
 
-Add depends_on option.
+Caching of long-running stuff via pg_dump, mysqldump, or other?  
 
 Document options (key_attribute, line, postprocess, etc)
 
@@ -258,7 +231,7 @@ Author
 License
 -------
 
-Copyright 2015 Brad Pardee
+Copyright 2015-2016 Brad Pardee
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/app/models/data_seeder/seed_file.rb

@@ -3,32 +3,72 @@ require 'digest'
 module DataSeeder
   class SeedFile < ActiveRecord::Base
     def self.file_hash
-      hash = {}
-      all.each do |seed_file|
-        hash[seed_file.path] = seed_file
+      @file_hash ||= begin
+        hash = Hash.new { |hash, path| hash[path] = new(path: path) }
+        all.each do |seed_file|
+          hash[seed_file.path] = seed_file
+        end
+        hash
       end
-      hash
     end
 
     def self.load(path)
-      seed_file = self.file_hash[path] || new(path: path)
-      seed_file.load
+      seed_file = self.file_hash[path]
+      return seed_file.load
+    end
+
+    def self.processed_set
+      @processed_set ||= Set.new
+    end
+
+    def self.add_processed(path)
+      self.processed_set.add(path)
+    end
+
+    def self.processed?(paths)
+      self.processed_set.proper_superset?(Array(paths).to_set)
     end
 
     def load
       new_sha256 = Digest::SHA256.file(path).hexdigest
+      dot_index      = path.rindex('.')
+      path_minus_ext = path[0, dot_index]
       if self.sha256 != new_sha256
-        self.sha256 = new_sha256
+        cfg_file       = "#{path_minus_ext}.cfg"
+        config         = {}
+        if File.exist?(cfg_file)
+          config = eval(File.read(cfg_file))
+        end
+        begin
+          config[:klass] ||= path_minus_ext.classify.constantize
+        rescue NameError => e
+        end
         ext = File.extname(self.path)[1..-1]
-        return unless ext
-        loader = DataSeeder.config.loaders[ext]
-        unless loader
-          DataSeeder.logger.info { "Warning: No loader for #{path}"}
-          return
+        return true unless ext
+        loader_klass = config[:loader] || DataSeeder.config.loaders[ext]
+        unless loader_klass
+          DataSeeder.config.logger.warn "Warning: No loader for #{path}"
+          return true
+        end
+        if loader_klass.respond_to?(:default_config)
+          config = loader_klass.default_config.merge(config)
+        end
+        config[:path]           = path
+        config[:path_minus_ext] = path_minus_ext
+        loader                  = loader_klass.new(config)
+        depends                 = loader.config[:depends]
+        return false if depends && !self.class.processed?(depends)
+        DataSeeder.config.logger.debug { "Loading #{path}" }
+        DataSeeder.config.log_indent do
+          File.open(path, 'r') do |io|
+            loader.process(io)
+          end
+          self.sha256 = new_sha256
+          save!
         end
-        loader.process(path)
-        save!
       end
+      self.class.add_processed(path_minus_ext)
+      return true
     end
   end
 end

data/lib/data_seeder.rb

@@ -1,7 +1,6 @@
 require 'data_seeder/config'
 require 'data_seeder/engine'
 require 'data_seeder/loader'
-require 'data_seeder/logger'
 
 module DataSeeder
   class << self
@@ -22,24 +21,45 @@ module DataSeeder
     yield(config)
   end
 
-  def self.logger
-    config.logger
-  end
-
   def self.run(new_config={})
     @@mutex.synchronize do
       msec = Benchmark.ms do
         new_config.each do |key, value|
           self.config.send("#{key}=", value)
         end
-        Dir.chdir(config.seed_dir) do
-          Dir['**/*'].each do |path|
-            next if path.end_with?('.cfg')
-            SeedFile.load(path) if File.file?(path)
+        # Keep track of the seed files that have dependencies that aren't fulfilled
+        pending = []
+        config.seed_dirs.each do |seed_dir|
+          Dir.chdir(seed_dir) do
+            Dir['**/*'].each do |path|
+              next if path.end_with?('.cfg')
+              if File.file?(path)
+                unless SeedFile.load(path)
+                  pending << [seed_dir, path]
+                end
+              end
+            end
+          end
+        end
+        # Loop thru the ones that couldn't be processed previously because they depended on another seed being loaded first
+        until pending.empty?
+          new_pending = []
+          pending.each do |seed_dir, path|
+            Dir.chdir(seed_dir) do
+              unless SeedFile.load(path)
+                new_pending << [seed_dir, path]
+              end
+            end
+          end
+          if pending.size == new_pending.size
+            msg = "Error: Circular dependency in DataSeeder, seeds=#{pending.inspect}"
+            config.logger.error msg
+            raise msg
           end
+          pending = new_pending
         end
       end
-      logger.info { "DataSeeder.run took #{msec.to_i} msec" }
+      config.logger.info "DataSeeder.run took #{msec.to_i} msec"
     end
   end
 

data/lib/data_seeder/config.rb

@@ -1,28 +1,25 @@
 module DataSeeder
   class Config
-    attr_accessor :seed_dir, :logger, :loaders
+    attr_accessor :seed_dirs, :logger, :loaders, :verbose
 
     def initialize
-      @seed_dir = 'db/seed'
-      @logger   = Logger.new
-      @loaders  = default_loaders
-    end
-
-    def verbose=(verbose)
-      @logger.verbose = verbose
-    end
-
-    def verbose
-      @logger.verbose
+      @seed_dirs        = ['db/seed'].freeze
+      @loaders          = default_loaders
+      @verbose          = true
+      @is_default       = true
+      @logger           = Logger.new($stdout)
+      @logger.formatter = ->(severity, datetime, progname, msg) { "#{@indent}#{msg}\n" }
+      @indent_level     = 0
+      @indent           = ''
     end
 
     def default_loaders
       {
-        'csv'  => Loader::CSV.new,
-        'json' => Loader::JSON.new,
-        'txt'  => Loader::Txt.new,
-        'yaml' => Loader::YAML.new,
-        'yml'  => Loader::YAML.new,
+        'csv'  => Loader::CSV,
+        'json' => Loader::JSON,
+        'txt'  => Loader::Txt,
+        'yaml' => Loader::YAML,
+        'yml'  => Loader::YAML,
       }
     end
 
@@ -37,5 +34,30 @@ module DataSeeder
     def add_loader(ext, loader)
       @loaders[ext] = loader
     end
+
+    def seed_dir=(seed_dir)
+      @seed_dirs = [seed_dir]
+    end
+
+    def seed_dir
+      @seed_dirs.first
+    end
+
+    def add_seed_dir(seed_dir)
+      if @seed_dirs.frozen?
+        @seed_dirs = [seed_dir]
+      else
+        @seed_dirs << seed_dir
+      end
+    end
+
+    def log_indent(&block)
+      @indent_level += 1
+      @indent = '  ' * @indent_level
+      yield
+    ensure
+      @indent_level -= 1
+      @indent = '  ' * @indent_level
+    end
   end
 end