config_scripts 0.4.1

data/lib/config_scripts/seeds/seed_set.rb

@@ -0,0 +1,321 @@
+module ConfigScripts
+  module Seeds
+    # This class represents a set of related seeds.
+    #
+    # These seeds will be stored as CSV files in a folder together.
+    class SeedSet
+      # @return [String]
+      # The name of the folder for this seed set.
+      attr_reader :name
+
+      # @return [Integer]
+      # A number identifying this set. Seed sets will be run from the one with
+      # the lowest number to the highest.
+      attr_accessor :set_number
+
+      # @return [String]
+      # The name of the folder to which we will write the seeds.
+      attr_reader :folder
+
+      # @return [Hash]
+      # Arbitrary extra data passed in when defining the seed set.
+      attr_reader :options
+
+      # @return [Hash]
+      # A hash mapping class names to the {SeedType} instances describing how
+      # to handle seeds for that class within this set.
+      attr_reader :seed_types
+
+      # @return [Proc]
+      # The block that will be run when resetting the records during a load.
+      attr_reader :reset_block
+
+      class << self
+        # @!group Registration
+
+        # @return [Hash<Integer, SeedSet>]
+        # The seed sets that have been defined.
+        attr_reader :registered_sets
+
+        # This method adds a new seed set to our registry.
+        #
+        # If there is already a registered seed set with this set's set_number,
+        # the number will be incremented until it is available.
+        #
+        # @param [SeedSet] set
+        #   The new seed set.
+        #
+        # @return [SeedSet]
+        def register_seed_set(set)
+          @registered_sets ||= {}
+          while @registered_sets[set.set_number]
+            return if @registered_sets[set.set_number] == set
+            set.set_number += 1
+          end
+          @registered_sets[set.set_number] = set
+        end
+
+        # This method wipes out our records of registered seed sets.
+        #
+        # @return [Hash]
+        #   The new list of seed sets.
+        def clear_registered_sets
+          @registered_sets = {}
+        end
+
+        # This method loads all of the seed definitions from the files in the
+        # +db/seeds/definitions+ directory.
+        def load_seed_sets
+          Dir[Rails.root.join('db', 'seeds', 'definitions', '*')].each do |file|
+            require file
+          end
+        end
+
+        # @!group Batch Operations
+
+        # This method writes the data for every seed set to its seed data
+        # folder.
+        #
+        # @param [Integer] set_number
+        #   The number of the set to write.
+        #
+        # @return [Array]
+        def write(set_number=nil)
+          self.each_set(set_number, &:write)
+        end
+
+        # This method loads the data from every seed set into the database.
+        #
+        # @param [Integer] set_number
+        #   The number of the set to read.
+        #
+        # @return [Array]
+        def read(set_number=nil)
+          self.each_set set_number do |set|
+            set.read(set_number.present?)
+          end
+        end
+
+        # This method lists every seed set, with its set number.
+        # @return [Array]
+        def list
+          self.each_set do |set|
+            puts "#{set.set_number}: #{set.name}"
+          end
+        end
+
+        # This method runs a block on each set that the app has defined.
+        #
+        # The block will be given one parameter, which is the seed set.
+        #
+        # @param [String] set_number
+        #   The number of the set that we should run.
+        #
+        # @return [Array]
+        def each_set(set_number=nil, &block)
+          @registered_sets ||= {}
+          self.load_seed_sets
+          if set_number
+            if self.registered_sets[set_number]
+              block.call(self.registered_sets[set_number])
+            end
+          else
+            self.registered_sets.keys.sort.each do |set_number|
+              block.call(self.registered_sets[set_number])
+            end
+          end
+        end
+      end
+
+      # @!group Creation
+
+      # This method creates a new seed set.
+      #
+      # It should be given a block, which will be run on the instance, and
+      # which should use the {#seeds_for} method to define seed types.
+      #
+      # @param [String] name
+      #   The name for the folder for the seeds.
+      #
+      # @param [Integer] set_number
+      #   The set_number in which this seed set should be run.
+      #
+      # @param [String] folder
+      #   The folder that we should use for this seed set. If this is not
+      #   provided, we will use the name.
+      #
+      # @param [Hash] options
+      #   Additional information that can be made accessible to the seed type
+      #   definitions.
+      def initialize(name, set_number=1, folder=nil, options = {}, &block)
+        @name = name.to_s
+        @set_number = set_number
+        @folder = folder || @name
+        @options = options
+        @seed_types = {}
+        self.instance_eval(&block) if block_given?
+        ConfigScripts::Seeds::SeedSet.register_seed_set(self)
+      end
+
+      # @!group Reading and Writing
+
+      # This method writes the data for this seed set to its seed folder.
+      #
+      # It will create the folder and then write the file for each seed type
+      # that has been defined.
+      def write
+        folder_path = Rails.root.join('db', 'seeds', 'data', self.folder)
+        FileUtils.mkdir_p(folder_path)
+        puts "Writing seeds for #{self.name} to #{folder_path}"
+        self.seed_types.each do |klass, seed_type|
+          seed_type.write_to_folder(folder_path)
+        end
+      end
+
+      # This method reads the data for this seed set from its seed folder.
+      #
+      # @param [Boolean] reset
+      #   Whether we should reset the existing records before loading the seeds.
+      #
+      # It will load the data for each seed type's file, enclosing all the
+      # seed types in a transaction block.
+      def read(reset=false)
+        folder_path = Rails.root.join('db', 'seeds', 'data', self.folder)
+        FileUtils.mkdir_p(folder_path)
+        puts "Reading seeds for #{self.name} from #{folder_path}"
+        ActiveRecord::Base.transaction do
+          self.reset_records if reset
+          self.seed_types.each do |klass, seed_type|
+            seed_type.read_from_folder(folder_path)
+          end
+        end
+      end
+
+      # This method resets all the existing records that could be populated by
+      # this seed set.
+      def reset_records
+        self.reset_block.call if self.reset_block
+      end
+
+      # @!group DSL
+
+      # This method defines a new seed type within this seed set.
+      #
+      # This method should be given a block, which will be passed to the
+      # initializer for the new seed type.
+      #
+      # @param [Class] klass
+      #   The model class whose seed data this stores.
+      #
+      # @param [String] filename
+      #   The name of the file in which the seed data should be stored.
+      #   If this is not provided, it will use the name of the class.
+      #   This should not include the file extension.
+      #
+      # @return [SeedType]
+      def seeds_for(klass, filename=nil, &block)
+        filename ||= klass.name.underscore.pluralize
+        @seed_types[klass] = SeedType.new(self, klass, filename, &block)
+      end
+
+      # This method defines a block that will be run when resetting existing
+      # records.
+      #
+      # This block will be run when loading a seed set as a one-off, but not
+      # when loading all the seed sets.
+      #
+      # @return [Proc]
+      def when_resetting(&block)
+        @reset_block = block
+      end
+
+      # @!group Seed Identifiers
+
+      # This method gets a seed type that we have on file for a class.
+      #
+      # @param [Class] klass
+      #   The class whose seeds we are dealing with.
+      #
+      # @return [SeedType]
+      def seed_type_for_class(klass)
+        while klass && klass != ActiveRecord::Base
+          seed_type = self.seed_types[klass]
+          if seed_type
+            return seed_type
+          end
+          klass = klass.superclass
+        end
+      end
+
+
+      # This method gets a unique identifier for a record when writing seeds
+      # that refer to it.
+      #
+      # It will look for a seed type that handles seeds for a class in the
+      # record's class heirarchy, and use that seed type's
+      # {#seed_identifier_for_record} method. If it cannot find a seed type,
+      # it will use the record's ID.
+      #
+      # @param [ActiveRecord::Base] record
+      #   The record whose identifier we are generating.
+      #
+      # @return [String]
+      def seed_identifier_for_record(record)
+        seed_type = self.seed_type_for_class(record.class)
+        if seed_type
+          seed_type.seed_identifier_for_record(record)
+        else
+          record.id rescue nil
+        end
+      end
+
+      # This method finds a record based on a unique identifier in the seed
+      # data.
+      #
+      # This will look for a seed type for the class, and use its
+      # {#record_for_seed_identifier} method to get the record.
+      #
+      # The result of this will be memoized so that we do not have to keep
+      # looking up the records.
+      #
+      # @param [Class] klass
+      #   The model class for the record we are finding.
+      #
+      # @param [Array<String>] identifier
+      #   The identifier components from the seed data.
+      #
+      # @return [ActiveRecord::Base]
+      #   The model record.
+      def record_for_seed_identifier(klass, identifier)
+        seed_type = self.seed_type_for_class(klass)
+        return nil unless seed_type
+
+        @record_cache ||= {}
+        @record_cache[klass] ||= {}
+
+        cache_identifier = identifier.dup
+
+        while cache_identifier.any?
+          record = @record_cache[klass][cache_identifier]
+          if record
+            cache_identifier.count.times { identifier.shift }
+            return record
+          end
+          cache_identifier.pop
+        end
+
+        if seed_type
+          cache_identifier = identifier.dup
+          record = seed_type.record_for_seed_identifier(identifier)
+
+          if identifier.any?
+            cache_identifier = cache_identifier[0...-1*identifier.count]
+          end
+
+          @record_cache[klass][cache_identifier] = record
+        end
+        record
+      end
+    end
+  end
+end

data/lib/config_scripts/seeds/seed_type.rb

@@ -0,0 +1,361 @@
+require 'csv'
+
+module ConfigScripts
+  module Seeds
+    # This class encapsulates information about how to write seeds for a class
+    # to a seed file.
+    class SeedType
+      # @!group Attributes
+
+      # @return [SeedSet]
+      # The seed set that this type has been defined within.
+      attr_reader :seed_set
+
+      # @return [Class]
+      # The model class whose records we are storing in this seed file.
+      attr_reader :klass
+
+      # @return [String]
+      # The name of the file that we will store these records in.
+      attr_reader :filename
+
+      # @return [Array<Symbol>]
+      # The names of the attributes on the model object that we store in the
+      # seed file.
+      attr_reader :attributes
+
+      # @return [Array<Symbol>]
+      # The names of the attributes used to compose a unique identifier for
+      # a record.
+      attr_reader :identifier_attributes
+
+      # @return [Array]
+      # The active record associations for the model class for this seed file.
+      attr_reader :associations
+
+      # @return [Hash<Symbol, Proc>]
+      # The attributes that we generate dynamically after loading the ones from
+      # the seed file.
+      attr_reader :dynamic_readers
+
+      # @return [Hash<Symbol, Proc>]
+      # The attributes that we generate dynamically when writing things to the
+      # seed file.
+      attr_reader :dynamic_writers
+
+      # @return [Array<Array>]
+      # The scopes that we apply when fetching items.
+      #
+      # Each entry will be an array. The first entry in the inner arrays will
+      # be a symbol, the name of a method that can be run on a relation. The 
+      # result of the array will be passed in when running the method on the
+      # scope.
+      attr_reader :scopes
+
+      # @!group Creation
+
+      # This method creates a new seed type.
+      #
+      # This method should be given a block, which will be run in the instance
+      # context of the new seed type. That block should use the DSL methods to
+      # fill in the details for the seed type.
+      #
+      # @param [SeedSet] seed_set
+      #   The seed set that this seed type is defined within.
+      #
+      # @param [Class] klass
+      #   The model class whose data we are running.
+      #
+      # @param [String] filename
+      #   The name of the file in which the seed data will be stored.
+      def initialize(seed_set, klass, filename, &block)
+        @seed_set = seed_set
+        @klass = klass
+        @filename = filename
+        @attributes = []
+        @identifier_attributes = [:id]
+        @scopes = []
+        @dynamic_writers = {}
+        @dynamic_readers = {}
+
+        @associations = {}
+        @klass.reflect_on_all_associations.each do |association|
+          @associations[association.name] = association.klass rescue nil
+        end
+
+        self.instance_eval(&block) if block_given?
+      end
+
+      # @!group DSL
+
+      # This method adds new attributes to the ones written in this seed type.
+      #
+      # @param [Array<Symbol>] new_attributes
+      #   The attributes to add.
+      #
+      # @return [Array<Symbol>]
+      #   The full list of attributes after the new ones are added.
+      def has_attributes(*new_attributes)
+        @attributes += new_attributes
+      end
+
+      # This method defines the attributes used to generate a unique identifier
+      # for a record.
+      #
+      # @param [Array<Symbol>] attributes
+      #   The attributes that form the unique identifier.
+      #
+      # @return [Array<Symbol>]
+      #   The attributes.
+      def has_identifier_attributes(*attributes)
+        @identifier_attributes = attributes
+      end
+
+      # This method adds a scope to the list used to filter the records for
+      # writing.
+      #
+      # @param [Symbol] method
+      #   The name of the method to call on the relation.
+      #
+      # @param [Array] args
+      #   The arguments that will be passed into the scope method on the
+      #   relation.
+      #
+      # @return [Array]
+      #   The full list of scopes.
+      def has_scope(method, *args)
+        @scopes << [method, args]
+      end
+
+      # This method registers a custom block that will be run when reading a
+      # value from the seed file.
+      #
+      # This method takes a block that will be run on the value from the seed
+      # file. The return value of the block will be used in place of the
+      # original value from the seed file.
+      #
+      # @param [Symbol] attribute
+      #   The attribute that we are reading.
+      def when_reading(attribute, &block)
+        @dynamic_readers[attribute] = block
+      end
+
+      # This method registers a custom block that will be run when writing a
+      # value to the seed file.
+      #
+      # This method takes a block that will be run on the item whose values we
+      # are writing. The return value of the block will be used instead of
+      # running the method on the record.
+      #
+      # @param [Symbol] attribute
+      #   The attribute we are writing.
+      def when_writing(attribute, &block)
+        @dynamic_writers[attribute] = block
+      end
+
+      # @!group Reading and Writing
+
+      # This method writes the seed data file to a folder.
+      #
+      # This will write a header row with the names of the attributes, and then
+      # write a row for each item from the {#items} method. It will use the
+      # {#write_value_for_attribute} method to get the values for the CSV file.
+      #
+      # If this seed type has no attributes, this method will not write
+      # anything.
+      #
+      # @param [String] folder
+      #   The full path to the folder to write to.
+      def write_to_folder(folder)
+        return unless attributes.any?
+        CSV.open(File.join(folder, "#{self.filename}.csv"), 'w') do |csv|
+          csv << self.attributes
+          self.items.each do |item|
+            data = self.attributes.collect { |attribute| self.write_value_for_attribute(item, attribute) }
+            csv << data
+          end
+        end
+      end
+
+      # This method reads the seed data from a file, and creates new records
+      # from it.
+      #
+      # This will extract all the rows from the CSV file, and use
+      # {#read_value_for_attribute} to get the attributes for the record from
+      # each cell in the CSV file.
+      #
+      # If this seed type has no attributes, this method will not try to read
+      # the file.
+      #
+      # @param [String] folder
+      #   The full path to the folder with the seed files.
+      def read_from_folder(folder)
+        return unless attributes.any?
+        CSV.open(File.join(folder, "#{self.filename}.csv"), headers: true) do |csv|
+          csv.each do |row|
+            record = self.klass.new
+            row.each do |attribute, value|
+              attribute = attribute.to_sym
+              value = self.read_value_for_attribute(value, attribute)
+              record.send("#{attribute}=", value)
+            end
+            record.save!
+          end
+        end
+      end
+
+      # This method gets the value that we should write into the CSV file for
+      # an attribute.
+      #
+      # If the value for that attribute is another model record, this will get
+      # its seed identifier from the seed set. Otherwise, it will just use the
+      # value.
+      #
+      # If the attribute is a polymorphic foreign key, this will prefix the seed
+      # identifier with the class name.
+      #
+      # @param [ActiveRecord::Base] item
+      #   The record whose value we are getting.
+      #
+      # @param [Symbol] attribute
+      #   The attribute we are getting.
+      #
+      # @return [String]
+      #   The value to write.
+      def write_value_for_attribute(item, attribute)
+        if @dynamic_writers[attribute]
+          value = @dynamic_writers[attribute].call(item)
+        else
+          value = item.send(attribute)
+        end
+
+        if value.is_a?(ActiveRecord::Base)
+          identifier = self.seed_set.seed_identifier_for_record(value)
+          if !self.associations[attribute]
+            identifier = "#{value.class.name}::#{identifier}"
+          end
+          value = identifier
+        end
+        value
+      end
+
+      # This method takes a value from the CSV file and gives back the value
+      # that should be set on the record.
+      #
+      # If the attribute is an association, this will pass it to the seed set
+      # as a seed identifier. If it is a polymorphic association, it will use
+      # the first part of the seed identifier as a class name.
+      #
+      # @param [String] value
+      #   The value from the CSV file.
+      #
+      # @param [Symbol] attribute
+      #   The name of the attribute that we are formatting.
+      #
+      # @return [Object]
+      #   The value to set on the record.
+      def read_value_for_attribute(value, attribute)
+        if @dynamic_readers[attribute]
+          value = @dynamic_readers[attribute].call(value)
+        end
+
+        if @associations.has_key?(attribute)
+          return nil if value.blank?
+          value = self.read_value_for_association(attribute, value.split("::"))
+        end
+        value
+      end
+
+      # This method gets the value that we will assign to an attribute for an
+      # association.
+      #
+      # This will remove as many entries from the identifier as it needs to get
+      # the value.
+      #
+      # @param [Symbol] attribute
+      #   The name of the association.
+      #
+      # @param [Array<String>] identifier
+      #   The components of the seed identifier.
+      #
+      # @return [ActiveRecord::Base]
+      def read_value_for_association(attribute, identifier)
+        klass = @associations[attribute]
+        unless klass
+          class_name = identifier.shift
+          klass = class_name.constantize
+        end
+        value = self.seed_set.record_for_seed_identifier(klass, identifier)
+      end
+
+      # @!group Fetching
+
+      # This method gets a relation encompassing all the records in the class.
+      #
+      # We encapsulate this here so that we can use different calls in Rails 3
+      # and Rails 4.
+      #
+      # @return [Relation]
+      def all
+        version = Rails.version[0]
+        records = version == '3' ? self.klass.scoped : self.klass.all
+      end
+      # This method gets the items that we should write to our seed file.
+      #
+      # It will start with the scoped list for the model class, and then apply
+      # all the scopes in our {#scopes} list.
+      #
+      # @return [Relation]
+      def items
+        records = self.all
+        self.scopes.each { |method, args| records = records.send(method, *args) }
+        records
+      end
+
+      # This method gets the additional information passed in when defining our
+      # seed set.
+      #
+      # @return [Hash]
+      def options
+        self.seed_set.options
+      end
+
+      # @!group Seed Identifiers
+
+      # This method gets the unique identifier for a record of the class that
+      # this seed type handles.
+      #
+      # @param [ActiveRecord::Base] record
+      #   The record
+      #
+      # @return [String]
+      #   The identifier for the seed files.
+      def seed_identifier_for_record(record)
+        self.identifier_attributes.collect { |param| self.write_value_for_attribute(record, param) }.join("::")
+      end
+
+      # This method finds a record for our model class based on the unique seed
+      # identifier.
+      #
+      # @param [Array<String>] identifier
+      #   The identifier from the CSV file.
+      #
+      # @return [ActiveRecord::Base]
+      #   The record
+      def record_for_seed_identifier(identifier)
+        return nil if identifier.blank?
+        records = self.all
+        self.identifier_attributes.each_with_index do |attribute, index|
+          if self.associations.has_key?(attribute)
+            value = self.read_value_for_association(attribute, identifier)
+            records = records.where("#{attribute}_id" => value.try(:id))
+          else
+            value = self.read_value_for_attribute(identifier.shift, attribute)
+            records = records.where(attribute => value)
+          end
+        end
+        records.first
+      end
+    end
+  end
+end

data/lib/config_scripts/seeds.rb

@@ -0,0 +1,8 @@
+module ConfigScripts
+  # This module encapsulates code for reading and writing seed data.
+  module Seeds
+  end
+end
+
+require 'config_scripts/seeds/seed_set'
+require 'config_scripts/seeds/seed_type'

data/lib/config_scripts/tasks/pending_migrations.rake

@@ -0,0 +1,11 @@
+namespace :config_scripts do
+  desc "Run pending config scripts"
+  task :run_pending => :environment do
+    ConfigScripts::Scripts::Script.run_pending_scripts
+  end
+
+  desc "List pending config scripts"
+  task :list_pending => :environment do
+    ConfigScripts::Scripts::Script.list_pending_scripts
+  end
+end