rseed 0.0.1 → 1.0.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ODgyMmI1OTUxYmZlMTBkOTY4M2I5ZDRmOWQ1ODA0MDNjM2M1MTFkZg==
+    YTljZGYyMzkyZjE0MzQyZjVhYTdiZGM5YmMwNjdiM2RmN2M3ODFkNw==
   data.tar.gz: !binary |-
-    YzRlMzE4YzM3ZjI4MzFjMmFmZjcwYjYxODAwMmQyMTUwNGNiY2FhMA==
+    NWM2ZDNmNmRhMmU0NjdkNzk5YjQ2ODAxMmZiYzM2YzYzYTU4MWI0ZQ==
 SHA512:
   metadata.gz: !binary |-
-    NDdhYWUyMTU2MzAyN2M0MjliYzliZWI1NjY5YmI1N2ZjNzdiMjcwMzU3Njc5
-    MjA1OTI4ZDFkZjg4MzZhN2UzZDNmYjIzYThkYzU2ZDQwNjhlZGEzNmVlMDk4
-    Yzg2ZGJhMmQxNDg4Mzg0Y2NhZTRjYTQ0NzI1ODQxNzllNjkwNDY=
+    MjMxM2IyY2VkZTI4YTg4MGQzNzk2MzMzMmYxYTdlYmQ4YzI5YTgyYmQ0NjQz
+    ZmYyMWJhYzkwN2QzYWUzOGRmZTdiNDIwMDQ1NTljOGQ2MjJjZjY0ZjRlYjAw
+    MDAyMDcyZmQ2M2IyZjA1ZjY4Y2QyYjQ1N2E5MjM2ZTRmOWEyNjU=
   data.tar.gz: !binary |-
-    NmZmNzFhYzRiMmEyYzYwNmEyMTVhYmI4OTk1MGRhNjNkMjBiYjU2ZWFlMGQz
-    NTA1NDNiMzU1YWM1MzIyMjUyYjgwYzBjNjQyYzBlOGEzMTI2NjZlZjVmMzEy
-    MzdkZTI5YjFhYWI4YmY4N2NhZjRmMTQxZmE2YzU5MDlmZTkxMTE=
+    ZmU5YWQ5ZmY1N2MwOWRlZWE0ZDI0OTNjMmEwYjAzMjE2NWI5ZjU4ZjZhOWQ4
+    YThkZThmYWFlNDAxNzNmZWQzMDk3Njg3NjgxZDI5YTQ3MWI0MDkyNzk1MzJk
+    YzE3ZGQ5MWEwNDkwNDIxY2MyYzYxNjkxODdiZTY0OTIxYjYyZGE=

data/README.rdoc

@@ -1,7 +1,17 @@
 = Rseed
 
-Rseed is a replacement for rseed. There are lots of improvements in order to make it easy to create and
-maintain converters.
+Rseed is a featureful library and bunch of utilities to assist in importing mass data (or even a few lines of data) into
+your Rails project.
+
+Rseed is a replacement for active_import (http://github.com/intrica/active_import). There are lots of improvements in
+order to make it easy to create and maintain converters.
+
+Rseed can also import from excel files with the optional rseed-roo gem (http://github.com/intrica/rseed-roo).
+
+== Requirements
+
+>= Rails 3.0
+>= Ruby 1.9
 
 == Installation
 
@@ -15,36 +25,47 @@ Then run:
 
 == Quick Example
 
-    rails g rseed:converter User
+    rails g rseed:converter HtmlColor
 
 This will create an model converter in the directory app/rseed. You can read through this import file to see how the import works.
 
-This also creates a default data file in db/rseed. This will be the CSV used for this converter.
+This also creates a default data file of html_colors.csv in db/rseed. This will be the CSV used for this converter.
+
+== Generators
+
+The generator will automatically create attribute lines for all of the attributes in the model except :id, :created_at, :updated_at.
+
+Generator options are as follows:
+
+* --attribute
+This will set up the converter to do a *first_or_initilize* on the specified attribute instead of a *new*
+
+* --minimal
+This will cause the generator to create a file with fewer
 
 == The Converter File
 
-=== Options
+=== Attribute Options
 
-:header
+* :header
 Defines the name of the attribute to be used for serialization. If there is no :match defined, it will also be used
 to match the attribute name of the input to the attribute being defined.
 
-:match
+* :match
 A regex string that is used to match the attribute name of the input to the attribute being defined. If this is not
 defined, a match will be checked against :header and then the attribute name.
 
-:type
+* :type
 Defines a type for the string.
 
-:model
+* :model
 This can be set to the name of a model that this attribute should resolve to. The model is classified so using a symbol
-works here. Alternately, if this is set to *true*, then the name of the attribute will be used as the model name. In
-order for this to work, :model_attribute must also be set.
+works here. Alternately, if only the :model_attribute is set, the name of the attribute will be used as the model name.
 
-:model_attribute
+* :model_attribute
 Specify which attribute on the model is used for lookup.
 
-:model_match
+* :model_match
 Specifies how the model should be resolved. The value here is called against the *where* that is used to look up the model.
 For example, this defaults to *:first*. If your model is *Person* and the :model_attribute is *:name* then this is what
 is called to set the attribute value:
@@ -53,7 +74,7 @@ is called to set the attribute value:
 
 You may use any active record method in this case, such as :first_or_create, or :last.
 
-:optional
+* :optional
 Defines the attribute as optionsal. This has no effect in the *HashAdapter*.
 
 == Rake Tasks
@@ -61,40 +82,21 @@ Defines the attribute as optionsal. This has no effect in the *HashAdapter*.
 These rake tasks allow you to run seeds manually:
 
     rake rseed:csv                   Load csv file into a model using a model converter
-    rake rseed:seed                  Seed a list of import files
 
 === Examples
 
-    rake rseed:csv FILE=user.csv CONVERTER=User CONVERTER_OPTIONS="give_admin_access=true"
+    rake rseed:csv file=users.csv converter=User converter_options="give_admin_access=true"
 
-In this case the file in db/rseed/user.csv would be run through the converter UserConverter. The options specified are available within the converter. In this case @options["give_admin_access"] will evaluate to true.
-
-The FILE parameter is not strictly necessary in this case either as the default file name will be an underscored value of the name of the converter.
+In this case the file in db/rseed/users.csv would be run through the converter UserConverter. The options specified are available within the converter. In this case options["give_admin_access"] will evaluate to "true".
 
 == Seeding
 
-Seeding allows you to import several files through different model converters in a single command. It involves the creation of a .seed file. Each file goes on a single line and the options are separated by pipe symbols.
-
-=== Example
-
-    user_info/user.xls | User | give_admin_access=false,send_email=true
-    user_info/roles.xls | Role
-    user_info/permissions.xls | UserPermission
-
-You could save this file as db/rseed/user_info.seed and run the command like this:
-
-    rake rseed:seed SET=user_info
-
-Note this will get the data files from a subdirectory: db/rseed/user_info. Also the top conversion uses options but as they are optional, the following two do not.
-
-If you do not specify a set, the rake task will look for a set based on the current development environment: ie development.seed.
-
+If you want to seed your Rails application using Rseed. The best method is to add lines like the following to db/seeds.rb
 
-== The Converter Class
+    Rseed::from_csv 'html_colors.csv', converter: :html_color, converter_options: { no_red: true }
 
-=== Column Setup
+You can always use *if* statements in this file to filter seeds by Rails.env or something similar.
 
-==== Mandatory Columns
-==== Column Types
+=== Custom Type Conversions
 
-=== Custom Type Conversions
+TODO

data/lib/generators/rseed/converter.rb

@@ -4,6 +4,8 @@ module Rseed
   module Generators
     class ConverterGenerator < Rails::Generators::NamedBase
       class_option :converter_name, :type => :string, :default => nil, :desc => "Names the converter file, defaults to the model name"
+      class_option :attribute, :type => :string, :default => nil, :desc => "If set will default to a first_or_initialize for creation of the model."
+      class_option :minimal, :type => :boolean, :default => false, :desc => "Create a converter with less template code and less comments"
 
       def self.source_root
         @source_root ||= File.join(File.dirname(__FILE__), 'templates')
@@ -15,12 +17,22 @@ module Rseed
         Dir.mkdir(converter_dir) unless File.directory?(converter_dir)
         @model_name = file_name
         @class_name = class_name
-        @model =  eval(@class_name)
+        @model = eval(@class_name)
         @columns = @model.columns_hash.except "id", "created_at", "updated_at"
+        @relationships = {}
+        @model.reflect_on_all_associations.each do |relationship|
+          if relationship.macro == :belongs_to
+            @relationships[relationship.name] = relationship
+            @columns.delete(relationship.foreign_key)
+          end
+        end
         @converter_name = options.converter_name || @class_name
+        @match_attribute = options.attribute
+        @minimal = options.minimal
+
         template 'converter.rb.erb', File.join(converter_dir, "#{@converter_name.underscore}_converter.rb")
         Dir.mkdir(seed_dir) unless File.directory?(seed_dir)
-        template 'data.csv.erb', File.join(seed_dir, "#{@converter_name.underscore}.csv")
+        template 'data.csv.erb', File.join(seed_dir, "#{@converter_name.underscore.pluralize}.csv")
       end
     end
   end

data/lib/generators/rseed/templates/converter.rb.erb

@@ -4,6 +4,9 @@ class <%= @converter_name.camelize %>Converter < Rseed::Converter
 <% @columns.each_pair do |attribute, meta| -%>
     attribute :<%= attribute %>, type: :<%= meta.type %>
 <% end -%>
+<% @relationships.each_pair do |attribute, meta| -%>
+    attribute :<%= attribute %><% unless @minimal %>, model: <%= meta.class_name %><% end %>, model_attribute: :<%= meta.foreign_key %><% unless @minimal %>, model_match: :first<% end %>
+<% end -%>
 
     def <%= @model_name %>_attributes values
         attributes = {}
@@ -14,26 +17,43 @@ class <%= @converter_name.camelize %>Converter < Rseed::Converter
 
         return attributes
     end
+<% unless @minimal -%>
 
     def before_deserialize
+        true
     end
 
     def after_deserialize
     end
+<% end -%>
 
     def deserialize values
         # Prevents nil values coming from the import overwriting the model attributes
         remove_nil_from values
+<% unless @minimal -%>
+
         # Prevents blank values coming from the import overwriting the model attributes
         # remove_blank_from values
 
         # For create only
-        <%= @model_name %> = <%= @class_name %>.new
+        # <%= @model_name %> = <%= @class_name %>.new
+        #
         # For create or update, use the following instead and change the match attribute name as required
-        # match_attribute = :id
-        # <%= @model_name %> = <%= @class_name %>.where(match_attribute => values[match_attribute]).first_or_initialize
+        # <%= @model_name %> = <%= @class_name %>.where(:id => values[:id]).first_or_initialize
+<% end -%>
+
+<% if @match_attribute -%>
+        <%= @model_name %> = <%= @class_name %>.where(:<%= @match_attribute %> => values[:<%= @match_attribute %>]).first_or_initialize
+<% else -%>
+        <%= @model_name %> = <%= @class_name %>.new
+<% end -%>
+
+        success = <%= @model_name %>.update_attributes <%= @model_name %>_attributes(values)
+
+        # This will return false and set errors if the record fails to update, signalling a failed import
+        return fail_with_error <%= @model_name %>.errors unless success
 
-        # This will return false if the record fails to update, signalling a failed import
-        <%= @model_name %>.update_attributes <%= @model_name %>_attributes(values)
+        # Return true signalling a successful import
+        true
     end
 end

data/lib/rseed/attribute.rb

@@ -25,10 +25,11 @@ module Rseed
       return nil if values[self.name].nil?
       value = values[self.name]
 
-      if options[:model] && options[:model_attribute]
+      if options[:model_attribute]
+        model = options[:model] || true
         # The attribute is a model, we look up the model via the specified attribute
-        model_name = options[:model] == true ? self.name : options[:model]
-        klass = model_name.to_s.classify.constantize
+        model_name = model == true ? self.name : model
+        klass = model_name.is_a?(Class) ? model_name : model_name.to_s.classify.constantize
         model_match = options[:model_match] || :first
         value = klass.where(options[:model_attribute] => value).send(model_match.to_s)
       elsif options[:type]

data/lib/rseed/converter.rb

@@ -32,6 +32,7 @@ module Rseed
     end
 
     def before_deserialize
+      true
     end
 
     def after_deserialize

data/lib/rseed/{utilities.rb → helpers.rb}

@@ -16,7 +16,7 @@ module Rseed
   end
 
   def process_with_status_bar processor, options = {}
-    title = options[:title] ? options[:title] : "Seed"
+    title = options[:title] ? options[:title].dup : "Seed"
     title = "#{processor.converter.name.cyan} #{title.blue}"
     record_count = 0
     progress_bar = ProgressBar.create(starting_at: nil, total: nil, format: "#{"Preprocessing".magenta} %t <%B>", title: title)
@@ -37,19 +37,21 @@ module Rseed
           end
         when :complete
           progress_bar.format "#{"Complete".green} %t <%B> %C (%a)"
-          progress_bar.finish
+          progress_bar.finish if progress_bar.total
         when :error
           processor.logger.error result[:message].to_s.red
           processor.logger.error result[:error]
-          processor.logger.error result[:backtrace].join('\n')
+          processor.logger.error result[:backtrace].join('\n') if result[:backtrace]
       end
     end
   end
 
   def from_file file, options = {}
+    f = import_file(file)
+    return false unless f
     p = Processor.new(options)
     return nil unless p
-    p.adapter.file = file
+    p.adapter.file = f
     process_with_status_bar p, title: file
   end
 
@@ -65,8 +67,20 @@ module Rseed
     process_with_status_bar p, title: "Hash"
   end
 
+  def import_file(file)
+    return file if File.exists? file
+    # Try it relative to the rseed directory
+    rseed_file = File.join(Rails.root, 'db', 'rseed', file)
+    return rseed_file if File.exists? rseed_file
+    # Try it relative to the Rails directory
+    rails_file = File.join(Rails.root, file)
+    return rails_file if File.exists? rails_file
+    nil
+  end
+
   module_function :from_file
   module_function :from_hash
   module_function :from_csv
   module_function :process_with_status_bar
+  module_function :import_file
 end

data/lib/rseed/processor.rb

@@ -10,7 +10,18 @@ module Rseed
 
       adapter = options[:adapter].is_a?(Adapter) ? options[:adapter] : Rseed.const_get("#{options[:adapter].to_s.classify}Adapter").new
       converter = options[:converter].is_a?(Converter) ? options[:converter] : Rseed.const_get("#{options[:converter].to_s.classify}Converter").new
-
+      if options[:converter_options]
+        converter_options = options[:converter_options]
+        if converter_options.is_a? String
+          options = converter_options.split(";")
+          converter_options = {}
+          options.each do |option|
+            s = option.split("=")
+            converter_options[s[0].strip] = s[1].strip
+          end
+        end
+        converter.options = converter_options
+      end
       @adapter = adapter
       @converter = converter
     end
@@ -24,39 +35,44 @@ module Rseed
       adapter.logger = logger
       adapter.converter = converter
 
-      yield :preprocessing
       begin
+        logger.info "Converter: #{@converter.name.cyan}"
+        logger.info "Converter Options: #{@converter.options.to_s.dup.cyan}"
+        yield :preprocessing
         if @adapter.preprocess
-          @converter.before_deserialize
-          yield :processing
-          start_time = Time.now
-          adapter.process do |values, meta|
-            result = {}
-            meta ||= {}
-            begin
-              if @converter.deserialize_raw(values)
-                result[:success] = true
-              else
+          if @converter.before_deserialize
+            yield :processing
+            start_time = Time.now
+            adapter.process do |values, meta|
+              result = {values: values}
+              meta ||= {}
+              begin
+                if @converter.deserialize_raw(values)
+                  result[:success] = true
+                else
+                  result[:success] = false
+                  result[:message] = "Failed to convert"
+                  result[:error] = @converter.error
+                end
+              rescue Exception => e
                 result[:success] = false
-                result[:message] = "Failed to convert"
-                result[:error] = @converter.error
+                result[:message] = "Exception during deserialize"
+                result[:error] = e.message
+                result[:backtrace] = e.backtrace
               end
-            rescue Exception => e
-              result[:success] = false
-              result[:message] = "Exception during deserialize"
-              result[:error] = e.message
-              result[:backtrace] = e.backtrace
-            end
 
-            # Calculate the ETA
-            if meta[:record_count] and meta[:total_records]
-              remaining = meta[:total_records] - meta[:record_count]
-              tpr = (Time.now - start_time)/meta[:record_count]
-              meta[:eta] = remaining * tpr
+              # Calculate the ETA
+              if meta[:record_count] and meta[:total_records]
+                remaining = meta[:total_records] - meta[:record_count]
+                tpr = (Time.now - start_time)/meta[:record_count]
+                meta[:eta] = remaining * tpr
+              end
+              yield :processing, result, meta
             end
-            yield :processing, result, meta
+            @converter.after_deserialize
+          else
+            yield :error, {success: false, message: 'Before deserialize failed', error: @converter.error}
           end
-          @converter.after_deserialize
         else
           yield :error, {success: false, message: 'Preprocessing failed', error: @adapter.error}
         end

data/lib/rseed/version.rb

@@ -1,3 +1,3 @@
 module Rseed
-  VERSION = "0.0.1"
+  VERSION = "1.0.0"
 end

data/lib/rseed.rb

@@ -7,7 +7,7 @@ require "rseed/hash_adapter"
 require "rseed/converter"
 require "rseed/processor"
 require "rseed/csv_adapter"
-require "rseed/utilities"
+require "rseed/helpers"
 
 module Rseed
   class << self

data/lib/tasks/rseed.rake

@@ -2,7 +2,17 @@ require "rseed"
 require "colorize"
 
 namespace :rseed do
-  desc "Load csv file into a model using the CSV adapter and a converter"
+  desc "Seed a CSV file using the CsvAdapter and a converter"
   task :csv => :environment do
+    converter = ENV["converter"] || ENV["CONVERTER"]
+    converter_options = ENV["converter_options"] || ENV["CONVERTER_OPTIONS"]
+    file = ENV["file"] || ENV["FILE"]
+    if file && converter
+      options = {converter: converter}
+      options[:converter_options] = converter_options if converter_options
+      Rseed::from_csv file, options
+    else
+      puts "You must specify file=<file> and converter=<converter>".red
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rseed
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 1.0.0
 platform: ruby
 authors:
 - David Monagle
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-07 00:00:00.000000000 Z
+date: 2013-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: colorize
@@ -88,8 +88,8 @@ files:
 - lib/rseed/converter.rb
 - lib/rseed/csv_adapter.rb
 - lib/rseed/hash_adapter.rb
+- lib/rseed/helpers.rb
 - lib/rseed/processor.rb
-- lib/rseed/utilities.rb
 - lib/rseed/version.rb
 - lib/tasks/rseed.rake
 - rseed.gemspec