simple_importer 0.1.0

data/.gitignore

@@ -0,0 +1,2 @@
+.DS_Store
+pkg/

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Justin Marney
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,160 @@
+= simple_importer
+
+Simple API for importing csv files.
+
+== Defining importers
+
+The basic importer has a name, file, and a foreach block.
+
+  importer :items do
+    file 'items.csv'
+
+    foreach do |row|
+      Item.create(:name => row[:name])
+    end
+  end
+
+simple_importer uses fastercsv, which replaces the csv standard lib in ruby1.9. Importers have configuration options that match those of fastercsv. The foreach block is called for each row in the csv file. The argument passed into the foreach block is a fastercsv row created with the specified configuration.
+
+  importer :items do
+    file 'items.csv'
+
+    # all fastercsv options are supported
+    headers false
+    converters :numeric
+    # etc..
+
+    foreach do |row|
+      Item.create(:name => row[:name])
+    end
+  end
+
+Importers have a default configuration that matches the following importer.
+
+  importer :items do
+    file 'items.csv'
+
+    # default configuration
+    col_sep ","
+    row_sep :auto
+    quote_char '"'
+    field_size_limit nil
+    converters :all
+    unconverted_fields nil
+    headers true
+    return_headers false
+    header_converters :symbol
+    skip_blanks false
+    force_quotes false
+    # end default configuration
+
+    foreach do |row|
+      Item.create(:name => row[:name])
+    end
+  end
+
+== Before callback
+
+Define before_filter-style callbacks in the form of the before block. This block is run once before the importer runs.
+
+  importer :items do
+    file 'items.csv'
+
+    before do
+      Item.delete_all
+    end
+
+    foreach do |row|
+      Item.create(:name => row[:name])
+    end
+  end
+
+== File processing
+
+Passing an array of file paths to the file configuration field will process all the rows in each file.
+
+  importer :items do
+    file Dir.glob("data/*.csv")
+
+    foreach do |row|
+      Item.create(:name => row[:name])
+    end
+  end
+
+Use the foreach_file block to add per-file behavior.
+
+  importer :items do
+    file Dir.glob("data/*.csv")
+
+    foreach_file do |file|
+      List.create(:name => File.basename(file, ".csv"))
+    end
+
+    foreach do |row|
+      Item.create(:name => row[:name])
+    end
+  end
+
+Nest the foreach block to give your row processing block access to foreach_file block variables.
+
+  importer :items do
+    file Dir.glob("data/*.csv")
+
+    foreach_file do |file|
+      list = List.create(:name => File.basename(file, ".csv"))
+
+      foreach do |row|
+        Item.create(:name => row[:name], :list => list)
+      end
+    end
+  end
+
+== Loading and running importers
+
+simple_importer provides a method that will find and load importer definitions in the importers, lib/importers, and app/importers directories.
+
+  # importer definition is in lib/importers/item.rb
+  SimpleImporter.find_importers
+
+SimpleImporter now has a collection of importers and each importer can be run now that it has been loaded.
+
+  # the following will fire off the actual importers
+  SimpleImporter.importers.each do |importer|
+    importer.run
+  end
+
+== Rake tasks and autoloaded importers
+
+simple_importer provides rake tasks to run importers individually or all at once.
+
+  # add this to your rakefile
+  # lib/tasks/simple_importer.rake is a good place in your rails app
+  require 'simple_importer/tasks'
+
+rake -T simple_importer will show a rake task for each importer as well as an import task which runs all of the importers. Descriptions defined in the importer will be used for the rake task description.
+
+  importer :items do
+    file 'items.csv'
+    desc 'Import all of the cool items'
+
+    ...
+  end
+
+== Requirements
+
+* fastercsv (if you are not using ruby 1.9)
+
+== Note on Patches/Pull Requests 
+
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but
+  bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2009 Justin Marney. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,54 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "simple_importer"
+    gem.summary = %Q{Simple api for importing from csv.}
+    gem.description = %Q{Provides a simple dsl for creating csv import tasks.}
+    gem.email = "gotascii@gmail.com"
+    gem.homepage = "http://github.com/vigetlabs/simple_importer"
+    gem.authors = ["Justin Marney"]
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/test_*.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "simple_importer #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+require 'lib/simple_importer/tasks'

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/data/example.csv

@@ -0,0 +1,2 @@
+name,lvl,screenname
+justin,11,gotascii

data/importers/example.rb

@@ -0,0 +1,11 @@
+importer :example do
+  file 'data/example.csv'
+
+  before do
+    puts 'before callback'
+  end
+
+  foreach do |row|
+    puts "#{row[:name]}, #{row[:lvl]}, #{row[:screenname]}"
+  end
+end

data/lib/simple_importer.rb

@@ -0,0 +1,106 @@
+require "csv"
+
+if CSV.const_defined? :Reader
+  require 'fastercsv'
+  Object.send(:remove_const, :CSV)
+  CSV = FasterCSV
+end
+
+module SimpleImporter
+  def self.included(base)
+    config_meths.each do |meth|
+      base.class_eval <<-METH
+        def #{meth}(val = nil)
+          config[:#{meth}] = val unless val.nil?
+          config[:#{meth}]
+        end
+      METH
+    end
+
+    [:before, :foreach, :foreach_file].each do |meth|
+      base.class_eval <<-METH
+        def #{meth}(&block)
+          config[:#{meth}] = block if block_given?
+          config[:#{meth}]
+        end
+      METH
+    end
+    
+    base.class_eval do
+      attr_accessor :name
+    end
+  end
+
+  def self.importer_paths
+    %w{importers app/importers lib/importers}
+  end
+
+  def self.find_importers
+    importer_paths.each do |path| 
+      Dir[File.join(path, '*.rb')].each do |file|
+        coedz = File.open(file).read
+        SimpleImporter.class_eval(coedz)
+      end
+    end
+  end
+
+  def self.csv_config_meths
+    CSV::DEFAULT_OPTIONS.keys
+  end
+
+  def self.config_meths
+    csv_config_meths + [:file, :callbacks, :desc]
+  end
+
+  def self.importers
+    @importers ||= []
+  end
+
+  def self.importer(name, &block)
+    importers << Importer.new(name, &block)
+  end
+
+  def self.[](name)
+    importers.select{|i| i.name == name}.first
+  end
+
+  def config
+    @config ||= {
+      :callbacks => true,
+      :headers => true,
+      :header_converters => :symbol,
+      :converters => :all,
+      :desc => 'a simple_import importer'
+    }
+  end
+
+  def csv_config
+    ret = {}
+    config.each do |k, v|
+      ret[k] = config[k] if SimpleImporter.csv_config_meths.include?(k)
+    end
+    ret
+    # config.select{|k,v| SimpleImporter.csv_config_meths.include?(k)}
+  end
+
+  def run
+    run_callbacks
+    [file].flatten.each do |f|
+      foreach_file.call(f) if foreach_file
+      CSV.foreach(f, csv_config, &foreach) if foreach
+    end
+  end
+
+  def run_callbacks
+    self.before.call if callbacks && before
+  end
+
+  class Importer
+    include SimpleImporter
+
+    def initialize(name, &block)
+      self.name = name
+      instance_eval(&block)
+    end
+  end
+end

data/lib/simple_importer/tasks.rb

@@ -0,0 +1,3 @@
+Dir[File.join(File.dirname(__FILE__), 'tasks', '*.rake')].each do |f|
+  load f
+end

data/lib/simple_importer/tasks/simple_importer.rake

@@ -0,0 +1,24 @@
+namespace :simple_importer do
+  def importers
+    require 'simple_importer'
+    SimpleImporter.find_importers
+    SimpleImporter.importers
+  end
+
+  importers.each do |i|
+    desc i.desc
+    task i.name do
+      Rake::Task[:environment].invoke if Rake::Task.task_defined?(:environment)
+      puts "Importing #{i.name}"
+      i.run
+      puts "Finished importing #{i.name}"
+    end
+  end
+
+  desc "run all importers"
+  task :import do
+    importers.each do |i|
+      Rake::Task["simple_importer:#{i.name}"].invoke
+    end
+  end
+end

data/simple_importer.gemspec

@@ -0,0 +1,54 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{simple_importer}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Justin Marney"]
+  s.date = %q{2009-10-22}
+  s.description = %q{Provides a simple dsl for creating csv import tasks.}
+  s.email = %q{gotascii@gmail.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".gitignore",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "data/example.csv",
+     "importers/example.rb",
+     "lib/simple_importer.rb",
+     "lib/simple_importer/tasks.rb",
+     "lib/simple_importer/tasks/simple_importer.rake",
+     "simple_importer.gemspec",
+     "test/test_helper.rb",
+     "test/test_simple_importer.rb"
+  ]
+  s.homepage = %q{http://github.com/vigetlabs/simple_importer}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{Simple api for importing from csv.}
+  s.test_files = [
+    "test/test_helper.rb",
+     "test/test_simple_importer.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end
+

data/test/test_helper.rb

@@ -0,0 +1,5 @@
+require 'rubygems'
+require 'shoulda'
+require 'matchy'
+require 'mocha'
+require 'simple_importer'

data/test/test_simple_importer.rb

@@ -0,0 +1,164 @@
+require 'test_helper'
+
+class CrazyImporter
+  include SimpleImporter
+end
+
+class SimpleImporterTest < Test::Unit::TestCase
+  context "The SimpleImporter module" do
+    setup do
+      SimpleImporter.importers.clear
+    end
+
+    should "have an importers hash field" do
+      SimpleImporter.importers.should == []
+    end
+
+    should "create a new importer" do
+      SimpleImporter::Importer.expects(:new).with(:bob)
+      SimpleImporter.importer(:bob)
+    end
+
+    should "add new importer to importers" do
+      SimpleImporter::Importer.stubs(:new).returns('hi')
+      SimpleImporter.importer(:bob)
+      SimpleImporter.importers.should == ['hi']
+    end
+
+    should "look in importer, app/importers, and lib/importers for importers" do
+      SimpleImporter.importer_paths.should == %w{importers app/importers lib/importers}
+    end
+
+    should "return csv default options for csv_config_methods" do
+      SimpleImporter.csv_config_meths.should == CSV::DEFAULT_OPTIONS.keys
+    end
+
+    should "return csv_config_meths plus file and callbacks for config_meths" do
+      SimpleImporter.stubs(:csv_config_meths).returns([:yo])
+      SimpleImporter.config_meths.should == [:yo, :file, :callbacks, :desc]
+    end
+
+    should "return first importer whos name matches arg for []" do
+      importer = stub(:name => :omg)
+      SimpleImporter.stubs(:importers).returns([importer])
+      SimpleImporter[:omg].should == importer
+    end
+  end
+
+  context "An instance of the CrazyImporter class" do
+    setup do
+      @importer = CrazyImporter.new
+    end
+
+    should "return kv pairs where key is in SimpleImporter.csv_config_meths for csv_config" do
+      @importer.csv_config.keys.each do |k|
+        SimpleImporter.csv_config_meths.include?(k).should be(true)
+      end
+    end
+
+    should "return the right gd hash for csv_config" do
+      SimpleImporter.stubs(:csv_config_meths).returns([:one, :two])
+      @importer.stubs(:config).returns({:one => 1, :two => 2, :three => 3})
+      @importer.csv_config.should == {:one => 1, :two => 2}
+    end
+
+    should "have fields for every CSV::DEFAULT_OPTIONS" do
+      CSV::DEFAULT_OPTIONS.keys.each do |meth|
+        @importer.send(meth, 'val')
+        @importer.send(meth).should == 'val'
+      end
+    end
+
+    should "have a file field" do
+      @importer.file 'file'
+      @importer.file.should == 'file'
+    end
+
+    should "have a callbacks field" do
+      @importer.callbacks true
+      @importer.callbacks.should be(true)
+    end
+    
+    should "execute callbacks by default" do
+      @importer.callbacks.should be(true)
+    end
+
+    should "create headers by default" do
+      @importer.headers.should be(true)
+    end
+
+    should "execute symbol header_converters by default" do
+      @importer.header_converters.should == :symbol
+    end
+
+    should "execute all converters by default" do
+      @importer.converters.should == :all
+    end
+
+    should "save block passed to before in before field" do
+      @importer.before { 'blazer' }
+      @importer.before.call.should == 'blazer'
+    end
+
+    should "save block passed to foreach in foreach field" do
+      @importer.foreach { 'blazer' }
+      @importer.foreach.call.should == 'blazer'
+    end
+
+    should "not run callbacks if callbacks is false" do
+      before = mock { expects(:call).never }
+      @importer.stubs(:before => before)
+      @importer.callbacks false
+      @importer.run_callbacks
+    end
+
+    should "run callbacks if callbacks is true" do
+      before = mock { expects(:call) }
+      @importer.stubs(:before => before)
+      @importer.callbacks true
+      @importer.run_callbacks
+    end
+
+    should "not run callbacks if callbacks is true and there are no callbacks" do
+      @importer.config[:before] = nil
+      @importer.callbacks true
+      @importer.run_callbacks
+    end
+
+    should "run callbacks when run" do
+      CSV.stubs(:foreach)
+      @importer.expects(:run_callbacks)
+      @importer.run
+    end
+
+    should "call CSV.foreach with file, csv_config when run" do
+      @importer.stubs(:foreach).returns(lambda{})
+      @importer.stubs(:file).returns('file')
+      @importer.stubs(:csv_config).returns('csv_config')
+      CSV.expects(:foreach).with('file', 'csv_config')
+      @importer.run
+    end
+
+    should "call CSV.foreach with each file if file is an array" do
+      @importer.stubs(:foreach).returns(lambda{})
+      @importer.stubs(:file).returns(['file1', 'file2'])
+      @importer.stubs(:csv_config).returns('csv_config')
+      CSV.expects(:foreach).with('file1', 'csv_config')
+      CSV.expects(:foreach).with('file2', 'csv_config')
+      @importer.run
+    end
+  end
+
+  context "The Importer class" do
+    should "include SimpleImporter" do
+      SimpleImporter::Importer.ancestors.include?(SimpleImporter)
+    end
+
+    should "instance eval block on initialize" do
+      SimpleImporter::Importer.any_instance.expects(:bobo)
+      SimpleImporter::Importer.new(:hi) do
+        bobo
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,69 @@
+--- !ruby/object:Gem::Specification 
+name: simple_importer
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Justin Marney
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-10-22 00:00:00 -04:00
+default_executable: 
+dependencies: []
+
+description: Provides a simple dsl for creating csv import tasks.
+email: gotascii@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- data/example.csv
+- importers/example.rb
+- lib/simple_importer.rb
+- lib/simple_importer/tasks.rb
+- lib/simple_importer/tasks/simple_importer.rake
+- simple_importer.gemspec
+- test/test_helper.rb
+- test/test_simple_importer.rb
+has_rdoc: true
+homepage: http://github.com/vigetlabs/simple_importer
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Simple api for importing from csv.
+test_files: 
+- test/test_helper.rb
+- test/test_simple_importer.rb