csv_importer 0.0.1

data/History.txt

@@ -0,0 +1,4 @@
+=== 0.0.1 2009-11-04
+
+* 1 major enhancement:
+  * Initial release

data/Manifest.txt

@@ -0,0 +1,16 @@
+History.txt
+Manifest.txt
+PostInstall.txt
+Rakefile
+README.rdoc
+lib/csv_importer.rb
+lib/csv_importer/csv_importer.rb
+script/console
+script/destroy
+script/generate
+script/txt2html
+spec/csv/sample.csv
+spec/csv_importer_spec.rb
+spec/spec.opts
+spec/spec_helper.rb
+tasks/rspec.rake

data/PostInstall.txt

@@ -0,0 +1 @@
+For more information on csv_importer, see http://csv_importer.rubyforge.org

data/README.rdoc

@@ -0,0 +1,69 @@
+== DESCRIPTION:
+
+http://github.com/sparkboxx/csv-importer
+
+Ever needed to import csv files where every row needs to be converted into a model?
+
+The CSV importer turns every row of a CSV file into an object. Each column is matched and tested against a given class.
+You can provide a dictionary with translations between the CSV column names and the object properties.
+
+== FEATURES/PROBLEMS:
+
+* Probably no windows support right now because of different line endings.
+
+== SYNOPSIS:
+
+  require 'csv_importer'
+
+  class Product < Struct.new(:title, :price, :brand, :image); end
+  file = File.open("filename.csv", "rb")
+  importer = CSV_Importer::Importer.new(file, Product)
+  products = importer.objects
+
+or
+
+  require 'csv_importer'
+
+  class Product < Struct.new(:title, :price, :brand, :image); end
+  csv_string = "title, the price, the brand \n jeans, 10, fashionable"
+  dictionary = {"the brand"=>"brand", "the price"=>"price"}
+  importer = CSV_Importer::Importer.new(csv_string, Product, dictionary)
+  products = importer.objects
+  
+For use with ActiveRecord objects you can provide an argument to call save on every product when created
+
+  products = importer.object(true)
+
+
+== REQUIREMENTS:
+
+* This gem uses the standard ruby CSV libary. Ruby 1.9 will have a new library which will probably be compatible.
+
+== INSTALL:
+
+  sudo gem install csv_importer
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2009 Sparkboxx
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,19 @@
+require 'rubygems'
+gem 'hoe', '>= 2.1.0'
+require 'hoe'
+require 'fileutils'
+require './lib/csv_importer'
+
+Hoe.plugin :newgem
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+$hoe = Hoe.spec 'csv_importer' do
+  self.developer 'Sparkboxx', 'wilco@sparkboxx.com'
+end
+
+require 'newgem/tasks'
+Dir['tasks/**/*.rake'].each { |t| load t }
+
+# TODO - want other tests/tasks run by default? Add them to the list
+# remove_task :default
+# task :default => [:spec, :features]

data/lib/csv_importer.rb

@@ -0,0 +1,4 @@
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+require 'csv_importer/csv_importer'

data/lib/csv_importer/csv_importer.rb

@@ -0,0 +1,79 @@
+module CSVImporter
+  VERSION = "0.0.1"
+  require 'csv'
+
+
+  class Importer
+    attr_reader :reader, :delimiter, :csv_columns, :columns, :klass, :dictionary, :objects, :save
+        
+    def initialize(str_or_readable, klass=nil, dictionary={})
+      @klass = klass
+      @save = save
+      @dictionary = {}
+      dictionary.each_pair{|k,v| @dictionary[k.downcase] = v.downcase}
+      @delimiter, @csv_columns = determine_delimiter_and_columns(str_or_readable)
+      @columns = match_columns
+      @reader = CSV::Reader.create(str_or_readable, @delimiter)
+    end
+    
+    def objects(save=false)
+      return @objects if !@objects.nil?
+      @objects = Array.new if @object.nil?
+      first = true
+      @reader.each do |row|
+        #TODO: Ruby 1.9 has header option would be nicer instead of this hack.
+        if !first
+          object = @klass.new
+          @columns.each_pair do |column, row_number|
+            object.send("#{column}=", row[row_number].strip)
+          end
+          object.save if save
+          @objects << object
+        else
+          first = false
+        end
+      end
+      return @objects
+    end
+    
+    private
+    def determine_delimiter_and_columns(str_or_readable)
+      case str_or_readable
+      when IO
+        first_line = str_or_readable.gets
+        str_or_readable.pos = 0 #Reset IO cursor position, else the first line will be skipped
+      when String
+        #TODO: Find efficient way to get the first line by stoping at first newline character
+        #TODO: Test for different newline characters on windows machines
+        first_line = str_or_readable.split("\n")[0]
+      else
+        #TODO: What's the best exception to raise?
+        raise ArgumentException, "No String or IO argument detected"
+      end
+
+      #TODO: Fix repitition in determine_delimiter_and_columns
+      comma = first_line.split(",")
+      semicolon = first_line.split(";")
+      if comma.count > semicolon.count
+        return ",",comma.map!{|c| c.strip.downcase}
+      else
+        return ";",semicolon.map!{|c| c.strip.downcase}
+      end
+    end
+    
+    def match_columns
+      columns = {} if @columns.nil?
+      klass = @klass.new
+      @csv_columns.each_with_index do |column, index|
+        if klass.respond_to?("#{column}=")
+          columns[column] = index
+        elsif klass.respond_to?("#{@dictionary[column.downcase]}=")
+          columns[@dictionary[column.downcase]] = index
+        end
+      end
+      return columns
+    end
+    
+    
+  end
+end

data/script/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# File: script/console
+irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
+
+libs =  " -r irb/completion"
+# Perhaps use a console_lib to store any extra methods I may want available in the cosole
+# libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}"
+libs <<  " -r #{File.dirname(__FILE__) + '/../lib/csv_importer.rb'}"
+puts "Loading csv_importer gem"
+exec "#{irb} #{libs} --simple-prompt"

data/script/destroy

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/destroy'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Destroy.new.run(ARGV)

data/script/generate

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/generate'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Generate.new.run(ARGV)

data/script/txt2html

@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby
+
+load File.dirname(__FILE__) + "/../Rakefile"
+require 'rubyforge'
+require 'redcloth'
+require 'syntax/convertors/html'
+require 'erb'
+
+download = "http://rubyforge.org/projects/#{$hoe.rubyforge_name}"
+version  = $hoe.version
+
+def rubyforge_project_id
+  RubyForge.new.configure.autoconfig["group_ids"][$hoe.rubyforge_name]
+end
+
+class Fixnum
+  def ordinal
+    # teens
+    return 'th' if (10..19).include?(self % 100)
+    # others
+    case self % 10
+    when 1: return 'st'
+    when 2: return 'nd'
+    when 3: return 'rd'
+    else    return 'th'
+    end
+  end
+end
+
+class Time
+  def pretty
+    return "#{mday}#{mday.ordinal} #{strftime('%B')} #{year}"
+  end
+end
+
+def convert_syntax(syntax, source)
+  return Syntax::Convertors::HTML.for_syntax(syntax).convert(source).gsub(%r!^<pre>|</pre>$!,'')
+end
+
+if ARGV.length >= 1
+  src, template = ARGV
+  template ||= File.join(File.dirname(__FILE__), '/../website/template.html.erb')
+else
+  puts("Usage: #{File.split($0).last} source.txt [template.html.erb] > output.html")
+  exit!
+end
+
+template = ERB.new(File.open(template).read)
+
+title = nil
+body = nil
+File.open(src) do |fsrc|
+  title_text = fsrc.readline
+  body_text_template = fsrc.read
+  body_text = ERB.new(body_text_template).result(binding)
+  syntax_items = []
+  body_text.gsub!(%r!<(pre|code)[^>]*?syntax=['"]([^'"]+)[^>]*>(.*?)</\1>!m){
+    ident = syntax_items.length
+    element, syntax, source = $1, $2, $3
+    syntax_items << "<#{element} class='syntax'>#{convert_syntax(syntax, source)}</#{element}>"
+    "syntax-temp-#{ident}"
+  }
+  title = RedCloth.new(title_text).to_html.gsub(%r!<.*?>!,'').strip
+  body = RedCloth.new(body_text).to_html
+  body.gsub!(%r!(?:<pre><code>)?syntax-temp-(\d+)(?:</code></pre>)?!){ syntax_items[$1.to_i] }
+end
+stat = File.stat(src)
+created = stat.ctime
+modified = stat.mtime
+
+$stdout << template.result(binding)

data/spec/csv/sample.csv

@@ -0,0 +1,6 @@
+Description;Brand;SellPrice;SalePrice;Image;Category;ArticleCode
+SINGLE JERSEY L.S R;PME Jeans;  59.95;  59.95;http://www.example.com/images/JTS91573.JPG;T-shirts;JTS91573
+MONTANA L.SL. GRAND;PME legend;  59.95;  59.95;http://www.example.com/images/PTS88510.JPG;T-shirts;PTS88510
+CO DOUBLE JERSEY, 1;Cast Iron;  29.95;  29.95;http://www.example.com/images/CAC91101.JPG;Belts/ Keyholders;CAC91101
+PLAYER L.SLV. VEST;Cast Iron;  99.95;  99.95;http://www.example.com/images/CKW91406.JPG;Knitwear;CKW91406
+SLUBOR L.SL POLO;Cast Iron;  79.95;  79.95;http://www.example.com/images/CPS91301.JPG;Polo l.sl;CPS91301

data/spec/csv_importer_spec.rb

@@ -0,0 +1,83 @@
+require File.dirname(__FILE__) + '/spec_helper.rb'
+
+describe "Initialze CSV" do
+  include CSVImporter
+
+  it "should open a reader on the given CSV file" do
+    importer = new_importer
+    importer.reader.class == CSV::Reader
+  end
+  
+  it "should open a reader given a string" do
+    importer = Importer.new("foo,bar,foo,bar", Product)
+    importer.reader.class == CSV::Reader
+  end
+  
+  it "should detect the ; file delimiter" do
+    importer = new_importer
+    importer.delimiter.should == ";"
+  end
+  
+  it "should detect the , file delimiter" do
+    importer = Importer.new("foo,bar,foo,bar,foo", Product)
+    importer.delimiter.should == ","
+  end
+  
+  it "should pick the delimiter with the maximum split count" do
+    importer = Importer.new("column,col;umn,column,col;umn,column,column", Product)
+    importer.delimiter.should == ","
+  end
+
+  it "should set the columns of the CSV file" do
+    importer = Importer.new("a,b,c,d,e\nfoo,bar,foo,bar,foo", Product)
+    importer.csv_columns.should == ["a","b","c","d","e"]
+  end
+  
+  it "should strip the column names" do
+    importer = Importer.new("a, b, c , d ", Product)
+    importer.csv_columns.should == ["a","b","c","d"]
+  end
+  
+  it "should memorize the class to handle the rows" do
+    importer = Importer.new("a,b", Product)
+    importer.klass.should == Product
+  end
+  
+  it "should match the CSV columns with the Object columns" do
+    importer = Importer.new("title, price, brand, foo, barred", Product)
+    importer.columns.should == {"title"=>0, "price"=>1, "brand"=>2}
+  end
+
+  it "should match the CSV columns with the Object columns through the dictionary" do
+    dictionary = {"the title"=>"title", "the price"=>"price", "the brand" => "brand"}
+    importer = Importer.new("the title, the price, the brand", Product, dictionary)
+    importer.columns.should == {"title"=>0, "price"=>1, "brand"=>2}
+  end
+  
+  it "should ignore case differences in the dictionary" do
+    dictionary = {"THE title"=>"title", "the PRice"=>"price", "The Brand" => "brand"}
+    importer = Importer.new("the title, the price, the brand", Product, dictionary)
+    importer.columns.should == {"title"=>0, "price"=>1, "brand"=>2}
+  end
+  
+  it "should be able to cope with partial dictionaries" do
+    dictionary = {"Description"=>"title", "Saleprice"=>"Price"}
+    importer = Importer.new(File.open(File.dirname(__FILE__)+ "/csv/sample.csv", "rb"), Product, dictionary)
+    importer.columns.should == {"title"=>0, "price"=>3, "brand"=>1, "image"=>4}
+  end
+  
+  it "should create the collection of objects" do
+    dictionary = {"Description"=>"title", "Saleprice"=>"Price"}
+    importer = Importer.new(File.open(File.dirname(__FILE__)+ "/csv/sample.csv", "rb"), Product, dictionary)
+    importer.objects.count.should == 5 #See spec/csv/sample.csv
+  end
+  
+  it "shoud not turn the title line into an object" do
+    importer = Importer.new("title,brand\nThis is a nice title!, belonging to a brand", Product)
+    importer.objects[0].title.should == "This is a nice title!"
+    importer.objects[0].brand.should == "belonging to a brand"
+    importer.objects.count.should == 1    
+  end   
+    
+  
+end

data/spec/spec.opts

@@ -0,0 +1 @@
+--colour

data/spec/spec_helper.rb

@@ -0,0 +1,18 @@
+begin
+  require 'spec'
+rescue LoadError
+  require 'rubygems' unless ENV['NO_RUBYGEMS']
+  gem 'rspec'
+  require 'spec'
+end
+
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+require 'csv_importer'
+include CSVImporter
+
+# Define a Product Struct for test purposes
+class Product < Struct.new(:title, :price, :brand, :image); end
+
+def new_importer
+  Importer.new(File.open(File.dirname(__FILE__)+ "/csv/sample.csv", "rb"), Product)
+end

data/tasks/rspec.rake

@@ -0,0 +1,21 @@
+begin
+  require 'spec'
+rescue LoadError
+  require 'rubygems' unless ENV['NO_RUBYGEMS']
+  require 'spec'
+end
+begin
+  require 'spec/rake/spectask'
+rescue LoadError
+  puts <<-EOS
+To use rspec for testing you must install rspec gem:
+    gem install rspec
+EOS
+  exit(0)
+end
+
+desc "Run the specs under spec/models"
+Spec::Rake::SpecTask.new do |t|
+  t.spec_opts = ['--options', "spec/spec.opts"]
+  t.spec_files = FileList['spec/**/*_spec.rb']
+end

metadata

@@ -0,0 +1,89 @@
+--- !ruby/object:Gem::Specification 
+name: csv_importer
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Sparkboxx
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-11-04 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.3.3
+    version: 
+description: |-
+  http://github.com/sparkboxx/csv-importer
+  
+  Ever needed to import csv files where every row needs to be converted into a model?
+  
+  The CSV importer turns every row of a CSV file into an object. Each column is matched and tested against a given class.
+  You can provide a dictionary with translations between the CSV column names and the object properties.
+email: 
+- wilco@sparkboxx.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- Manifest.txt
+- PostInstall.txt
+files: 
+- History.txt
+- Manifest.txt
+- PostInstall.txt
+- Rakefile
+- README.rdoc
+- lib/csv_importer.rb
+- lib/csv_importer/csv_importer.rb
+- script/console
+- script/destroy
+- script/generate
+- script/txt2html
+- spec/csv/sample.csv
+- spec/csv_importer_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+- tasks/rspec.rake
+has_rdoc: true
+homepage: http://github.com/sparkboxx/csv-importer
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: csv_importer
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: http://github.com/sparkboxx/csv-importer  Ever needed to import csv files where every row needs to be converted into a model?  The CSV importer turns every row of a CSV file into an object
+test_files: []
+