cheap_imports 0.0.2

data/.bzr/README

@@ -0,0 +1,3 @@
+This is a Bazaar control directory.
+Do not change any files in this directory.
+See http://bazaar-vcs.org/ for more information about Bazaar.

data/.bzr/branch/branch.conf

@@ -0,0 +1,2 @@
+parent_location = http://ananelson.com/code/cheap_imports/
+push_location = sftp://anaslist@ananelson.com/~/sites/ananelson.com/code/cheap_imports/

data/.bzr/branch/format

@@ -0,0 +1 @@
+Bazaar Branch Format 6 (bzr 0.15)

data/.bzr/branch/last-revision

@@ -0,0 +1 @@
+4 ana@ananelson.com-20091103102943-5xrd13o5d0grzb4j

data/.bzr/branch-format

@@ -0,0 +1 @@
+Bazaar-NG meta directory, format 1

data/.bzr/checkout/conflicts

@@ -0,0 +1 @@
+BZR conflict list format 1

data/.bzr/checkout/format

@@ -0,0 +1 @@
+Bazaar Working Tree Format 4 (bzr 0.15)

data/.bzr/checkout/merge-hashes

@@ -0,0 +1,6 @@
+BZR merge-modified list format 1
+file_id: cheap_imports.rb-20090721140243-7zd6ayu1gjqkmv5d-22
+hash: 8e31b226cd9163bcc12089d289659a8938b11212
+
+file_id: cheap_imports.rb-20090721140243-7zd6ayu1gjqkmv5d-16
+hash: 6d4bcc8c2461ffbf20b3535190cf8995002959da

data/.bzr/repository/format

@@ -0,0 +1 @@
+Bazaar pack repository format 1 (needs bzr 0.92)

data/.bzr/repository/indices/3b0e60c570a8cfc21b7d22db489579e7.six

@@ -0,0 +1,5 @@
+Bazaar Graph Index 1
+node_ref_lists=0
+key_elements=1
+len=0
+

data/.bzr/repository/indices/d139719d6bdd037de4ec3718dcf1c0c4.six

@@ -0,0 +1,5 @@
+Bazaar Graph Index 1
+node_ref_lists=0
+key_elements=1
+len=0
+

data/.bzr/repository/indices/d5dffeca0774033e54854302570cb330.six

@@ -0,0 +1,5 @@
+Bazaar Graph Index 1
+node_ref_lists=0
+key_elements=1
+len=0
+

data/.bzr/repository/indices/e5571628132ac072d270e0a249095c14.six

@@ -0,0 +1,5 @@
+Bazaar Graph Index 1
+node_ref_lists=0
+key_elements=1
+len=0
+

data/.bzrignore

@@ -0,0 +1,2 @@
+pkg
+coverage

data/History.txt

@@ -0,0 +1,4 @@
+== 1.0.0 / 2009-07-20
+
+* 1 major enhancement
+  * Birthday!

data/README.txt

@@ -0,0 +1,47 @@
+cheap_imports
+    by Ana Nelson
+
+== DESCRIPTION:
+
+Imports delimited text files to Active Record or Datamapper, with validations.
+
+== FEATURES/PROBLEMS:
+
+* FIXME (list of features or problems)
+
+== SYNOPSIS:
+
+  FIXME (code sample of usage)
+
+== REQUIREMENTS:
+
+* FIXME (list of requirements)
+
+== INSTALL:
+
+* FIXME (sudo gem install, anything else)
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2008 FIXME (different license?)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,38 @@
+# Look in the tasks/setup.rb file for the various options that can be
+# configured in this Rakefile. The .rake files in the tasks directory
+# are where the options are used.
+
+begin
+  require 'bones'
+  Bones.setup
+rescue LoadError
+  begin
+    load 'tasks/setup.rb'
+  rescue LoadError
+    raise RuntimeError, '### please install the "bones" gem ###'
+  end
+end
+
+ensure_in_path 'lib'
+require 'cheap_imports'
+
+PROJ.name = 'cheap_imports'
+PROJ.authors = 'Ana Nelson'
+PROJ.email = 'ana@ananelson.com'
+PROJ.url = 'http://ananelson.com'
+PROJ.version = CheapImports::VERSION
+PROJ.rubyforge.name = 'cheap_imports'
+
+PROJ.spec.opts << '--color'
+
+require 'rake'
+require 'spec/rake/spectask'
+require 'spec/rake/verify_rcov'
+
+task :default => :verify_rcov
+
+desc "Verify RCOV coverage above threshold"
+RCov::VerifyTask.new(:verify_rcov => 'spec:rcov') do |t|
+  t.threshold = 87.7
+  t.index_html = 'coverage/index.html'
+end

data/bin/cheap_imports

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require File.expand_path(
+    File.join(File.dirname(__FILE__), %w[.. lib cheap_imports]))
+
+# Put your code here
+
+# EOF

data/lib/cheap_imports/cheap_imports.rb

@@ -0,0 +1,154 @@
+module CheapImports
+  def self.included(base)
+    base.extend CheapImportsClassMethods
+    
+    # Detect what ORM we are using.
+    if base.superclass.name == "ActiveRecord::Base"
+      @cheap_imports_orm = "ActiveRecord"
+    elsif base.included_modules.collect {|m| m.name }.include?("DataMapper::Resource")
+      @cheap_imports_orm = "DataMapper"
+      base.extend DataMapperMethods
+    else
+      raise "unknown ORM!"
+    end
+  end
+  
+  module DataMapperMethods
+    def column_names
+      properties.collect {|p| p.name.to_s }
+    end
+    
+    def columns
+      properties
+    end
+  end
+  
+  module CheapImportsClassMethods
+    def init_recognizable_hashes
+      default_hash = {}
+      column_names.each do |n|
+        next if n === 'id'
+        default_hash[n.to_sym] = n
+      end
+      raise "default hash is empty" if default_hash.empty?
+      @recognizable_hashes ||= {:default => default_hash}
+    end
+    
+    # Add a new import definition.
+    # 
+    # imports :my_descriptive_hash_name => {
+    #   :database_column_name => "CORRESPONDING_TABLE_HEADER_TEXT"
+    # }
+    def imports(hash)
+      init_recognizable_hashes
+      @recognizable_hashes.merge!(hash)
+    end
+    
+    # Don't override this. You probably want to override import_rash instead.
+    def import(raw, args)
+      style = recognize_hash_style(raw)
+
+      # Want to be able to access this data in the raw hash using either the native
+      # key or our nicer symbolic key.
+      @recognizable_hashes[style].each do |k, v|
+        raw[k] = raw[v]
+      end
+
+      hash = fetch_default_hash(raw, style)
+      hash[:imported_at] = args[:imported_at] if column_names.include?("imported_at")
+      hash[:history] = "Imported from #{style.to_s} at #{args[:imported_at].to_s}." if column_names.include?("history")
+      hash[:source] = style.to_s if column_names.include?("source")
+      import_rash(raw, args, style, hash)
+    end
+    
+    def import_rash(raw, args, style, hash)
+      delete_prior hash[:imported_at]
+      create(hash)
+    end
+    
+    def delete_prior(imported_at)
+      delete_all "imported_at < '#{imported_at}'" if column_names.include?("imported_at")
+    end
+    
+    def fetch_default_hash(raw, style)
+      hash = {}
+      columns.each do |c|
+        column_name = c.name.to_sym
+        next if column_name == :id
+        keys = @recognizable_hashes[style].keys
+        
+        raw_value = nil
+        if keys.include?(column_name)
+          raw_value = value_of_from(column_name, raw)
+        elsif keys.include?("#{name.downcase}_#{column_name}".to_sym)
+          raw_value = value_of_from("#{name.downcase}_#{column_name}".to_sym, raw)
+        end
+        next if raw_value.nil?
+
+        value = nil
+
+        type = c.type.to_s.downcase
+        case type
+        when "string", "text", "integer", "boolean"
+          if raw_value === ""
+            value = nil
+          else
+            value = raw_value
+          end
+        when "date", "datetime", "time"
+          date_format_string = @recognizable_hashes[style]["#{column_name}_format".to_sym]
+          date_format_string ||= @recognizable_hashes[style]["#{name.downcase}_#{column_name}_format".to_sym]
+          raise "nil date_format_string for #{style} #{column_name}" if date_format_string.empty?
+          begin
+            value = Date.strptime(raw_value, date_format_string) if type === "date"
+            value = DateTime.strptime(raw_value, date_format_string) if type === "datetime"
+            value = Time.strptime(raw_value, date_format_string) if type === "time"
+          rescue ArgumentError
+            value = nil
+          end
+        when "float", "decimal"
+          value = BigDecimal.new(raw_value.gsub(/[^0-9\.\-]/, ''))
+        else
+          raise "unhandled type: #{c.type.to_s.downcase}"
+        end
+        hash[column_name] = value
+      end
+      hash
+    end
+    
+    # Fetch a row from the raw value hash.
+    def value_of_from(field_name, value_hash)
+      specification = @recognizable_hashes[recognize_hash_style(value_hash)]
+      raise "specification hash should not be nil!" if specification.nil?
+      value = value_hash[specification[field_name]].to_s.strip
+      block_given? ? yield(value) : value
+    end
+
+    # Can we import this type of hash?
+    # All classes have a default hash so we can always import data which was previously exported from the database.
+    # TODO Get rid of nasty workaround for dated_on_format keys.
+    # Return the name of style, or false if no style is matched.
+    def recognize_hash_style(raw_hash, debug = false)
+      @recognizable_hashes.each do |s, h|
+        puts "trying style: #{s} in class #{self.name}" if debug
+        
+        match = true
+        h.each do |k, v|
+          this_key_matches = (raw_hash.has_key?(v.to_s) || v.to_s =~ /^\%/)
+          puts "key: #{k} matches: #{this_key_matches}" if debug
+          match = match && this_key_matches
+        end
+        
+        puts "overall match: #{match}" if debug
+        return s if match
+      end
+      false
+    end
+  end
+end
+
+class NilClass
+  def id_or_nil
+    nil
+  end
+end

data/lib/cheap_imports/import.rb

@@ -0,0 +1,64 @@
+module Import
+  # Feel free to add extra delimiters to this array.
+  DELIMITERS = ["\t", '|', ","]
+  
+  def Import.import(klasses, params_data, args = {}, debug = false)
+    args.merge!({:imported_at => Time.now.strftime("%Y-%m-%d %H:%M:%S")}) unless args.has_key?(:imported_at)
+    klasses = [klasses] unless klasses.is_a?(Array)
+    delim = auto_detect_delimiter(params_data.to_s)
+    header_row = nil
+    
+    objects = []
+    
+    FasterCSV.parse(params_data, {:col_sep => delim, :row_sep => :auto}) do |row|
+      next if row.join("") !~ /\w/ # Skip blank rows.
+      
+      header_row ||= row
+      next if header_row.join("") === row.join("") # Skip over duplicate header rows.
+      
+      # Clean up the data, get rid of leading/trailing spaces.
+      row.collect {|str| str.to_s.strip!}
+      
+      h = Hash[*header_row.zip(row).flatten]
+      klasses.each do |c|
+        style = c.recognize_hash_style(h, debug)
+        objects << c.import(h, args) if style
+      end
+    end
+    
+    # Return an array of everything that was imported in this batch.
+    objects
+  end
+  
+  def Import.import_from_file(klasses, filename, debug = false)
+    args = {}
+    args[:date] = Date.strptime($1) if filename =~ /([0-9]{4}-[0-9]{2}-[0-9]{2})/
+    import(klasses, File.read(filename), args, debug)
+  end
+
+  private
+  
+  def Import.import_if_valid_else_ignore
+    begin
+      yield
+    rescue Exception => e
+      puts e.inspect
+      return nil
+    end
+  end
+  
+  # Guess the correct text file column delimiter. We look at the first row
+  # of data and the most common character from the DELIMITERS array is
+  # assumed to be the delimiter. You can add additional characters to the
+  # DELIMITERS array but to avoid the risk of false positives it's probably
+  # best not to add too many.
+  def Import.auto_detect_delimiter(sample_text)
+    sample_text = sample_text.split(/$/)[0] || ""
+    delim = DELIMITERS[0]
+    len = DELIMITERS.length
+    DELIMITERS[1-len, len-1].each do |char|
+      delim = char if sample_text.count(char) > sample_text.count(delim)
+    end
+    delim
+  end
+end

data/lib/cheap_imports.rb

@@ -0,0 +1,52 @@
+module CheapImports
+
+  # :stopdoc:
+  VERSION = '0.0.2'
+  LIBPATH = ::File.expand_path(::File.dirname(__FILE__)) + ::File::SEPARATOR
+  PATH = ::File.dirname(LIBPATH) + ::File::SEPARATOR
+  # :startdoc:
+
+  # Returns the version string for the library.
+  #
+  def self.version
+    VERSION
+  end
+
+  # Returns the library path for the module. If any arguments are given,
+  # they will be joined to the end of the libray path using
+  # <tt>File.join</tt>.
+  #
+  def self.libpath( *args )
+    args.empty? ? LIBPATH : ::File.join(LIBPATH, args.flatten)
+  end
+
+  # Returns the lpath for the module. If any arguments are given,
+  # they will be joined to the end of the path using
+  # <tt>File.join</tt>.
+  #
+  def self.path( *args )
+    args.empty? ? PATH : ::File.join(PATH, args.flatten)
+  end
+
+  # Utility method used to require all files ending in .rb that lie in the
+  # directory below this file that has the same name as the filename passed
+  # in. Optionally, a specific _directory_ name can be passed in such that
+  # the _filename_ does not have to be equivalent to the directory.
+  #
+  def self.require_all_libs_relative_to( fname, dir = nil )
+    dir ||= ::File.basename(fname, '.*')
+    search_me = ::File.expand_path(
+        ::File.join(::File.dirname(fname), dir, '**', '*.rb'))
+
+    Dir.glob(search_me).sort.each {|rb| require rb}
+  end
+
+end  # module CheapImports
+
+CheapImports.require_all_libs_relative_to(__FILE__)
+
+begin
+  require "fastercsv"
+rescue LoadError
+  raise "FasterCSV is required!" unless RUBY_VERSION >= "1.9.0"
+end

data/spec/cheap_imports_spec.rb

@@ -0,0 +1,7 @@
+
+require File.join(File.dirname(__FILE__), %w[spec_helper])
+
+describe CheapImports do
+end
+
+# EOF

data/spec/orms/active_record_spec.rb

@@ -0,0 +1,20 @@
+# class NewClass < ActiveRecord::Base
+#   validates_presence_of :code
+#   validates_uniqueness_of :code
+# end
+# 
+# context "is_recognized_hash_style" do
+#   setup do
+#     NewClass.connection.execute("CREATE TABLE new_classes (id INT AUTO_INCREMENT, code TEXT, name TEXT, PRIMARY KEY(id));")
+#     @obj = NewClass.create(:code => 'ABC', :name => 'Full Object Name')
+#     @hsh = {:newclass_code => "DEF", :newclass_name => "Another Object Name"}
+#   end
+#   
+#   # specify "should raise an error when called on a class with no recognize_hash_style method" do
+#   #   lambda { NewClass.is_recognized_hash_style?(@hsh) }.should raise_error(RuntimeError, "NewClass does not have a recognize_hash_style method")
+#   # end
+#   
+#   teardown do
+#     NewClass.connection.execute("DROP TABLE new_classes;")
+#   end
+# end

data/spec/orms/datamapper_spec.rb

@@ -0,0 +1,53 @@
+require File.join(File.dirname(__FILE__), '..',  %w[spec_helper])
+
+require "dm-core"
+DataMapper.setup(:default, 'sqlite3::memory:')
+
+class Cat
+  include DataMapper::Resource
+  property :id, Serial
+  property :name, String
+  property :age, Integer
+
+  include CheapImports
+end
+
+class Appointment
+  include DataMapper::Resource
+  property :id, Serial
+  property :cat_id, Integer
+  property :starts_at, DateTime
+  
+  include CheapImports
+  imports :legacy_data => {
+    :starts_at => "APPOINTMENT START TIME",
+    :starts_at_format => "%d %b %Y %H:%M",
+    :cat_name => "ANIMAL IDENTIFIER"
+  }
+end
+
+DataMapper.auto_migrate!
+
+describe Cat, "importing default" do
+  it "should import without any config" do
+    data = %{name\tage\nFluffy\t7}
+    imported_cats = Import.import(Cat, data)
+    
+    cats_named_fluffy = Cat.all(:name => "Fluffy")
+    
+    cats_named_fluffy.should have(1).items
+    cats_named_fluffy.first.age.should eql(7)
+  end
+end
+
+describe Appointment, "importing a specified config" do
+  it "should description" do
+    data = %{ANIMAL IDENTIFIER,APPOINTMENT START TIME\n}
+    data << %{Fluffy,5 Jan 2009 10:30\n}
+    
+    appointments = Import.import(Appointment, data)
+    a = appointments.first
+
+    a.starts_at.should eql(DateTime.new(2009, 1, 5, 10, 30))
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,16 @@
+
+require File.expand_path(
+    File.join(File.dirname(__FILE__), %w[.. lib cheap_imports]))
+
+Spec::Runner.configure do |config|
+  # == Mock Framework
+  #
+  # RSpec uses it's own mocking framework by default. If you prefer to
+  # use mocha, flexmock or RR, uncomment the appropriate line:
+  #
+  # config.mock_with :mocha
+  # config.mock_with :flexmock
+  # config.mock_with :rr
+end
+
+# EOF

data/tasks/ann.rake

@@ -0,0 +1,80 @@
+
+begin
+  require 'bones/smtp_tls'
+rescue LoadError
+  require 'net/smtp'
+end
+require 'time'
+
+namespace :ann do
+
+  # A prerequisites task that all other tasks depend upon
+  task :prereqs
+
+  file PROJ.ann.file do
+    ann = PROJ.ann
+    puts "Generating #{ann.file}"
+    File.open(ann.file,'w') do |fd|
+      fd.puts("#{PROJ.name} version #{PROJ.version}")
+      fd.puts("    by #{Array(PROJ.authors).first}") if PROJ.authors
+      fd.puts("    #{PROJ.url}") if PROJ.url.valid?
+      fd.puts("    (the \"#{PROJ.release_name}\" release)") if PROJ.release_name
+      fd.puts
+      fd.puts("== DESCRIPTION")
+      fd.puts
+      fd.puts(PROJ.description)
+      fd.puts
+      fd.puts(PROJ.changes.sub(%r/^.*$/, '== CHANGES'))
+      fd.puts
+      ann.paragraphs.each do |p|
+        fd.puts "== #{p.upcase}"
+        fd.puts
+        fd.puts paragraphs_of(PROJ.readme_file, p).join("\n\n")
+        fd.puts
+      end
+      fd.puts ann.text if ann.text
+    end
+  end
+
+  desc "Create an announcement file"
+  task :announcement => ['ann:prereqs', PROJ.ann.file]
+
+  desc "Send an email announcement"
+  task :email => ['ann:prereqs', PROJ.ann.file] do
+    ann = PROJ.ann
+    from = ann.email[:from] || Array(PROJ.authors).first || PROJ.email
+    to   = Array(ann.email[:to])
+
+    ### build a mail header for RFC 822
+    rfc822msg =  "From: #{from}\n"
+    rfc822msg << "To: #{to.join(',')}\n"
+    rfc822msg << "Subject: [ANN] #{PROJ.name} #{PROJ.version}"
+    rfc822msg << " (#{PROJ.release_name})" if PROJ.release_name
+    rfc822msg << "\n"
+    rfc822msg << "Date: #{Time.new.rfc822}\n"
+    rfc822msg << "Message-Id: "
+    rfc822msg << "<#{"%.8f" % Time.now.to_f}@#{ann.email[:domain]}>\n\n"
+    rfc822msg << File.read(ann.file)
+
+    params = [:server, :port, :domain, :acct, :passwd, :authtype].map do |key|
+      ann.email[key]
+    end
+
+    params[3] = PROJ.email if params[3].nil?
+
+    if params[4].nil?
+      STDOUT.write "Please enter your e-mail password (#{params[3]}): "
+      params[4] = STDIN.gets.chomp
+    end
+
+    ### send email
+    Net::SMTP.start(*params) {|smtp| smtp.sendmail(rfc822msg, from, to)}
+  end
+end  # namespace :ann
+
+desc 'Alias to ann:announcement'
+task :ann => 'ann:announcement'
+
+CLOBBER << PROJ.ann.file
+
+# EOF