theman 0.0.3 → 0.0.4

data/README.rdoc

@@ -2,16 +2,26 @@
 
 The man getting you down?
 
-FasterCSV is great and all but when you get to 100mb files it takes a while and you may only be looking for certain records that match some criteria, enter theman.
+FasterCSV is great and all but when you get to 100MB files it takes a 
+while and you may only be looking for certain records that match some 
+criteria, enter Theman.
 
 == Installation
 
+=== Rails3
+
+  gem 'theman'
+
+=== Rails2.x
+
   config.gem 'theman'
 
 Or
 
   gem install 'theman'
 
+Only needs active record and postgresql gem.
+
 == Basic Usage
 
   my_agent = ::Theman::Agency.new 'pretty.csv'
@@ -20,10 +30,10 @@ Or
 
 == Advanced Usage
     
-  my_agent = ::Theman::Agency.new 'ugly.csv' do |agent|
-    agent.nulls /"N"/, /"UNKNOWN"/, /""/
-    agent.seds "-n -e :a -e '1,15!{P;N;D;};N;ba'"
-    agent.table do |t|
+  my_agent = ::Theman::Agency.new 'ugly.csv' do |smith|
+    smith.nulls /"N"/, /"UNKNOWN"/, /""/
+    smith.seds "-n -e :a -e '1,15!{P;N;D;};N;ba'"
+    smith.table do |t|
       t.date :date
       t.integer :ext_id
       t.float :amount
@@ -33,15 +43,68 @@ Or
   temp_model = my_agent.instance
   temp_model.where(:exited => true).count    
 
-In the above example we ommitted the last 15 rows and made some things null.
+In the above example we omitted the last 15 rows and made some things null.
+
+If you do not provide a table block your columns will be VARCHAR(255), you 
+can cherry pick cols to change data types.
+
+The temp table has no id column but you could add one after if you wanted.
+
+If you want to call this procedural just don't pass in the path to the file 
+and Theman will not create a table in which case 
+you will need to call everything explicitly:
+
+  smith = ::Theman::Agency.new
+  smith.stream 'real_ugly.csv'
+  smith.seds "-n -e :a -e '1,15!{P;N;D;};N;ba'"
+  smith.nulls /"XXXX"/
+  smith.date :date
+
+  smith.create_table
+  smith.pipe_it
+
+== Dates
+
+Ah dates, everybodys joy. Use datestyle to tell Theman to tell postgresql:
+
+    my_agent = ::Theman::Agency.new 'uber_foie_gras.csv' do |schmit|
+      schmit.datestyle 'European'
+      schmit.table do |t|
+        t.date :col_date
+      end
+    end
+
+Refer to postgrsql docs for more info but here is some copy and paste:
+
+ISO
+
+* Use ISO 8601-style dates and times (YYYY-MM-DD HH:MM:SS). This is the default. 
+
+SQL
+
+* Use Oracle/Ingres-style dates and times. Note that this style has nothing to do with SQL 
+(which mandates ISO 8601 style), the naming of this option is a historical accident. 
+
+PostgreSQL
+
+* Use traditional PostgreSQL format. 
+
+German
+
+    dd.mm.yyyy
+
+European
+
+    dd/mm/yyyy
 
-If you do not provide a table block your columns will be VARCHAR(255), you can cherry pick cols to change data types.
+US
 
-The temp table has no id col but you could add one after if you wanted
+    mm/dd/yyyy
 
 == Troubles
 
-Table empty? the man has given you crappy data and PostgresSQL has silently dissed your data
+Table empty? the man (the real life one) has given you crappy data and PostgresSQL 
+has silently dissed it.
 
 == Copyright
 

data/lib/theman/themans_agency.rb

@@ -2,8 +2,6 @@ module Theman
   class Agency
     attr_reader :instance, :column_names, :null_replacements, :sed_commands
 
-    attr_writer :stream
-    
     def initialize(stream = nil, parent = ::ActiveRecord::Base)
       # source of the data
       @stream = stream
@@ -70,12 +68,27 @@ module Theman
         end
       end
     end
-    
+
+    def stream(path)
+      @stream = path
+    end
+
+    def datestyle(local)
+      @psql_datestyle = local
+    end
+
+    def psql_command
+      psql = []
+      psql << "SET DATESTYLE TO #{@psql_datestyle}" unless @psql_datestyle.nil?
+      psql << "COPY #{instance.table_name} FROM STDIN WITH CSV HEADER"
+      psql.join("; ")
+    end
+
     # use postgress COPY command using STDIN with CSV HEADER
     # reads chunks of 8192 bytes to save memory
     def pipe_it(l = "")
       raw = instance.connection.raw_connection
-      raw.query "COPY #{instance.table_name} FROM STDIN WITH CSV HEADER"
+      raw.query psql_command
       command = "cat #{@stream} #{seds_join}"
       f = IO.popen(command)
       begin

data/lib/theman/version.rb

@@ -1,3 +1,3 @@
 module Theman
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

data/spec/fixtures/temp_five.csv

@@ -0,0 +1,5 @@
+"COL DATE"
+"2010-12-25"
+"2010-11-19"
+"2010-11-05"
+"2010-11-05"

data/spec/fixtures/temp_four.csv

@@ -0,0 +1,5 @@
+"COL DATE"
+"12/25/2010"
+"11/19/2010"
+"05/11/2010"
+"05/11/2010"

data/spec/fixtures/temp_three.csv

@@ -0,0 +1,5 @@
+"COL DATE"
+"25/12/2010"
+"19/11/2010"
+"05/11/2010"
+"05/11/2010"

data/spec/theman_spec.rb

@@ -89,3 +89,81 @@ describe Theman::Agency, "data types" do
     @instance.where(:col_three => nil).count.should == 2
   end
 end
+
+describe Theman::Agency, "european date styles" do
+  before do
+    @csv = File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec', 'fixtures', 'temp_three.csv'))
+    @agent = ::Theman::Agency.new @csv do |smith|
+      smith.datestyle 'European'
+      smith.table do |t|
+        t.date :col_date
+      end
+    end
+    @instance = @agent.instance
+  end
+  
+  it "should have correct date" do
+    date = @instance.first.col_date
+    date.day.should == 25
+    date.month.should == 12
+  end
+end
+
+describe Theman::Agency, "US date styles" do
+  before do
+    @csv = File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec', 'fixtures', 'temp_four.csv'))
+    @agent = ::Theman::Agency.new @csv do |smith|
+      smith.datestyle 'US'
+      smith.table do |t|
+        t.date :col_date
+      end
+    end
+    @instance = @agent.instance
+  end
+  
+  it "should have correct date" do
+    date = @instance.first.col_date
+    date.day.should == 25
+    date.month.should == 12
+  end
+end
+
+describe Theman::Agency, "ISO date styles" do
+  before do
+    @csv = File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec', 'fixtures', 'temp_five.csv'))
+    @agent = ::Theman::Agency.new @csv do |smith|
+      smith.datestyle 'ISO'
+      smith.table do |t|
+        t.date :col_date
+      end
+    end
+    @instance = @agent.instance
+  end
+  
+  it "should have correct date" do
+    date = @instance.first.col_date
+    date.day.should == 25
+    date.month.should == 12
+  end
+end
+
+describe Theman::Agency, "procedural" do
+  before do
+    @csv = File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec', 'fixtures', 'temp_two.csv'))
+  end
+
+  it "should be able to be called procedural" do
+    smith = ::Theman::Agency.new
+    smith.stream @csv
+    smith.datestyle "European"
+    smith.seds "-n -e :a -e '1,15!{P;N;D;};N;ba'"
+    smith.nulls /"XXXX"/
+    smith.date :date
+    smith.create_table
+    smith.pipe_it
+    my_model = smith.instance
+    my_model.first.date.class.should == Date
+    my_model.first.org_code.class.should == NilClass
+    my_model.count.should == 5
+  end
+end

data/theman.gemspec

@@ -17,6 +17,10 @@ Gem::Specification.new do |s|
   s.add_development_dependency "bundler", ">= 1.0.0"
   s.add_development_dependency "rspec", ">= 2.0.0"
   s.add_development_dependency "activerecord", ">= 3.0.0"
+  s.add_development_dependency "pg"
+  
+  s.add_runtime_dependency "activerecord"
+  s.add_runtime_dependency "pg"
 
   s.files        = `git ls-files`.split("\n")
   s.executables  = `git ls-files`.split("\n").map{|f| f =~ /^bin\/(.*)/ ? $1 : nil}.compact

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 0
-  - 3
-  version: 0.0.3
+  - 4
+  version: 0.0.4
 platform: ruby
 authors: 
 - Rufus Post
@@ -62,6 +62,45 @@ dependencies:
         version: 3.0.0
   type: :development
   version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: pg
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: pg
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id006
 description: FasterCSV is great and all but when you get to 100mb files it takes a while and you may only be looking for certain records that match some criteria, enter theman
 email: 
 - rufuspost@gmail.com
@@ -79,7 +118,10 @@ files:
 - lib/theman.rb
 - lib/theman/themans_agency.rb
 - lib/theman/version.rb
+- spec/fixtures/temp_five.csv
+- spec/fixtures/temp_four.csv
 - spec/fixtures/temp_one.csv
+- spec/fixtures/temp_three.csv
 - spec/fixtures/temp_two.csv
 - spec/spec_helper.rb
 - spec/theman_spec.rb