theman 0.0.4 → 0.0.5

data/README.rdoc

@@ -2,109 +2,115 @@
 
 The man getting you down?
 
-FasterCSV is great and all but when you get to 100MB files it takes a 
-while and you may only be looking for certain records that match some 
-criteria, enter Theman.
+FasterCSV is great but when you get to 100MB files it takes too long and 
+you may only be looking for certain records that match some criteria, 
+enter Theman.
 
 == Installation
 
-=== Rails3
-
-  gem 'theman'
+  gem install 'theman'
 
-=== Rails2.x
+== Basic Usage
 
-  config.gem 'theman'
+  my_agent = ::Theman::Agency.new 'pretty.csv'
+  my_agent.instance.count
 
-Or
+Say if you wanted to analyse some data the man has given you, you could make the 
+table persist by doing the following in an IRB session:
 
-  gem install 'theman'
+  require 'active_record'
+  require 'theman'
 
-Only needs active record and postgresql gem.
+  ActiveRecord::Base.establish_connection :database => $USER, :host => "localhost", :adapter => "postgresql"
 
-== Basic Usage
+  my_agent = ::Theman::Agency.new('pretty.csv', ActiveRecord::Base, :temporary => false) {|a| a.date :date_col }
 
-  my_agent = ::Theman::Agency.new 'pretty.csv'
-  temp_model = my_agent.instance
-  temp_model.count
+After you have figured out what the hell is going on with the data, you can get the man of 
+your back (for now).
 
 == Advanced Usage
     
-  my_agent = ::Theman::Agency.new 'ugly.csv' do |smith|
-    smith.nulls /"N"/, /"UNKNOWN"/, /""/
-    smith.seds "-n -e :a -e '1,15!{P;N;D;};N;ba'"
-    smith.table do |t|
+  my_agent = ::Theman::Agency.new 'ugly.csv', ActiveRecord::Base, :primary_key => true do |s|
+    s.nulls /"N"/, /"UNKNOWN"/, /""/
+    s.seds "-n -e :a -e '1,15!{P;N;D;};N;ba'"
+    s.delimiter "|"
+    s.table do |t|
+      t.string :name, :limit => 50
       t.date :date
       t.integer :ext_id
       t.float :amount
       t.boolean :exited
     end
   end
-  temp_model = my_agent.instance
-  temp_model.where(:exited => true).count    
 
-In the above example we omitted the last 15 rows and made some things null.
+  my_agent.instance.where(:exited => true).count
+
+In the above example we omitted the last 15 rows, made some things null, 
+added a primary key and changed some column data types to something else.
 
-If you do not provide a table block your columns will be VARCHAR(255), you 
-can cherry pick cols to change data types.
+If you do not provide a table block your columns will be VARCHAR(255); you 
+can cherry pick the columns that you want to change the data types for.
 
-The temp table has no id column but you could add one after if you wanted.
+The temp table has no id column by default, but you can add one using the options 
+hash or calling add_primary_key, this will add the agents_pkey column.
 
-If you want to call this procedural just don't pass in the path to the file 
-and Theman will not create a table in which case 
-you will need to call everything explicitly:
+If you want to call this procedural style just don't pass in the path to the file 
+and Theman will not create a table, in this case you will need to call everything 
+explicitly:
 
   smith = ::Theman::Agency.new
   smith.stream 'real_ugly.csv'
-  smith.seds "-n -e :a -e '1,15!{P;N;D;};N;ba'"
-  smith.nulls /"XXXX"/
-  smith.date :date
-
   smith.create_table
   smith.pipe_it
 
+WARNING: if you have user input in your sed commands, don't.
+
+NOTE: When passing in extra sed commands each one will be given it's own subprocess, 
+this makes it much faster if you have more than one core.
+
 == Dates
 
-Ah dates, everybodys joy. Use datestyle to tell Theman to tell postgresql:
+Ah dates, the joy! Use datestyle to tell Theman to then tell PostgreSQL:
 
-    my_agent = ::Theman::Agency.new 'uber_foie_gras.csv' do |schmit|
-      schmit.datestyle 'European'
-      schmit.table do |t|
-        t.date :col_date
-      end
+  my_agent = ::Theman::Agency.new 'uber_foie_gras.csv' do |schmit|
+    schmit.datestyle 'European'
+    schmit.table do |t|
+      t.date :start_date
+      t.date :end_date
     end
+  end
 
-Refer to postgrsql docs for more info but here is some copy and paste:
+Refer to PostgreSQL docs for more info in the mean time here is some 
+copy and paste action:
 
-ISO
+=== ISO
 
-* Use ISO 8601-style dates and times (YYYY-MM-DD HH:MM:SS). This is the default. 
+Use ISO 8601-style dates and times (YYYY-MM-DD HH:MM:SS). This is the default. 
 
-SQL
+=== SQL
 
-* Use Oracle/Ingres-style dates and times. Note that this style has nothing to do with SQL 
-(which mandates ISO 8601 style), the naming of this option is a historical accident. 
+Use Oracle/Ingres-style dates and times. 
 
-PostgreSQL
+=== PostgreSQL
 
-* Use traditional PostgreSQL format. 
+Use traditional PostgreSQL format. 
 
-German
+=== German
 
-    dd.mm.yyyy
+dd.mm.yyyy
 
-European
+=== European
 
-    dd/mm/yyyy
+dd/mm/yyyy
 
-US
+=== US
 
-    mm/dd/yyyy
+mm/dd/yyyy
 
 == Troubles
 
-Table empty? the man (the real life one) has given you crappy data and PostgresSQL 
-has silently dissed it.
+Table empty? the man (the real life one) has given you crappy data and 
+PostgresSQL has silently dissed it.
 
 == Copyright
 

data/lib/theman/agency.rb

@@ -0,0 +1,157 @@
+module Theman
+  class Agency
+    attr_reader :instance, :column_names
+
+    def initialize(stream = nil, parent = ::ActiveRecord::Base, options = {})
+      @options = options
+      @stream  = stream
+
+      agent_id = sprintf "agent%010d", rand(100000000)
+      @column_names = {}
+      @instance = Class.new(parent) do
+        instance_eval <<-EOV, __FILE__, __LINE__ + 1
+          set_table_name "#{agent_id}"
+          def table_name
+            "#{agent_id}"
+          end
+          def inspect
+            "Agent (#{agent_id})"
+          end
+        EOV
+      end
+
+      yield self if block_given?
+      return unless stream
+      create_table
+      pipe_it
+      if @options[:primary_key]
+        add_primary_key
+      end
+    end
+
+    def table
+      yield self if block_given?
+    end
+
+    # columnn data type methods
+    %w( string text integer float decimal datetime timestamp time date binary boolean ).each do |column_type|
+      class_eval <<-EOV, __FILE__, __LINE__ + 1
+        def #{column_type}(column_name, *args)
+          column(column_name, '#{column_type}', *args)
+        end
+      EOV
+    end
+
+    # overides the default string type column
+    def column(column_name, column_type, *args)
+      @column_names.merge! column_name.to_sym => [column_name, column_type, *args]
+    end
+
+    def stream(arg)
+      @stream = arg
+    end
+
+    def datestyle(arg)
+      @datestyle = arg
+    end
+
+    def nulls(*args)
+      @nulls = args
+    end
+
+    def seds(*args)
+      @seds = args
+    end
+
+    def delimiter(arg)
+      @delimiter = arg
+    end
+    
+    def symbolize(name)
+      name.gsub(/ /,"_").gsub(/\W/, "").downcase.to_sym
+    end
+    
+    def psql_copy(psql = [])
+      psql << "COPY #{@instance.table_name} FROM STDIN WITH"
+      psql << "DELIMITER '#{@delimiter}'" unless @delimiter.nil?
+      psql << "CSV HEADER"
+      psql
+    end
+
+    def psql_command(psql = [])
+      psql << "SET DATESTYLE TO #{@datestyle}" unless @datestyle.nil?
+      psql << psql_copy.join(" ")
+      psql
+    end
+
+    def sed_command(sed = [])
+      sed << nulls_to_sed unless @nulls.nil?
+      sed << @seds unless @seds.nil?
+      sed
+    end
+
+    def nulls_to_sed
+      @nulls.map do |regex|
+        "-e 's/#{regex.source}//g'"
+      end
+    end
+
+    # creates a delimiter regular expresion
+    def delimiter_regexp
+      Regexp.new(@delimiter.nil? ? "," : "\\#{@delimiter}")
+    end
+
+    # read the first line from the stream to create a table with
+    def create_table
+      f = File.open(@stream, 'r')
+      options = {:id => false}
+      options.merge!(:temporary => true) if @options[:temporary].nil?
+      instance.connection.create_table(instance.table_name, options) do |t|
+        f.each_line do |line|
+          line.split(delimiter_regexp).each do |col|
+            column_name = symbolize(col)
+            if custom = @column_names.fetch(column_name, nil)
+              t.column(*custom) 
+            else
+              t.string column_name
+            end
+          end
+          break
+        end
+      end
+    end
+    
+    # system command for IO subprocesses, commands are piped to 
+    # take advantage of multi cores
+    def system_command
+      unless sed_command.empty?
+        "cat #{@stream} | sed #{sed_command.join(" | sed ")}" 
+      else
+        "cat #{@stream}"
+      end
+    end
+
+    # addition of a primary key after the data has been piped to 
+    # the table
+    def add_primary_key
+      instance.connection.raw_connection.query "ALTER TABLE #{instance.table_name} ADD COLUMN agents_pkey serial PRIMARY KEY;"
+    end
+
+    # use postgress COPY command using STDIN with CSV HEADER
+    # reads chunks of 8192 bytes to save memory
+    def pipe_it(l = "")
+      raise "table does not exist" unless instance.table_exists?
+      raw = instance.connection.raw_connection
+      raw.query psql_command.join("; ")
+      f = IO.popen(system_command)
+      begin
+        while f.read(8192, l)
+          raw.put_copy_data l
+        end
+      rescue EOFError
+        f.close
+      end
+      raw.put_copy_end
+    end
+  end
+end

data/lib/theman/version.rb

@@ -1,3 +1,3 @@
 module Theman
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

data/lib/theman.rb

@@ -1,4 +1,4 @@
-require 'theman/themans_agency'
+require 'theman/agency'
 
 module Theman
 end

data/spec/fixtures/temp_six.txt

@@ -0,0 +1,5 @@
+"COL DATE"|"COL TWO"|"COL THREE"|"COL FOUR"|"COL FIVE"
+"05/12/2010"|"some \"text\""|"123134343"|"Y"|"2.0"
+"05/11/2010"|"some other \"text\""|"987983134343"|"Y"|"11.5"
+"05/11/2010"|""|"N"|""|"10.5"
+"05/11/2010"|""|"UNKNOWN"|""|"3.0"

data/spec/theman_spec.rb

@@ -14,7 +14,7 @@ describe Theman::Agency, "instance object" do
   end
 
   it "should have a table name" do
-    @instance.table_name.should match /c[0-9]{10}/
+    @instance.table_name.should match /agent[0-9]{10}/
   end
 
   it "should have an ispect method" do
@@ -81,7 +81,7 @@ describe Theman::Agency, "data types" do
   end
 
   it "should have an array of nulls" do
-    @agent.null_replacements.should == [/"N"/, /"UNKNOWN"/, /""/]
+    @agent.nulls_to_sed.should == ["-e 's/\"N\"//g'", "-e 's/\"UNKNOWN\"//g'", "-e 's/\"\"//g'"]
   end
   
   it "should have nulls not strings" do
@@ -167,3 +167,46 @@ describe Theman::Agency, "procedural" do
     my_model.count.should == 5
   end
 end
+
+describe Theman::Agency, "create table" do
+  before do
+    @csv = File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec', 'fixtures', 'temp_one.csv'))
+    @agent = ::Theman::Agency.new @csv do |agent|
+      agent.nulls /"N"/, /"UNKNOWN"/, /""/
+      agent.table do |t|
+        t.string :col_two, :limit => 50
+      end
+    end
+    @instance = @agent.instance
+  end
+
+  it "should have" do
+    @instance.first.col_two.should == "some \\text\\"
+  end
+end
+
+describe Theman::Agency, "add primary key" do
+  before do
+    @csv = File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec', 'fixtures', 'temp_one.csv'))
+    @agent = ::Theman::Agency.new @csv, ActiveRecord::Base, :primary_key => true
+    @instance = @agent.instance
+  end
+
+  it "should have serial primary key" do
+    @instance.first.agents_pkey.should == 1
+  end
+end
+
+describe Theman::Agency, "delimiters" do
+  before do
+    @csv = File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec', 'fixtures', 'temp_six.txt'))
+    @agent = ::Theman::Agency.new @csv do |agent|
+      agent.delimiter "|"
+    end
+    @instance = @agent.instance
+  end
+
+  it "should have imported pipe delimited txt file" do
+    @instance.count.should == 4
+  end
+end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 0
-  - 4
-  version: 0.0.4
+  - 5
+  version: 0.0.5
 platform: ruby
 authors: 
 - Rufus Post
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-28 00:00:00 +11:00
+date: 2010-11-08 00:00:00 +11:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -116,11 +116,12 @@ files:
 - README.rdoc
 - Rakefile
 - lib/theman.rb
-- lib/theman/themans_agency.rb
+- lib/theman/agency.rb
 - lib/theman/version.rb
 - spec/fixtures/temp_five.csv
 - spec/fixtures/temp_four.csv
 - spec/fixtures/temp_one.csv
+- spec/fixtures/temp_six.txt
 - spec/fixtures/temp_three.csv
 - spec/fixtures/temp_two.csv
 - spec/spec_helper.rb

data/lib/theman/themans_agency.rb

@@ -1,133 +0,0 @@
-module Theman
-  class Agency
-    attr_reader :instance, :column_names, :null_replacements, :sed_commands
-
-    def initialize(stream = nil, parent = ::ActiveRecord::Base)
-      # source of the data
-      @stream = stream
-
-      # create a new class that extends an active record model
-      # use instance_parent(klass) if not ActiveRecord::Base
-      cabinet_id = "c#{10.times.map{rand(9)}.join}" 
-      @column_names = {}
-      @instance = Class.new(parent) do
-        instance_eval <<-EOV, __FILE__, __LINE__ + 1
-          set_table_name "#{cabinet_id}"
-          def table_name
-            "#{cabinet_id}"
-          end
-          def inspect
-            "Agent (#{cabinet_id})"
-          end
-        EOV
-      end
-
-      # if stream given table will be created
-      # other wise create_table and pipe_it will need to called
-      # proceduraly
-      if stream
-        if block_given?
-          yield self
-        end
-        create_table
-        pipe_it
-      end
-    end
-
-    def table
-      yield self if block_given?
-    end
-
-    # overide ActiveRecord column types to be used in a block
-    %w( string text integer float decimal datetime timestamp time date binary boolean ).each do |column_type|
-      class_eval <<-EOV, __FILE__, __LINE__ + 1
-        def #{column_type}(*args)
-          column(args[0], '#{column_type}', args[1].nil? ? {} : args[1])
-        end
-      EOV
-    end
-
-    # overides the default string type column
-    def column(name, type, options)
-      @column_names.merge! name.to_sym => [name, type, options]
-    end
-
-    def create_table
-      f = File.open(@stream, 'r')
-      instance.connection.create_table(instance.table_name, :temporary => true, :id => false) do |t|
-        f.each_line do |line|
-          line.split(/,/).each do |col|
-            column_name = symbolize(col)
-            if custom = @column_names.fetch(column_name, nil)
-              t.column(*custom) 
-            else
-              t.string column_name
-            end
-          end
-          break
-        end
-      end
-    end
-
-    def stream(path)
-      @stream = path
-    end
-
-    def datestyle(local)
-      @psql_datestyle = local
-    end
-
-    def psql_command
-      psql = []
-      psql << "SET DATESTYLE TO #{@psql_datestyle}" unless @psql_datestyle.nil?
-      psql << "COPY #{instance.table_name} FROM STDIN WITH CSV HEADER"
-      psql.join("; ")
-    end
-
-    # use postgress COPY command using STDIN with CSV HEADER
-    # reads chunks of 8192 bytes to save memory
-    def pipe_it(l = "")
-      raw = instance.connection.raw_connection
-      raw.query psql_command
-      command = "cat #{@stream} #{seds_join}"
-      f = IO.popen(command)
-      begin
-        while f.read(8192, l)
-          raw.put_copy_data l
-        end
-      rescue EOFError
-        f.close
-      end
-      raw.put_copy_end
-    end
-
-    def nulls(*args)
-      @null_replacements = args
-    end
-
-    def seds(*args)
-      @sed_commands = args
-    end
-    
-    def symbolize(name)
-      name.gsub(/ /,"_").gsub(/\W/, "").downcase.to_sym
-    end
-    
-    # join together the sed commands to apply to stream
-    def seds_join(commands = [])
-      unless null_replacements.nil?
-        commands << "| sed #{nulls_to_sed.join(" ")}"
-      end
-      unless sed_commands.nil?
-        commands << "| sed #{sed_commands.join("| sed ")}"
-      end
-      commands.join(" ")
-    end
-
-    def nulls_to_sed
-      @null_replacements.map do |null|
-        "-e 's/#{null.source}//g'"
-      end
-    end
-  end
-end