theman 0.0.1

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,25 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+
+## PROJECT::SPECIFIC
+spec/database.yml
+log/
+spec/fixtures/fucking_huge.csv
+spec/fucking_huge_spec.rb

data/README.rdoc

@@ -0,0 +1,69 @@
+= theman
+
+The man getting you down?
+
+FasterCSV is great and all but when you get to 100mb files it takes a while and you may only be looking for certain records that match some criteria, enter theman.
+
+== Installation
+
+  config.gem 'theman'
+
+Or
+
+  gem install 'theman'
+
+== Basic Usage
+
+  cabinet = ::Theman::Agency.new 'pretty.csv'
+  temp_model = cabinet.instance
+  temp_model.count
+
+== Advanced Usage
+    
+  cabinet = ::Theman::Agency.new 'ugly.csv' do |cabinet|
+    cabinet.nulls /"N"/, /"UNKNOWN"/, /""/
+    cabinet.seds "-n -e :a -e '1,15!{P;N;D;};N;ba'"
+    cabinet.table do |t|
+      t.date :date
+      t.integer :ext_id
+      t.float :amount
+      t.boolean :exited
+    end
+  end
+  temp_model = cabinet.instance
+  temp_model.where(:exited => true).count    
+
+In the above example we ommitted the last 15 rows and made some things null.
+
+If you do not provide a table block your columns will be VARCHAR(255), you can cherry pick cols to change data types.
+
+The temp table has no id col but you could add one after if you wanted
+
+== Troubles
+
+Table empty? the man has given you crappy data and PostgresSQL has silently dissed your data
+
+== Copyright
+
+(The MIT License)
+
+Copyright (c) 2010 {mynameisrufus (Rufus Post)}[http://github.com/mynameisrufus]
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "theman"
+    gem.summary = %Q{PostgreSQL AR temporary table generator using PostgreSQL COPY}
+    gem.description = %Q{longer description of your gem}
+    gem.email = "rufuspost@gmail.com"
+    gem.homepage = "http://github.com/mynameisrufus/theman"
+    gem.authors = ["Rufus Post"]
+    gem.add_development_dependency "rspec", ">= 2.0.0.beta.20"
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/lib/theman/themans_agency.rb

@@ -0,0 +1,109 @@
+module Theman
+  class Agency
+    attr_reader :instance, :table_proc, :null_replacements, :stream_location, :column_names
+
+    def initialize(*args)
+      cabinet_id = "c#{10.times.map{rand(9)}.join}" 
+      @column_names = {}
+      @instance = Class.new(parent) do
+        instance_eval <<-EOV, __FILE__, __LINE__ + 1
+          set_table_name "#{cabinet_id}"
+          def table_name
+            "#{cabinet_id}"
+          end
+          def inspect
+            "Agent (#{cabinet_id})"
+          end
+        EOV
+      end
+      yield self if block_given?
+      unless args.first.nil?
+        @stream_location = args.first
+        create_table
+        pipe_it
+      end
+    end
+    
+    def instance_parent(klass)
+      @parent = klass
+    end
+
+    def table(&block)
+      yield self if block_given?
+    end
+
+    def nulls(*args)
+      @null_replacements = args
+    end
+
+    def seds(*args)
+      @extra_seds = args
+    end
+    
+    def symbolize(name)
+      name.gsub(/ /,"_").gsub(/\W/, "").downcase.to_sym
+    end
+
+    %w( string text integer float decimal datetime timestamp time date binary boolean ).each do |column_type|
+      class_eval <<-EOV, __FILE__, __LINE__ + 1
+        def #{column_type}(*args)
+          column(args[0], '#{column_type}', args[1].nil? ? {} : args[1])
+        end
+      EOV
+    end
+    
+    protected
+    def sed_to_s
+      seds = []
+      seds << "| sed #{nulls_to_sed.join(" ")}" unless @null_replacements.nil?
+      seds << "| sed #{@extra_seds.join("| sed ")}" unless @extra_seds.nil?
+      return seds.join(" ") unless seds.empty?
+    end
+
+    def nulls_to_sed
+      @null_replacements.map do |null|
+        "-e 's/#{null.source}//g'"
+      end
+    end
+
+    def parent
+      @parent ||= ::ActiveRecord::Base
+    end
+
+    def create_table
+      f = File.open(stream_location, 'r')
+      instance.connection.create_table(instance.table_name, :temporary => true, :id => false) do |t|
+        f.each_line do |line|
+          line.split(/,/).each do |col|
+            column_name = col.is_a?(Hash) ? col : symbolize(col)
+            if custom = @column_names.fetch(column_name, nil)
+              t.column(*custom) 
+            else
+              t.string column_name
+            end
+          end
+          break
+        end
+      end
+    end
+    
+    def column(name, type, options)
+      @column_names.merge! name.to_sym => [name, type, options]
+    end
+
+    def pipe_it(l = "")
+      raw = instance.connection.raw_connection
+      raw.query "COPY #{instance.table_name} FROM STDIN WITH CSV HEADER"
+      command = "cat #{stream_location} #{sed_to_s}"
+      f = IO.popen(command)
+      begin
+        while f.read(8192, l)
+          raw.put_copy_data l
+        end
+      rescue EOFError
+        f.close
+      end
+      raw.put_copy_end
+    end
+  end
+end

data/lib/theman.rb

@@ -0,0 +1,4 @@
+require 'theman/themans_agency'
+
+module Theman
+end

data/spec/fixtures/temp_one.csv

@@ -0,0 +1,5 @@
+"COL DATE","COL TWO","COL THREE","COL FOUR","COL FIVE"
+"05/12/2010","some \"text\"","123134343","Y","2.0"
+"05/11/2010","some other \"text\"","987983134343","Y","11.5"
+"05/11/2010","","N","","10.5"
+"05/11/2010","","UNKNOWN","","3.0"

data/spec/fixtures/temp_two.csv

@@ -0,0 +1,21 @@
+"DATE","ORG_CODE","JOB_SEEKER_ID"
+"23/09/2010","XXXX","342310094609"
+"23/09/2010","XXXX","234583366709"
+"23/09/2010","XXXX","342939870209"
+"23/09/2010","XXXX","234098040009"
+"23/09/2010","XXXX","987919906509"
+
+"@!!@"
+"Subscription ID: XXXXXXXXXXXXXXXXXXXXXX"
+"Managed By: XXXXX"
+"Record Count: XXXXX"
+"Report  Date: XXXXXXX"
+"Extract Date: XXXXXXX"
+"Data Load Freq: DAILY"
+"Report Name: XXXXX"
+"Report Desc: XXX XXXX XXXXX XXX XXX XX XX"
+"Report Message: "
+"Solution Name: XXXXXX"
+"Solution Desc: "
+"REPORT CRITERIA BELOW"
+"Organisation:  XXXX"

data/spec/spec_helper.rb

@@ -0,0 +1,15 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'rails'
+
+require 'active_record'
+Rails.env = 'test'
+ActiveRecord::Base.configurations = YAML.load_file(File.join("spec", "database.yml"))
+FileUtils.mkdir_p "#{Dir.pwd}/log"
+logfile= File.new("#{Dir.pwd}/log/database.log", "w")
+ActiveRecord::Base.logger = Logger.new(File.open(logfile, 'w'))
+ActiveRecord::Base.establish_connection(ActiveRecord::Base.configurations.fetch(Rails.env))
+
+require 'theman'
+require 'rspec'
+require 'rspec/autorun'

data/spec/theman_spec.rb

@@ -0,0 +1,91 @@
+require 'spec_helper'
+
+describe Theman::Agency, "instance object" do
+  before do
+    @instance = ::Theman::Agency.new.instance
+  end
+
+  it "should superclass active record" do
+    @instance.superclass.should == ActiveRecord::Base 
+  end
+
+  it "should have connection" do
+    @instance.connection.class.should == ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
+  end
+
+  it "should have a table name" do
+    @instance.table_name.should match /c[0-9]{10}/
+  end
+
+  it "should have an ispect method" do
+    @instance.inspect.should match /Agent/
+  end
+end
+
+describe Theman::Agency, "instance methods" do
+  it "should downcase and symbolize" do
+    Theman::Agency.new.symbolize("STRANGE NAME").should == :strange_name
+  end
+end
+
+describe Theman::Agency, "basic" do
+  before do
+    @csv = File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec', 'fixtures', 'temp_one.csv'))
+    @cabinet = ::Theman::Agency.new @csv
+    @instance = @cabinet.instance
+  end
+  
+  it "should have all the records from the csv" do
+    @instance.count.should == 4
+  end
+end
+
+describe Theman::Agency, "sed chomp" do
+  before do
+    @csv = File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec', 'fixtures', 'temp_two.csv'))
+    @cabinet = ::Theman::Agency.new @csv do |cabinet|
+      cabinet.seds "-n -e :a -e '1,15!{P;N;D;};N;ba'"
+    end
+    @instance = @cabinet.instance
+  end
+  
+  it "should have all the records from the csv" do
+    @instance.count.should == 5
+  end
+end
+
+describe Theman::Agency, "data types" do
+  before do
+    @csv = File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec', 'fixtures', 'temp_one.csv'))
+    @cabinet = ::Theman::Agency.new @csv do |cabinet|
+      cabinet.nulls /"N"/, /"UNKNOWN"/, /""/
+      cabinet.table do |t|
+        t.date :col_date
+        t.boolean :col_four
+        t.float :col_five
+      end
+    end
+    @instance = @cabinet.instance
+  end
+
+  it "should create date col" do
+    @instance.first.col_date.class.should == Date
+  end
+
+  it "should create boolean col" do
+    @instance.where(:col_four => true).count.should == 2
+  end
+
+  it "should create float col" do
+    @instance.where("col_five > 10.0").count.should == 2
+  end
+
+  it "should have an array of nulls" do
+    @cabinet.null_replacements.should == [/"N"/, /"UNKNOWN"/, /""/]
+  end
+  
+  it "should have nulls not strings" do
+    @instance.where(:col_two => nil).count.should == 2
+    @instance.where(:col_three => nil).count.should == 2
+  end
+end

data/theman.gemspec

@@ -0,0 +1,55 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{theman}
+  s.version = "0.0.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Rufus Post"]
+  s.date = %q{2010-09-28}
+  s.description = %q{longer description of your gem}
+  s.email = %q{rufuspost@gmail.com}
+  s.extra_rdoc_files = [
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "lib/theman.rb",
+     "lib/theman/themans_agency.rb",
+     "spec/fixtures/temp_one.csv",
+     "spec/fixtures/temp_two.csv",
+     "spec/spec_helper.rb",
+     "spec/theman_spec.rb",
+     "theman.gemspec"
+  ]
+  s.homepage = %q{http://github.com/mynameisrufus/theman}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{PostgreSQL AR temporary table generator using PostgreSQL COPY}
+  s.test_files = [
+    "spec/theman_spec.rb",
+     "spec/spec_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<rspec>, [">= 2.0.0.beta.20"])
+    else
+      s.add_dependency(%q<rspec>, [">= 2.0.0.beta.20"])
+    end
+  else
+    s.add_dependency(%q<rspec>, [">= 2.0.0.beta.20"])
+  end
+end
+

metadata

@@ -0,0 +1,92 @@
+--- !ruby/object:Gem::Specification 
+name: theman
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Rufus Post
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-09-28 00:00:00 +10:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 0
+        - 0
+        - beta
+        - 20
+        version: 2.0.0.beta.20
+  type: :development
+  version_requirements: *id001
+description: longer description of your gem
+email: rufuspost@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- .document
+- .gitignore
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/theman.rb
+- lib/theman/themans_agency.rb
+- spec/fixtures/temp_one.csv
+- spec/fixtures/temp_two.csv
+- spec/spec_helper.rb
+- spec/theman_spec.rb
+- theman.gemspec
+has_rdoc: true
+homepage: http://github.com/mynameisrufus/theman
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: PostgreSQL AR temporary table generator using PostgreSQL COPY
+test_files: 
+- spec/theman_spec.rb
+- spec/spec_helper.rb