honkster-fasterer-csv 1.7.1

data/LICENSE

@@ -0,0 +1,3 @@
+Mockyright (m) 2010 Mason
+
+Permission is hereby granted.

data/README.rdoc

@@ -0,0 +1,26 @@
+= fasterer-csv
+
+Normal FasterCSV was just too slow for the project I was working on taking a billion trallion years to read in these tons of huge CSV files, so: this.
+
+Depends on various stuff and such it almost always reads at least twice as fast, sometimes up to 40x as fast.
+
+It's got some caveats, though... which you'll discover in fun and entertaining ways.
+
+Also, it's not *completely* a drop in replacement, but it's really really close.  Why do it this way?  Because I hate you and want to make you life a little more difficult whenever possible.
+
+
+== Note on Patches/Pull Requests
+ 
+* Spoon the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request.
+* Delete everything you've done at the point because I'm probably going to reimplement it anyways.
+
+== Monkyright
+
+Monkeyright (m) 2010 Mason. See LICENSE for details.
+

data/Rakefile

@@ -0,0 +1,43 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+
+    gem.name                      = %Q{#{ENV["GEM_PREFIX"] && "#{ENV["GEM_PREFIX"]}-"}fasterer-csv}
+    gem.authors                   = ["Mason"]
+    gem.email                     = %q{mason@chipped.net}
+    gem.date                      = Time.now.strftime("%Y-%m-%d")
+    gem.description               = %q{CSV parsing awesomeness}
+    gem.summary                   = %q{Even fasterer than FasterCSV!}
+    gem.homepage                  = %q{http://github.com/gnovos/fasterer-csv}
+    gem.post_install_message      = <<-POST
+Kernel Panic!  System32 deleted!  Klaxons klaxoning!  Dogs and Cats living together!!!!  We're doooomed!  Everything is...
+oh, no wait, it installed fine.  My bad.
+POST
+    gem.files = Dir["lib/**/*.rb", "LICENSE", "Rakefile", "VERSION", "**/*.rdoc", "spec/**/*.rb"]
+
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler you doof"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "FastererCSV #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+1.7.0

data/lib/fasterer_csv.rb

@@ -0,0 +1,429 @@
+require 'rubygems'
+require 'stringio'
+
+module FastererCSV
+
+  class Table < Array
+
+    class << self
+      def format_headers(unformatted)
+        unformatted.map { |header| Row.to_key(header) }
+      end
+    end
+
+    attr_reader :headers, :lines, :line_block
+
+    def initialize(headers, fail_on_malformed_columns = true, &line_block)
+      @headers = Table.format_headers(headers)
+      @fail_on_malformed_columns = fail_on_malformed_columns
+      @line_block = line_block
+      @lines = 0
+      @indexes = {}
+    end
+
+    def <<(row)
+      @lines += 1
+      if !row.is_a?(Row)
+        row = Row.new(self, row, @lines)
+      end
+      if @headers.length != row.length
+        error = "*** WARNING - COLUMN COUNT MISMATCH - WARNING ***\n*** ROW #{size} : EXPECTED #{@headers.length} : FOUND #{row.length}\n\n"
+        len = 0
+        headers.each do |header|
+          len = header.to_s.length if header.to_s.length > len
+        end
+        headers.each_with_index do |header, i|
+          error << sprintf("%-32s : %s\n", header, row[i])
+        end
+        puts error
+        raise error if @fail_on_malformed_columns
+      end
+      if line_block
+        line_block.call(row)
+      else
+        super(row)
+      end
+    end
+    alias_method :push, :<<
+
+    def merge(*tables)
+
+      tables.each do |table|
+        matching = self.headers & table.headers
+
+        key = {}
+
+        table.each do |row|
+          matching.each do |match|
+            key[match] = row[match]
+          end
+
+          self.lookup(key) { |r| r.merge(row) }
+        end
+      end
+
+      self
+
+    end
+
+    def index(columns, reindex = false)
+      columns = columns.compact.uniq.sort { |a, b| a.to_s <=> b.to_s }.map { |column| Row.to_key(column) }
+
+      key = columns.join('|#|')
+
+      @indexes[key] ||= {}
+
+      index = @indexes[key]
+
+      if reindex || index.empty?
+
+        self.each do |row|
+          vkey = columns.map { |column| row[column] }
+          index[vkey] ||= []
+          index[vkey] << row
+        end
+      end
+      index
+    end
+
+    def lookup(key)
+
+      values  = []
+      columns = key.keys.compact.uniq.sort { |a, b| a.to_s <=> b.to_s }.map do |column|
+        values << key[column]
+        Row.to_key(column)
+      end
+
+      rows = index(columns)[values]
+      if rows && block_given?
+        rows.each do |row|
+          yield(row)
+        end
+      end
+
+      rows
+    end
+
+    def write(file, quot = '~', sep = ',')
+      FastererCSV.write(file, quot, sep) do |out|
+        out << headers
+        each do |row|
+          out << row
+        end
+      end
+    end
+
+    alias_method :rows, :to_a
+    alias_method :merge!, :merge
+
+  end
+
+  class Row < Array
+
+    class << self
+      def to_key(key)
+        key = "#{key}".downcase.gsub(/\s+/, '_')
+        key.empty? ? :_ : key.to_sym
+      end
+    end
+
+    def headers
+      @table.headers
+    end
+
+    attr_reader :line
+
+    def initialize(table, array, line=-1)
+      @table = table
+      @line = line
+      super(array)
+    end
+
+    def [](*is)
+      is.each do |i|
+        val = if i.is_a? Fixnum
+          super
+        else
+          found = headers.index(Row::to_key(i))
+          found ? super(found) : nil
+        end
+        return val unless val.nil?
+      end
+      nil
+    end
+
+    def []=(key, val)
+      if key.is_a? Fixnum
+        super
+      else
+        key = Row::to_key(key)
+        headers << key unless headers.include? key
+        found = headers.index(key)
+        super(found, val)
+      end
+    end
+
+    def pull(*columns)
+      columns.map do |column|
+        self[*column]
+      end
+    end
+
+    def merge(row)
+      if row.is_a? Row
+        row.headers.each do |header|
+          self[header] = row[header]
+        end
+      else
+        row.each do |key, value|
+          self[key] = value
+        end
+      end
+      self
+    end
+
+    def to_hash
+      headers.inject({}) do |memo, h|
+        memo[h] = self[h]
+        memo
+      end
+    end
+
+    def key?(key)
+      keys.include?(Row.to_key(key))
+    end
+
+    def value?(value)
+      values.include?(value)
+    end
+
+    def method_missing(method, *args, &block)
+      to_hash.send(method, *args, &block)
+    end
+
+    alias_method :keys, :headers
+    alias_method :values, :to_a
+
+    alias_method :has_key?, :key?
+    alias_method :member?, :key?
+    alias_method :include?, :key?
+
+    alias_method :has_value?, :value?
+    alias_method :merge!, :merge
+
+  end
+
+  class NumericConversion < Array
+
+    def initialize
+      @int = @float = true
+      @dot = false
+    end
+
+    def clear
+      @int = @float = true
+      @dot = false
+      super
+    end
+
+    def <<(ch)
+      if ch == ?-
+        @float = @int = size == 0
+      elsif (ch > ?9 || ch < ?0) && ch != ?.
+        @int = @float = false
+      elsif ch == ?. && @dot
+        @int = @float = false
+      elsif ch == ?.
+        @int = false
+        @dot = true
+      end
+
+      super(ch.chr)
+    end
+
+    def convert(as_string = false)
+      if as_string
+        join
+      elsif empty?
+        nil
+      elsif @int
+        join.to_i
+      elsif @float
+        join.to_f
+      else
+        join
+      end
+    end
+
+  end
+
+  class NoConversion < Array
+
+    def <<(ch)
+      super(ch.chr)
+    end
+
+    def convert(as_string = false)
+      if as_string
+        join
+      elsif empty?
+        nil
+      else
+        join
+      end
+    end
+
+  end
+
+  class IOWriter
+    def initialize(file, quot = '~', sep = ',', quotenum = false)
+      @first = true; @io = file; @quot = quot; @sep = sep; @quotenum = quotenum
+    end
+
+    def <<(row)
+      raise "can only write arrays! #{row.class} #{row.inspect}" unless row.is_a? Array
+      if @first && row.is_a?(Row)
+        self.<<(row.headers)
+      end
+      @first = false
+      @io.syswrite FastererCSV::quot_row(row, @quot, @sep, @quotenum)
+      row
+    end
+  end
+
+  class << self
+
+    def headers(file, quot = '~', sep = ',', fail_on_malformed = true, column = NoConversion.new, &block)
+      parse_headers(File.open(file, 'r') { |io| io.gets }, quot, sep, fail_on_malformed, column, &block)
+    end
+
+    def read(file, quot = '~', sep = ',', fail_on_malformed = true, column = NoConversion.new, &block)
+      File.open(file, 'r') do |io|
+        parse(io, quot, sep, fail_on_malformed, column, &block)
+      end
+    end
+
+    def convread(file, quot = '~', sep = ',', fail_on_malformed = true, column = NumericConversion.new, &block)
+      File.open(file, 'r') do |io|
+        parse(io, quot, sep, fail_on_malformed, column, &block)
+      end
+    end
+
+    def parse_headers(data, quot = '~', sep = ',', fail_on_malformed = true, column = NoConversion.new, &block)
+      parse(data, quot, sep, fail_on_malformed, column, &block).headers
+    end
+
+    def parse(io, quot = '~', sep = ',', fail_on_malformed = true, column = NoConversion.new, &block)
+      q, s, row, inquot, clean, maybe, table, field, endline = quot[0], sep[0], [], false, true, false, nil, true, false
+
+      io.each_byte do |c|
+        next if c == ?\r
+
+        if maybe && c == s
+          row << column.convert(true)
+          column.clear
+          clean, inquot, maybe, field, endline = true, false, false, true, false
+        elsif maybe && c == ?\n && table.nil?
+          row << column.convert(true) unless (column.empty? && endline)
+          column.clear
+          table = Table.new(row, fail_on_malformed, &block) unless row.empty?
+          row, clean, inquot, maybe, field, endline = [], true, false, false, false, true
+        elsif maybe && c == ?\n
+          row << column.convert(true) unless (column.empty? && endline)
+          column.clear
+          table << row unless row.empty?
+          row, clean, inquot, maybe, field, endline = [], true, false, false, false, true
+        elsif clean && c == q
+          inquot, clean, endline = true, false, false
+        elsif maybe && c == q
+          column << c
+          clean, maybe, endline = false, false, false
+        elsif c == q
+          maybe, endline = true, false
+        elsif inquot
+          column << c
+          clean, endline = false, false
+        elsif c == s
+          row << column.convert(false)
+          column.clear
+          clean, field, endline = true, true, false
+        elsif c == ?\n && table.nil?
+
+          row << column.convert(false) unless column.empty? && endline
+
+          column.clear
+          table = Table.new(row, fail_on_malformed, &block) unless row.empty?
+          row, clean, inquot, field, endline = [], true, false, false, true
+        elsif c == ?\n
+
+          row << column.convert(false) unless column.empty? && endline
+
+          column.clear
+          table << row unless row.empty?
+          row, clean, inquot, field, endline = [], true, false, false, true
+        else
+          column << c
+          clean, endline = false, false
+        end
+      end
+
+      if !clean
+        row << column.convert(maybe)
+        if table
+          table << row unless row.empty?
+        else
+          table = Table.new(row, fail_on_malformed, &block) unless row.empty?
+        end
+      elsif field
+        row << column.convert(maybe)
+      end
+
+      table
+    end
+
+    def quot_row(row, q = '~', s = ',', numquot = false)
+      num_quot = /(?:[#{q}#{s}\n]|^\d+$)/
+      need_quot = /[#{q}#{s}\n]/
+      row.map do |val|
+        if val.nil?
+          ""
+        elsif val.is_a? Numeric
+          val.to_s
+        else
+          quot = (val.is_a?(Symbol) || !numquot) ? need_quot : num_quot
+          val = String(val)
+          if val.length == 0
+            q * 2
+          else
+            val[quot] ? q + val.gsub(q, q * 2) + q : val
+          end
+        end
+      end.join(s) + "\n"
+    end
+
+    def generate(quot = '~', sep = ',', &block)
+      builder = StringIO.new
+      write(builder, quot, sep, &block)
+      builder.string
+    end
+
+    def write(data, quot = '~', sep = ',', quotenum = false, &block)
+      out(data, 'w', quot, sep, quotenum, &block)
+    end
+
+    def append(data, quot = '~', sep = ',', quotenum = false, &block)
+      out(data, 'a', quot, sep, quotenum, &block)
+    end
+
+    def out(data, mode = 'w', quot = '~', sep = ',', quotenum = false, &block)
+      if data.class == String
+        File.open(data, mode) do |io|
+          out(io, mode, quot, sep, quotenum, &block)
+        end
+      else
+        yield(IOWriter.new(data, quot, sep, quotenum))
+      end
+    end
+
+  end
+end

data/spec/fasterer_csv_spec.rb

@@ -0,0 +1,166 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "FastererCSV" do
+
+  describe "fiddly bits" do
+    describe "Table" do
+      it "works" do
+
+      end
+    end
+
+    describe "Row" do
+      it "works" do
+
+      end
+    end
+  end
+
+  describe "Converters" do
+    describe "NumericConverters" do
+      it "works" do
+
+        conv = FastererCSV::NumericConversion.new
+        conv << ?1
+        conv.convert(true).class.should == String
+        conv.convert(true).should == "1"
+
+        conv.convert(false).class.should == Fixnum
+        conv.convert(false).should == 1
+
+        conv.clear
+        conv << ?-
+        conv << ?1
+        conv.convert(false).class.should == Fixnum
+        conv.convert(false).should == -1
+
+        conv.clear
+        conv << ?1
+        conv << ?.
+        conv << ?1
+        conv.convert(false).class.should == Float
+        conv.convert(false).should == 1.1
+
+        conv.clear
+        conv << ?-
+        conv << ?1
+        conv << ?.
+        conv << ?1
+        conv.convert(false).class.should == Float
+        conv.convert(false).should == -1.1
+
+        conv.clear
+        conv << ?1
+        conv << ?.
+        conv << ?1
+        conv << ?.
+        conv << ?1
+        conv.convert(false).class.should == String
+        conv.convert(false).should == "1.1.1"
+
+        conv.clear
+        conv << ?a
+        conv.convert(false).class.should == String
+        conv.convert(false).should == "a"
+
+        conv.clear
+        conv.should be_empty
+        conv.convert(false).should be_nil
+        conv.convert(true).should == ""
+
+      end
+    end
+    describe "NoConverter" do
+      it "works" do
+
+        conv = FastererCSV::NoConversion.new
+        conv << ?1
+        conv.convert(true).class.should == String
+        conv.convert(false).class.should == String
+
+        conv.convert(true).should == "1"
+        conv.convert(false).should == "1"
+
+        conv.clear
+        conv.should be_empty
+        conv.convert(false).should be_nil
+        conv.convert(true).should == ""
+
+      end
+    end
+  end
+
+  describe "important stuff" do
+
+    before do
+      @data = <<-CSV
+a,b,c,d,e,f,g,h,i,j,k,l,m,
+,,1,1.1,-1,-1.1,1.1.1,~1~,a,~a~,~a~~a~,~a
+~~a~,~,~,
+
+0,1,2,3,4,5,6,7,8,9,10,11,12,14
+      CSV
+    end
+
+    describe "parse" do
+      it "works" do
+        table = FastererCSV.parse(@data)
+        table.headers.should == [:a, :b, :c,:d,:e,:f,:g,:h,:i,:j,:k,:l,:m,:_]
+        table.lines.should == 2
+
+        table[0].should == [nil, nil, "1", "1.1", "-1", "-1.1", "1.1.1", "1", "a", "a", "a~a", "a\n~a", ",", nil]
+        table[1].should == ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "14"]
+
+        row = table[1]
+        row.pull(:a, nil, 'd').should == ["0","14","3"]
+        row[:b].should == "1"
+        row["b"].should == "1"
+
+      end
+    end
+
+    describe "read" do
+      it "" do
+
+      end
+    end
+
+    describe "convread" do
+      it "" do
+
+      end
+    end
+
+    describe "headers" do
+      it "" do
+
+      end
+    end
+
+    describe "parse_headers" do
+      it "" do
+
+      end
+    end
+
+    describe "quot_row" do
+      it "" do
+
+      end
+    end
+
+    describe "generate" do
+      it "" do
+
+      end
+    end
+
+    describe "write" do
+      it "" do
+
+      end
+    end
+
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,10 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'fasterer_csv'
+ARGV.push("-b")
+require 'spec'
+require 'spec/autorun'
+
+Spec::Runner.configure do |config|
+
+end

metadata

@@ -0,0 +1,78 @@
+--- !ruby/object:Gem::Specification 
+name: honkster-fasterer-csv
+version: !ruby/object:Gem::Version 
+  hash: 9
+  prerelease: false
+  segments: 
+  - 1
+  - 7
+  - 1
+  version: 1.7.1
+platform: ruby
+authors: 
+- Mason
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-09-07 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: CSV parsing awesomeness
+email: mason@chipped.net
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/fasterer_csv.rb
+- spec/fasterer_csv_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/gnovos/fasterer-csv
+licenses: []
+
+post_install_message: |
+  Kernel Panic!  System32 deleted!  Klaxons klaxoning!  Dogs and Cats living together!!!!  We're doooomed!  Everything is...
+  oh, no wait, it installed fine.  My bad.
+
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Even fasterer than FasterCSV!
+test_files: 
+- spec/spec_helper.rb
+- spec/fasterer_csv_spec.rb