podcsv 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c2063aefdf28a7cc548de6b13a26c4aac2c1383e
+  data.tar.gz: ce128de96cd979ade07e762f509b486fcda30948
+SHA512:
+  metadata.gz: e86f4aa504004fe9dac8d9874264cb351c3c8ed30d1416fa75116ff55a858a13c077687eb14d99e0f1d8ee03517d79b5fb1948f10b16fd09e979ddb38b22b87b
+  data.tar.gz: 3684b1c99503f9a5c32bb6730dd68c07256c94d62b99dbc399beb875013a1853ba835358a27a19a4357f241b5c1cba17b1c300765c1943a0dbceba2bea71c2e0

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.2.0
+before_install: gem install bundler -v 1.10.2

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in podcsv.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 YAMAMOTO, Masayuki
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,111 @@
+# PodCSV another lazy solution for `csv`
+
+podcsv.gem provides fast access to big CSV file such as several 10 thousand
+records. This introduces two classes: Parse-on-demand CSV and Array (PodCSV,
+PodArray) and is around 10 times faster than library default 'csv'. You can
+also randomly access to the elements (records).
+
+This gem may be useful if you use only very small part (fg. 1 %) of records
+in big CSV.
+
+
+## 1. Benchmark
+
+```
+$ bundle exec rake bm
+
+# of records
+csv: 40000
+podcsv: 40000
+
+Benchmark
+read:
+Rehearsal --------------------------------------------
+csv        4.780000   0.090000   4.870000 (  5.135495)
+podcsv     0.270000   0.040000   0.310000 (  0.319921)
+----------------------------------- total: 5.180000sec
+
+               user     system      total        real
+csv        4.650000   0.070000   4.720000 (  5.041622)
+podcsv     0.240000   0.030000   0.270000 (  0.272924)
+
+access:
+Rehearsal --------------------------------------------
+csv        4.620000   0.060000   4.680000 (  4.919373)
+podcsv     0.400000   0.030000   0.430000 (  0.460540)
+----------------------------------- total: 5.110000sec
+
+               user     system      total        real
+csv        4.660000   0.100000   4.760000 (  4.921194)
+podcsv     0.360000   0.020000   0.380000 (  0.399690)
+```
+
+### 1.1. Trick
+
+This gem defines two classes: `PodCSV` and `PodArray`.
+PodCSV does not parse any strings on reading CSV file and just returns an
+array (PodArray).
+
+When you access elements (records) of that array via `[]`, `each`, etc.,
+strings are parsed and changed into fields (PodArray cache mechanism).
+
+
+## 2. Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'podcsv'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install podcsv
+
+
+## 3. Usage
+
+### (1) Load CSV
+
+Same as `CSV.read`.
+
+```
+ret = PodCSV.read( file [, opt_file] )
+```
+
+### (2) Access records
+
+Same as Array: `[]`, `each`, `first, last`, etc.
+
+```
+ret = PodCSV.read( file [, opt_file] )
+puts ret[-1]
+```
+
+
+### (3) Custom Line Parser
+
+```
+ary = PodCSV.read( file, {},
+                   lambda{|s| s.split(/"/) } )
+```
+
+
+## 4. Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+
+## 5. Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/mephistobooks/podcsv.
+
+
+## 6. License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,12 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec
+
+desc "execute benchmark script"
+task :bm do
+  load 'spec/csvbm.rb'
+end
+

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "podcsv"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/podcsv/version.rb

@@ -0,0 +1,3 @@
+class PodCSV
+  VERSION = "0.0.1"
+end

data/lib/podcsv.rb

@@ -0,0 +1,212 @@
+#
+#
+#
+require 'csv'
+
+
+class PodArray < Array
+
+  #PARSER_DEFAULT = lambda{|s,opt| CSV.parse_line(s,opt)}
+  #PARSER_DEFAULT = lambda{|s| CSV.parse_line(s,@opt)}
+  #PARSER_DEFAULT = lambda{|s| CSV.parse_line(s,@opt.to_h)}
+
+  def initialize(args)
+    super
+
+    # we can also distinguish which are cached by the type of elements
+    # (String or not).
+    # But the advantage of having cache structure is that
+    # we can know which are the cached elements without traversing
+    # whole array.
+    #
+    @_cache = {}
+    if args.class == self.class
+      @_cache = args._cache
+    end
+
+    # opt for CSV.parse.
+    # default is {}, which means that dont change the default behaviour
+    # of CSV.parse_line.
+    #@opt = {}
+    @_lazy_parser_default = lambda{|s| CSV.parse_line(s)}
+
+    #
+    #
+    #
+    #@_lazy_parser = lambda{|x,o| CSV.parse(x,o).first}
+    @_lazy_parser = @_lazy_parser_default
+
+  end
+
+  def cached?(i)
+    not(@_cache[i].nil?)
+  end
+  attr_reader   :_cache
+  attr_reader   :_lazy_parser_default
+  attr_accessor :_lazy_parser
+  #attr_accessor :opt
+
+  #
+  def [](args)
+    ret = nil
+
+    # .
+    if args.is_a?(Integer)
+      tmp = super
+
+      # cache function.
+      if self.cached?(args)
+
+        # do nothing.
+
+      else
+        #tmp = super
+        #$stderr.puts "*tmp (class:#{tmp.class}):#{tmp}"
+        if tmp.class == String
+
+          #@_cache[args] = @_lazy_parser.call(tmp, @opt)
+          @_cache[args] = @_lazy_parser.call(tmp)
+             self[args] = @_cache[args]
+
+        else
+          @_cache[args] = tmp unless self.cached?(args)
+        end
+      end
+
+      #$stderr.puts "_cache[#{args}]: #{@_cache[args]}"
+      @_cache[args]   # return the value of self[args].
+      #self[args]
+      #self[args] = @_cache[args]
+    else
+      # Range, etc.
+      # $stderr.puts "[INFO] access for #{args} (#{args.class})."
+
+      # copy partial array.
+      idxs = Array(args)
+      pary = idxs.map{|ii| self[ii]}
+      ret = PodArray.new(
+        # Integer
+        pary
+      )
+
+      # copy partial cache.
+      ini_cache = self._cache
+      idxs.each {|ii|
+        ret.instance_eval{ @_cache[ii] = ini_cache[ii] }
+      }
+
+      ret
+    end
+  end
+
+  def index(x)
+    #$stderr.puts "#{self.class}##{__method__} (#{x.class})"
+    self[x]
+  end
+
+  def first
+    #$stderr.puts "#{self.class}##{__method__} ()"
+    # self.index(0)
+    self[0]
+  end
+
+  def last
+    #$stderr.puts "#{self.class}##{__method__} ()"
+    self[self.size-1]
+  end
+
+  def reverse
+    #$stderr.puts "#{self.class}##{__method__} ()"
+    ret = PodArray.new(super)
+    #$stderr.puts "#{self.class}##{__method__} ret: #{ret.class}"
+    #$stderr.puts "#{self.class}##{__method__} ret.first: #{ret.first.class}"
+
+    ret
+  end
+
+  #
+  def each
+    s = self
+    a = Array(s.to_a)   # cast to Array.
+
+    #$stderr.puts "s: #{s.class}"
+    #$stderr.puts "a: #{a.class}"
+    #pp a
+
+    # do cache.
+    a.each.with_index {|_,i|
+      # $stderr.puts "i: #{i}"
+      self[i]   # do cache.
+      yield(self[i]) if block_given?
+    }
+
+    # return
+    if block_given?
+      self
+    else
+      self.to_enum
+    end
+  end
+
+  def map
+    tmp = self.each
+    ret = []
+
+    tmp.each{|e|
+      ret << yield(e) if block_given?
+    }
+
+    # return
+    if block_given?
+      ret
+    else
+      tmp
+    end
+
+  end
+
+  def select
+    tmp = self.each
+    ret = []
+
+    tmp.each{|e|
+      # $stderr.puts "#{e} (#{e.class})"
+      if block_given?
+        ret << e if yield(e)
+      end
+    }
+
+    # return
+    if block_given?
+      PodArray.new(ret)
+    else
+      tmp
+    end
+
+  end
+
+
+end
+
+
+#
+#
+#
+class PodCSV
+
+  #
+  # ==== Return
+  #
+  def self.read( file, opt_file = {}, line_parser = nil )
+    #@opt_csv = opt_csv
+    ret = PodArray.new( File.open(file,opt_file).read.split(/\n/) )
+    #ret.opt = @opt_csv
+    ret._lazy_parser = line_parser unless line_parser.nil?
+    ret
+  end
+end
+
+require 'podcsv/version'
+
+
+####

data/podcsv.gemspec

@@ -0,0 +1,33 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'podcsv/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "podcsv"
+  spec.version       = PodCSV::VERSION
+  spec.authors       = ["YAMAMOTO, Masayuki"]
+  spec.email         = ["martin.route66.blues+github@gmail.com"]
+
+  spec.summary       = %q{Parse-on-demand CSV.}
+  spec.description   = %q{This gem defines PodCSV and PodArray which are available to cache and parse data on-demand. These are useful when you need to read a big CSV file (around thousand records) but use very small part of it.}
+  spec.homepage      = "https://github.com/mephistobooks/podcsv"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
+  # delete this section to allow pushing this gem to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = "https://rubygems.org"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: podcsv
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- YAMAMOTO, Masayuki
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-06-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem defines PodCSV and PodArray which are available to cache and
+  parse data on-demand. These are useful when you need to read a big CSV file (around
+  thousand records) but use very small part of it.
+email:
+- martin.route66.blues+github@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/podcsv.rb
+- lib/podcsv/version.rb
+- podcsv.gemspec
+homepage: https://github.com/mephistobooks/podcsv
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: Parse-on-demand CSV.
+test_files: []