table_utils 0.0.1

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/.rspec

@@ -0,0 +1 @@
+-rspec_helper

data/Gemfile

@@ -0,0 +1,2 @@
+source "https://rubygems.org"
+gemspec

data/Gemfile.lock

@@ -0,0 +1,88 @@
+PATH
+  remote: .
+  specs:
+    table_utils (0.0.1)
+      activerecord
+      activerecord-import
+      ruby-progressbar
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (4.0.2)
+      activesupport (= 4.0.2)
+      builder (~> 3.1.0)
+    activerecord (4.0.2)
+      activemodel (= 4.0.2)
+      activerecord-deprecated_finders (~> 1.0.2)
+      activesupport (= 4.0.2)
+      arel (~> 4.0.0)
+    activerecord-deprecated_finders (1.0.3)
+    activerecord-import (0.4.1)
+      activerecord (>= 3.0)
+    activesupport (4.0.2)
+      i18n (~> 0.6, >= 0.6.4)
+      minitest (~> 4.2)
+      multi_json (~> 1.3)
+      thread_safe (~> 0.1)
+      tzinfo (~> 0.3.37)
+    arel (4.0.1)
+    atomic (1.1.14)
+    builder (3.1.4)
+    coderay (1.1.0)
+    diff-lcs (1.2.5)
+    ffi (1.9.3)
+    formatador (0.2.4)
+    guard (1.8.3)
+      formatador (>= 0.2.4)
+      listen (~> 1.3)
+      lumberjack (>= 1.0.2)
+      pry (>= 0.9.10)
+      thor (>= 0.14.6)
+    guard-rspec (3.1.0)
+      guard (>= 1.8)
+      rspec (~> 2.13)
+    i18n (0.6.9)
+    listen (1.3.1)
+      rb-fsevent (>= 0.9.3)
+      rb-inotify (>= 0.9)
+      rb-kqueue (>= 0.2)
+    lumberjack (1.0.4)
+    method_source (0.8.2)
+    minitest (4.7.5)
+    multi_json (1.8.2)
+    pry (0.9.12.4)
+      coderay (~> 1.0)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    rb-fsevent (0.9.3)
+    rb-inotify (0.9.3)
+      ffi (>= 0.5.0)
+    rb-kqueue (0.2.0)
+      ffi (>= 0.5.0)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.7)
+    rspec-expectations (2.14.4)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.4)
+    ruby-progressbar (1.3.2)
+    slop (3.4.7)
+    sqlite3 (1.3.8)
+    thor (0.18.1)
+    thread_safe (0.1.3)
+      atomic
+    tzinfo (0.3.38)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.3)
+  guard (< 2.0)
+  guard-rspec
+  rspec
+  sqlite3
+  table_utils!

data/Guardfile

@@ -0,0 +1,9 @@
+
+guard :rspec, cmd: "bundle exec rspec", all_after_pass: true  do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch("spec/spec_helper.rb")  { "spec" }
+
+  watch("lib/data_source.rb")   { "spec/lib/rake/table_import_spec.rb" }
+end
+

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Artem Baguinski
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,4 @@
+table_utils
+===========
+
+utilities for working with real world tabular data

data/lib/data_source.rb

@@ -0,0 +1,63 @@
+require "active_record"
+require "activerecord-import"
+require "table_utils/progress"
+
+class DataSource
+  attr_reader :table_name, :input_path
+
+  def initialize table_name, input_path, csv_opts = {}, &block
+    @table_name, @input_path = table_name, input_path
+    @csv_opts = csv_opts ? csv_opts.dup : {}
+    @block = block
+  end
+
+  def import
+    batch = []
+    TableUtils::Progress.over csv do |row, bar|
+      batch << common_columns.map{ |c| row[c] }
+      if batch.count >= 1000 || bar.finished?
+        model.import common_columns, batch, validate: false
+        batch = []
+      end
+    end
+  end
+
+  def table_exists?
+    conn.table_exists? table_name
+  end
+
+  def drop_table
+    conn.drop_table table_name
+  end
+
+  private
+
+  attr_reader :csv_opts, :block
+
+  def conn
+    @conn ||= ActiveRecord::Base.connection
+  end
+
+  def model
+    conn.create_table table_name, &block unless table_exists?
+    _table_name = table_name
+    @model ||= Class.new(ActiveRecord::Base) { self.table_name = _table_name }
+  end
+
+  def csv
+    @csv ||= begin
+               csv_opts[:headers] ||= model.column_names - %w(id created_at updated_at)
+               csv_opts[:return_headers] = true
+               CSV.open(input_path, csv_opts).tap do |csv|
+                 csv.shift
+                 @common_columns = model.column_names & csv.headers
+                 csv.shift if Array === csv_opts[:headers]
+               end
+             end
+  end
+
+  def common_columns
+    csv
+    @common_columns
+  end
+end

data/lib/rake/logger.rb

@@ -0,0 +1,7 @@
+module Rake::DSL
+  attr_writer :logger
+
+  def logger
+    @logger ||= Logger.new STDOUT
+  end
+end

data/lib/rake/table_import.rb

@@ -0,0 +1,23 @@
+require "data_source"
+require "rake/logger"
+
+module Rake::DSL
+  def table_import args, &block
+    table_name = args.keys[0]
+    task args.extract!(table_name) do |t|
+      source = DataSource.new table_name, t.prerequisites[0], args[:csv], &block
+      unless source.table_exists?
+        begin
+          logger.info { "Importing #{source.table_name} from #{source.input_path}" }
+          source.import
+        rescue
+          if source.table_exists?
+            logger.warn { "dropping #{table_name} due to errors during import" }
+            source.drop_table
+          end
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/table_utils.rb

@@ -0,0 +1,3 @@
+require "table_utils/limit_loops"
+require "table_utils/progress"
+

data/lib/table_utils/limit_loops.rb

@@ -0,0 +1,21 @@
+module TableUtils
+  class LimitLoops
+    attr_reader :total
+
+    def initialize max_count
+      @total = @left_count = max_count
+    end
+
+    def limit
+      catch(self) { yield self }
+    end
+
+    def check!
+      throw self if @left_count and (@left_count -= 1) <= 0
+    end
+
+    def self.to max_count, &block
+      LimitLoops.new(max_count).limit &block
+    end
+  end
+end

data/lib/table_utils/progress.rb

@@ -0,0 +1,52 @@
+require "ruby-progressbar"
+
+module TableUtils
+  module Progress
+    DefaultOptions = {
+      format: "%a |%b %c/%C =>%i| %E",
+      throttle_rate: 0.1,
+    }
+
+    def self.bar options = {}
+      bar = ProgressBar.create DefaultOptions.merge options
+      bar.format("%a: |%i| %c") if bar.total == nil
+      if block_given?
+        begin
+          yield bar
+        ensure
+          if bar.total
+            bar.finish unless bar.finished?
+          end
+        end
+      else
+        bar
+      end
+    end
+
+    def self.over enum, options = {}
+      options = options.dup
+
+      unless options.include? :total
+        options[:total] = enum.count
+      end
+
+      Progress.bar options do |bar|
+        enum.each do |i|
+          bar.increment
+          yield i, bar
+        end
+      end
+    end
+
+  end
+end
+
+require "csv"
+class CSV
+  def count
+    oldpos = tell if respond_to?(:tell)
+    super
+  ensure
+    seek oldpos if respond_to?(:seek)
+  end
+end

data/lib/table_utils/version.rb

@@ -0,0 +1,3 @@
+module TableUtils
+  VERSION = "0.0.1"
+end

data/spec/fixtures/table.csv

@@ -0,0 +1,9 @@
+planet,earth_masses,jupiter_masses
+Mercury,0.0553,0.000174
+Venus,0.815,0.00256
+Earth,1,0.00315
+Mars,0.107,0.000338
+Jupiter,318,1
+Saturn,95.2,0.299
+Uranus,14.5,0.0457
+Neptune,17.2,0.0540

data/spec/lib/data_source_spec.rb

@@ -0,0 +1,42 @@
+require "data_source"
+
+describe DataSource do
+  include_context "planets"
+  include_context "silent_progress"
+  include_context "in-memory db"
+
+  let(:more_args) { [] }
+  let(:source) {
+    DataSource.new :planets, fixture("table.csv"), *more_args do |t|
+      t.string :planet
+      t.float :earth_masses
+      t.float :jupiter_masses
+    end
+  }
+  before { source.import }
+  let(:model) { Class.new(ActiveRecord::Base) { self.table_name = :planets } }
+  let(:records_without_id) { model.all.map{|r| r.attributes.tap{|h| h.delete("id")}} }
+
+  shared_examples "imports a table" do
+    it "imports a table" do
+      expect( conn.table_exists? :planets ).to be_true
+      expect( model.column_names ).to match_array planet_headers + %w(id)
+      expect( records_without_id ).to match_array planets_table
+    end
+  end
+
+  context "with implicit csv headers" do
+    include_examples "imports a table"
+  end
+
+  context "with explicit csv headers" do
+    let(:more_args) { [headers: planet_headers] }
+    include_examples "imports a table"
+  end
+
+  context "with csv headers from the first row" do
+    let(:more_args) { [headers: true] }
+    include_examples "imports a table"
+  end
+
+end

data/spec/lib/rake/table_import_spec.rb

@@ -0,0 +1,68 @@
+require "rake"
+require "rake/table_import"
+include Rake::DSL
+
+describe "Rake::DSL.table_import" do
+  let(:rake) { Rake::Application.new }
+
+  before(:each) do
+    Rake.application = rake
+    logger.level = Logger::UNKNOWN
+    task "path"
+  end
+
+  it "unrolls the arguments and passes them to DataSource constructor" do
+    catch :done do
+      DataSource.should_receive(:new).with(:table_name, "path", :csv_options).and_throw :done
+      table_import :table_name => "path", csv: :csv_options
+      rake[:table_name].invoke
+    end
+  end
+
+  it "passes nil for csv_options if not supplied" do
+    catch :done do
+      DataSource.should_receive(:new).with(:table_name, "path", nil).and_throw :done
+      table_import :table_name => "path"
+      rake[:table_name].invoke
+    end
+  end
+
+  shared_context "mock source" do
+    let(:source) { double() }
+    before { DataSource.stub(:new) { source } }
+  end
+
+  context "table doesn't exist" do
+    include_context "mock source"
+    before { source.stub(:table_exists?) { false } }
+    it "import from the source" do
+      expect(source).to receive(:import).with()
+      table_import :table_name => "path"
+      rake[:table_name].invoke
+    end
+
+    context "import fails" do
+      before {
+        source.stub(:import) {
+          source.stub(:table_exists?) { true }
+          raise RuntimeError, "mock import failure"
+        }
+      }
+      it "should drop the table and reraise the exception" do
+        expect(source).to receive(:drop_table)
+        table_import :table_name => "path"
+        expect{rake[:table_name].invoke}.to raise_error("mock import failure")
+      end
+    end
+  end
+
+  context "table exists" do
+    include_context "mock source"
+    before { source.stub(:table_exists?) { true } }
+    it "doesn't import" do
+      expect(source).not_to receive(:import)
+      table_import :table_name => "path"
+      rake[:table_name].invoke
+    end
+  end
+end

data/spec/lib/table_utils/limit_loops_spec.rb

@@ -0,0 +1,35 @@
+require "table_utils/limit_loops"
+include TableUtils
+
+describe LimitLoops do
+  let(:limit_value) { 5 }
+  let(:default_count) { 10 }
+
+  describe "#total" do
+    subject { LimitLoops.new limit_value }
+    its(:total) { should == limit_value }
+  end
+
+  describe "::limit_loops + #check!" do
+    it "limits number of iterations to limit value" do
+      counter = 0
+      LimitLoops.to limit_value do |limit|
+        default_count.times do
+          counter += 1
+          limit.check!
+        end
+      end
+      expect(counter).to eq limit_value
+    end
+    it "doesn't limit number of iterations when given nil" do
+      counter = 0
+      LimitLoops.to nil do |limit|
+        default_count.times do
+          counter += 1
+          limit.check!
+        end
+      end
+      expect(counter).to eq default_count
+    end
+  end
+end

data/spec/lib/table_utils/progress_spec.rb

@@ -0,0 +1,37 @@
+require "table_utils/progress"
+
+include TableUtils
+
+describe Progress do
+  include_context "silent_progress"
+
+  describe "::bar" do
+    context "without block" do
+      subject { Progress.bar }
+      it { should be_a ProgressBar::Base }
+    end
+
+    context "with block" do
+      it "should yield a ProgressBar::Base instance" do
+        yielded = nil
+        Progress.bar do |bar|
+          yielded = bar
+        end
+        expect( yielded ).to be_a ProgressBar::Base
+      end
+    end
+  end
+
+  describe "::over" do
+    let(:enum) { [:foo,:bar] }
+    let(:yielded) { [] }
+    it "iterates over supplied enum" do
+      Progress.over enum, output: dummy_io do |item,bar|
+        expect(bar).to be_a ProgressBar::Base
+        expect(bar.total).to eq enum.count
+        yielded << item
+      end
+      expect(yielded).to eq enum
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,35 @@
+require "active_record"
+require "csv"
+require "ruby-progressbar"
+
+def fixture basename
+  File.expand_path("../fixtures/#{basename}", __FILE__)
+end
+
+shared_context "planets" do
+  let(:planets_table) {
+    CSV.read( fixture("table.csv"),
+              headers: true,
+              converters: :numeric )
+       .map &:to_hash
+  }
+  let(:planets) { planets_table.map{|row| row["planet"]} }
+  let(:planet_headers) { planets_table[0].keys }
+end
+
+shared_context "silent_progress" do
+  let(:dummy_io) { StringIO.new }
+  before(:each) { ProgressBar::Base.any_instance.stub(:output) { dummy_io } }
+end
+
+shared_context "in-memory db" do
+  around(:each) do |example|
+    ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
+    ActiveRecord::Base.transaction {
+      example.run
+      raise ActiveRecord::Rollback, "clear the database"
+    }
+  end
+  let(:conn) { ActiveRecord::Base.connection }
+end
+

data/table_utils.gemspec

@@ -0,0 +1,30 @@
+$LOAD_PATH << File.expand_path("../lib", __FILE__)
+
+require  "table_utils/version"
+
+Gem::Specification.new do |s|
+  s.name = "table_utils"
+  s.version = TableUtils::VERSION
+  s.summary = "utilities for working with real world tabular data"
+  s.description = "make dealing with real life tables simpler"
+  s.author = "Artem Baguinski"
+  s.email = "femistofel@gmail.com"
+  s.homepage = "https://github.com/artm/table_utils"
+  s.license = "MIT"
+
+  s.files = `git ls-files`.split($/)
+  s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency "activerecord"
+  s.add_runtime_dependency "activerecord-import"
+  s.add_runtime_dependency "ruby-progressbar"
+
+  s.add_development_dependency "bundler", "~> 1.3"
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "guard", "< 2.0"
+  s.add_development_dependency "guard-rspec"
+  s.add_development_dependency "sqlite3"
+end
+

metadata

@@ -0,0 +1,200 @@
+--- !ruby/object:Gem::Specification
+name: table_utils
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Artem Baguinski
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-12-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord-import
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - <
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - <
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: make dealing with real life tables simpler
+email: femistofel@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- Gemfile.lock
+- Guardfile
+- LICENSE
+- README.md
+- lib/data_source.rb
+- lib/rake/logger.rb
+- lib/rake/table_import.rb
+- lib/table_utils.rb
+- lib/table_utils/limit_loops.rb
+- lib/table_utils/progress.rb
+- lib/table_utils/version.rb
+- spec/fixtures/table.csv
+- spec/lib/data_source_spec.rb
+- spec/lib/rake/table_import_spec.rb
+- spec/lib/table_utils/limit_loops_spec.rb
+- spec/lib/table_utils/progress_spec.rb
+- spec/spec_helper.rb
+- table_utils.gemspec
+homepage: https://github.com/artm/table_utils
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: utilities for working with real world tabular data
+test_files:
+- spec/fixtures/table.csv
+- spec/lib/data_source_spec.rb
+- spec/lib/rake/table_import_spec.rb
+- spec/lib/table_utils/limit_loops_spec.rb
+- spec/lib/table_utils/progress_spec.rb
+- spec/spec_helper.rb