icebox 0.0.2 → 0.0.3

data/Gemfile

@@ -11,5 +11,6 @@ group :development do
   gem "shoulda", ">= 0"
   gem "bundler", "~> 1.0.0"
   gem "jeweler", "~> 1.6.4"
+  gem "rdoc", "~> 3.11"
   gem "rcov", ">= 0"
 end

data/Rakefile

@@ -47,8 +47,8 @@ end
 
 task :default => :test
 
-require 'rake/rdoctask'
-Rake::RDocTask.new do |rdoc|
+require 'rdoc/task'
+RDoc::Task.new do |rdoc|
   version = File.exist?('VERSION') ? File.read('VERSION') : ""
 
   rdoc.rdoc_dir = 'rdoc'

data/VERSION

@@ -1 +1 @@
-0.0.2
+0.0.3

data/icebox.gemspec

@@ -4,14 +4,15 @@
 # -*- encoding: utf-8 -*-
 
 Gem::Specification.new do |s|
-  s.name = "icebox"
-  s.version = "0.0.2"
+  s.name = %q{icebox}
+  s.version = "0.0.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Riley Goodside"]
-  s.date = "2011-10-05"
-  s.description = "Ruby library and command-line utility for working with Infobright Community Edition"
-  s.email = "riley.goodside@gmail.com"
+  s.date = %q{2011-10-25}
+  s.default_executable = %q{icebox}
+  s.description = %q{Ruby library and command-line utility for working with Infobright Community Edition}
+  s.email = %q{riley.goodside@gmail.com}
   s.executables = ["icebox"]
   s.extra_rdoc_files = [
     "LICENSE.txt",
@@ -32,11 +33,11 @@ Gem::Specification.new do |s|
     "test/helper.rb",
     "test/test_icebox.rb"
   ]
-  s.homepage = "http://github.com/goodside/icebox"
+  s.homepage = %q{http://github.com/goodside/icebox}
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.10"
-  s.summary = "Multitool for working with Infobright Community Edition"
+  s.rubygems_version = %q{1.6.2}
+  s.summary = %q{Multitool for working with Infobright Community Edition}
 
   if s.respond_to? :specification_version then
     s.specification_version = 3
@@ -47,6 +48,7 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<shoulda>, [">= 0"])
       s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_development_dependency(%q<rdoc>, ["~> 3.11"])
       s.add_development_dependency(%q<rcov>, [">= 0"])
     else
       s.add_dependency(%q<mysql>, ["~> 2.8.1"])
@@ -54,6 +56,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<shoulda>, [">= 0"])
       s.add_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_dependency(%q<rdoc>, ["~> 3.11"])
       s.add_dependency(%q<rcov>, [">= 0"])
     end
   else
@@ -62,6 +65,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<shoulda>, [">= 0"])
     s.add_dependency(%q<bundler>, ["~> 1.0.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+    s.add_dependency(%q<rdoc>, ["~> 3.11"])
     s.add_dependency(%q<rcov>, [">= 0"])
   end
 end

data/lib/icebox.rb

@@ -1,37 +1,88 @@
 require 'sequel'
+require 'yaml'
 
 module Icebox
+  def self.new(*args, &block)
+    Icebox.new(*args, &block)
+  end
+
   class Icebox
-    def initialize(db)
-      @db = db || Sequel.mysql('test', socket: '/tmp/mysql-ib.sock')
+    attr_accessor :db
+
+    def initialize(db = nil)
+      # Icebox is configurable using a YAML file in /etc/icebox.conf
+      # The root-level key :connections should be an array of option sets to be
+      # passed to the Sequel#mysql method. Right now, only the first is used.
+
+      @db = db
+      if @db.nil?
+        begin
+          opts = YAML.load(File.open("/etc/icebox.conf"))[:connections][0]
+        rescue 
+          opts = {database: 'test', socket: '/tmp/mysql-ib.sock'}
+        end
+        @db = Sequel.mysql(opts)
+      end
+    end
+
+    def load_data_infile(table, path=nil, opts={})
+      # If a handle object is given, start writing it out into a new FIFO pipe
+      # from within a forked subprocess before we start loading:
+      if opts[:handle]
+        pipe = "/tmp/icebox_pipe_for_#{table}_at_#{Time.now.to_i}"
+        system "mkfifo #{pipe}"
+        opts[:fifo] = pipe
+        pid = fork do
+          File.open(pipe, 'a') do |p|
+            p.write opts[:handle].read
+          end
+        end
+      end
+
+      path = path || opts[:path] || opts[:fifo]
+      raise "load_data_infile called with no input source" unless path
+
+      load_sql = <<-SQL
+        #{"SET @bh_pipemode = 'server';" if opts[:fifo]}
+        LOAD DATA INFILE '#{path}' INTO TABLE #{table}
+        FIELDS TERMINATED BY ',' ENCLOSED BY '"' ESCAPED BY '\\\\';
+      SQL
+      # Force value retrieval to delay function return until query finishes:
+      @db[load_sql].all
+      Process.waitpid(pid) if pid # Wait for writer too if it exists
+    end
+
+    def insert_into(table, sql)
+      pipe = "/tmp/icebox_pipe_for_#{table}"
+      sql.sub! /\s*;\s*/, ''  # Chop off semicolon if present
+
+      # Simultaneously export and re-import CSV through FIFO pipe:
+      @db.disconnect  # Needed for fork; Reconnects automatically
+      pid1 = fork do
+        @db << <<-SQL
+          SET @bh_pipemode = 'client';
+          #{sql} INTO OUTFILE '#{pipe}'
+          FIELDS TERMINATED BY ',' ENCLOSED BY '"' ESCAPED BY '\\\\'
+        SQL
+      end
+      pid2 = fork do
+        load_data_infile table, pipe, fifo: true
+      end
+      Process.waitpid(pid1)
+      Process.waitpid(pid2)
     end
 
     def create_table_as(table, sql)
       view = "icebox_view_for_#{table}"
-      pipe = "/tmp/icebox_pipe_for_#{table}"
 
       # Create view from SQL, extract layout, and make new table from it:
       @db.create_or_replace_view view, sql
       fields_definition = @db.schema(view).map do |col|
         "#{col[0]} #{col[1][:db_type]}"
       end.join(',')
-      @db.run "CREATE TABLE #{table} (#{fields_definition})"
-
-      # Simultaneously export and re-import CSV through FIFO pipe:
-      @db.disconnect  # Needed for fork; Reconnects automatically
-      fork do
-        @db.run <<-SQL
-          SET @bh_pipemode = 'server';
-          LOAD DATA INFILE '#{pipe}' INTO TABLE #{table}
-          FIELDS TERMINATED BY ',' ENCLOSED BY '"' ESCAPED BY '\\\\';
-        SQL
-      end
-      @db.run <<-SQL
-        SET @bh_pipemode = 'client';
-        SELECT * FROM #{view} INTO OUTFILE '#{pipe}'
-        FIELDS TERMINATED BY ',' ENCLOSED BY '"' ESCAPED BY '\\\\'
-      SQL
+      @db << "CREATE TABLE #{table} (#{fields_definition})"
 
+      insert_into table, "SELECT * FROM #{view}"
       @db.drop_view view
     end
   end

data/test/test_icebox.rb

@@ -1,29 +1,90 @@
 require 'helper'
 
 class TestIcebox < Test::Unit::TestCase
-  context "a basic CSV file" do
+  context "A clean, default Icebox instance" do
     setup do
-      @db = Sequel.mysql('test', socket: '/tmp/mysql-ib.sock')
-      csv_body = <<-CSV.gsub(/^\s*/, '')
-        0,"Every good boy deserves fudge."
-        1,"A stitch in time saves nine."
-        2,""
-      CSV
-      File.open('/tmp/input_table.csv', 'w') { |f| f.write csv_body }
-      @db.run <<-SQL
-        DROP TABLE IF EXISTS input_table;
-        CREATE TABLE input_table (number INTEGER, line TEXT);
-        LOAD DATA INFILE '/tmp/input_table.csv' INTO TABLE input_table
-        FIELDS TERMINATED BY ',' ENCLOSED BY '\\"';
+      @box = Icebox.new
+    end
+    should "have an internal Sequel MySQL DB accessible via #db" do
+      assert_equal @box.db.class, Sequel::MySQL::Database
+    end
+  end
+
+  context "Given a CSV file," do
+    setup do
+      File.open('/tmp/test_table.csv', 'w') do |f|
+        f.write <<-CSV.gsub(/^\s*/, '')
+          -127,-127,-32767,-8388608,-2147483647,-9223372036854775806,-3.402823466E+38,-1.7976931348623157E+308,-8675309.8675309,"Negative"
+          127,127,32767,8388608,2147483647,9223372036854775806,3.402823466E+38,1.7976931348623157E+308,8675309.8675309,"Positive"
+          0,0,0,0,0,0,0.0,0.0,0.0,"Zero"
+        CSV
+      end
+      @box = Icebox.new
+      @db = @box.db
+      @db << <<-SQL
+        CREATE TABLE test_table (
+          ti TINYINT,
+          b  BOOLEAN,
+          si SMALLINT,
+          mi MEDIUMINT,
+          i  INTEGER,
+          bi BIGINT,
+          f  FLOAT,
+          dp DOUBLE PRECISION,
+          dc DEC(14,7),
+          tx TEXT
+        )
       SQL
     end
 
-    should "copy a table with create_table_as" do
-      box = Icebox::Icebox.new(@db)
-      @db.run "DROP TABLE IF EXISTS output_table"
-      box.create_table_as "output_table", "SELECT * FROM input_table"
-      result = @db["SELECT COUNT(*) AS ct FROM output_table"].first[:ct]
-      assert_equal result, 3
+    should "load data from CSV and copy it with #create_table_as" do
+      @box.load_data_infile 'test_table', '/tmp/test_table.csv'
+      @box.create_table_as 'test_ctas_output', 'SELECT * FROM test_table'
+
+      t = @db[:test_ctas_output]
+      assert_equal 3, t.count
+      assert_equal -127, t.order(:ti).get(:ti)
+      assert_equal "Negative", t.order(:si).get(:tx)
+      assert_equal BigDecimal.new("8675309.8675309"), t.order(:dc.desc).get(:dc)
+    end
+
+    should "append loaded CSV data to an existing table" do
+      @box.load_data_infile 'test_table', '/tmp/test_table.csv'
+      count_1 = @db[:test_table].count
+      @box.load_data_infile 'test_table', '/tmp/test_table.csv'
+      count_2 = @db[:test_table].count
+
+      assert_equal 3, count_1
+      assert_equal 6, count_2
+    end
+
+    should "append a query result onto existing data via #insert_into" do
+      @box.load_data_infile 'test_table', '/tmp/test_table.csv'
+      count_1 = @db[:test_table].count
+      @box.insert_into 'test_table', 'SELECT * FROM test_table'
+      count_2 = @db[:test_table].count
+
+      assert_equal 3, count_1
+      assert_equal 6, count_2
+    end
+
+    should "not have race conditions when #insert_into is called rapidly" do
+      @box.load_data_infile 'test_table', '/tmp/test_table.csv'
+      count_1 = @db[:test_table].count
+      5.times do
+        @box.insert_into 'test_table', 'SELECT * FROM test_table'
+      end
+      count_2 = @db[:test_table].count
+
+      assert_equal 3, count_1
+      assert_equal 3 * (2**5), count_2
+    end
+
+    teardown do
+      File.delete '/tmp/test_table.csv'
+      @db.drop_table 'test_table'
+      @db.drop_table 'test_ctas_output' rescue Sequel
     end
   end
+
 end

metadata

@@ -1,92 +1,105 @@
---- !ruby/object:Gem::Specification
+--- !ruby/object:Gem::Specification 
 name: icebox
-version: !ruby/object:Gem::Version
-  version: 0.0.2
+version: !ruby/object:Gem::Version 
   prerelease: 
+  version: 0.0.3
 platform: ruby
-authors:
+authors: 
 - Riley Goodside
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-10-05 00:00:00.000000000Z
-dependencies:
-- !ruby/object:Gem::Dependency
+
+date: 2011-10-25 00:00:00 -04:00
+default_executable: icebox
+dependencies: 
+- !ruby/object:Gem::Dependency 
   name: mysql
-  requirement: &17081840 !ruby/object:Gem::Requirement
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
+      - !ruby/object:Gem::Version 
         version: 2.8.1
   type: :development
   prerelease: false
-  version_requirements: *17081840
-- !ruby/object:Gem::Dependency
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
   name: sequel
-  requirement: &17081320 !ruby/object:Gem::Requirement
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
+      - !ruby/object:Gem::Version 
         version: 3.27.0
   type: :development
   prerelease: false
-  version_requirements: *17081320
-- !ruby/object:Gem::Dependency
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
   name: shoulda
-  requirement: &17080800 !ruby/object:Gem::Requirement
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
   type: :development
   prerelease: false
-  version_requirements: *17080800
-- !ruby/object:Gem::Dependency
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
   name: bundler
-  requirement: &17080280 !ruby/object:Gem::Requirement
+  requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
+      - !ruby/object:Gem::Version 
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *17080280
-- !ruby/object:Gem::Dependency
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
   name: jeweler
-  requirement: &17079760 !ruby/object:Gem::Requirement
+  requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
+      - !ruby/object:Gem::Version 
         version: 1.6.4
   type: :development
   prerelease: false
-  version_requirements: *17079760
-- !ruby/object:Gem::Dependency
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: rdoc
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: "3.11"
+  type: :development
+  prerelease: false
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
   name: rcov
-  requirement: &17079220 !ruby/object:Gem::Requirement
+  requirement: &id007 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
   type: :development
   prerelease: false
-  version_requirements: *17079220
-description: Ruby library and command-line utility for working with Infobright Community
-  Edition
+  version_requirements: *id007
+description: Ruby library and command-line utility for working with Infobright Community Edition
 email: riley.goodside@gmail.com
-executables:
+executables: 
 - icebox
 extensions: []
-extra_rdoc_files:
+
+extra_rdoc_files: 
 - LICENSE.txt
 - README.rdoc
-files:
+files: 
 - .document
 - Gemfile
 - LICENSE.txt
@@ -100,32 +113,36 @@ files:
 - tasks/scratch.rb
 - test/helper.rb
 - test/test_icebox.rb
+has_rdoc: true
 homepage: http://github.com/goodside/icebox
-licenses:
+licenses: 
 - MIT
 post_install_message: 
 rdoc_options: []
-require_paths:
+
+require_paths: 
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement
+required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
-      segments:
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: -170128870566158022
+      segments: 
       - 0
-      hash: 2168865579570218248
-required_rubygems_version: !ruby/object:Gem::Requirement
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
 requirements: []
+
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: Multitool for working with Infobright Community Edition
 test_files: []
+