sequel_load_data_infile 0.0.1

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,12 @@
+source "http://rubygems.org"
+
+gem "sequel"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "mysql", "2.8.1"
+  gem "rspec", "~> 2"
+  gem "bundler"
+  gem "jeweler", "~> 1.8.4"
+end

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2013 Roland Swingler
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,29 @@
+= sequel_load_data_infile
+
+Provides LOAD DATA INFILE support for mysql datasets.
+
+To use:
+
+    require 'sequel/load_data_infile'
+
+    DB[:some_table].load_data_infile(filepath, [:column, :names])
+
+    # Or, for CSV data:
+
+    DB[:some_table].load_csv_infile(filepath, [:column, :names])
+
+== Contributing to sequel_load_data_infile
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
+* Fork the project.
+* Start a feature/bugfix branch.
+* Commit and push until you are happy with your contribution.
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2013 Roland Swingler. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,34 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "sequel_load_data_infile"
+  gem.homepage = "http://github.com/knaveofdiamonds/sequel_load_data_infile"
+  gem.license = "MIT"
+  gem.summary = "Provides LOAD DATA INFILE support for mysql datasets"
+  gem.description = "Provides LOAD DATA INFILE support for mysql datasets"
+  gem.email = "roland.swingler@gmail.com"
+  gem.authors = ["Roland Swingler"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+task :default => :spec

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/lib/sequel/load_data_infile.rb

@@ -0,0 +1,159 @@
+require 'sequel'
+
+module Sequel
+  # @api private
+  class LoadDataInfileExpression
+    attr_reader :path, :table, :columns, :ignore, :character_set
+    
+    def initialize(path, table, columns, opts={})
+      @path    = path
+      @table   = table
+      @columns = columns
+      @ignore  = opts[:ignore]
+      @update  = opts[:update]
+      @set     = opts[:set] || {}
+      @character_set = opts[:character_set] || "utf8"
+      if opts[:format] == :csv
+        @field_terminator = ","
+        @enclosed_by = '"'
+        @escaped_by = '"'
+      end
+    end
+    
+    def replace?
+      @update == :replace
+    end
+
+    def ignore?
+      @update == :ignore
+    end
+
+    def to_sql(db)
+      @db = db
+      
+      [load_fragment,
+       replace_fragment,
+       table_fragment,
+       character_set_fragment,
+       field_terminator_fragment,
+       field_enclosure_fragment,
+       escape_fragment,
+       ignore_fragment, 
+       column_fragment,
+       set_fragment].compact.join(" ")
+    end
+
+    private
+
+    def load_fragment
+      "LOAD DATA INFILE '#{path}'"
+    end
+
+    def replace_fragment
+      @update.to_s.upcase if replace? || ignore?
+    end
+
+    def table_fragment
+      "INTO TABLE `#{table}`"
+    end
+
+    def character_set_fragment
+      "CHARACTER SET '#{character_set}'"
+    end
+
+    def field_terminator_fragment
+      "FIELDS TERMINATED BY '#{@field_terminator}'" if @field_terminator
+    end
+
+    def field_enclosure_fragment
+      "OPTIONALLY ENCLOSED BY '#{@enclosed_by}'" if @enclosed_by
+    end
+
+    def escape_fragment
+      "ESCAPED BY '#{@escaped_by}'" if @escaped_by
+    end
+
+    def ignore_fragment
+      "IGNORE #{ignore} LINES" if ignore
+    end
+
+    def column_fragment
+      "(" + columns.map {|c| format_column(c) }.join(",") + ")"
+    end
+
+    def set_fragment
+      unless set_columns.empty?
+        "SET " + set_columns.map do |k, v|
+          "#{@db.literal(k)} = #{@db.literal(v)}"
+        end.join(", ")
+      end
+    end
+
+    def format_column(column)
+      if binary_columns.include?(column)
+        "@#{column}"
+      elsif column.to_s[0..0] == "@"
+        column
+      else
+        "`#{column}`"
+      end
+    end
+
+    def binary_columns
+      @binary_columns ||= @db.schema(@table).
+        select {|a| a[1][:type] == :blob }.map {|a| a.first.to_s }
+    end
+    
+    def set_columns
+      binary_columns.inject({}) do |hash, column|
+        hash[column.to_sym] = :unhex.sql_function("@#{column}".lit)
+        hash
+      end.merge(@set)
+    end
+  end
+
+  module LoadDataInfile
+    # Load data in file specified at path.
+    #
+    # Columns is a list of columns to load - column names starting
+    # with an @ symbol will be treated as variables.
+    #
+    # By default, this will generate a REPLACE INTO TABLE
+    # statement.
+    #
+    # Options:
+    # :ignore - the number of lines to ignore in the source file
+    # :update - nil, :ignore or :replace
+    # :set - a hash specifying autopopulation of columns
+    # :character_set - the character set of the file, UTF8 default
+    # :format - either nil or :csv
+    def load_infile(path, columns, options={})
+      execute_dui(load_infile_sql(path, columns, options))
+    end
+
+    # Returns the SQL for a LOAD DATA INFILE statement.
+    def load_infile_sql(path, columns, options={})
+      replacement = opts[:insert_ignore] ? :ignore : :replace
+      options = {:update => replacement}.merge(options)
+      LoadDataInfileExpression.new(path, 
+                                   opts[:from].first, 
+                                   columns, 
+                                   options).
+        to_sql(db)
+    end
+
+    # Loads the CSV data columns in path into this dataset's
+    # table.
+    #
+    # See load_infile for more options.
+    def load_csv_infile(path, columns, options={})
+      execute_dui(load_csv_infile_sql(path, columns, options))
+    end
+    
+    def load_csv_infile_sql(path, columns, options={})
+      load_infile_sql(path, columns, options.merge(:format => :csv))
+    end
+  end
+end
+
+Sequel::Dataset.send :include, Sequel::LoadDataInfile

data/spec/db_connections.yml.dist

@@ -0,0 +1,4 @@
+adapter: mysql
+username: root
+socket: /tmp/mysql.sock
+database: load_data_infile_test_db

data/spec/load_data_infile_expression_spec.rb

@@ -0,0 +1,71 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe Sequel::LoadDataInfileExpression do
+  before :each do
+    TEST_DB.stub(:schema).and_return([])
+  end
+
+  it "loads the data in the file into the table" do
+    described_class.new("bar.csv", :foo, ['bar', 'quux']).
+      to_sql(TEST_DB).should include("LOAD DATA INFILE 'bar.csv' INTO TABLE `foo`")
+  end
+
+  it "loads the data with replacment" do
+    described_class.new("bar.csv", :foo, ['bar', 'quux'],
+                        :update => :replace).
+      to_sql(TEST_DB).should include("REPLACE INTO TABLE")
+  end
+
+  it "loads the data ignoring rows" do
+    described_class.new("bar.csv", :foo, ['bar', 'quux'], :update => :ignore).
+      to_sql(TEST_DB).should include("IGNORE INTO TABLE")
+  end
+
+  it "should be in UTF-8 character set by default" do
+    described_class.new("bar.csv", :foo, ['bar', 'quux']).
+      to_sql(TEST_DB).should include("CHARACTER SET 'utf8'")
+  end
+
+  it "may be in other character sets" do
+    described_class.new("bar.csv", :foo, ['bar', 'quux'], :character_set => "ascii").
+      to_sql(TEST_DB).should include("CHARACTER SET 'ascii'")
+  end
+
+  it "should load columns" do
+    described_class.new("bar.csv", :foo, ['bar', 'quux']).
+      to_sql(TEST_DB).should include("(`bar`,`quux`)")
+  end
+
+  it "should load into variables if column begins with @" do
+    described_class.new("bar.csv", :foo, ['@bar', 'quux']).
+      to_sql(TEST_DB).should include("(@bar,`quux`)")
+  end
+
+  it "can ignore lines" do
+    described_class.new("bar.csv", :foo, ['bar', 'quux'], :ignore => 2).
+      to_sql(TEST_DB).should include("IGNORE 2 LINES")
+  end
+
+  it "can be in csv format" do
+    described_class.new("bar.csv", :foo, ['bar', 'quux'], :format => :csv).
+      to_sql(TEST_DB).should include("FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' ESCAPED BY '\"'")
+  end
+
+  it "can set column values" do
+    sql = described_class.new("bar.csv", :foo, ['@bar', 'quux'], 
+                        :set => {:bar => :unhex.sql_function("@bar".lit),
+                        :etl_batch_id => 3}).
+      to_sql(TEST_DB)
+
+    
+    sql.should include("`etl_batch_id` = 3")
+    sql.should include("`bar` = unhex(@bar)")
+  end
+
+  it "unhexes binary columns automatically via set" do
+    TEST_DB.stub(:schema).and_return([[:bar, {:type => :blob}]])
+    sql = described_class.new("bar.csv", :foo, ['bar', 'quux']).to_sql(TEST_DB)
+    sql.should include("(@bar,`quux`)")
+    sql.should include("SET `bar` = unhex(@bar)")
+  end
+end

data/spec/load_data_infile_spec.rb

@@ -0,0 +1,39 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe Sequel::LoadDataInfile do
+  before :each do
+    dataset = TEST_DB[:foo]
+    dataset.db.stub(:schema).and_return([])
+    @sql = dataset.load_csv_infile_sql("bar.csv", [:bar, :baz])
+  end
+
+  it "loads the data in the file" do
+    @sql.should include("LOAD DATA INFILE 'bar.csv'")
+  end
+
+  it "replaces rows currently in the table" do
+    @sql.should include("REPLACE INTO TABLE `foo`")
+  end
+
+  it "should be in the UTF 8 character set" do
+    @sql.should include("CHARACTER SET 'utf8'")
+  end
+
+  it "should escape with the \" character" do
+    @sql.should include("ESCAPED BY '\"'")
+  end
+
+  it "supports standard csv, with optional quoting" do
+    @sql.should include("FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"'")
+  end
+
+  it "loads into the columns specified" do
+    @sql.should include("(`bar`,`baz`)")
+  end
+
+  it "can ignore instead of replacing rows" do
+    @sql = TEST_DB[:foo].insert_ignore.
+      load_csv_infile_sql("bar.csv", [:bar, :baz])
+    @sql.should include("IGNORE INTO TABLE `foo`")
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,17 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'rspec'
+require 'yaml'
+require 'sequel/load_data_infile'
+
+unless defined? TEST_DB
+  TEST_DB = Sequel.connect(YAML.load(File.read(File.dirname(__FILE__) + "/db_connections.yml")))
+end
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+
+RSpec.configure do |config|
+  
+end

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: sequel_load_data_infile
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Roland Swingler
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mysql
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.8.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.8.1
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.4
+description: Provides LOAD DATA INFILE support for mysql datasets
+email: roland.swingler@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.rdoc
+files:
+- .document
+- .rspec
+- Gemfile
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/sequel/load_data_infile.rb
+- spec/db_connections.yml.dist
+- spec/load_data_infile_expression_spec.rb
+- spec/load_data_infile_spec.rb
+- spec/spec_helper.rb
+homepage: http://github.com/knaveofdiamonds/sequel_load_data_infile
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -2195219335310673832
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Provides LOAD DATA INFILE support for mysql datasets
+test_files: []