bulk_record 0.0.1

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+database.yml

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in bulk_record.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Shunsuke Mikami
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# BulkRecord
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'bulk_record'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install bulk_record
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/benchmark/benchmark.rb

@@ -0,0 +1,48 @@
+here = File.dirname(__FILE__)
+$LOAD_PATH << File.expand_path(File.join(here, '../lib'))
+require 'bulk_record'
+require 'yaml'
+require 'benchmark'
+require 'active_record'
+
+dbconfig = YAML::load_file('database.yml')
+ActiveRecord::Base.configurations = dbconfig
+ActiveRecord::Base.establish_connection "development"
+
+ActiveRecord::Schema.define do
+  create_table :counts, :options=>'ENGINE=InnoDb', :force=>true, :id => false do |t|
+    t.column :name, :string, :null=>false
+    t.column :count, :integer, :null => false, :default => 0
+    t.column :count2, :integer, :null => false, :default => 0
+  end
+
+  execute("ALTER TABLE `counts` ADD PRIMARY KEY(`name`)")
+end
+
+BulkRecord::Base.configurations = dbconfig
+BulkRecord::Base.establish_connection('development')
+
+class Count < BulkRecord::Base
+end
+
+count = Count.new(:fix_columns => [:name])
+row = { :name => 'test', :count => 1}
+
+number = 10
+start_time = Time.now
+Benchmark.bm do |x|
+  x.report("import") {
+    count.add({ :name => 'test', :count2 => nil})
+    number.times.each do |i|
+      count.add(row)
+    end
+    count.add({ :name => 'test', :count2 => nil})
+    count.import(:on_duplicate_key_update => true)
+  }
+end
+
+puts "#{number / (Time.now - start_time)} req / sec"
+
+ActiveRecord::Schema.define do
+  drop_table :counts
+end

data/bulk_record.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/bulk_record/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Shunsuke Mikami"]
+  gem.email         = ["shun0102@gmail.com"]
+  gem.description   = "library for mysql bulk insert"
+  gem.summary       = "library for mysql bulk insert"
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "bulk_record"
+  gem.require_paths = ["lib"]
+  gem.version       = BulkRecord::VERSION
+
+  gem.add_runtime_dependency "mysql2"
+  gem.add_runtime_dependency "activesupport"
+  gem.add_development_dependency "mysql2"
+  gem.add_development_dependency "activesupport"
+end

data/lib/bulk_record.rb

@@ -0,0 +1,142 @@
+require "bulk_record/version"
+require 'active_support/all'
+require 'mysql2'
+
+module BulkRecord
+  class Base
+    mattr_accessor :configurations, instance_writer: false
+    self.configurations = {}
+
+    class_attribute :connection, instance_writer: false
+
+    class_attribute :table_name_prefix
+    self.table_name_prefix = ""
+
+    class_attribute :table_name
+    self.table_name = ""
+
+    class_attribute :connection
+
+    class_attribute :primary_keys
+
+    class_attribute :fix_columns
+    self.fix_columns = []
+
+    class_attribute :columns
+    self.columns = []
+
+    class_attribute :rows
+    self.rows = []
+
+    class_attribute :columns_in_rows
+    self.columns_in_rows = []
+
+    module ConnectionHandling
+      def establish_connection(env)
+        conf = self.configurations[env]
+        self.connection = mysql2_connection(conf)
+      end
+
+      def mysql2_connection(config)
+        config[:username] = 'root' if config[:username].nil?
+        return Mysql2::Client.new(config.symbolize_keys)
+      end
+    end
+
+    module ClassMethods
+      def initialize(option = nil)
+        set_schema(option)
+        set_primary_keys
+      end
+
+      def set_primary_keys
+        query = "SHOW KEYS FROM #{self.full_table_name} WHERE Key_name = 'PRIMARY'"
+        result = connection.query(query)
+        self.primary_keys = []
+        result.each do |row|
+          self.primary_keys << row['Column_name']
+        end
+      end
+
+      def set_table_name(name)
+        self.table_name = name
+      end
+
+      def add(row)
+        self.columns_in_rows = columns_in_rows | row.keys
+        self.rows << row
+      end
+
+      def full_table_name
+        if @table_name_prefix.nil?
+          @table_name
+        else
+          @table_name_prefix + @table_name
+        end
+      end
+
+      def set_schema(option)
+        self.table_name = self.class.name.underscore.pluralize
+        unless option.blank?
+          unless option[:table_name_prefix].blank?
+            self.table_name_prefix = option[:table_name_prefix]
+          end
+          unless option[:fix_columns].blank?
+            self.fix_columns = option[:fix_columns]
+          end
+        end
+        query = "DESC #{self.full_table_name}"
+        connection.query(query).each do |r|
+          unless r["Extra"] && r["Extra"] == "auto_increment"
+            self.columns << r["Field"].to_sym
+          end
+        end
+      end
+
+      def import(option = nil)
+        values = []
+        update_columns = columns_in_rows & columns
+
+        self.rows.each do |row|
+          value = []
+          self.columns.each do |col|
+            value << format(row[col])
+          end
+          values << "(#{value.join(",")})"
+        end
+        query = "INSERT INTO #{self.full_table_name} (#{columns.join(',')}) VALUES #{values.join(",")}"
+
+        if (!option.nil? && option[:on_duplicate_key_update])
+          counter_columns = update_columns.select { |x| !fix_columns.include?(x) }
+          update = " ON DUPLICATE KEY UPDATE " + counter_columns.map{ |x| "#{x} = #{x} + VALUES(#{x})"}.join(',')
+          query += update
+          connection.query(query)
+        else
+          connection.query(query)
+        end
+      end
+
+      def format(value)
+        formalized = ""
+        if value.nil?
+          formalized = "NULL"
+        elsif value.respond_to?(:strftime)
+          formalized = "'" + rawvalue.strftime('%Y-%m-%d %H:%M:%S') + "'"
+        elsif value.is_a?(Array)
+          formalized = value.map{|v| "'" + Mysql2::Client.escape(v.to_s) + "'" }.join(",")
+        else
+          formalized = "'" + Mysql2::Client.escape(value.to_s) + "'"
+        end
+      end
+
+      # TODO: bulk insert useing LOAD DATA INFILE
+      def load
+      end
+
+    end
+
+    extend ConnectionHandling
+    include ClassMethods
+  end
+
+end

data/lib/bulk_record/version.rb

@@ -0,0 +1,3 @@
+module BulkRecord
+  VERSION = "0.0.1"
+end

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification
+name: bulk_record
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Shunsuke Mikami
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-01-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: library for mysql bulk insert
+email:
+- shun0102@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- benchmark/benchmark.rb
+- bulk_record.gemspec
+- lib/bulk_record.rb
+- lib/bulk_record/version.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.19
+signing_key: 
+specification_version: 3
+summary: library for mysql bulk insert
+test_files: []
+has_rdoc: