mysql_import 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1dc22640b21bf3b9cf9c765dbab33d8c6da11fd1
-  data.tar.gz: 94e4ec0f279005ca04db5b69164db542dc1dcdd9
+  metadata.gz: fa4a531d2bdd0606ab8ef9d54301a33a066cda2c
+  data.tar.gz: 8c10dbdb843d9be5edf3b0c49e7153318fa3cf51
 SHA512:
-  metadata.gz: eff179fed36fd5d1b24818099057ea604cdda72a6c67576c3e261f7e56a5861ab97d55a052ae7cb790fcc017f020b1d948761bf12ad424974adcc1a97756ce75
-  data.tar.gz: 44f5534efac69c3d292170bcb12032ea6b18176945ee04d0e190cd50185e314109c8da4d51ff9ad88e743bc306600a0a058adb5f148474a2cb033707bdd08209
+  metadata.gz: 741420989677a2c37e21bdbbfaf8184c272f10fc68f0ad4590d799a14b12dc8304db2d79a9266e0f81a9525b27b67db8706e40795cb6fa1effe4e951bd2d4023
+  data.tar.gz: 7fbb7a69ece5555bbbd64eb93931e339500001a855a0f2bdfd78585084dd01a220d8f4a0bceb886bdfaa7ac96288d174d323b9754aa1b16f0733a4b26c2221ad

data/.travis.yml

@@ -1,5 +1,23 @@
-sudo: false
+dist: trusty
+sudo: required
 language: ruby
 rvm:
-  - 2.3.1
-before_install: gem install bundler -v 1.12.4
+- 2.2.5
+- 2.3.1
+before_install: gem install bundler
+cache:
+- bundler
+- apt
+before_script:
+- bin/db_setup
+addons:
+  apt:
+    packages:
+    - mysql-server-5.6
+    - mysql-client-core-5.6
+    - mysql-client-5.6
+  code_climate:
+    repo_token: 7ccd2178b58732b43c4968b9472ee62fada44441a474183b0571a74d9c0db877
+notifications:
+  slack:
+    secure: qLm5VTc7heQ+Ch73rKjrcX7VpP6T/RakKfzvK75c79z8ItpZNr8zCY+l3benZh8IvXde0+uXPIBHq3eaTzz8IEpeqK28TLlMHNCdpxGPaDkgdOtTqVYF9lQQ0MnJmgc8IUj0ESiIPCLxp258zY9zGg6LDmdf2OlwND5/Bvcqs2HJQWO8f9QcHzWonYGmsFTifHOSfz7LqA7/pPQ0yjQ3b2JUieMFy32x/h97/DackSNwn0nWawX5Z3850PFKK0ZBPi47J2GlF0xlZh20TEY0a8mTeS7NikNeBn5MKDtYqE2MDq9CllahaSLna86AP2+VL8QGvBX6seBIg3Y/bL6c+R6t1rT1iy4etd9IVFdjQw87NyMLef4QgdyVxVKtjHoj9KaRrz+7ghshHqlaNFmyo6IFwS0lQJ8aL8m5UGfcQ5yx3Kap1VbUZqRb5nQ1+uQJ3T0Sef5cLpUnvwhJ3ZophXhc8NT+juM5Ho/oaRQu57gKt3jAYeDO0kydbVcnChfEsValFxNTs8iB5hFbAZzkvCB2lUv+j+HhDt1E+g/sgpntQrGjs3T2Vb1XPvK+vjU+UQdNTJ+MHj5tkC6CmUZApES3+rTK5NXeFrJoMBK8yO2h3p7uYpKUrn5bQ/MnvluFowzpCDevP1jj54bVgzOGGUEQJg1LlxkfG9CM71r2T1k=

data/README.md

@@ -1,38 +1,178 @@
 # MysqlImport
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/mysql_import`. To experiment with that code, run `bin/console` for an interactive prompt.
+[![Gem Version](https://badge.fury.io/rb/mysql_import.svg)](https://badge.fury.io/rb/mysql_import)
+[![Build Status](https://travis-ci.org/nalabjp/mysql_import.svg?branch=master)](https://travis-ci.org/nalabjp/mysql_import)
+[![Code Climate](https://codeclimate.com/github/nalabjp/mysql_import/badges/gpa.svg)](https://codeclimate.com/github/nalabjp/mysql_import)
+[![Test Coverage](https://codeclimate.com/github/nalabjp/mysql_import/badges/coverage.svg)](https://codeclimate.com/github/nalabjp/mysql_import/coverage)
+[![Dependency Status](https://gemnasium.com/badges/github.com/nalabjp/mysql_import.svg)](https://gemnasium.com/github.com/nalabjp/mysql_import)
 
-TODO: Delete this and the text above, and describe your gem
+Simple concurrent importer for MySQL using [load_data_infile2](https://github.com/nalabjp/load_data_infile2).
 
 ## Installation
 
-Add this line to your application's Gemfile:
+Add to your application's Gemfile:
 
 ```ruby
 gem 'mysql_import'
 ```
 
-And then execute:
+And bundle.
 
-    $ bundle
+## Examples
+### Basic Usage
 
-Or install it yourself as:
+For exampole, if you want to import to `users` table from `/path/to/users.csv`:
+```ruby
+db_config = {
+  host: 'localhost'
+  database: 'mysql_import_test'
+  username: 'root'
+}
+importer = MysqlImport.new(db_config)
+importer.add('/path/to/users.csv')
+importer.import
+# => Import to `users` tables
+```
+
+Multiple import:
+```ruby
+importer = MysqlImport.new(db_config)
+importer.add('/path/to/users.csv')
+importer.add('/path/to/groups.csv')
+importer.add('/path/to/departments.csv')
+importer.import
+# => Import to three tables from three csv files
+```
+
+MysqlImport has the concurrency because it uses the [parallel](https://github.com/grosser/parallel) gem.
+
+With import options:
+
+```ruby
+importer = MysqlImport.new(db_config)
+importer.add('/path/to/users1.csv', table: 'users')
+importer.add('/path/to/users2.csv', table: 'users')
+importer.add('/path/to/users3.csv', table: 'users')
+importer.import
+# => Import to `users` table from three csv files
+```
+
+See more details for import options.
+
+https://github.com/nalabjp/load_data_infile2#sql-options
+
+### Filter
+
+If you want to import only a specific file, you can specify the file.
+
+The specification of the file will be used regular expression
+
+```ruby
+importer = MysqlImport.new(db_config)
+importer.add('/path/to/users.csv')
+importer.add('/path/to/groups.csv')
+importer.add('/path/to/departments.csv')
+importer.import('users')
+# => Only import to `users` table
+
+importer.import('users', 'groups')
+# => Import to `users` and `groups` table
+```
 
-    $ gem install mysql_import
+### Hook
 
-## Usage
+You are able to set the hook immediately before and after import.
 
-TODO: Write usage instructions here
+The hook will accept either String or Proc or Array.
 
-## Development
+#### String
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+String is evaluated directly as SQL.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+```ruby
+importer = MysqlImport.new(db_config)
+importer.add(
+  '/path/to/users.csv',
+  {
+    before: 'TRUNCATE TABLE users;'
+  }
+)
+importer.import
+# => Truncate query is always executed before import.
+```
+
+#### Proc
+
+If you want to make the subsequent processing based on the execution result of SQL, you should use Proc.
+
+Arguments that are passed to Proc is an instance of `LoadDataInfile2::Client`, which is a subclass of `Mysql2::Client`.
+
+```ruby
+importer = MysqlImport.new(db_config)
+importer.add(
+  '/path/to/users.csv',
+  {
+    before: ->(cli) {
+      res = cli.query('SELECT COUNT(*) AS c FROM users;')
+      cli.query('TRUNCATE TABLE users;') if res.first['c'] > 0
+    }
+  }
+)
+importer.import
+# => If there is one or more records in `users` table, truncate query is executed.
+```
+
+#### Array
+
+Array of elements you need to use String or Proc.
+
+```ruby
+importer = MysqlImport.new(db_config)
+importer.add(
+  '/path/to/users.csv',
+  {
+    before: [
+      "SET sql_mode = 'STRICT_TRANS_TABLES';",
+      ->(cli) {
+        res = cli.query('SELECT COUNT(*) AS c FROM users;')
+        cli.query('TRUNCATE TABLE users;') if res.first['c'] > 0
+      }
+    ],
+    after: [
+      'SET @i = 0;',
+      'UPDATE users SET order = (@i := @i + 1) ORDER BY name, email ASC;',
+    ]
+  }
+)
+importer.import
+```
+
+#### Skip all subsequent processing
+
+If you want to skip all subsequent processing, you will need to raise `MysqlImport::Break` in Proc.
+
+```ruby
+importer = MysqlImport.new(db_config)
+importer.add(
+  '/path/to/users.csv',
+  {
+    before: ->(cli) {
+      res = cli.query('SELECT COUNT(*) AS c FROM users;')
+      raise MysqlImport::Break if res.first['c'] > 0
+    },
+    after: [
+      'SET @i = 0;',
+      'UPDATE users SET order = (@i := @i + 1) ORDER BY name, email ASC;',
+    ]
+  }
+)
+importer.import
+# => If there is one or more records in `users` table, import and after hook will be skipped.
+```
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/mysql_import.
+Bug reports and pull requests are welcome on GitHub at https://github.com/nalabjp/mysql_import.
 
 
 ## License

data/lib/mysql_import/logger.rb

@@ -10,7 +10,7 @@ class MysqlImport
         obj = ::Logger.new(nil)
       when STDOUT, STDERR
         obj = ::Logger.new(out)
-        obj.formatter = ->(seveity, datetime, progname, message) { "#{String === message ? message : message.inspect}\n" }
+        obj.formatter = ->(_, _, _, message) { "#{String === message ? message : message.inspect}\n" }
       else
         obj = out
       end
@@ -21,7 +21,7 @@ class MysqlImport
   end
 
   module Logging
-    def initialize(config, opts ={}, sql_opts = {})
+    def initialize(config, opts = {})
       @logger = Logger.new(opts[:log], opts.fetch(:debug, false))
       embed_logger
       super
@@ -30,36 +30,34 @@ class MysqlImport
     def import(*filters)
       super
     ensure
-      logger.info('Imported tables:')
-      if result.imported.size > 0
-        result.imported.sort.each {|t| logger.info("  #{t[0]} (#{t[1]} sec)") }
+      @logger.info('Imported tables:')
+      if @result.imported.size > 0
+        @result.imported.sort.each {|t| @logger.info("  #{t[0]} (#{t[1]} sec)") }
       else
-        logger.info('  nothing...')
+        @logger.info('  nothing...')
       end
-      if result.skipped.size > 0
-        logger.info('Skipped tables:')
-        result.skipped.sort.each {|t| logger.info("  #{t}") }
+      if @result.skipped.size > 0
+        @logger.info('Skipped tables:')
+        @result.skipped.sort.each {|t| @logger.info("  #{t}") }
       end
 
-      result.clear
+      @result.clear
     end
 
     private
 
-    attr_reader :logger
-
     def parallel_opts
       @parallel_opts ||= super.merge(
         finish: proc do |item, index, _result|
-          logger.debug("parallel_item: #{item.inspect}")
-          logger.debug("parallel_index: #{index}")
+          @logger.debug("parallel_item: #{item.inspect}")
+          @logger.debug("parallel_index: #{index}")
         end
       )
     end
 
     def embed_logger
       unless LoadDataInfile2::Client.instance_methods.include?(:build_sql_with_logging)
-        LoadDataInfile2::Client.class_exec(logger) do |logger|
+        LoadDataInfile2::Client.class_exec(@logger) do |logger|
           define_method :build_sql_with_logging do |file, options = {}|
             build_sql_without_logging(file, options).tap {|sql| logger.debug("sql: #{sql}") }
           end

data/lib/mysql_import/version.rb

@@ -1,3 +1,3 @@
 class MysqlImport
-  VERSION = '0.2.1'
+  VERSION = '0.3.0'
 end

data/lib/mysql_import.rb

@@ -5,23 +5,23 @@ require 'connection_pool'
 require 'parallel'
 
 class MysqlImport
-  def initialize(config, opts ={}, sql_opts = {})
+  def initialize(config, opts = {})
     @stash = []
-    @fileters = []
     @concurrency = opts.has_key?(:concurrency) ? opts[:concurrency].to_i : 2
-    pool = concurrency.zero? ? 1 : concurrency
+    pool = @concurrency.zero? ? 1 : @concurrency
+    sql_opts = opts.fetch(:sql_opts, {})
 
     @client = ConnectionPool.new(size: pool) { LoadDataInfile2::Client.new(config, sql_opts) }
     @result = Result.new
   end
 
-  def add(file_path, options = {})
-    stash.push([file_path, options])
+  def add(file_path, opts = {})
+    @stash.push([file_path, opts])
   end
 
   def import(*filters)
     Parallel.each(filtered_list(filters), parallel_opts) do |args|
-      client.with do |cli|
+      @client.with do |cli|
         run_import(cli, *args)
       end
     end
@@ -29,45 +29,42 @@ class MysqlImport
 
   private
 
-  attr_reader :stash, :filters, :concurrency, :client, :result
-
   def filtered_list(filters)
-    return stash if filters.empty?
+    return @stash if filters.empty?
 
     regexps = filters.map{|f| Regexp.new(f) }
-    stash.map{|row| row if regexps.any?{|r| r.match(row[0]) } }.compact
+    @stash.map{|row| row if regexps.any?{|r| r.match(row[0]) } }.compact
   end
 
   def parallel_opts
-    { in_threads: concurrency }
+    { in_threads: @concurrency }
   end
 
   def run_import(cli, fpath, opts)
     t = Time.now
 
-    before = opts.delete(:before)
-    after = opts.delete(:after)
-    table = opts[:table] || File.basename(fpath, '.*')
+    sql_opts = Marshal.load(Marshal.dump(opts.reject {|k, _| %i(before after).include?(k) }))
+    table = sql_opts[:table] || File.basename(fpath, '.*')
 
-    if before
+    if opts[:before]
       begin
-        run_action(before, cli)
+        run_action(opts[:before], cli)
       rescue Break
-        result.add(:skipped, table)
+        @result.add(:skipped, table)
         return
       end
     end
 
-    cli.import(fpath, opts)
+    cli.import(fpath, sql_opts)
 
-    if after
+    if opts[:after]
       begin
-        run_action(after, cli)
+        run_action(opts[:after], cli)
       rescue Break
       end
     end
 
-    result.add(:imported, [table, (Time.now - t)])
+    @result.add(:imported, [table, (Time.now - t)])
   end
 
   def run_action(action, cli)

data/mysql_import.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'load_data_infile2'
+  spec.add_dependency 'load_data_infile2', '~> 0.2'
   spec.add_dependency 'connection_pool'
   spec.add_dependency 'parallel'
 

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: mysql_import
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - nalabjp
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-07-30 00:00:00.000000000 Z
+date: 2016-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: load_data_infile2
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: connection_pool
   requirement: !ruby/object:Gem::Requirement