mysql_import 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 87f855e324bbd56af3b9ebd3652a752c8c5ddad9
4
+ data.tar.gz: 64166a6f3265a9e33e47f4a102c2e9c967a8e3b3
5
+ SHA512:
6
+ metadata.gz: f01b3bac43469acc2b0a5d14eae8c981e71085f38b9721dab09fcde3eacb0ad65889564ab6ec71870636d6ed1984078eb849f8f74e6d3adbff5ae18fe4c98e99
7
+ data.tar.gz: 70287c5d1b20f527ad67317c026ee323db26fc028f6a089d97447967f9233b74327388115e0037e2f54e36a7156782cc233977907a82da68769a3eef2bd840cd
data/.gitignore ADDED
@@ -0,0 +1,9 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
data/.travis.yml ADDED
@@ -0,0 +1,5 @@
1
+ sudo: false
2
+ language: ruby
3
+ rvm:
4
+ - 2.3.1
5
+ before_install: gem install bundler -v 1.12.4
data/Gemfile ADDED
@@ -0,0 +1,15 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in mysql_import.gemspec
4
+ gemspec
5
+
6
+ group :development, :test do
7
+ gem 'pry'
8
+ gem 'pry-power_assert'
9
+ gem 'pry-doc'
10
+ gem 'pry-byebug'
11
+ end
12
+
13
+ group :test do
14
+ gem "codeclimate-test-reporter", require: false
15
+ end
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2016 nalabjp
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,41 @@
1
+ # MysqlImport
2
+
3
+ Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/mysql_import`. To experiment with that code, run `bin/console` for an interactive prompt.
4
+
5
+ TODO: Delete this and the text above, and describe your gem
6
+
7
+ ## Installation
8
+
9
+ Add this line to your application's Gemfile:
10
+
11
+ ```ruby
12
+ gem 'mysql_import'
13
+ ```
14
+
15
+ And then execute:
16
+
17
+ $ bundle
18
+
19
+ Or install it yourself as:
20
+
21
+ $ gem install mysql_import
22
+
23
+ ## Usage
24
+
25
+ TODO: Write usage instructions here
26
+
27
+ ## Development
28
+
29
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
30
+
31
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
32
+
33
+ ## Contributing
34
+
35
+ Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/mysql_import.
36
+
37
+
38
+ ## License
39
+
40
+ The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
41
+
data/Rakefile ADDED
@@ -0,0 +1,10 @@
1
+ require "bundler/gem_tasks"
2
+ require "rake/testtask"
3
+
4
+ Rake::TestTask.new(:test) do |t|
5
+ t.libs << "test"
6
+ t.libs << "lib"
7
+ t.test_files = FileList['test/**/*_test.rb']
8
+ end
9
+
10
+ task :default => :test
data/bin/console ADDED
@@ -0,0 +1,7 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "mysql_import"
5
+
6
+ require "pry"
7
+ Pry.start
data/bin/db_setup ADDED
@@ -0,0 +1,9 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ cd $(dirname "$0")
7
+
8
+ mysql -u root < ../test/sql/database.sql
9
+ mysql -u root < ../test/sql/tables.sql
data/bin/setup ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,99 @@
1
+ require 'mysql_import/version'
2
+ require 'load_data_infile2'
3
+ require 'connection_pool'
4
+ require 'parallel'
5
+
6
+ class MysqlImport
7
+ def initialize(config, opts ={}, sql_opts = {})
8
+ @stash = []
9
+ @fileters = []
10
+ @concurrency = opts.has_key?(:concurrency) ? opts[:concurrency].to_i : 2
11
+ pool = concurrency == 0 ? 1 : concurrency
12
+
13
+ @client = ConnectionPool.new(size: pool) { LoadDataInfile2::Client.new(config, sql_opts) }
14
+ @result = Result.new
15
+ end
16
+
17
+ def add(file_path, options = {})
18
+ stash.push([file_path, options])
19
+ end
20
+
21
+ def import(*filters)
22
+ Parallel.each(filtered_list(filters), in_threads: concurrency) do |args|
23
+ client.with do |cli|
24
+ run_import(cli, *args)
25
+ end
26
+ end
27
+ end
28
+
29
+ private
30
+
31
+ attr_reader :stash, :filters, :concurrency, :client, :result
32
+
33
+ def filtered_list(filters)
34
+ return stash if filters.empty?
35
+
36
+ regexps = filters.map{|f| Regexp.new(f) }
37
+ stash.map{|row| row if regexps.any?{|r| r.match(row[0]) } }.compact
38
+ end
39
+
40
+ def run_import(cli, fpath, opts)
41
+ before = opts.delete(:before)
42
+ after = opts.delete(:after)
43
+ table = opts[:table] || File.basename(fpath, '.*')
44
+
45
+ if before
46
+ begin
47
+ run_action(before, cli)
48
+ rescue Break
49
+ result.add(:skipped, table)
50
+ return
51
+ end
52
+ end
53
+
54
+ res = cli.import(fpath, opts)
55
+ result.add(:imported, table)
56
+
57
+ if after
58
+ begin
59
+ run_action(after, cli, res)
60
+ rescue Break
61
+ end
62
+ end
63
+ end
64
+
65
+ def run_action(action, cli, res = nil)
66
+ case action
67
+ when Array
68
+ action.map { |act| run_action(act, cli, res) }
69
+ when String
70
+ cli.query(action)
71
+ else
72
+ if res
73
+ action.call(cli, res)
74
+ else
75
+ action.call(cli)
76
+ end
77
+ end
78
+ end
79
+
80
+ class Result
81
+ def imported
82
+ @imported ||= []
83
+ end
84
+
85
+ def skipped
86
+ @skipped ||= []
87
+ end
88
+
89
+ def mutex
90
+ @mutext ||= Mutex.new
91
+ end
92
+
93
+ def add(meth, res)
94
+ mutex.synchronize { __send__(meth).push(res) }
95
+ end
96
+ end
97
+
98
+ class Break < StandardError; end
99
+ end
@@ -0,0 +1,23 @@
1
+ require 'logger'
2
+
3
+ class MysqlImport::Logger < SimpleDelegator
4
+ def initialize(logger, debug = false)
5
+ case logger
6
+ when String
7
+ obj = ::Logger.new(arg)
8
+ when NilClass
9
+ obj = ::Logger.new(STDOUT)
10
+ obj.formatter = ->(seveity, datetime, progname, message) { "#{String === message ? message : message.inspect}\n" }
11
+ when FalseClass
12
+ obj = ::Logger.new('/dev/null')
13
+ else
14
+ obj = logger
15
+ end
16
+
17
+ if obj.respond_to?(:level=)
18
+ obj.level = debug ? Logger::DEBUG : Logger::INFO
19
+ end
20
+
21
+ __setobj__(obj)
22
+ end
23
+ end
@@ -0,0 +1,3 @@
1
+ class MysqlImport
2
+ VERSION = '0.1.0'
3
+ end
@@ -0,0 +1,28 @@
1
+ lib = File.expand_path('../lib', __FILE__)
2
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
+ require 'mysql_import/version'
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = 'mysql_import'
7
+ spec.version = MysqlImport::VERSION
8
+ spec.authors = ['nalabjp']
9
+ spec.email = ['nalabjp@gmail.com']
10
+
11
+ spec.summary = 'Concurrent importer for MySQL'
12
+ spec.description = 'Concurrent importer for MySQL'
13
+ spec.homepage = 'https://github.com/nalabjp/mysql_import'
14
+ spec.license = 'MIT'
15
+
16
+ spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
17
+ spec.bindir = 'exe'
18
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
19
+ spec.require_paths = ['lib']
20
+
21
+ spec.add_dependency 'load_data_infile2'
22
+ spec.add_dependency 'connection_pool'
23
+ spec.add_dependency 'parallel'
24
+
25
+ spec.add_development_dependency 'bundler', '~> 1.12'
26
+ spec.add_development_dependency 'rake'
27
+ spec.add_development_dependency 'test-unit'
28
+ end
metadata ADDED
@@ -0,0 +1,142 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: mysql_import
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - nalabjp
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2016-07-18 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: load_data_infile2
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: connection_pool
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: parallel
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: bundler
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '1.12'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '1.12'
69
+ - !ruby/object:Gem::Dependency
70
+ name: rake
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: test-unit
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ description: Concurrent importer for MySQL
98
+ email:
99
+ - nalabjp@gmail.com
100
+ executables: []
101
+ extensions: []
102
+ extra_rdoc_files: []
103
+ files:
104
+ - ".gitignore"
105
+ - ".travis.yml"
106
+ - Gemfile
107
+ - LICENSE.txt
108
+ - README.md
109
+ - Rakefile
110
+ - bin/console
111
+ - bin/db_setup
112
+ - bin/setup
113
+ - lib/mysql_import.rb
114
+ - lib/mysql_import/logger.rb
115
+ - lib/mysql_import/version.rb
116
+ - mysql_import.gemspec
117
+ homepage: https://github.com/nalabjp/mysql_import
118
+ licenses:
119
+ - MIT
120
+ metadata: {}
121
+ post_install_message:
122
+ rdoc_options: []
123
+ require_paths:
124
+ - lib
125
+ required_ruby_version: !ruby/object:Gem::Requirement
126
+ requirements:
127
+ - - ">="
128
+ - !ruby/object:Gem::Version
129
+ version: '0'
130
+ required_rubygems_version: !ruby/object:Gem::Requirement
131
+ requirements:
132
+ - - ">="
133
+ - !ruby/object:Gem::Version
134
+ version: '0'
135
+ requirements: []
136
+ rubyforge_project:
137
+ rubygems_version: 2.5.1
138
+ signing_key:
139
+ specification_version: 4
140
+ summary: Concurrent importer for MySQL
141
+ test_files: []
142
+ has_rdoc: