naginegi 0.4.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: a76fe38b7173ab84b9ab04d60af0ddd4d09ba19494a8b89b9800152485d098ae
4
+ data.tar.gz: fec84aa30a5e9877fa86effc5f891d3c8ec3f23ef626abfda8dff9069a94e97a
5
+ SHA512:
6
+ metadata.gz: a8b447fa09122b4ee88bfcbd230d6b176fb6fcf00e1bc9a996c49523f12de29a5348a8c6156a2787a402b939d81b24e3f69afbb3e7af575045a7621410e9e8ed
7
+ data.tar.gz: 0adf72690e15bdeeea4389c4acb3f6284f3860451ae4b514e35be7cbbbd3c0e33767791a3cd7a4de54fc6911e69931f9766e7a5db5c7ae60514536954c22ea09
@@ -0,0 +1,41 @@
1
+ version: 2
2
+ jobs:
3
+ build:
4
+ working_directory: ~/naginegi
5
+ docker:
6
+ - image: circleci/ruby:2.6.3
7
+ environment:
8
+ TZ: /usr/share/zoneinfo/Asia/Tokyo
9
+ steps:
10
+ - checkout
11
+
12
+ - restore_cache:
13
+ name: Restore bundle cache
14
+ key: naginegi-{{ checksum "Gemfile.lock" }}
15
+
16
+ - run:
17
+ name: Install Bundler
18
+ command: gem install bundler -v 2.1.4
19
+
20
+ - run:
21
+ name: Run bundle install
22
+ command: bundle install --path vendor/bundle
23
+
24
+ - save_cache:
25
+ name: Store bundle cache
26
+ key: naginegi-{{ checksum "Gemfile.lock" }}
27
+ paths:
28
+ - vendor/bundle
29
+
30
+ - run:
31
+ name: chmod +x ./lint.sh
32
+ command: chmod +x ./lint.sh
33
+
34
+ - run:
35
+ name: ./lint.sh
36
+ command: ./lint.sh
37
+
38
+ - run:
39
+ name: Run rspec
40
+ command: bundle exec rspec spec/
41
+
@@ -0,0 +1,41 @@
1
+ *.rbc
2
+ capybara-*.html
3
+ .rspec
4
+ .rspec_status
5
+ *.log
6
+ /log
7
+ /tmp
8
+ /db/*.sqlite3
9
+ /db/*.sqlite3-journal
10
+ /db/schema.rb
11
+ /public/system
12
+ /coverage/
13
+ /spec/tmp
14
+ *.orig
15
+ rerun.txt
16
+ pickle-email-*.html
17
+
18
+ .project
19
+
20
+ config/initializers/secret_token.rb
21
+
22
+ .env
23
+
24
+ /.bundle
25
+ /vendor/bundle
26
+
27
+ .rvmrc
28
+
29
+ /vendor/assets/bower_components
30
+ *.bowerrc
31
+ bower.json
32
+
33
+ .powenv
34
+
35
+ .byebug_history
36
+
37
+ .ruby-version
38
+
39
+ .DS_Store
40
+
41
+ *.gem
@@ -0,0 +1,105 @@
1
+ AllCops:
2
+ TargetRubyVersion: 2.4.1
3
+ Include:
4
+ - '**/Gemfile'
5
+ - '**/Rakefile'
6
+ - '**/config.ru'
7
+ Exclude:
8
+ - 'db/**/*'
9
+ - 'config/**/*'
10
+ - 'script/**/*'
11
+ - 'vendor/**/*'
12
+ - 'bin/*'
13
+ - !ruby/regexp /old_and_unused\.rb$/
14
+
15
+ AlignParameters:
16
+ Enabled: false
17
+
18
+ AsciiComments:
19
+ Enabled: false
20
+
21
+ BracesAroundHashParameters:
22
+ Enabled: false
23
+
24
+ Bundler/OrderedGems:
25
+ Enabled: false
26
+
27
+ ClassAndModuleChildren:
28
+ Enabled: false
29
+
30
+ ClassLength:
31
+ Enabled: false
32
+
33
+ Layout/AlignHash:
34
+ Enabled: false
35
+
36
+ Layout/EndOfLine:
37
+ Enabled: false
38
+
39
+ Layout/IndentHash:
40
+ Enabled: false
41
+
42
+ Layout/MultilineArrayBraceLayout:
43
+ Enabled: false
44
+
45
+ Layout/MultilineHashBraceLayout:
46
+ Enabled: false
47
+
48
+ Layout/MultilineMethodCallIndentation:
49
+ Enabled: false
50
+
51
+ Metrics/BlockLength:
52
+ Max: 50
53
+ Exclude:
54
+ - 'spec/**/*'
55
+ - 'config/routes.rb'
56
+ - 'app/jobs/**/*.rb'
57
+ - 'lib/tasks/**/*.rb'
58
+
59
+ Metrics/AbcSize:
60
+ Enabled: false
61
+
62
+ Metrics/CyclomaticComplexity:
63
+ Enabled: false
64
+
65
+ Metrics/LineLength:
66
+ Enabled: false
67
+
68
+ Metrics/MethodLength:
69
+ Enabled: false
70
+
71
+ Metrics/PerceivedComplexity:
72
+ Enabled: false
73
+
74
+ ModuleLength:
75
+ Enabled: false
76
+
77
+ Style/ClassAndModuleCamelCase:
78
+ Enabled: false
79
+
80
+ Style/Documentation:
81
+ Enabled: false
82
+
83
+ Style/FrozenStringLiteralComment:
84
+ Enabled: false
85
+
86
+ Style/GuardClause:
87
+ Enabled: false
88
+
89
+ Style/Next:
90
+ Enabled: false
91
+
92
+ Style/NumericLiterals:
93
+ Enabled: false
94
+
95
+ Style/RaiseArgs:
96
+ Enabled: false
97
+
98
+ Style/RedundantBegin:
99
+ Enabled: false
100
+
101
+ Style/Lambda:
102
+ Enabled: false
103
+
104
+ Rails/SkipsModelValidations:
105
+ Enabled: false
@@ -0,0 +1,7 @@
1
+ ---
2
+ sudo: false
3
+ language: ruby
4
+ cache: bundler
5
+ rvm:
6
+ - 2.6.3
7
+ before_install: gem install bundler -v 2.0.2
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gemspec
@@ -0,0 +1,118 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ naginegi (0.3.0)
5
+ google-cloud-bigquery (= 1.18.1)
6
+ mysql2 (= 0.5.3)
7
+ mysql2-cs-bind (= 0.0.7)
8
+ pg (= 1.2.2)
9
+ unindent (= 1.0)
10
+
11
+ GEM
12
+ remote: https://rubygems.org/
13
+ specs:
14
+ addressable (2.7.0)
15
+ public_suffix (>= 2.0.2, < 5.0)
16
+ ast (2.4.0)
17
+ concurrent-ruby (1.1.5)
18
+ declarative (0.0.10)
19
+ declarative-option (0.1.0)
20
+ diff-lcs (1.3)
21
+ faraday (0.17.3)
22
+ multipart-post (>= 1.2, < 3)
23
+ google-api-client (0.36.4)
24
+ addressable (~> 2.5, >= 2.5.1)
25
+ googleauth (~> 0.9)
26
+ httpclient (>= 2.8.1, < 3.0)
27
+ mini_mime (~> 1.0)
28
+ representable (~> 3.0)
29
+ retriable (>= 2.0, < 4.0)
30
+ signet (~> 0.12)
31
+ google-cloud-bigquery (1.18.1)
32
+ concurrent-ruby (~> 1.0)
33
+ google-api-client (~> 0.33)
34
+ google-cloud-core (~> 1.2)
35
+ googleauth (~> 0.9)
36
+ mini_mime (~> 1.0)
37
+ google-cloud-core (1.5.0)
38
+ google-cloud-env (~> 1.0)
39
+ google-cloud-errors (~> 1.0)
40
+ google-cloud-env (1.3.0)
41
+ faraday (~> 0.11)
42
+ google-cloud-errors (1.0.0)
43
+ googleauth (0.10.0)
44
+ faraday (~> 0.12)
45
+ jwt (>= 1.4, < 3.0)
46
+ memoist (~> 0.16)
47
+ multi_json (~> 1.11)
48
+ os (>= 0.9, < 2.0)
49
+ signet (~> 0.12)
50
+ httpclient (2.8.3)
51
+ jwt (2.2.1)
52
+ memoist (0.16.2)
53
+ mini_mime (1.0.2)
54
+ multi_json (1.14.1)
55
+ multipart-post (2.1.1)
56
+ mysql2 (0.5.3-x64-mingw32)
57
+ mysql2-cs-bind (0.0.7)
58
+ mysql2
59
+ os (1.0.1)
60
+ parallel (1.19.1)
61
+ parser (2.7.0.1)
62
+ ast (~> 2.4.0)
63
+ pg (1.2.2-x64-mingw32)
64
+ powerpack (0.1.2)
65
+ public_suffix (4.0.3)
66
+ rainbow (2.2.2)
67
+ rake
68
+ rake (12.3.3)
69
+ representable (3.0.4)
70
+ declarative (< 0.1.0)
71
+ declarative-option (< 0.2.0)
72
+ uber (< 0.2.0)
73
+ retriable (3.1.2)
74
+ rspec (3.8.0)
75
+ rspec-core (~> 3.8.0)
76
+ rspec-expectations (~> 3.8.0)
77
+ rspec-mocks (~> 3.8.0)
78
+ rspec-core (3.8.2)
79
+ rspec-support (~> 3.8.0)
80
+ rspec-expectations (3.8.6)
81
+ diff-lcs (>= 1.2.0, < 2.0)
82
+ rspec-support (~> 3.8.0)
83
+ rspec-mocks (3.8.2)
84
+ diff-lcs (>= 1.2.0, < 2.0)
85
+ rspec-support (~> 3.8.0)
86
+ rspec-support (3.8.3)
87
+ rubocop (0.49.1)
88
+ parallel (~> 1.10)
89
+ parser (>= 2.3.3.1, < 3.0)
90
+ powerpack (~> 0.1)
91
+ rainbow (>= 1.99.1, < 3.0)
92
+ ruby-progressbar (~> 1.7)
93
+ unicode-display_width (~> 1.0, >= 1.0.1)
94
+ ruby-progressbar (1.10.1)
95
+ signet (0.12.0)
96
+ addressable (~> 2.3)
97
+ faraday (~> 0.9)
98
+ jwt (>= 1.5, < 3.0)
99
+ multi_json (~> 1.10)
100
+ timecop (0.9.1)
101
+ uber (0.1.0)
102
+ unicode-display_width (1.6.0)
103
+ unindent (1.0)
104
+
105
+ PLATFORMS
106
+ ruby
107
+ x64-mingw32
108
+
109
+ DEPENDENCIES
110
+ bundler (= 2.1.4)
111
+ naginegi!
112
+ rake (= 12.3.3)
113
+ rspec (= 3.8.0)
114
+ rubocop (= 0.49.1)
115
+ timecop (= 0.9.1)
116
+
117
+ BUNDLED WITH
118
+ 2.1.4
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2019 cobot00
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,222 @@
1
+ # Naginegi
2
+
3
+ Generate Embulk config and BigQuery schema from MySQL and PostgreSQL schema and run Embulk.
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ ```ruby
10
+ gem 'naginegi'
11
+ ```
12
+
13
+ And then execute:
14
+
15
+ $ bundle
16
+
17
+ Or install it yourself as:
18
+
19
+ $ gem install naginegi
20
+
21
+ ## Embulk setup
22
+ `Naginegi` is utility for `Embulk` .
23
+ You need to install `Embulk` and install some gems like below.
24
+
25
+ ```bash
26
+ embulk gem install embulk-input-mysql --version 0.10.1
27
+ embulk gem install embulk-input-postgresql --version 0.10.1
28
+ embulk gem install embulk-output-bigquery --version 0.6.4
29
+ embulk gem install embulk-parser-jsonl --version 0.2.0
30
+ embulk gem install embulk-formatter-jsonl --version 0.1.4
31
+ ```
32
+
33
+ ## Usage
34
+ Require `database.yml` and `table.yml`.
35
+ `database.yml` requires `db_type` (mysql or postgresql).
36
+
37
+ |RDBMS|db_type|
38
+ |---|---|
39
+ |MySQL|mysql|
40
+ |PostgreSQL|postgresql|
41
+
42
+ Below is a sample config file.
43
+
44
+ ### database.yml
45
+ ```yml
46
+ db01:
47
+ db_type: mysql
48
+ host: localhost
49
+ username: root
50
+ password: pswd
51
+ database: production
52
+ bq_dataset: mysql
53
+
54
+ db02:
55
+ db_type: postgresql
56
+ host: localhost
57
+ username: root
58
+ password: pswd
59
+ database: production
60
+ bq_dataset: pg
61
+
62
+ ```
63
+
64
+ **Caution: Embulk doesn't allow no password for MySQL**
65
+
66
+ ### table.yml
67
+ ```yml
68
+ db01:
69
+ tables:
70
+ - name: users
71
+ - name: events
72
+ - name: hobbies
73
+
74
+ db02:
75
+ tables:
76
+ - name: administrators
77
+ - name: configs
78
+ ```
79
+
80
+ ### sample
81
+ Naginegi requires BigQuery parameters like below.
82
+
83
+ #### using json key file path
84
+ ```ruby
85
+ [sample.rb]
86
+ require 'naginegi'
87
+
88
+ config = {
89
+ 'project_id' => 'BIGQUERY_PROJECT_ID',
90
+ 'service_email' => 'SERVICE_ACCOUNT_EMAIL',
91
+ 'auth_method' => 'json_key',
92
+ 'json_keyfile' => 'JSON_KEYFILE_PATH',
93
+ 'schema_dir' => '/var/tmp/embulk/schema',
94
+ 'config_dir' => '/var/tmp/embulk/config'
95
+ }
96
+
97
+ client = Naginegi::EmbulkRunner.new
98
+ client.generate_config(config)
99
+ client.run(config)
100
+ ```
101
+
102
+ ```bash
103
+ ruby sample.rb
104
+ ```
105
+
106
+ #### using key values
107
+ ```ruby
108
+ [sample.rb]
109
+ require 'naginegi'
110
+
111
+ json_key = {
112
+ "type" => "...",
113
+ "project_id" => "...",
114
+ "private_key_id" => "...",
115
+ "private_key" => "...",
116
+ ...
117
+ }
118
+
119
+ config = {
120
+ 'project_id' => 'BIGQUERY_PROJECT_ID',
121
+ 'service_email' => 'SERVICE_ACCOUNT_EMAIL',
122
+ 'auth_method' => 'json_key',
123
+ 'json_key' => json_key,
124
+ 'schema_dir' => '/var/tmp/embulk/schema',
125
+ 'config_dir' => '/var/tmp/embulk/config'
126
+ }
127
+
128
+ client = Naginegi::EmbulkRunner.new
129
+ client.generate_config(config)
130
+ client.run(config)
131
+ ```
132
+
133
+ ```bash
134
+ ruby sample.rb
135
+ ```
136
+
137
+ ## Features
138
+ ### process status
139
+ `Naginegi` returns process status as boolean.
140
+ If all tables are succeed, then returns `true`, else `false` .
141
+ It is useful to control system flow.
142
+
143
+ ```ruby
144
+ process_status = Naginegi::EmbulkClient.new.run(config)
145
+ exit 1 unless process_status
146
+ ```
147
+
148
+ ### narrow tables
149
+ You can narrow actual target tables from `table.yml` for test or to retry.
150
+ If no target tables is given, `Naginegi` will execute all tables.
151
+
152
+ ```ruby
153
+ # in case, all tables are ['users', 'purchases', 'items']
154
+ target_tables = ['users', 'purchases']
155
+ Naginegi::EmbulkClient.new.run(config, target_tables)
156
+ ```
157
+
158
+ ### retry
159
+ You can set retry count.
160
+ If any table failed, only failed table will be retried until retry count.
161
+ If no retry count is given, `Naginegi` dosen't retry.
162
+
163
+ ```ruby
164
+ # 2 times retry will execute
165
+ Naginegi::EmbulkClient.new.run(config, [], 2)
166
+ ```
167
+
168
+ ### SQL condition
169
+ If you set `condition` to a table in `table.yml` , SQL is generated like below.
170
+ It is useful for large size table.
171
+
172
+ ```yml
173
+ [table.yml]
174
+ production:
175
+ tables:
176
+ - name: users
177
+ - name: events
178
+ conditon: created_at < CURRENT_DATE()
179
+ ```
180
+
181
+ ```sql
182
+ SELECT * FROM users
183
+ SELECT * FROM events WHERE created_at < CURRENT_DATE()
184
+ ```
185
+
186
+ ### daily snapshot
187
+ BigQuery supports table wildcard expression of a specific set of daily tables, for example, `sales20150701` .
188
+ If you need daily snapshot of a table for BigQuery, use `daily_snapshot` option to `database.yml` or `table.yml` like below.
189
+ `daily_snapshot` option effects all tables in case of `database.yml` .
190
+ On the other hand, only target table in `table.yml` .
191
+ **Daily part is determined by execute date.**
192
+
193
+ ```yml
194
+ [database.yml]
195
+ production:
196
+ host: localhost
197
+ username: root
198
+ password: pswd
199
+ database: production
200
+ bq_dataset: mysql
201
+ daily_snapshot: true
202
+ ```
203
+
204
+ ```yml
205
+ [table.yml]
206
+ production:
207
+ tables:
208
+ - name: users
209
+ - name: events
210
+ daily_snapshot: true
211
+ - name: hobbies
212
+
213
+ Only `events` is renamed to `eventsYYYYMMDD` for BigQuery.
214
+ ```
215
+
216
+ ## Contributing
217
+
218
+ 1. Fork it ( https://github.com/[my-github-username]/naginegi/fork )
219
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
220
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
221
+ 4. Push to the branch (`git push origin my-new-feature`)
222
+ 5. Create a new Pull Request