naginegi 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 2a23dc48ebec8810113d0b5a3a085c106048bb8aa24cfffc25d7e9c420ff5504
4
+ data.tar.gz: 3cdfdf331f116492d7457072fc16f93bb6ab88fb7b8c8efdfb286f745f8dd11d
5
+ SHA512:
6
+ metadata.gz: 4af85f8bdad42f90f06478954ccda166f35028a4660b11b08dbfe7a3ef94a9e8c8652212673549d1d2bac485cebd4dd05102cf021328a63fb67ccf97e6e4dfef
7
+ data.tar.gz: 803c5fe510fccc0e98dd9fec3294a576f58c2f25c3df531b4f53e1bad42e90fd68754bca8b5076dd6894c919112577ff3c5101de16cb6c68592ce59de3e42daa
@@ -0,0 +1,41 @@
1
+ version: 2
2
+ jobs:
3
+ build:
4
+ working_directory: ~/naginegi
5
+ docker:
6
+ - image: circleci/ruby:2.6.3
7
+ environment:
8
+ TZ: /usr/share/zoneinfo/Asia/Tokyo
9
+ steps:
10
+ - checkout
11
+
12
+ - restore_cache:
13
+ name: Restore bundle cache
14
+ key: naginegi-{{ checksum "Gemfile.lock" }}
15
+
16
+ - run:
17
+ name: Install Bundler
18
+ command: gem install bundler -v 2.1.4
19
+
20
+ - run:
21
+ name: Run bundle install
22
+ command: bundle install --path vendor/bundle
23
+
24
+ - save_cache:
25
+ name: Store bundle cache
26
+ key: naginegi-{{ checksum "Gemfile.lock" }}
27
+ paths:
28
+ - vendor/bundle
29
+
30
+ - run:
31
+ name: chmod +x ./lint.sh
32
+ command: chmod +x ./lint.sh
33
+
34
+ - run:
35
+ name: ./lint.sh
36
+ command: ./lint.sh
37
+
38
+ - run:
39
+ name: Run rspec
40
+ command: bundle exec rspec spec/
41
+
@@ -0,0 +1,41 @@
1
+ *.rbc
2
+ capybara-*.html
3
+ .rspec
4
+ .rspec_status
5
+ *.log
6
+ /log
7
+ /tmp
8
+ /db/*.sqlite3
9
+ /db/*.sqlite3-journal
10
+ /db/schema.rb
11
+ /public/system
12
+ /coverage/
13
+ /spec/tmp
14
+ *.orig
15
+ rerun.txt
16
+ pickle-email-*.html
17
+
18
+ .project
19
+
20
+ config/initializers/secret_token.rb
21
+
22
+ .env
23
+
24
+ /.bundle
25
+ /vendor/bundle
26
+
27
+ .rvmrc
28
+
29
+ /vendor/assets/bower_components
30
+ *.bowerrc
31
+ bower.json
32
+
33
+ .powenv
34
+
35
+ .byebug_history
36
+
37
+ .ruby-version
38
+
39
+ .DS_Store
40
+
41
+ *.gem
@@ -0,0 +1,105 @@
1
+ AllCops:
2
+ TargetRubyVersion: 2.4.1
3
+ Include:
4
+ - '**/Gemfile'
5
+ - '**/Rakefile'
6
+ - '**/config.ru'
7
+ Exclude:
8
+ - 'db/**/*'
9
+ - 'config/**/*'
10
+ - 'script/**/*'
11
+ - 'vendor/**/*'
12
+ - 'bin/*'
13
+ - !ruby/regexp /old_and_unused\.rb$/
14
+
15
+ AlignParameters:
16
+ Enabled: false
17
+
18
+ AsciiComments:
19
+ Enabled: false
20
+
21
+ BracesAroundHashParameters:
22
+ Enabled: false
23
+
24
+ Bundler/OrderedGems:
25
+ Enabled: false
26
+
27
+ ClassAndModuleChildren:
28
+ Enabled: false
29
+
30
+ ClassLength:
31
+ Enabled: false
32
+
33
+ Layout/AlignHash:
34
+ Enabled: false
35
+
36
+ Layout/EndOfLine:
37
+ Enabled: false
38
+
39
+ Layout/IndentHash:
40
+ Enabled: false
41
+
42
+ Layout/MultilineArrayBraceLayout:
43
+ Enabled: false
44
+
45
+ Layout/MultilineHashBraceLayout:
46
+ Enabled: false
47
+
48
+ Layout/MultilineMethodCallIndentation:
49
+ Enabled: false
50
+
51
+ Metrics/BlockLength:
52
+ Max: 50
53
+ Exclude:
54
+ - 'spec/**/*'
55
+ - 'config/routes.rb'
56
+ - 'app/jobs/**/*.rb'
57
+ - 'lib/tasks/**/*.rb'
58
+
59
+ Metrics/AbcSize:
60
+ Enabled: false
61
+
62
+ Metrics/CyclomaticComplexity:
63
+ Enabled: false
64
+
65
+ Metrics/LineLength:
66
+ Enabled: false
67
+
68
+ Metrics/MethodLength:
69
+ Enabled: false
70
+
71
+ Metrics/PerceivedComplexity:
72
+ Enabled: false
73
+
74
+ ModuleLength:
75
+ Enabled: false
76
+
77
+ Style/ClassAndModuleCamelCase:
78
+ Enabled: false
79
+
80
+ Style/Documentation:
81
+ Enabled: false
82
+
83
+ Style/FrozenStringLiteralComment:
84
+ Enabled: false
85
+
86
+ Style/GuardClause:
87
+ Enabled: false
88
+
89
+ Style/Next:
90
+ Enabled: false
91
+
92
+ Style/NumericLiterals:
93
+ Enabled: false
94
+
95
+ Style/RaiseArgs:
96
+ Enabled: false
97
+
98
+ Style/RedundantBegin:
99
+ Enabled: false
100
+
101
+ Style/Lambda:
102
+ Enabled: false
103
+
104
+ Rails/SkipsModelValidations:
105
+ Enabled: false
@@ -0,0 +1,7 @@
1
+ ---
2
+ sudo: false
3
+ language: ruby
4
+ cache: bundler
5
+ rvm:
6
+ - 2.6.3
7
+ before_install: gem install bundler -v 2.0.2
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gemspec
@@ -0,0 +1,120 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ naginegi (0.1.0)
5
+ google-cloud-bigquery (= 1.18.1)
6
+ mysql2 (= 0.5.3)
7
+ mysql2-cs-bind (= 0.0.7)
8
+ pg (= 1.2.2)
9
+ unindent (= 1.0)
10
+
11
+ GEM
12
+ remote: https://rubygems.org/
13
+ specs:
14
+ addressable (2.7.0)
15
+ public_suffix (>= 2.0.2, < 5.0)
16
+ ast (2.4.0)
17
+ concurrent-ruby (1.1.5)
18
+ declarative (0.0.10)
19
+ declarative-option (0.1.0)
20
+ diff-lcs (1.3)
21
+ faraday (0.17.3)
22
+ multipart-post (>= 1.2, < 3)
23
+ google-api-client (0.36.4)
24
+ addressable (~> 2.5, >= 2.5.1)
25
+ googleauth (~> 0.9)
26
+ httpclient (>= 2.8.1, < 3.0)
27
+ mini_mime (~> 1.0)
28
+ representable (~> 3.0)
29
+ retriable (>= 2.0, < 4.0)
30
+ signet (~> 0.12)
31
+ google-cloud-bigquery (1.18.1)
32
+ concurrent-ruby (~> 1.0)
33
+ google-api-client (~> 0.33)
34
+ google-cloud-core (~> 1.2)
35
+ googleauth (~> 0.9)
36
+ mini_mime (~> 1.0)
37
+ google-cloud-core (1.5.0)
38
+ google-cloud-env (~> 1.0)
39
+ google-cloud-errors (~> 1.0)
40
+ google-cloud-env (1.3.0)
41
+ faraday (~> 0.11)
42
+ google-cloud-errors (1.0.0)
43
+ googleauth (0.10.0)
44
+ faraday (~> 0.12)
45
+ jwt (>= 1.4, < 3.0)
46
+ memoist (~> 0.16)
47
+ multi_json (~> 1.11)
48
+ os (>= 0.9, < 2.0)
49
+ signet (~> 0.12)
50
+ httpclient (2.8.3)
51
+ jwt (2.2.1)
52
+ memoist (0.16.2)
53
+ mini_mime (1.0.2)
54
+ multi_json (1.14.1)
55
+ multipart-post (2.1.1)
56
+ mysql2 (0.5.3)
57
+ mysql2 (0.5.3-x64-mingw32)
58
+ mysql2-cs-bind (0.0.7)
59
+ mysql2
60
+ os (1.0.1)
61
+ parallel (1.19.1)
62
+ parser (2.7.0.1)
63
+ ast (~> 2.4.0)
64
+ pg (1.2.2)
65
+ pg (1.2.2-x64-mingw32)
66
+ powerpack (0.1.2)
67
+ public_suffix (4.0.3)
68
+ rainbow (2.2.2)
69
+ rake
70
+ rake (12.3.3)
71
+ representable (3.0.4)
72
+ declarative (< 0.1.0)
73
+ declarative-option (< 0.2.0)
74
+ uber (< 0.2.0)
75
+ retriable (3.1.2)
76
+ rspec (3.8.0)
77
+ rspec-core (~> 3.8.0)
78
+ rspec-expectations (~> 3.8.0)
79
+ rspec-mocks (~> 3.8.0)
80
+ rspec-core (3.8.2)
81
+ rspec-support (~> 3.8.0)
82
+ rspec-expectations (3.8.6)
83
+ diff-lcs (>= 1.2.0, < 2.0)
84
+ rspec-support (~> 3.8.0)
85
+ rspec-mocks (3.8.2)
86
+ diff-lcs (>= 1.2.0, < 2.0)
87
+ rspec-support (~> 3.8.0)
88
+ rspec-support (3.8.3)
89
+ rubocop (0.49.1)
90
+ parallel (~> 1.10)
91
+ parser (>= 2.3.3.1, < 3.0)
92
+ powerpack (~> 0.1)
93
+ rainbow (>= 1.99.1, < 3.0)
94
+ ruby-progressbar (~> 1.7)
95
+ unicode-display_width (~> 1.0, >= 1.0.1)
96
+ ruby-progressbar (1.10.1)
97
+ signet (0.12.0)
98
+ addressable (~> 2.3)
99
+ faraday (~> 0.9)
100
+ jwt (>= 1.5, < 3.0)
101
+ multi_json (~> 1.10)
102
+ timecop (0.9.1)
103
+ uber (0.1.0)
104
+ unicode-display_width (1.6.0)
105
+ unindent (1.0)
106
+
107
+ PLATFORMS
108
+ ruby
109
+ x64-mingw32
110
+
111
+ DEPENDENCIES
112
+ bundler (= 2.1.4)
113
+ naginegi!
114
+ rake (= 12.3.3)
115
+ rspec (= 3.8.0)
116
+ rubocop (= 0.49.1)
117
+ timecop (= 0.9.1)
118
+
119
+ BUNDLED WITH
120
+ 2.1.4
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2019 cobot00
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,189 @@
1
+ # Naginegi
2
+
3
+ Generate Embulk config and BigQuery schema from MySQL and PostgreSQL schema and run Embulk.
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ ```ruby
10
+ gem 'naginegi'
11
+ ```
12
+
13
+ And then execute:
14
+
15
+ $ bundle
16
+
17
+ Or install it yourself as:
18
+
19
+ $ gem install naginegi
20
+
21
+ ## Embulk setup
22
+ `Naginegi` is utility for `Embulk` .
23
+ You need to install `Embulk` and install some gems like below.
24
+
25
+ ```bash
26
+ embulk gem install embulk-input-mysql --version 0.10.1
27
+ embulk gem install embulk-input-postgresql --version 0.10.1
28
+ embulk gem install embulk-output-bigquery --version 0.6.4
29
+ embulk gem install embulk-parser-jsonl --version 0.2.0
30
+ embulk gem install embulk-formatter-jsonl --version 0.1.4
31
+ ```
32
+
33
+ ## Usage
34
+ Require `database.yml` and `table.yml`.
35
+ `database.yml` requires `db_type` (mysql or postgresql).
36
+
37
+ |RDBMS|db_type|
38
+ |---|---|
39
+ |MySQL|mysql|
40
+ |PostgreSQL|postgresql|
41
+
42
+ Below is a sample config file.
43
+
44
+ ### database.yml
45
+ ```yml
46
+ db01:
47
+ db_type: mysql
48
+ host: localhost
49
+ username: root
50
+ password: pswd
51
+ database: production
52
+ bq_dataset: mysql
53
+
54
+ db02:
55
+ db_type: postgresql
56
+ host: localhost
57
+ username: root
58
+ password: pswd
59
+ database: production
60
+ bq_dataset: pg
61
+
62
+ ```
63
+
64
+ **Caution: Embulk doesn't allow no password for MySQL**
65
+
66
+ ### table.yml
67
+ ```yml
68
+ db01:
69
+ tables:
70
+ - name: users
71
+ - name: events
72
+ - name: hobbies
73
+
74
+ db02:
75
+ tables:
76
+ - name: administrators
77
+ - name: configs
78
+ ```
79
+
80
+ Naginegi requires BigQuery parameters like below.
81
+
82
+ ```ruby
83
+ [sample.rb]
84
+ require 'naginegi'
85
+
86
+ config = {
87
+ 'project_id' => 'BIGQUERY_PROJECT_ID',
88
+ 'service_email' => 'SERVICE_ACCOUNT_EMAIL',
89
+ 'auth_method' => 'json_key',
90
+ 'json_keyfile' => 'JSON_KEYFILE_PATH',
91
+ 'schema_dir' => '/var/tmp/embulk/schema',
92
+ 'config_dir' => '/var/tmp/embulk/config'
93
+ }
94
+
95
+ client = Naginegi::EmbulkRunner.new
96
+ client.generate_config(config)
97
+ client.run(config)
98
+ ```
99
+
100
+ ```bash
101
+ ruby sample.rb
102
+ ```
103
+
104
+ ## Features
105
+ ### process status
106
+ `Naginegi` returns process status as boolean.
107
+ If all tables are succeed, then returns `true`, else `false` .
108
+ It is useful to control system flow.
109
+
110
+ ```ruby
111
+ process_status = Naginegi::EmbulkClient.new.run(config)
112
+ exit 1 unless process_status
113
+ ```
114
+
115
+ ### narrow tables
116
+ You can narrow actual target tables from `table.yml` for test or to retry.
117
+ If no target tables is given, `Naginegi` will execute all tables.
118
+
119
+ ```ruby
120
+ # in case, all tables are ['users', 'purchases', 'items']
121
+ target_tables = ['users', 'purchases']
122
+ Naginegi::EmbulkClient.new.run(config, target_tables)
123
+ ```
124
+
125
+ ### retry
126
+ You can set retry count.
127
+ If any table failed, only failed table will be retried until retry count.
128
+ If no retry count is given, `Naginegi` dosen't retry.
129
+
130
+ ```ruby
131
+ # 2 times retry will execute
132
+ Naginegi::EmbulkClient.new.run(config, [], 2)
133
+ ```
134
+
135
+ ### SQL condition
136
+ If you set `condition` to a table in `table.yml` , SQL is generated like below.
137
+ It is useful for large size table.
138
+
139
+ ```yml
140
+ [table.yml]
141
+ production:
142
+ tables:
143
+ - name: users
144
+ - name: events
145
+ conditon: created_at < CURRENT_DATE()
146
+ ```
147
+
148
+ ```sql
149
+ SELECT * FROM users
150
+ SELECT * FROM events WHERE created_at < CURRENT_DATE()
151
+ ```
152
+
153
+ ### daily snapshot
154
+ BigQuery supports table wildcard expression of a specific set of daily tables, for example, `sales20150701` .
155
+ If you need daily snapshot of a table for BigQuery, use `daily_snapshot` option to `database.yml` or `table.yml` like below.
156
+ `daily_snapshot` option effects all tables in case of `database.yml` .
157
+ On the other hand, only target table in `table.yml` .
158
+ **Daily part is determined by execute date.**
159
+
160
+ ```yml
161
+ [database.yml]
162
+ production:
163
+ host: localhost
164
+ username: root
165
+ password: pswd
166
+ database: production
167
+ bq_dataset: mysql
168
+ daily_snapshot: true
169
+ ```
170
+
171
+ ```yml
172
+ [table.yml]
173
+ production:
174
+ tables:
175
+ - name: users
176
+ - name: events
177
+ daily_snapshot: true
178
+ - name: hobbies
179
+
180
+ Only `events` is renamed to `eventsYYYYMMDD` for BigQuery.
181
+ ```
182
+
183
+ ## Contributing
184
+
185
+ 1. Fork it ( https://github.com/[my-github-username]/naginegi/fork )
186
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
187
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
188
+ 4. Push to the branch (`git push origin my-new-feature`)
189
+ 5. Create a new Pull Request