naginegi 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: bffe8bf28a51050e58ef2ccb8d25da01691d27f1e28aec92a42c98c1d4a69248
+  data.tar.gz: 5e47d30b633d323c58c4dfc8cbcad5f6cd594567e13d3ae52b62717e9b733c5e
+SHA512:
+  metadata.gz: b967eedb71e86f4cf20a65bcb479fe008faf80cd3709e46c22cad69d9e310b92900105a1208db94cdacc0fc603bf2e7b5c17e043bce87ba504029d71ee2a83f6
+  data.tar.gz: 981dd5ebda7890cbc6b5cfba8dc8d2b5e1dd531dc7614225b2115beb235ef212bfb7a450c3efea275aa7058084fb3408cdc2e52addff1184a434b5381941efdc

data/.circleci/config.yml

@@ -0,0 +1,41 @@
+version: 2
+jobs:
+  build:
+    working_directory: ~/naginegi
+    docker:
+      - image: circleci/ruby:2.6.3
+        environment:
+          TZ: /usr/share/zoneinfo/Asia/Tokyo
+    steps:
+      - checkout
+
+      - restore_cache:
+          name: Restore bundle cache
+          key: naginegi-{{ checksum "Gemfile.lock" }}
+
+      - run:
+          name: Install Bundler
+          command: gem install bundler -v 2.1.4
+
+      - run:
+          name: Run bundle install
+          command: bundle install --path vendor/bundle
+
+      - save_cache:
+          name: Store bundle cache
+          key: naginegi-{{ checksum "Gemfile.lock" }}
+          paths:
+            - vendor/bundle
+
+      - run:
+          name: chmod +x ./lint.sh
+          command: chmod +x ./lint.sh
+
+      - run:
+          name: ./lint.sh
+          command: ./lint.sh
+
+      - run:
+          name: Run rspec
+          command: bundle exec rspec spec/
+

data/.gitignore

@@ -0,0 +1,41 @@
+*.rbc
+capybara-*.html
+.rspec
+.rspec_status
+*.log
+/log
+/tmp
+/db/*.sqlite3
+/db/*.sqlite3-journal
+/db/schema.rb
+/public/system
+/coverage/
+/spec/tmp
+*.orig
+rerun.txt
+pickle-email-*.html
+
+.project
+
+config/initializers/secret_token.rb
+
+.env
+
+/.bundle
+/vendor/bundle
+
+.rvmrc
+
+/vendor/assets/bower_components
+*.bowerrc
+bower.json
+
+.powenv
+
+.byebug_history
+
+.ruby-version
+
+.DS_Store
+
+*.gem

data/.rubocop.yml

@@ -0,0 +1,105 @@
+AllCops:
+  TargetRubyVersion: 2.4.1
+  Include:
+    - '**/Gemfile'
+    - '**/Rakefile'
+    - '**/config.ru'
+  Exclude:
+    - 'db/**/*'
+    - 'config/**/*'
+    - 'script/**/*'
+    - 'vendor/**/*'
+    - 'bin/*'
+    - !ruby/regexp /old_and_unused\.rb$/
+
+AlignParameters:
+  Enabled: false
+
+AsciiComments:
+  Enabled: false
+
+BracesAroundHashParameters:
+  Enabled: false
+
+Bundler/OrderedGems:
+  Enabled: false
+
+ClassAndModuleChildren:
+  Enabled: false
+
+ClassLength:
+  Enabled: false
+
+Layout/AlignHash:
+  Enabled: false
+
+Layout/EndOfLine:
+  Enabled: false
+
+Layout/IndentHash:
+  Enabled: false
+
+Layout/MultilineArrayBraceLayout:
+  Enabled: false
+
+Layout/MultilineHashBraceLayout:
+  Enabled: false
+
+Layout/MultilineMethodCallIndentation:
+  Enabled: false
+
+Metrics/BlockLength:
+  Max: 50
+  Exclude:
+    - 'spec/**/*'
+    - 'config/routes.rb'
+    - 'app/jobs/**/*.rb'
+    - 'lib/tasks/**/*.rb'
+
+Metrics/AbcSize:
+  Enabled: false
+
+Metrics/CyclomaticComplexity:
+  Enabled: false
+
+Metrics/LineLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/PerceivedComplexity:
+  Enabled: false
+
+ModuleLength:
+  Enabled: false
+
+Style/ClassAndModuleCamelCase:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/FrozenStringLiteralComment:
+  Enabled: false
+
+Style/GuardClause:
+  Enabled: false
+
+Style/Next:
+  Enabled: false
+
+Style/NumericLiterals:
+  Enabled: false
+
+Style/RaiseArgs:
+  Enabled: false
+
+Style/RedundantBegin:
+  Enabled: false
+
+Style/Lambda:
+  Enabled: false
+
+Rails/SkipsModelValidations:
+  Enabled: false

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.6.3
+before_install: gem install bundler -v 2.0.2

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,118 @@
+PATH
+  remote: .
+  specs:
+    naginegi (0.3.0)
+      google-cloud-bigquery (= 1.18.1)
+      mysql2 (= 0.5.3)
+      mysql2-cs-bind (= 0.0.7)
+      pg (= 1.2.2)
+      unindent (= 1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    ast (2.4.0)
+    concurrent-ruby (1.1.5)
+    declarative (0.0.10)
+    declarative-option (0.1.0)
+    diff-lcs (1.3)
+    faraday (0.17.3)
+      multipart-post (>= 1.2, < 3)
+    google-api-client (0.36.4)
+      addressable (~> 2.5, >= 2.5.1)
+      googleauth (~> 0.9)
+      httpclient (>= 2.8.1, < 3.0)
+      mini_mime (~> 1.0)
+      representable (~> 3.0)
+      retriable (>= 2.0, < 4.0)
+      signet (~> 0.12)
+    google-cloud-bigquery (1.18.1)
+      concurrent-ruby (~> 1.0)
+      google-api-client (~> 0.33)
+      google-cloud-core (~> 1.2)
+      googleauth (~> 0.9)
+      mini_mime (~> 1.0)
+    google-cloud-core (1.5.0)
+      google-cloud-env (~> 1.0)
+      google-cloud-errors (~> 1.0)
+    google-cloud-env (1.3.0)
+      faraday (~> 0.11)
+    google-cloud-errors (1.0.0)
+    googleauth (0.10.0)
+      faraday (~> 0.12)
+      jwt (>= 1.4, < 3.0)
+      memoist (~> 0.16)
+      multi_json (~> 1.11)
+      os (>= 0.9, < 2.0)
+      signet (~> 0.12)
+    httpclient (2.8.3)
+    jwt (2.2.1)
+    memoist (0.16.2)
+    mini_mime (1.0.2)
+    multi_json (1.14.1)
+    multipart-post (2.1.1)
+    mysql2 (0.5.3-x64-mingw32)
+    mysql2-cs-bind (0.0.7)
+      mysql2
+    os (1.0.1)
+    parallel (1.19.1)
+    parser (2.7.0.1)
+      ast (~> 2.4.0)
+    pg (1.2.2-x64-mingw32)
+    powerpack (0.1.2)
+    public_suffix (4.0.3)
+    rainbow (2.2.2)
+      rake
+    rake (12.3.3)
+    representable (3.0.4)
+      declarative (< 0.1.0)
+      declarative-option (< 0.2.0)
+      uber (< 0.2.0)
+    retriable (3.1.2)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.2)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.6)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.3)
+    rubocop (0.49.1)
+      parallel (~> 1.10)
+      parser (>= 2.3.3.1, < 3.0)
+      powerpack (~> 0.1)
+      rainbow (>= 1.99.1, < 3.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.0, >= 1.0.1)
+    ruby-progressbar (1.10.1)
+    signet (0.12.0)
+      addressable (~> 2.3)
+      faraday (~> 0.9)
+      jwt (>= 1.5, < 3.0)
+      multi_json (~> 1.10)
+    timecop (0.9.1)
+    uber (0.1.0)
+    unicode-display_width (1.6.0)
+    unindent (1.0)
+
+PLATFORMS
+  ruby
+  x64-mingw32
+
+DEPENDENCIES
+  bundler (= 2.1.4)
+  naginegi!
+  rake (= 12.3.3)
+  rspec (= 3.8.0)
+  rubocop (= 0.49.1)
+  timecop (= 0.9.1)
+
+BUNDLED WITH
+   2.1.4

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 cobot00
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,222 @@
+# Naginegi
+
+Generate Embulk config and BigQuery schema from MySQL and PostgreSQL schema and run Embulk.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'naginegi'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install naginegi
+
+## Embulk setup
+`Naginegi` is utility for `Embulk` .
+You need to install `Embulk` and install some gems like below.
+
+```bash
+embulk gem install embulk-input-mysql --version 0.10.1
+embulk gem install embulk-input-postgresql --version 0.10.1
+embulk gem install embulk-output-bigquery --version 0.6.4
+embulk gem install embulk-parser-jsonl --version 0.2.0
+embulk gem install embulk-formatter-jsonl --version 0.1.4
+```
+
+## Usage
+Require `database.yml` and `table.yml`.  
+`database.yml` requires `db_type` (mysql or postgresql).
+
+|RDBMS|db_type|
+|---|---|
+|MySQL|mysql|
+|PostgreSQL|postgresql|
+
+Below is a sample config file.
+
+### database.yml
+```yml
+db01:
+  db_type: mysql
+  host: localhost
+  username: root
+  password: pswd
+  database: production
+  bq_dataset: mysql
+
+db02:
+  db_type: postgresql
+  host: localhost
+  username: root
+  password: pswd
+  database: production
+  bq_dataset: pg
+
+```
+
+**Caution: Embulk doesn't allow no password for MySQL**
+
+### table.yml
+```yml
+db01:
+  tables:
+    - name: users
+    - name: events
+    - name: hobbies
+
+db02:
+  tables:
+    - name: administrators
+    - name: configs
+```
+
+### sample
+Naginegi requires BigQuery parameters like below.
+
+#### using json key file path
+```ruby
+[sample.rb]
+require 'naginegi'
+
+config = {
+ 'project_id' => 'BIGQUERY_PROJECT_ID',
+ 'service_email' => 'SERVICE_ACCOUNT_EMAIL',
+ 'auth_method' => 'json_key',
+ 'json_keyfile' => 'JSON_KEYFILE_PATH',
+ 'schema_dir' => '/var/tmp/embulk/schema',
+ 'config_dir' => '/var/tmp/embulk/config'
+}
+
+client = Naginegi::EmbulkRunner.new
+client.generate_config(config)
+client.run(config)
+```
+
+```bash
+ruby sample.rb
+```
+
+#### using key values
+```ruby
+[sample.rb]
+require 'naginegi'
+
+json_key = {
+  "type" => "...",
+  "project_id" => "...",
+  "private_key_id" => "...",
+  "private_key" => "...",
+  ...
+}
+
+config = {
+ 'project_id' => 'BIGQUERY_PROJECT_ID',
+ 'service_email' => 'SERVICE_ACCOUNT_EMAIL',
+ 'auth_method' => 'json_key',
+ 'json_key' => json_key,
+ 'schema_dir' => '/var/tmp/embulk/schema',
+ 'config_dir' => '/var/tmp/embulk/config'
+}
+
+client = Naginegi::EmbulkRunner.new
+client.generate_config(config)
+client.run(config)
+```
+
+```bash
+ruby sample.rb
+```
+
+## Features
+### process status
+`Naginegi` returns process status as boolean.  
+If all tables are succeed, then returns `true`, else `false` .  
+It is useful to control system flow.
+
+```ruby
+process_status = Naginegi::EmbulkClient.new.run(config)
+exit 1 unless process_status
+```
+
+### narrow tables
+You can narrow actual target tables from `table.yml` for test or to retry.  
+If no target tables is given, `Naginegi` will execute all tables.
+
+```ruby
+# in case, all tables are ['users', 'purchases', 'items']
+target_tables = ['users', 'purchases']
+Naginegi::EmbulkClient.new.run(config, target_tables)
+```
+
+### retry
+You can set retry count.  
+If any table failed, only failed table will be retried until retry count.  
+If no retry count is given, `Naginegi` dosen't retry.
+
+```ruby
+# 2 times retry will execute
+Naginegi::EmbulkClient.new.run(config, [], 2)
+```
+
+### SQL condition
+If you set `condition` to a table in `table.yml` , SQL is generated like below.  
+It is useful for large size table.
+
+```yml
+[table.yml]
+production:
+  tables:
+    - name: users
+    - name: events
+      conditon: created_at < CURRENT_DATE()
+```
+
+```sql
+SELECT * FROM users
+SELECT * FROM events WHERE created_at < CURRENT_DATE()
+```
+
+### daily snapshot
+BigQuery supports table wildcard expression of a specific set of daily tables, for example, `sales20150701` .  
+If you need daily snapshot of a table for BigQuery, use `daily_snapshot` option to `database.yml` or `table.yml` like below.  
+`daily_snapshot` option effects all tables in case of  `database.yml` .  
+On the other hand, only target table in `table.yml` .  
+**Daily part is determined by execute date.**
+
+```yml
+[database.yml]
+production:
+  host: localhost
+  username: root
+  password: pswd
+  database: production
+  bq_dataset: mysql
+  daily_snapshot: true
+```
+
+```yml
+[table.yml]
+production:
+  tables:
+    - name: users
+    - name: events
+      daily_snapshot: true
+    - name: hobbies
+
+Only `events` is renamed to `eventsYYYYMMDD` for BigQuery.
+```
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/naginegi/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request