naginegi 0.4.1

data/lib/naginegi/postgresql.rb

@@ -0,0 +1,93 @@
+require 'pg'
+require 'json'
+require 'yaml'
+require 'fileutils'
+require 'naginegi/bigquery'
+
+module Naginegi
+  module PostgreSQL
+    class PgClient
+      COLUMN_SQL = <<-SQL.freeze
+        SELECT column_name, data_type
+        FROM INFORMATION_SCHEMA.COLUMNS
+        WHERE table_name = $1
+        ORDER BY ordinal_position
+      SQL
+
+      def initialize(db_config)
+        @db_config = db_config
+      end
+
+      def client
+        @client ||= PG::Connection.new(
+          host: @db_config['host'],
+          user: @db_config['username'],
+          password: @db_config['password'],
+          dbname: @db_config['database']
+        )
+      end
+
+      def generate_bq_schema(table_name)
+        infos = columns(table_name)
+        BigQuery.generate_schema(infos)
+      end
+
+      def columns(table_name)
+        rows = client.exec_params(COLUMN_SQL, [table_name])
+        rows.map { |row| Column.new(row['column_name'], row['data_type']) }
+      end
+    end
+
+    class Column
+      attr_reader :column_name, :data_type
+
+      TYPE_MAPPINGS = {
+        'smallint' => 'INT64',
+        'integer' => 'INT64',
+        'bigint' => 'INT64',
+        'smallserial' => 'INT64',
+        'serial' => 'INT64',
+        'bigserial' => 'INT64',
+        'decimal' => 'FLOAT64',
+        'numeric' => 'FLOAT64',
+        'real' => 'FLOAT64',
+        'double precision' => 'FLOAT64',
+        'character' => 'STRING',
+        'character varying' => 'STRING',
+        'text' => 'STRING',
+        'date' => 'TIMESTAMP',
+        'timestamp' => 'TIMESTAMP',
+        'timestamp with time zone' => 'TIMESTAMP',
+        'boolean' => 'BOOL'
+      }.freeze
+
+      def initialize(column_name, data_type)
+        @column_name = column_name
+        @data_type = data_type
+      end
+
+      def bigquery_data_type
+        TYPE_MAPPINGS[@data_type] || 'STRING'
+      end
+
+      def converted_value
+        if bigquery_data_type == 'TIMESTAMP'
+          # time zone translate to UTC
+          "EXTRACT(EPOCH FROM #{escaped_column_name}) AS #{escaped_column_name}"
+        else
+          escaped_column_name
+        end
+      end
+
+      def to_json(*a)
+        { 'name' => @column_name, 'type' => bigquery_data_type }.to_json(*a)
+      end
+
+      private
+
+      def escaped_column_name
+        "\"#{@column_name}\""
+      end
+    end
+  end
+end

data/lib/naginegi/version.rb

@@ -0,0 +1,3 @@
+module Naginegi
+  VERSION = '0.4.1'.freeze
+end

data/lint.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+echo 'CIRCLE_BRANCH: ' ${CIRCLE_BRANCH}
+TARGET_BRANCH=${CIRCLE_BRANCH}
+
+# ローカルでの実行用にカレントブランチをセットする
+if [ "$TARGET_BRANCH" = '' ]; then
+  TARGET_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+fi
+echo 'TARGET_BRANCH: ' $TARGET_BRANCH
+
+echo 'CIRCLE_BASE_BRANCH: ' ${CIRCLE_BASE_BRANCH}
+BASE_BRANCH=${CIRCLE_BASE_BRANCH}
+
+# ローカルでの実行時に環境変数をセットしていない時はmasterブランチと比較する
+if [ "$BASE_BRANCH" = '' ]; then
+  BASE_BRANCH=origin/master
+fi
+echo 'BASE_BRANCH: ' $BASE_BRANCH
+
+files=$(git diff --name-only $TARGET_BRANCH $BASE_BRANCH | grep -E '.rb' | egrep -v 'db/migrate|db/schema.rb|spec/factories')
+
+error=false
+for file in ${files}; do
+  # u-motion-api から同期されるファイルは確認不要
+  if [[ $file == *"app/models"* ]]; then
+    if [[ $file != *"app/models/admin"* ]]; then
+      continue
+    fi
+  fi
+
+  if [[ $file == *"app/serializers"* ]]; then
+    if [[ $file != *"app/serializers/admin"* ]]; then
+      continue
+    fi
+  fi
+
+  if [ -e $file ]; then
+    result=$(bundle exec rubocop ${file})
+    rubocop_error=$(echo "$result" | grep 'Offenses:')
+    if [ "$rubocop_error" != '' ]; then
+      error=true
+      echo ''
+      echo 'ERROR:' $file
+      echo "$result"
+    fi
+  fi
+done
+
+if $error; then
+  exit 1
+fi
+
+exit 0

data/naginegi.gemspec

@@ -0,0 +1,34 @@
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'naginegi/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'naginegi'
+  spec.version       = Naginegi::VERSION
+  spec.authors       = ['cobot00']
+  spec.email         = ['kobori75@gmail.com']
+
+  spec.summary       = %q{Embulk utility for MySQL and PostgreSQL to BigQuery}
+  spec.description   = %q{Generate Embulk config and BigQuery schema from RDBMS schema}
+  spec.homepage      = 'https://github.com/cobot00/naginegi'
+  spec.license       = 'MIT'
+
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '2.1.4'
+  spec.add_development_dependency 'rake', '12.3.3'
+  spec.add_development_dependency 'rspec', '3.8.0'
+  spec.add_development_dependency 'rubocop', '0.49.1'
+  spec.add_development_dependency 'timecop', '0.9.1'
+
+  spec.add_dependency 'google-cloud-bigquery', '1.18.1'
+  spec.add_dependency 'mysql2', '0.5.3'
+  spec.add_dependency 'mysql2-cs-bind', '0.0.7'
+  spec.add_dependency 'pg', '1.2.2'
+  spec.add_dependency 'unindent', '1.0'
+end

metadata

@@ -0,0 +1,203 @@
+--- !ruby/object:Gem::Specification
+name: naginegi
+version: !ruby/object:Gem::Version
+  version: 0.4.1
+platform: ruby
+authors:
+- cobot00
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2020-02-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.1.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.1.4
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 12.3.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 12.3.3
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.8.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.8.0
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.49.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.49.1
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+- !ruby/object:Gem::Dependency
+  name: google-cloud-bigquery
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.18.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.18.1
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.5.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.5.3
+- !ruby/object:Gem::Dependency
+  name: mysql2-cs-bind
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.7
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.2
+- !ruby/object:Gem::Dependency
+  name: unindent
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: Generate Embulk config and BigQuery schema from RDBMS schema
+email:
+- kobori75@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".circleci/config.yml"
+- ".gitignore"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/naginegi.rb
+- lib/naginegi/bigquery.rb
+- lib/naginegi/embulk.rb
+- lib/naginegi/embulk_config.rb
+- lib/naginegi/mysql.rb
+- lib/naginegi/postgresql.rb
+- lib/naginegi/version.rb
+- lint.sh
+- naginegi.gemspec
+homepage: https://github.com/cobot00/naginegi
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.6
+signing_key: 
+specification_version: 4
+summary: Embulk utility for MySQL and PostgreSQL to BigQuery
+test_files: []