naginegi 0.1.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5f1407ac3ed8008b5fe9016cc6b5497d7e4670c6a3053e82717bbca7ddcafe31
-  data.tar.gz: db73972c63544d2ee87ae2df508f38b813cddce5e6e6a8bf8c83012a5fa96a3c
+  metadata.gz: 8d04a1b61b8907c2b7d03ec4be2ea36097f44c52fca562681481d154314ac87f
+  data.tar.gz: 963141aa4c7ca6144dd9fc553e518af27be576907e8cb7e95865bed87bda8dc3
 SHA512:
-  metadata.gz: 96793bf87478c3ae07b1d6338813a2fd1f49e89d9b016c02f94bbfd7d250885ca67e24c2851bb52ed067c62cea4f317f8431577569c7f3dc461c325f549c1eec
-  data.tar.gz: 5c9d7a190cd1afd1d3561c90ca7d1c5b1491ef886f4a5ddadfdfbfb2371ddcea46241758b7b328569e7d105f400364131258eb65bef389f20f2aac819605d1dc
+  metadata.gz: 1368397fccb115f3785c485e8ef9b456e99ea7ca4310c0961a9f83dd67cbcec51295561cfe7d74a87f3b62adbd006a552c31bf99c80b72f7e65abc8dfc9f543d
+  data.tar.gz: 9fa489733086e3b59e46b27c9c3aaa48b5ab520a29ddffb3bfc9fb945766d37dd791ae4eabcf7759d7c24e6be4c0596cad67b2d17ec97ad30b94adae380ae30f

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    naginegi (0.1.0)
+    naginegi (0.3.0)
       google-cloud-bigquery (= 1.18.1)
       mysql2 (= 0.5.3)
       mysql2-cs-bind (= 0.0.7)
@@ -53,7 +53,6 @@ GEM
     mini_mime (1.0.2)
     multi_json (1.14.1)
     multipart-post (2.1.1)
-    mysql2 (0.5.3)
     mysql2 (0.5.3-x64-mingw32)
     mysql2-cs-bind (0.0.7)
       mysql2
@@ -61,7 +60,6 @@ GEM
     parallel (1.19.1)
     parser (2.7.0.1)
       ast (~> 2.4.0)
-    pg (1.2.2)
     pg (1.2.2-x64-mingw32)
     powerpack (0.1.2)
     public_suffix (4.0.3)

data/README.md

@@ -77,8 +77,10 @@ db02:
     - name: configs
 ```
 
+### sample
 Naginegi requires BigQuery parameters like below.
 
+#### using json key file path
 ```ruby
 [sample.rb]
 require 'naginegi'
@@ -92,7 +94,38 @@ config = {
  'config_dir' => '/var/tmp/embulk/config'
 }
 
-client = Naginegi::EmbulkClient.new
+client = Naginegi::EmbulkRunner.new
+client.generate_config(config)
+client.run(config)
+```
+
+```bash
+ruby sample.rb
+```
+
+#### using key values
+```ruby
+[sample.rb]
+require 'naginegi'
+
+json_key = {
+  "type" => "...",
+  "project_id" => "...",
+  "private_key_id" => "...",
+  "private_key" => "...",
+  ...
+}
+
+config = {
+ 'project_id' => 'BIGQUERY_PROJECT_ID',
+ 'service_email' => 'SERVICE_ACCOUNT_EMAIL',
+ 'auth_method' => 'json_key',
+ 'json_key' => json_key,
+ 'schema_dir' => '/var/tmp/embulk/schema',
+ 'config_dir' => '/var/tmp/embulk/config'
+}
+
+client = Naginegi::EmbulkRunner.new
 client.generate_config(config)
 client.run(config)
 ```

data/lib/naginegi.rb

@@ -7,13 +7,17 @@ require 'logger'
 
 module Naginegi
   class EmbulkRunner
-    def initialize
+    def initialize(db_configs: nil, log_level: 'warn', embulk_run_option: '')
       @logger = Logger.new(STDOUT)
       @logger.datetime_format = '%Y-%m-%d %H:%M:%S'
+
+      @db_configs = db_configs || YAML.load_file('database.yml')
+      @log_level = log_level
+      @embulk_run_option = embulk_run_option
     end
 
     def generate_config(bq_config)
-      Naginegi::EmbulkConfig.new.generate_config(db_configs, bq_config)
+      Naginegi::EmbulkConfig.new.generate_config(@db_configs, bq_config)
     end
 
     def run(bq_config, target_table_names = [], retry_max = 0)
@@ -31,8 +35,8 @@ module Naginegi
     private
 
     def run_and_retry(bq_config, target_table_names, retry_max, retry_count)
-      error_tables = Naginegi::Embulk.new.run(
-        db_configs,
+      error_tables = Naginegi::Embulk.new(@log_level, @embulk_run_option).run(
+        @db_configs,
         table_configs,
         bq_config,
         target_table_names
@@ -46,12 +50,8 @@ module Naginegi
       error_tables
     end
 
-    def db_configs
-      @db_configs ||= YAML.load_file('database.yml')
-    end
-
     def table_configs
-      @table_configs ||= Naginegi::MySQL::TableConfig.generate_table_configs
+      @table_configs ||= Naginegi::TableConfig.generate_table_configs
     end
   end
 end

data/lib/naginegi/bigquery.rb

@@ -9,17 +9,20 @@ module Naginegi
     CONTENTS = <<-EOS.unindent
     in:
       type: <%= db_type %>
+      host: <%= host %>
       user: <%= user %>
       password: <%= password %>
       database: <%= database %>
-      host: <%= host %>
+      ssl: <%= ssl %>
       query: |
         <%= query %>
       <%= options %>
     out:
       type: bigquery
+      mode: replace
       auth_method: <%= auth_method %>
       json_keyfile: <%= json_keyfile %>
+        <%= json_key_content %>
       project: <%= project %>
       service_account_email: <%= service_account_email %>
       dataset: <%= dataset %>
@@ -61,6 +64,7 @@ module Naginegi
       user = db_config['username']
       password = db_config['password']
       database = db_config['database']
+      ssl = db_config['embulk_ssl_enable'] || false
       options = if db_type == 'mysql'
                   "options: {useLegacyDatetimeCode: false, serverTimezone: #{db_config['timezone']}}"
                 else
@@ -69,7 +73,15 @@ module Naginegi
       query = Naginegi::BigQuery.generate_sql(table_config, columns)
 
       auth_method = @config['auth_method']
-      json_keyfile = @config['json_keyfile']
+      if @config['json_key']
+        values = @config['json_key'].map do |k, v|
+          value = v.gsub("\n", '\\n')
+          "\"#{k}\": \"#{value}\""
+        end
+        json_key_content = "content: |\n      {#{values.join(',')}}"
+      else
+        json_keyfile = @config['json_keyfile']
+      end
       project = @config['project_id']
       service_account_email = @config['service_email']
       dataset = db_config['bq_dataset']
@@ -81,9 +93,17 @@ module Naginegi
     end
 
     def delete_table(dataset, table_name)
+      keyfile = if @config['json_key']
+                  value = @config['json_key'].dup
+                  value['private_key'] = value['private_key'].gsub('\\n', "\n")
+                  value
+                else
+                  @config['json_keyfile']
+                end
+
       bq = Google::Cloud::Bigquery.new(
         project: @config['project_id'],
-        keyfile: @config['json_keyfile']
+        keyfile: keyfile
       )
       bq.service.delete_table(dataset, table_name)
     end

data/lib/naginegi/embulk.rb

@@ -2,9 +2,12 @@ require 'logger'
 
 module Naginegi
   class Embulk
-    def initialize
+    def initialize(log_level, embulk_run_option)
       @logger = Logger.new(STDOUT)
       @logger.datetime_format = '%Y-%m-%d %H:%M:%S'
+
+      @log_level = log_level
+      @embulk_run_option = embulk_run_option
     end
 
     def run(db_configs, all_table_configs, bq_config, target_table_names = [])
@@ -45,7 +48,7 @@ module Naginegi
           @logger.warn(e.message)
         end
 
-        cmd = "embulk run #{bq_config['config_dir']}/#{db_name}/#{table_config.name}.yml"
+        cmd = "embulk run #{@embulk_run_option} #{bq_config['config_dir']}/#{db_name}/#{table_config.name}.yml --log-level #{@log_level}"
         @logger.info("cmd: #{cmd}")
 
         if system(cmd)

data/lib/naginegi/embulk_config.rb

@@ -43,7 +43,31 @@ module Naginegi
     end
 
     def all_table_configs
-      @all_table_configs ||= MySQL::TableConfig.generate_table_configs
+      @all_table_configs ||= Naginegi::TableConfig.generate_table_configs
+    end
+  end
+
+  class TableConfig
+    attr_reader :name, :daily_snapshot, :condition
+
+    def initialize(config)
+      @name = config['name']
+      @daily_snapshot = config['daily_snapshot'] || false
+      @condition = config['condition']
+    end
+
+    def self.generate_table_configs(file_path = 'table.yml')
+      configs = YAML.load_file(file_path)
+      configs.each_with_object({}) do |(db, database_config), table_configs|
+        table_configs[db] = database_config['tables'].map { |config| TableConfig.new(config) }
+        table_configs
+      end
+    end
+
+    def ==(other)
+      instance_variables.all? do |v|
+        instance_variable_get(v) == other.instance_variable_get(v)
+      end
     end
   end
 end

data/lib/naginegi/mysql.rb

@@ -39,30 +39,6 @@ module Naginegi
       end
     end
 
-    class TableConfig
-      attr_reader :name, :daily_snapshot, :condition
-
-      def initialize(config)
-        @name = config['name']
-        @daily_snapshot = config['daily_snapshot'] || false
-        @condition = config['condition']
-      end
-
-      def self.generate_table_configs(file_path = 'table.yml')
-        configs = YAML.load_file(file_path)
-        configs.each_with_object({}) do |(db, database_config), table_configs|
-          table_configs[db] = database_config['tables'].map { |config| TableConfig.new(config) }
-          table_configs
-        end
-      end
-
-      def ==(other)
-        instance_variables.all? do |v|
-          instance_variable_get(v) == other.instance_variable_get(v)
-        end
-      end
-    end
-
     class Column
       attr_reader :column_name, :data_type
 
@@ -90,7 +66,7 @@ module Naginegi
       end
 
       def bigquery_data_type
-        TYPE_MAPPINGS[@data_type]
+        TYPE_MAPPINGS[@data_type] || 'STRING'
       end
 
       def converted_value

data/lib/naginegi/postgresql.rb

@@ -38,30 +38,6 @@ module Naginegi
       end
     end
 
-    class TableConfig
-      attr_reader :name, :daily_snapshot, :condition
-
-      def initialize(config)
-        @name = config['name']
-        @daily_snapshot = config['daily_snapshot'] || false
-        @condition = config['condition']
-      end
-
-      def self.generate_table_configs(file_path = 'table.yml')
-        configs = YAML.load_file(file_path)
-        configs.each_with_object({}) do |(db, db_config), table_configs|
-          table_configs[db] = db_config['tables'].map { |config| TableConfig.new(config) }
-          table_configs
-        end
-      end
-
-      def ==(other)
-        instance_variables.all? do |v|
-          instance_variable_get(v) == other.instance_variable_get(v)
-        end
-      end
-    end
-
     class Column
       attr_reader :column_name, :data_type
 
@@ -91,7 +67,7 @@ module Naginegi
       end
 
       def bigquery_data_type
-        TYPE_MAPPINGS[@data_type]
+        TYPE_MAPPINGS[@data_type] || 'STRING'
       end
 
       def converted_value

data/lib/naginegi/version.rb

@@ -1,3 +1,3 @@
 module Naginegi
-  VERSION = '0.1.0'.freeze
+  VERSION = '0.5.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: naginegi
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.5.0
 platform: ruby
 authors:
 - cobot00
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-01-14 00:00:00.000000000 Z
+date: 2020-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler