samidare 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 63ca3ecfca7bdd2d9928119a4565520a6cb43e3a
-  data.tar.gz: 9f8ba50115e2d24ec87cf0bf18a049a8cdf2cc0e
+  metadata.gz: b5773b739c82c8896501f93a28c8f9f16780fbcb
+  data.tar.gz: dcdf41eb57238dc34b6f399a0b7fc707d25549c4
 SHA512:
-  metadata.gz: 6b215d706b0ba77f13587597e5c79268176e79410b171f17a8d6f49c755569432699ddedb406d57d75ff6d86d357cc0829e2f6f1c5b4a1e57f1bfbc084509c6c
-  data.tar.gz: 0b171bee1aa063719c4e4c87c7d311df09f611c9153ea11f4202ddc91777e1a035b91c3aa689ad12c6cd8c02001776daa3ef3bd866d8adb39c4bd54f7f46d44c
+  metadata.gz: 0bf4586c4a43e98bd24e36515ae2ac2cdebc072e6c089ca0ac71914bece5266c2ec6f338fedd16cbddaf87f6fae86acb57c9851c8a46b1af34db4ebdc4540da0
+  data.tar.gz: 116af87aeae963a54116730a1d14e54d62323a96d39f5790028fd9e33e681ff2d1d3d5f793a746b54946bd0f3b62893c854f2be57118d175a1ab26a49cb804fd

data/README.md

@@ -31,8 +31,9 @@ config = {
  'auth_method' => 'private_key'
 }
 
-Samidare.generate_config(config)
-Samidare.embulk_run(config)
+client = Samidare::EmbulkClient.new
+client.generate_config(config)
+client.run(config)
 ```
 
 ## Contributing

data/lib/samidare.rb

@@ -2,12 +2,12 @@ require 'samidare/version'
 require 'samidare/embulk_utility'
 
 module Samidare
-  class << self
+  class EmbulkClient
     def generate_config(config)
       Samidare::EmbulkUtility::ConfigGenerator.new(config).generate_config
     end
 
-    def embulk_run(config)
+    def run(config)
       db_infos = Samidare::EmbulkUtility::DBInfo.generate_db_infos
       table_infos = Samidare::EmbulkUtility::TableInfo.generate_table_infos
       db_infos.keys.each do |db_name|

data/lib/samidare/bigquery_utility.rb

@@ -1,9 +1,38 @@
 require 'json'
 require 'erb'
 require 'big_query'
+require 'unindent'
 
 module Samidare
   class BigQueryUtility
+    CONTENTS = <<-EOS.unindent
+    in:
+      type: mysql
+      user: <%= user %>
+      password: <%= password %>
+      database: <%= database %>
+      host: <%= host %>
+      query: |
+        <%= query %>
+    out:
+      type: bigquery
+      project: <%= project %>
+      p12_keyfile_path: <%= p12_keyfile_path %>
+      service_account_email: <%= service_account_email %>
+      dataset: <%= dataset %>
+      table: <%= table_name %>
+      schema_path: <%= schema_path %>
+      auto_create_table: 1
+      path_prefix: <%= path_prefix %>
+      source_format: NEWLINE_DELIMITED_JSON
+      file_ext: .json.gz
+      delete_from_local_when_job_end: 1
+      formatter:
+        type: jsonl
+      encoders:
+      - {type: gzip}
+    EOS
+
     def initialize(config)
       @config = config.dup
     end
@@ -33,7 +62,8 @@ module Samidare
       schema_path = "#{@config['schema_dir']}/#{db_name}/#{table_info.name}.json"
       path_prefix = "/var/tmp/embulk_#{db_name}_#{table_info.name}"
 
-      File.open('lib/samidare/embulk_config.erb') { |f| ERB.new(f.read).result(binding) }
+      #File.open('lib/samidare/embulk_config.erb') { |f| ERB.new(f.read).result(binding) }
+      ERB.new(CONTENTS).result(binding)
     end
 
     def delete_table(dataset, table_name)

data/lib/samidare/embulk_utility.rb

@@ -1,6 +1,7 @@
 require 'mysql2-cs-bind'
 require 'json'
 require 'yaml'
+require 'fileutils'
 require 'samidare/bigquery_utility'
 
 module Samidare

data/lib/samidare/version.rb

@@ -1,3 +1,3 @@
 module Samidare
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: samidare
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Ryoji Kobori
@@ -180,7 +180,6 @@ files:
 - Rakefile
 - lib/samidare.rb
 - lib/samidare/bigquery_utility.rb
-- lib/samidare/embulk_config.erb
 - lib/samidare/embulk_utility.rb
 - lib/samidare/version.rb
 - samidare.gemspec
@@ -217,4 +216,3 @@ test_files:
 - spec/samidare/embulk_utility_spec.rb
 - spec/samidare_spec.rb
 - spec/spec_helper.rb
-has_rdoc: 

data/lib/samidare/embulk_config.erb

@@ -1,25 +0,0 @@
-in:
-  type: mysql
-  user: <%= user %>
-  password: <%= password %>
-  database: <%= database %>
-  host: <%= host %>
-  query: |
-    <%= query %>
-out:
-  type: bigquery
-  project: <%= project %>
-  p12_keyfile_path: <%= p12_keyfile_path %>
-  service_account_email: <%= service_account_email %>
-  dataset: <%= dataset %>
-  table: <%= table_name %>
-  schema_path: <%= schema_path %>
-  auto_create_table: 1
-  path_prefix: <%= path_prefix %>
-  source_format: NEWLINE_DELIMITED_JSON
-  file_ext: .json.gz
-  delete_from_local_when_job_end: 1
-  formatter:
-    type: jsonl
-  encoders:
-  - {type: gzip}