bricolage 5.9.6 → 5.9.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9905f46f89b86160ac6075a3c3f1f17c1d2df765
-  data.tar.gz: 0f0cbb87d71873ee799edba4201a5d49f34d9e82
+  metadata.gz: c19aa7e310f6f40ab0b9ce66b1782a81e82f5001
+  data.tar.gz: d5c3b3c822fb8f987ca49ef6f2b0e7091c194064
 SHA512:
-  metadata.gz: 755cc0f36123b24a461b3dc7b553604817c588fd4d25f1584eafbfcdd075a2185a1ed3e3020587315f7b5ad8b443746e171f09356ef1cfe316af080f731f6871
-  data.tar.gz: 41dce6356cb69bb7ca8d8c19da87d4c9278aadb754bf82a2445bad3c23f24a403478a67db5b2fcb38ead654841fc9ed3830a1bcfe8eface885d904ce966284eb
+  metadata.gz: 28d0ffd735a81cbc9d48ea66f468567d7825809bf89c229c0310f7540feb6084157832d000fb0562ebe5d4ee3ec240156d2b3bedad7386904af3d1c7f6668ed6
+  data.tar.gz: 19cae6930b0717ac2efed5bc37644ed29b5c9b019abd7a04586b254457eb12d1b7267ca3f0b3887f6ac0f36dc90ffaf3b3ee370d054aad62272a41e097defa44

data/jobclass/my-migrate.rb

@@ -12,6 +12,7 @@ JobClass.define('my-migrate') {
     params.add DestFileParam.new('s3-file', 'PATH', 'Temporary S3 file path.')
     params.add DataSourceParam.new('s3', 's3-ds', 'Temporary file storage.')
     params.add OptionalBoolParam.new('override', 'If true, overwrite s3 target file.  Otherwise causes error.')
+    params.add OptionalBoolParam.new('remove-tmp', 'Removes temporary local files after S3-PUT is succeeded.')
 
     # Load
     params.add DestTableParam.new(optional: false)
@@ -60,6 +61,11 @@ JobClass.define('my-migrate') {
       script.task(params['s3-ds']) {|task|
         task.put params['tmp-file'], params['s3-file'], check_args: false
       }
+      if params['remove-tmp']
+        script.task(params.file_ds) {|task|
+          task.remove params['tmp-file']
+        }
+      end
     end
 
     # Load

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.9.6'
+  VERSION = '5.9.7'
 end

data/test/home/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ../..
   specs:
-    bricolage (5.9.2)
+    bricolage (5.9.6)
       aws-sdk (< 2)
       mysql2
       pg
@@ -10,9 +10,9 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-sdk (1.64.0)
-      aws-sdk-v1 (= 1.64.0)
-    aws-sdk-v1 (1.64.0)
+    aws-sdk (1.66.0)
+      aws-sdk-v1 (= 1.66.0)
+    aws-sdk-v1 (1.66.0)
       json (~> 1.4)
       nokogiri (>= 1.4.4)
     fluent-logger (0.4.10)
@@ -20,25 +20,26 @@ GEM
       yajl-ruby (~> 1.0)
     hirb (0.7.3)
     httpclient (2.5.3.3)
-    json (1.8.2)
+    json (1.8.3)
     mini_portile (0.6.2)
-    msgpack (0.5.12)
-    mysql2 (0.3.18)
+    msgpack (0.5.11)
+    mysql2 (0.4.0)
     nokogiri (1.6.6.2)
       mini_portile (~> 0.6.0)
     parallel (0.6.5)
-    pg (0.18.2)
-    rubyzip (0.9.9)
-    td (0.11.10)
+    pg (0.18.3)
+    rubyzip (1.1.7)
+    td (0.12.0)
       hirb (>= 0.4.5)
-      msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
+      msgpack (>= 0.4.4, < 0.5.12, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
       parallel (~> 0.6.1)
-      rubyzip (~> 0.9.9)
-      td-client (~> 0.8.70)
+      rubyzip (~> 1.1.7)
+      td-client (~> 0.8.75)
       td-logger (~> 0.3.21)
       yajl-ruby (~> 1.1)
-    td-client (0.8.70)
-      httpclient (>= 2.4.0, < 2.6.0)
+      zip-zip (~> 0.3)
+    td-client (0.8.75)
+      httpclient (>= 2.5.2, < 2.6.0)
       json (>= 1.7.6)
       msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
     td-logger (0.3.24)
@@ -46,6 +47,8 @@ GEM
       msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
       td-client (~> 0.8.66)
     yajl-ruby (1.2.1)
+    zip-zip (0.3)
+      rubyzip (>= 1.0.0)
 
 PLATFORMS
   ruby
@@ -54,4 +57,4 @@ DEPENDENCIES
   bricolage!
 
 BUNDLED WITH
-   1.10.2
+   1.10.6

data/test/home/config/development/database.yml

@@ -1,27 +1,27 @@
 sql: &redshift_parameters
   type: psql
-  host: <%= ENV['RSPROXYHOST'] ? 'localhost' : (ENV['RSHOST'] || 'cookpad-dwh-301.cbzwjkf60uat.ap-northeast-1.redshift.amazonaws.com') %>
+  host: <%= ENV['RSPROXYHOST'] ? 'localhost' : (ENV['RSHOST'] || 'cookpad-dwh-401.cbzwjkf60uat.ap-northeast-1.redshift.amazonaws.com') %>
   port: <%= ENV['RSPROXYPORT'] || ENV['RSPORT'] || 5439 %>
   database: production
-  username: tabemirudev
+  username: <%= ENV['RSUSER'] || 'tabemirudev' %>
   pgpass: <%= user_home_relative_path '.pgpass' %>
   encoding: utf8
 
-sql_dev:
-  <<: *redshift_parameters
-
-sql_prod:
-  <<: *redshift_parameters
-  username: tabemiru_batch
-
-sql_app:
-  type: psql
-  host: localhost   #cookpad-dwh-301.cbzwjkf60uat.ap-northeast-1.redshift.amazonaws.com
-  port: 5445
-  database: production
-  username: tabemiru_batch
-  pgpass: <%= user_home_relative_path '.pgpass' %>
-  encoding: utf8
+#sql_dev:
+#  <<: *redshift_parameters
+#
+#sql_prod:
+#  <<: *redshift_parameters
+#  username: tabemiru_batch
+
+#sql_app:
+#  type: psql
+#  host: localhost   #cookpad-dwh-401.cbzwjkf60uat.ap-northeast-1.redshift.amazonaws.com
+#  port: 5445
+#  database: production
+#  username: tabemiru_batch
+#  pgpass: <%= user_home_relative_path '.pgpass' %>
+#  encoding: utf8
 
 sql_app2:
   type: psql
@@ -53,8 +53,10 @@ s3:
 
 mysql:
   type: mysql
-  host: db-main-slave-free-001
+  #host: db-main-slave-free-001
+  host: 127.0.0.1
+  port: 9011
   database: main
   username: work_readonly
-  password: <%= password 'mysql_local_work_readonly' %>
+  password: <%= password 'mysql_shared_work_readonly' %>
   encoding: utf8

data/test/home/config/development/password.yml

@@ -1,2 +1,4 @@
 td_tabemiru: "121/c1376bb5e3972fc4a4efc942c1df100f7bc156e0"
+mysql_shared_work_readonly: "s0uL/M@9r1"
+mysql_shared_ckpd_readonly: "no3lnar;"
 mysql_local_work_readonly: "ckpdtest"

data/test/home/config/development/variable.yml

@@ -0,0 +1,3 @@
+data_dir: /tmp/bricolagedev
+test_schema: aamine
+test_group: aamine

data/test/home/subsys/migrate.job

@@ -0,0 +1,32 @@
+class: my-migrate
+src-ds: mysql
+src-tables:
+    search_backends: main.search_backends
+tmp-file: $data_dir/search_backends.json.gz
+
+s3-ds: s3
+s3-file: /search_backends/search_backends.json.gz
+override: true
+remove-tmp: true
+
+dest-ds: sql
+dest-table: $test_schema.search_backends
+table-def: search_backends.ct
+options:
+    statupdate: false
+    compupdate: false
+    #maxerror: 10
+    ## string
+    #acceptinvchars: " "
+    emptyasnull: true
+    #trimblanks: true
+    #truncatecolumns: true
+    ## datetime
+    #acceptanydate: true
+    #dateformat: "auto"
+    #timeformat: "auto"
+vacuum-sort: true
+analyze: true
+grant:
+    privilege: select
+    to: "$test_group"

data/test/home/subsys/search_backends.ct

@@ -0,0 +1,13 @@
+--dest-table: search_backends
+
+create table $dest_table
+( id int
+, role varchar(10)
+, group_id smallint
+, index_type varchar(32)
+, config_key varchar(32)
+, updated_at timestamp
+)
+distkey (id)
+sortkey (id)
+;

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.9.6
+  version: 5.9.7
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-10 00:00:00.000000000 Z
+date: 2015-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -164,6 +164,7 @@ files:
 - test/home/Gemfile.lock
 - test/home/config/development/database.yml
 - test/home/config/development/password.yml
+- test/home/config/development/variable.yml
 - test/home/data/20141002-1355_00.txt
 - test/home/data/20141002-1355_01.txt
 - test/home/data/20141002-1355_02.txt
@@ -172,8 +173,10 @@ files:
 - test/home/subsys/d.ct
 - test/home/subsys/load_test.ct
 - test/home/subsys/load_test.job
+- test/home/subsys/migrate.job
 - test/home/subsys/raw-vacuum.sql.job
 - test/home/subsys/rebuild.sql.job
+- test/home/subsys/search_backends.ct
 - test/home/subsys/separated.job
 - test/home/subsys/separated.sql
 - test/home/subsys/unified.jobnet
@@ -203,7 +206,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: SQL batch framework