bricolage 5.24.4 → 5.24.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 832c3083b42f0c549ed2dbc8441716b16b093d3e
-  data.tar.gz: 50fd92b977334362676bd587bbd02fed46327213
+  metadata.gz: 4b71b15a2d239fd8a8b169bea0f99c7880c2aec1
+  data.tar.gz: 446fbe3dda930b01c6e5345fa0183ada08fac6ed
 SHA512:
-  metadata.gz: 1f0ed17c1a0e745cc5b54beb0bc434a5922a2f2767607002d40ba2f3599c31258b8dfb54e5b45a2cbd530cd49302d1a0b35c22d19815e77bb96946a79af7514e
-  data.tar.gz: e3ba6c8d1020d46e96e1e110e759e4022f3d63243f9d621e88278c0067337c574dab9062df3124c667a915ce4496cb6c260d8b741b33c46e199d48c636cd33d7
+  metadata.gz: bc8c94df189b0576f807def6842a11f4503c0b4aefdd62516d3b9ad5d203e5f8908ffad424e1cc187fc47a6516ffbb227b784654b87060dd3c2196dc5a626359
+  data.tar.gz: 30361db5eda0a5d0ca7893b43463caa47f0aa0460d5dc36f65e6e7fb20d6893931f87fe5d80ca8a066ed02f461fc6450f22c44423a4837045c93a68b3db22430

data/lib/bricolage/commandlineapplication.rb

@@ -24,6 +24,7 @@ module Bricolage
     end
 
     attr_reader :name
+    attr_reader :options
 
     def define_default_options
       @options.on_tail('-e', '--environment=ENV', "Bricolage execution environment. (default: #{Context.environment})") {|env|
@@ -41,6 +42,7 @@ module Bricolage
 
     extend Forwardable
     def_delegator '@options', :on
+    def_delegator '@options', :help
 
     DataSourceOpt = Struct.new(:kind, :name, :ds)
 

data/lib/bricolage/context.rb

@@ -49,7 +49,7 @@ module Bricolage
 
     def load_configurations
       @filesystem.config_pathes('prelude.rb').each do |path|
-        EmbeddedCodeAPI.module_eval(File.read(path)) if path.exist?
+        EmbeddedCodeAPI.module_eval(File.read(path), path.to_s, 1) if path.exist?
       end
       @data_sources = DataSourceFactory.load(self, @logger)
     end

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.24.4'
+  VERSION = '5.24.6'
 end

data/test/home/Gemfile.lock

@@ -0,0 +1,41 @@
+PATH
+  remote: ../..
+  specs:
+    bricolage (5.24.2)
+      aws-sdk (~> 2)
+      mysql2
+      pg (~> 0.18.0)
+      redis (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aws-sdk (2.10.9)
+      aws-sdk-resources (= 2.10.9)
+    aws-sdk-core (2.10.9)
+      aws-sigv4 (~> 1.0)
+      jmespath (~> 1.0)
+    aws-sdk-resources (2.10.9)
+      aws-sdk-core (= 2.10.9)
+    aws-sigv4 (1.0.0)
+    coderay (1.1.1)
+    jmespath (1.3.1)
+    method_source (0.8.2)
+    mysql2 (0.4.8)
+    pg (0.18.4)
+    pry (0.10.4)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    redis (3.3.3)
+    slop (3.6.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bricolage!
+  pry
+
+BUNDLED WITH
+   1.14.3

data/test/home/config/bricolage.yml

@@ -0,0 +1 @@
+enable-queue: true

data/test/home/config/development/database.yml

@@ -1,147 +1,113 @@
-#
-# Shared
-#
-
-redshift_base: &redshift_base
+sql: &redshift_parameters
   type: psql
-  host: <%= ENV['RSPROXYHOST'] ? 'localhost' : (ENV['RSHOST'] || 'cookpad-dwh-401.cbzwjkf60uat.ap-northeast-1.redshift.amazonaws.com') %>
-  port: <%= ENV['RSPROXYPORT'] || ENV['RSPORT'] || 5439 %>
+  host: <%= ENV['RSHOST'] || 'dwh.ckpd.co' %>
+  port: <%= ENV['RSPORT'] || 5439 %>
   database: production
-  username: <%= ENV['RSUSER'] || 'dwhdev' %>
+  username: <%= ENV['RSUSER'] || 'tabemirudev' %>
   pgpass: <%= user_home_relative_path '.pgpass' %>
-
-s3_base: &s3_base
-  type: s3
-  endpoint: s3-ap-northeast-1.amazonaws.com
-  bucket: redshift-copy-buffer
-  prefix: development/tmp
-  s3cfg: <%= user_home_relative_path '.s3cfg' %>
-  #access_key_id: <%%= password 's3_access_key_id' %>
-  #secret_access_key: <%%= password 's3_secret_access_key' %>
-
-mysql_base: &mysql_base
-  type: mysql
-  username: work_readonly
-  password: <%= password 'mysql_shared_work_readonly' %>
   encoding: utf8
 
+#sql_dev:
+#  <<: *redshift_parameters
 #
-# Redis
-#
-redis:
-  type: redis
+#sql_prod:
+#  <<: *redshift_parameters
+#  username: tabemiru_batch
+
+#sql_app:
+#  type: psql
+#  host: localhost   #cookpad-dwh-401.cbzwjkf60uat.ap-northeast-1.redshift.amazonaws.com
+#  port: 5445
+#  database: production
+#  username: tabemiru_batch
+#  pgpass: <%= user_home_relative_path '.pgpass' %>
+#  encoding: utf8
+
+sql_app2:
+  type: psql
   host: localhost
-  port: 6379
-  db: 0
-
-#
-# Redshift
-#
-
-dwh_db: &dwh_db
-  <<: *redshift_base
-
-sql:
-  <<: *dwh_db
-
-figlog_db:
-  <<: *redshift_base
+  port: 5446
+  database: production
+  username: tabemiru_batch
+  pgpass: <%= user_home_relative_path '.pgpass' %>
+  encoding: utf8
 
-#
-# S3
-#
+postgres:
+  type: psql
+  host: localhost
+  port: 5432
+  database: bricolage
+  username: bricolage
+  password: bricolage
+  encoding: utf8
+  sql_log_level: DEBUG
 
 s3:
-  <<: *s3_base
-  bucket: redshift-copy-buffer
-
-figlog_s3:
-  <<: *s3_base
-  bucket: redshift-copy-buffer
-  prefix: development/figlog
-
-# Read only
-pv_log_s3:
-  <<: *s3_base
-  bucket: pv-log.ap-northeast-1
-  prefix: ""
-
-search_log_s3:
-  <<: *s3_base
-  bucket: redshift-copy-buffer
-  prefix: production/search_log
-
-dmp_s3:
-  <<: *s3_base
-  bucket: redshift-copy-buffer
-  prefix: production/dmp
-
-intage_s3:
-  <<: *s3_base
-  bucket: cookpad-intage
-  prefix: ""
-
-td_s3:
-  <<: *s3_base
-  bucket: redshift-copy-buffer
-  prefix: td
-
-#
-# MySQL
-#
-
-% is_local = (`uname -s`.strip == 'Darwin')
-
-cookpad_main_db: &cookpad_main_db
-  <<: *mysql_base
-  host: <%= is_local ? '127.0.0.1' : 'db-main-slave-free-001.vpc.ap-northeast-1.compute.internal' %>
-  port: <%= is_local ? 9011 : 3306 %>
-  database: main
+  type: s3
+  endpoint: "s3-ap-northeast-1.amazonaws.com"
+  region: "ap-northeast-1"
+  bucket: tabemiru-data.ap-northeast-1
+  prefix: "dev"
+  #s3cfg: <%= user_home_relative_path '.s3cfg' %>
+  #access_key_id: "<%= password 'aws_access_key_id' %>"
+  #secret_access_key: "<%= password 'aws_secret_access_key' %>"
+  #master_symmetric_key: "12345678901234567890123456789012"
+  iam_role: "arn:aws:iam::789035092620:role/RedshiftDWHTeamS3Access"
+
+s3_log:
+  type: s3
+  endpoint: "s3-ap-northeast-1.amazonaws.com"
+  region: "ap-northeast-1"
+  bucket: "redshift-copy-buffer"
+  prefix: "development/bricolage/log"
 
-cookpad_bargain_db:
-  <<: *mysql_base
-  host: <%= is_local ? '127.0.0.1' : 'db-bargain-kuroko-101.vpc.ap-northeast-1.compute.internal' %>
-  port: <%= is_local ? 9131 : 3306 %>
+mysql:
+  type: mysql
+  host: 127.0.0.1
+  port: 9011
   database: main
-
-cookpad_device_guest_db:
-  <<: *mysql_base
-  host: <%= is_local ? '127.0.0.1' : 'db-device-guest-102.vpc.ap-northeast-1.compute.internal' %>
-  port: <%= is_local ? 9142 : 3306 %>
-  database: device_guest
-
-cookpad_ad_db:
-  <<: *mysql_base
-  host: <%= is_local ? '127.0.0.1' : 'db-ad-slave-staging-001.vpc.ap-northeast-1.compute.internal' %>
-  port: <%= is_local ? 9153 : 3306 %>
-  database: ad
-  username: ckpd_readonly
-  password: <%= password 'mysql_shared_ckpd_readonly' %>
-
-cookpad_ad4_db:
-  <<: *mysql_base
-  host: <%= is_local ? '127.0.0.1' : 'db-ad4-master-001.vpc.ap-northeast-1.compute.internal' %>
-  port: <%= is_local ? 40205 : 3306 %>
-  database: ad4
   username: work_readonly
   password: <%= password 'mysql_shared_work_readonly' %>
+  encoding: utf8
 
-cookpad_summary_db:
-  <<: *mysql_base
-  host: <%= is_local ? '127.0.0.1' : 'db-summary-slave-mysql56-free-001.vpc.ap-northeast-1.compute.internal' %>
-  port: <%= is_local ? 40201 : 3306 %>
+mysql_summary:
+  type: mysql
+  host: 127.0.0.1
+  port: 40201
   database: summarized_data
   username: work_readonly
   password: <%= password 'mysql_shared_work_readonly' %>
+  encoding: utf8
 
-mysql:
-  <<: *cookpad_main_db
-
-#
-# TD
-#
+sqs_preproc:
+  type: sqs
+  url: "https://sqs.ap-northeast-1.amazonaws.com/789035092620/bricolage-preproc-dev"
+  max_number_of_messages: 10
+  visibility_timeout: 60
+
+sqs_dispatch:
+  type: sqs
+  url: "https://sqs.ap-northeast-1.amazonaws.com/789035092620/log-stream-dev"
+  max_number_of_messages: 10
+  visibility_timeout: 60
+
+sqs_task:
+  type: sqs
+  url: "https://sqs.ap-northeast-1.amazonaws.com/789035092620/bricolage-load-tasks-dev"
+  max_number_of_messages: 1
+  visibility_timeout: 180
+
+sns:
+  type: sns
+  region: "ap-northeast-1"
+  topic_arn: "arn:aws:sns:ap-northeast-1:789035092620:dwh-service-notification"
 
 td:
   type: td
   database: logs
   apikey: <%= password 'td_tabemiru' %>
+
+td_search_log:
+  type: td
+  database: search_log
+  apikey: <%= password 'td_tabemiru' %>

data/test/home/config/development/password.yml

@@ -1,3 +1,6 @@
-mysql_shared_work_readonly: s0uL/M@9r1
-mysql_shared_ckpd_readonly: hoge
-td_tabemiru: 121/e9d758221f5ad5e2747337d1d79f974389e6cc0a
+td_tabemiru: "121/c1376bb5e3972fc4a4efc942c1df100f7bc156e0"
+mysql_shared_work_readonly: "s0uL/M@9r1"
+mysql_shared_ckpd_readonly: "no3lnar;"
+mysql_local_work_readonly: "ckpdtest"
+aws_access_key_id: "AKIAI5R5BL2W73PR3FXQ"
+aws_secret_access_key: "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"

data/test/home/config/development/variable.yml

@@ -1,21 +1,7 @@
-% data_date = Date.today - 1
-data_date: <%= data_date.strftime("%Y-%m-%d") %>
-data_date_ymd: <%= data_date.strftime("%Y%m%d") %>
-data_date_endx: <%= Date.today.strftime("%Y-%m-%d") %>
+data_dir: /tmp/bricolagedev
+test_schema: aamine
+test_group: aamine
+activity_schema: aamine
 
-data_dir: <%= ENV['TMPDIR'] || '/tmp' %>
-shared_data_dir: "/tmp"
-trigger_dir: "$shared_data_dir/bricolage.trigger"
-
-% dwh_dev_schema = ENV['RSUSER'] || 'dwh_dev'
-test_schema: <%= dwh_dev_schema %>
-
-secure_activity_schema: <%= dwh_dev_schema %>
-secure_cookpad_schema: <%= dwh_dev_schema %>
-secure_dwh_schema: <%= dwh_dev_schema %>
-
-tabemiru_schema: tabemiru
-
-test_group: "tabemirudev"
-
-ad_mirror_db: dwh_dev
+defaults:
+    analyze: true

data/test/home/config/streamingload.yml

@@ -0,0 +1,8 @@
+redshift-ds: sql
+event-queue-ds: sqs_strload_event
+task-queue-ds: sqs_strload_task
+url_patterns:
+  -
+    url: "s3://redshift-copy-buffer/development/log_stream/\\d{8}_\\d{4}_\\d+.*\\.gz"
+    schema: "aamine"
+    table: "pv_log"

data/test/home/log/20170117/subsys::job1/20170117_202014082/subsys-job1.log

@@ -0,0 +1,3 @@
+2017-01-17 20:20:14 +0900: INFO: development environment
+2017-01-17 20:20:14 +0900: INFO: 0.00 secs
+2017-01-17 20:20:14 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::job1/20170117_202334852/subsys-job1.log

@@ -0,0 +1,3 @@
+2017-01-17 20:23:34 +0900: INFO: development environment
+2017-01-17 20:23:34 +0900: INFO: 0.00 secs
+2017-01-17 20:23:34 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::job1/20170117_202405056/subsys-job1.log

@@ -0,0 +1,3 @@
+2017-01-17 20:24:05 +0900: INFO: development environment
+2017-01-17 20:24:05 +0900: INFO: 0.00 secs
+2017-01-17 20:24:05 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::migrate/20170117_202548409/subsys-migrate.log

@@ -0,0 +1,80 @@
+2017-01-17 20:25:48 +0900: INFO: development environment
+2017-01-17 20:25:48 +0900: INFO: [CMD] "/Users/minero-aoki/c/bricolage/libexec/sqldump" "--json" "127.0.0.1" "9011" "work_readonly" "main" "select * from main.search_backends;" | "gzip" "{:out=>"/tmp/bricolagedev/search_backends.json.gz"}"
+2017-01-17 20:25:48.540819891 +0900 JST: [SQL] select * from main.search_backends;
+2017-01-17 20:25:48.597928759 +0900 JST: query returned
+2017-01-17 20:25:48.598038993 +0900 JST: Total 11 records
+2017-01-17 20:25:48 +0900: INFO: aws s3 cp /tmp/bricolagedev/search_backends.json.gz s3://tabemiru-data.ap-northeast-1/dev/search_backends/search_backends.json.gz
+2017-01-17 20:25:51 +0900: INFO: 3.28 secs
+2017-01-17 20:25:51 +0900: ERROR: Aws::Errors::MissingCredentialsError: unable to sign request without credentials set
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/request_signer.rb:104:in `require_credentials'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_request_signer.rb:14:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/xml/error_handler.rb:8:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/helpful_socket_errors.rb:10:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_request_signer.rb:65:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_redirects.rb:15:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/retry_errors.rb:87:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_dualstack.rb:32:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_accelerate.rb:49:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_md5s.rb:31:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_expect_100_continue.rb:21:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_bucket_name_restrictions.rb:12:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_bucket_dns.rb:31:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/rest/handler.rb:7:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/user_agent.rb:12:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/seahorse/client/plugins/endpoint.rb:41:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/param_validator.rb:21:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/seahorse/client/plugins/raise_response_errors.rb:14:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_sse_cpk.rb:19:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_dualstack.rb:24:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_accelerate.rb:34:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/idempotency_token.rb:18:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/param_converter.rb:20:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/seahorse/client/plugins/response_target.rb:21:in `call'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/seahorse/client/request.rb:70:in `send_request'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/seahorse/client/base.rb:207:in `block (2 levels) in define_operation_methods'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-resources-2.6.36/lib/aws-sdk-resources/services/s3/file_uploader.rb:42:in `block in put_object'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-resources-2.6.36/lib/aws-sdk-resources/services/s3/file_uploader.rb:49:in `open_file'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-resources-2.6.36/lib/aws-sdk-resources/services/s3/file_uploader.rb:41:in `put_object'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-resources-2.6.36/lib/aws-sdk-resources/services/s3/file_uploader.rb:34:in `upload'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-resources-2.6.36/lib/aws-sdk-resources/services/s3/object.rb:251:in `upload_file'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/s3datasource.rb:188:in `block in run'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/s3datasource.rb:165:in `block in each_src_dest'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/s3datasource.rb:163:in `each'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/s3datasource.rb:163:in `each_src_dest'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/s3datasource.rb:186:in `run'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:82:in `block (2 levels) in run'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:81:in `each'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:81:in `block in run'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/datasource.rb:136:in `open'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/datasource.rb:140:in `open_for_batch'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:80:in `run'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:47:in `block in run'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:46:in `each'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:46:in `run'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/job.rb:143:in `block in execute'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/logger.rb:44:in `with_elapsed_time'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/job.rb:142:in `execute'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/application.rb:68:in `block in main'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/application.rb:97:in `redirect_log_to_file'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/application.rb:67:in `main'
+	/Users/minero-aoki/c/bricolage/lib/bricolage/application.rb:28:in `main'
+	/Users/minero-aoki/c/bricolage/bin/bricolage:6:in `<top (required)>'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/bin/bricolage:23:in `load'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/bin/bricolage:23:in `<top (required)>'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli/exec.rb:74:in `load'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli/exec.rb:74:in `kernel_load'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli/exec.rb:27:in `run'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli.rb:332:in `exec'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/vendor/thor/lib/thor/command.rb:27:in `run'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/vendor/thor/lib/thor/invocation.rb:126:in `invoke_command'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/vendor/thor/lib/thor.rb:359:in `dispatch'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli.rb:20:in `dispatch'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/vendor/thor/lib/thor/base.rb:440:in `start'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli.rb:11:in `start'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/exe/bundle:34:in `block in <top (required)>'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/friendly_errors.rb:100:in `with_friendly_errors'
+	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/exe/bundle:26:in `<top (required)>'
+	/Users/minero-aoki/.rbenv/versions/2.3/bin/bundle:23:in `load'
+	/Users/minero-aoki/.rbenv/versions/2.3/bin/bundle:23:in `<main>'
+
+2017-01-17 20:25:51 +0900: ERROR: error: Aws::Errors::MissingCredentialsError: unable to sign request without credentials set

data/test/home/log/20170117/subsys::migrate/20170117_202636115/subsys-migrate.log

@@ -0,0 +1,79 @@
+2017-01-17 20:26:36 +0900: INFO: development environment
+2017-01-17 20:26:36 +0900: INFO: [CMD] "/Users/minero-aoki/c/bricolage/libexec/sqldump" "--json" "127.0.0.1" "9011" "work_readonly" "main" "select * from main.search_backends;" | "gzip" "{:out=>"/tmp/bricolagedev/search_backends.json.gz"}"
+2017-01-17 20:26:36.214888001 +0900 JST: [SQL] select * from main.search_backends;
+2017-01-17 20:26:36.27827597 +0900 JST: query returned
+2017-01-17 20:26:36.278393544 +0900 JST: Total 11 records
+2017-01-17 20:26:36 +0900: INFO: aws s3 cp /tmp/bricolagedev/search_backends.json.gz s3://tabemiru-data.ap-northeast-1/dev/search_backends/search_backends.json.gz
+rm -f /tmp/bricolagedev/search_backends.json.gz
+2017-01-17 20:26:36 +0900: INFO: command: psql --no-psqlrc --host=dwh.ckpd.co --port=5439 --username=aamine production --echo-all -v ON_ERROR_STOP=true -f /var/folders/_t/f8rv7p995rq8m3j3tf45w4rm0000gp/T/1484652396_69460_3fd54d43f9d8_14594 --no-password
+\timing on
+Timing is on.
+begin transaction;
+BEGIN
+Time: 8.269 ms
+drop table if exists aamine.search_backends_old cascade;
+psql:/var/folders/_t/f8rv7p995rq8m3j3tf45w4rm0000gp/T/1484652396_69460_3fd54d43f9d8_14594:5: INFO:  Table "search_backends_old" does not exist and will be skipped
+DROP TABLE
+Time: 8.647 ms
+drop table if exists aamine.search_backends_wk cascade;
+psql:/var/folders/_t/f8rv7p995rq8m3j3tf45w4rm0000gp/T/1484652396_69460_3fd54d43f9d8_14594:7: INFO:  Table "search_backends_wk" does not exist and will be skipped
+DROP TABLE
+Time: 8.031 ms
+/* /Users/minero-aoki/c/bricolage/test/home/subsys/search_backends.ct */
+create table aamine.search_backends_wk
+( id int
+, role varchar(10)
+, group_id smallint
+, index_type varchar(32)
+, config_key varchar(32)
+, updated_at timestamp
+)
+distkey (id)
+sortkey (id)
+;
+CREATE TABLE
+Time: 521.167 ms
+copy aamine.search_backends_wk
+from 's3://tabemiru-data.ap-northeast-1/dev/search_backends/search_backends.json.gz'
+credentials 'aws_iam_role=arn:aws:iam::789035092620:role/RedshiftDWHTeamS3Access'
+json 'auto'
+statupdate false
+compupdate false
+emptyasnull
+gzip
+;
+psql:/var/folders/_t/f8rv7p995rq8m3j3tf45w4rm0000gp/T/1484652396_69460_3fd54d43f9d8_14594:30: INFO:  Load into table 'search_backends_wk' completed, 11 record(s) loaded successfully.
+COPY
+Time: 1235.569 ms
+grant select on aamine.search_backends_wk to aamine;
+GRANT
+Time: 8.580 ms
+analyze aamine.search_backends_wk;
+ANALYZE
+Time: 43.441 ms
+create table if not exists aamine.search_backends (x int);
+psql:/var/folders/_t/f8rv7p995rq8m3j3tf45w4rm0000gp/T/1484652396_69460_3fd54d43f9d8_14594:36: INFO:  Relation "search_backends" already exists and will be skipped
+CREATE TABLE
+Time: 7.768 ms
+alter table aamine.search_backends rename to search_backends_old;
+ALTER TABLE
+Time: 10.037 ms
+alter table aamine.search_backends_wk rename to search_backends;
+ALTER TABLE
+Time: 11.686 ms
+commit;
+COMMIT
+Time: 8482.452 ms
+drop table if exists aamine.search_backends_old cascade;
+DROP TABLE
+Time: 4801.919 ms
+;
+Time: 8.088 ms
+vacuum sort only aamine.search_backends;
+VACUUM
+Time: 36970.832 ms
+;
+Time: 8.746 ms
+2017-01-17 20:27:29 +0900: INFO: status: 0 (pid 69478 exit 0)
+2017-01-17 20:27:29 +0900: INFO: 53.05 secs
+2017-01-17 20:27:29 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job1.log

@@ -0,0 +1,3 @@
+2017-01-17 20:30:22 +0900: INFO: development environment
+2017-01-17 20:30:22 +0900: INFO: 0.00 secs
+2017-01-17 20:30:22 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job2.log

@@ -0,0 +1,3 @@
+2017-01-17 20:30:22 +0900: INFO: development environment
+2017-01-17 20:30:22 +0900: INFO: 0.00 secs
+2017-01-17 20:30:22 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job3.log

@@ -0,0 +1,3 @@
+2017-01-17 20:30:22 +0900: INFO: development environment
+2017-01-17 20:30:22 +0900: INFO: 0.00 secs
+2017-01-17 20:30:22 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job4.log

@@ -0,0 +1,3 @@
+2017-01-17 20:30:22 +0900: INFO: development environment
+2017-01-17 20:30:22 +0900: INFO: 0.00 secs
+2017-01-17 20:30:22 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job1.log

@@ -0,0 +1,3 @@
+2017-01-17 20:31:02 +0900: INFO: development environment
+2017-01-17 20:31:02 +0900: INFO: 0.00 secs
+2017-01-17 20:31:02 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job2.log

@@ -0,0 +1,3 @@
+2017-01-17 20:31:02 +0900: INFO: development environment
+2017-01-17 20:31:02 +0900: INFO: 0.00 secs
+2017-01-17 20:31:02 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job3.log

@@ -0,0 +1,3 @@
+2017-01-17 20:31:02 +0900: INFO: development environment
+2017-01-17 20:31:02 +0900: INFO: 0.00 secs
+2017-01-17 20:31:02 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job4.log

@@ -0,0 +1,3 @@
+2017-01-17 20:31:02 +0900: INFO: development environment
+2017-01-17 20:31:02 +0900: INFO: 0.00 secs
+2017-01-17 20:31:02 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job1.log

@@ -0,0 +1,3 @@
+2017-01-17 20:32:32 +0900: INFO: development environment
+2017-01-17 20:32:32 +0900: INFO: 0.00 secs
+2017-01-17 20:32:32 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job2.log

@@ -0,0 +1,3 @@
+2017-01-17 20:32:32 +0900: INFO: development environment
+2017-01-17 20:32:32 +0900: INFO: 0.00 secs
+2017-01-17 20:32:32 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job3.log

@@ -0,0 +1,3 @@
+2017-01-17 20:32:32 +0900: INFO: development environment
+2017-01-17 20:32:32 +0900: INFO: 0.00 secs
+2017-01-17 20:32:32 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job4.log

@@ -0,0 +1,3 @@
+2017-01-17 20:32:32 +0900: INFO: development environment
+2017-01-17 20:32:32 +0900: INFO: 0.00 secs
+2017-01-17 20:32:32 +0900: INFO: SUCCESS

data/test/home/log/20170629/subsys::job1/20170629_155543985/subsys-job1.log

@@ -0,0 +1,3 @@
+2017-06-29 15:55:43 +0900: INFO: development environment
+2017-06-29 15:55:43 +0900: INFO: 0.00 secs
+2017-06-29 15:55:43 +0900: INFO: SUCCESS

data/test/home/log/20170629/subsys::net1/20170629_154308811/subsys-job1.log

@@ -0,0 +1,3 @@
+2017-06-29 15:43:08 +0900: INFO: development environment
+2017-06-29 15:43:08 +0900: INFO: 0.00 secs
+2017-06-29 15:43:08 +0900: INFO: SUCCESS

data/test/home/log/20170629/subsys::net1/20170629_154308811/subsys-job2.log

@@ -0,0 +1,3 @@
+2017-06-29 15:43:08 +0900: INFO: development environment
+2017-06-29 15:43:08 +0900: INFO: 0.00 secs
+2017-06-29 15:43:08 +0900: INFO: SUCCESS

data/test/home/log/20170629/subsys::net1/20170629_154308811/subsys-job3.log

@@ -0,0 +1,3 @@
+2017-06-29 15:43:08 +0900: INFO: development environment
+2017-06-29 15:43:08 +0900: INFO: 0.00 secs
+2017-06-29 15:43:08 +0900: INFO: SUCCESS