bricolage 5.24.2 → 5.24.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 593e33c7e02661fc8769f5d1f5118415ea59b52b
-  data.tar.gz: e0f9c04913dccb58723308841150e4a2a129462b
+  metadata.gz: d7ca2ab62113b959d79f3102daa165cd7e843253
+  data.tar.gz: 9bac20ab47ac97242583a9075b5799a63ef07f52
 SHA512:
-  metadata.gz: 54b569b9721ad2a8e7340cc7570a739aaba51f31e24d8440bcfa295d50734cc8fb33050451d2433df58828bfd9344bae85507e96c21475854d60bf79eafa68ca
-  data.tar.gz: 1137ac538ab8e5bf9b2736100441946dd872154b8c3246236abd70e08d7192e7295c907823cf80f5d673abd1c1d13a4076db4965554e5c6c7a8a16f52ff5d1a0
+  metadata.gz: 4250720a52ba659add07a834c4a91f5d4d87165cf51bb4caeff78a93902173c2b2c55e244d42ad5f5fadeb90404743fbc63bd9b969f30241aa45ee19e8547fc7
+  data.tar.gz: 6b52d33d3dd7f0484248f6dbf3b0edc20f8509c81c63efcfae600eabe85b5efc95d29d7f97c57d5d2d0814d6b7f5a65bdb6d61d0d9fe161a0c20ab5e3cadda7c

data/lib/bricolage/job.rb

@@ -4,6 +4,7 @@ require 'bricolage/jobresult'
 require 'bricolage/parameters'
 require 'bricolage/variables'
 require 'bricolage/configloader'
+require 'bricolage/loglocator'
 require 'bricolage/exception'
 require 'fileutils'
 

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.24.2'
+  VERSION = '5.24.3'
 end

data/test/home/config/development/database.yml

@@ -1,113 +1,147 @@
-sql: &redshift_parameters
+#
+# Shared
+#
+
+redshift_base: &redshift_base
   type: psql
-  host: <%= ENV['RSHOST'] || 'dwh.ckpd.co' %>
-  port: <%= ENV['RSPORT'] || 5439 %>
+  host: <%= ENV['RSPROXYHOST'] ? 'localhost' : (ENV['RSHOST'] || 'cookpad-dwh-401.cbzwjkf60uat.ap-northeast-1.redshift.amazonaws.com') %>
+  port: <%= ENV['RSPROXYPORT'] || ENV['RSPORT'] || 5439 %>
   database: production
-  username: <%= ENV['RSUSER'] || 'tabemirudev' %>
+  username: <%= ENV['RSUSER'] || 'dwhdev' %>
   pgpass: <%= user_home_relative_path '.pgpass' %>
+
+s3_base: &s3_base
+  type: s3
+  endpoint: s3-ap-northeast-1.amazonaws.com
+  bucket: redshift-copy-buffer
+  prefix: development/tmp
+  s3cfg: <%= user_home_relative_path '.s3cfg' %>
+  #access_key_id: <%%= password 's3_access_key_id' %>
+  #secret_access_key: <%%= password 's3_secret_access_key' %>
+
+mysql_base: &mysql_base
+  type: mysql
+  username: work_readonly
+  password: <%= password 'mysql_shared_work_readonly' %>
   encoding: utf8
 
-#sql_dev:
-#  <<: *redshift_parameters
 #
-#sql_prod:
-#  <<: *redshift_parameters
-#  username: tabemiru_batch
-
-#sql_app:
-#  type: psql
-#  host: localhost   #cookpad-dwh-401.cbzwjkf60uat.ap-northeast-1.redshift.amazonaws.com
-#  port: 5445
-#  database: production
-#  username: tabemiru_batch
-#  pgpass: <%= user_home_relative_path '.pgpass' %>
-#  encoding: utf8
-
-sql_app2:
-  type: psql
+# Redis
+#
+redis:
+  type: redis
   host: localhost
-  port: 5446
-  database: production
-  username: tabemiru_batch
-  pgpass: <%= user_home_relative_path '.pgpass' %>
-  encoding: utf8
+  port: 6379
+  db: 0
 
-postgres:
-  type: psql
-  host: localhost
-  port: 5432
-  database: bricolage
-  username: bricolage
-  password: bricolage
-  encoding: utf8
-  sql_log_level: DEBUG
+#
+# Redshift
+#
+
+dwh_db: &dwh_db
+  <<: *redshift_base
+
+sql:
+  <<: *dwh_db
+
+figlog_db:
+  <<: *redshift_base
+
+#
+# S3
+#
 
 s3:
-  type: s3
-  endpoint: "s3-ap-northeast-1.amazonaws.com"
-  region: "ap-northeast-1"
-  bucket: tabemiru-data.ap-northeast-1
-  prefix: "dev"
-  #s3cfg: <%= user_home_relative_path '.s3cfg' %>
-  #access_key_id: "<%= password 'aws_access_key_id' %>"
-  #secret_access_key: "<%= password 'aws_secret_access_key' %>"
-  #master_symmetric_key: "12345678901234567890123456789012"
-  iam_role: "arn:aws:iam::789035092620:role/RedshiftDWHTeamS3Access"
-
-s3_log:
-  type: s3
-  endpoint: "s3-ap-northeast-1.amazonaws.com"
-  region: "ap-northeast-1"
-  bucket: "redshift-copy-buffer"
-  prefix: "development/bricolage/log"
+  <<: *s3_base
+  bucket: redshift-copy-buffer
 
-mysql:
-  type: mysql
-  host: 127.0.0.1
-  port: 9011
+figlog_s3:
+  <<: *s3_base
+  bucket: redshift-copy-buffer
+  prefix: development/figlog
+
+# Read only
+pv_log_s3:
+  <<: *s3_base
+  bucket: pv-log.ap-northeast-1
+  prefix: ""
+
+search_log_s3:
+  <<: *s3_base
+  bucket: redshift-copy-buffer
+  prefix: production/search_log
+
+dmp_s3:
+  <<: *s3_base
+  bucket: redshift-copy-buffer
+  prefix: production/dmp
+
+intage_s3:
+  <<: *s3_base
+  bucket: cookpad-intage
+  prefix: ""
+
+td_s3:
+  <<: *s3_base
+  bucket: redshift-copy-buffer
+  prefix: td
+
+#
+# MySQL
+#
+
+% is_local = (`uname -s`.strip == 'Darwin')
+
+cookpad_main_db: &cookpad_main_db
+  <<: *mysql_base
+  host: <%= is_local ? '127.0.0.1' : 'db-main-slave-free-001.vpc.ap-northeast-1.compute.internal' %>
+  port: <%= is_local ? 9011 : 3306 %>
+  database: main
+
+cookpad_bargain_db:
+  <<: *mysql_base
+  host: <%= is_local ? '127.0.0.1' : 'db-bargain-kuroko-101.vpc.ap-northeast-1.compute.internal' %>
+  port: <%= is_local ? 9131 : 3306 %>
   database: main
+
+cookpad_device_guest_db:
+  <<: *mysql_base
+  host: <%= is_local ? '127.0.0.1' : 'db-device-guest-102.vpc.ap-northeast-1.compute.internal' %>
+  port: <%= is_local ? 9142 : 3306 %>
+  database: device_guest
+
+cookpad_ad_db:
+  <<: *mysql_base
+  host: <%= is_local ? '127.0.0.1' : 'db-ad-slave-staging-001.vpc.ap-northeast-1.compute.internal' %>
+  port: <%= is_local ? 9153 : 3306 %>
+  database: ad
+  username: ckpd_readonly
+  password: <%= password 'mysql_shared_ckpd_readonly' %>
+
+cookpad_ad4_db:
+  <<: *mysql_base
+  host: <%= is_local ? '127.0.0.1' : 'db-ad4-master-001.vpc.ap-northeast-1.compute.internal' %>
+  port: <%= is_local ? 40205 : 3306 %>
+  database: ad4
   username: work_readonly
   password: <%= password 'mysql_shared_work_readonly' %>
-  encoding: utf8
 
-mysql_summary:
-  type: mysql
-  host: 127.0.0.1
-  port: 40201
+cookpad_summary_db:
+  <<: *mysql_base
+  host: <%= is_local ? '127.0.0.1' : 'db-summary-slave-mysql56-free-001.vpc.ap-northeast-1.compute.internal' %>
+  port: <%= is_local ? 40201 : 3306 %>
   database: summarized_data
   username: work_readonly
   password: <%= password 'mysql_shared_work_readonly' %>
-  encoding: utf8
 
-sqs_preproc:
-  type: sqs
-  url: "https://sqs.ap-northeast-1.amazonaws.com/789035092620/bricolage-preproc-dev"
-  max_number_of_messages: 10
-  visibility_timeout: 60
-
-sqs_dispatch:
-  type: sqs
-  url: "https://sqs.ap-northeast-1.amazonaws.com/789035092620/log-stream-dev"
-  max_number_of_messages: 10
-  visibility_timeout: 60
-
-sqs_task:
-  type: sqs
-  url: "https://sqs.ap-northeast-1.amazonaws.com/789035092620/bricolage-load-tasks-dev"
-  max_number_of_messages: 1
-  visibility_timeout: 180
-
-sns:
-  type: sns
-  region: "ap-northeast-1"
-  topic_arn: "arn:aws:sns:ap-northeast-1:789035092620:dwh-service-notification"
+mysql:
+  <<: *cookpad_main_db
+
+#
+# TD
+#
 
 td:
   type: td
   database: logs
   apikey: <%= password 'td_tabemiru' %>
-
-td_search_log:
-  type: td
-  database: search_log
-  apikey: <%= password 'td_tabemiru' %>

data/test/home/config/development/password.yml

@@ -1,6 +1,3 @@
-td_tabemiru: "121/c1376bb5e3972fc4a4efc942c1df100f7bc156e0"
-mysql_shared_work_readonly: "s0uL/M@9r1"
-mysql_shared_ckpd_readonly: "no3lnar;"
-mysql_local_work_readonly: "ckpdtest"
-aws_access_key_id: "AKIAI5R5BL2W73PR3FXQ"
-aws_secret_access_key: "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"
+mysql_shared_work_readonly: s0uL/M@9r1
+mysql_shared_ckpd_readonly: hoge
+td_tabemiru: 121/e9d758221f5ad5e2747337d1d79f974389e6cc0a

data/test/home/config/development/variable.yml

@@ -1,7 +1,21 @@
-data_dir: /tmp/bricolagedev
-test_schema: aamine
-test_group: aamine
-activity_schema: aamine
+% data_date = Date.today - 1
+data_date: <%= data_date.strftime("%Y-%m-%d") %>
+data_date_ymd: <%= data_date.strftime("%Y%m%d") %>
+data_date_endx: <%= Date.today.strftime("%Y-%m-%d") %>
 
-defaults:
-    analyze: true
+data_dir: <%= ENV['TMPDIR'] || '/tmp' %>
+shared_data_dir: "/tmp"
+trigger_dir: "$shared_data_dir/bricolage.trigger"
+
+% dwh_dev_schema = ENV['RSUSER'] || 'dwh_dev'
+test_schema: <%= dwh_dev_schema %>
+
+secure_activity_schema: <%= dwh_dev_schema %>
+secure_cookpad_schema: <%= dwh_dev_schema %>
+secure_dwh_schema: <%= dwh_dev_schema %>
+
+tabemiru_schema: tabemiru
+
+test_group: "tabemirudev"
+
+ad_mirror_db: dwh_dev

data/test/home/subsys/streaming_load.ct

@@ -0,0 +1,4 @@
+create table streaming_load (
+    col1 int
+    , col2 varchar(32)
+)

data/test/home/subsys/streaming_load.job

@@ -0,0 +1,21 @@
+class: streaming_load
+
+s3-ds: s3
+keep-ctl: true
+ctl-prefix: /strload/ctl/
+queue-path: /strload/queue/
+persistent-path: /strload/year=%Y/month=%m/day=%d/
+file-name: "%Y%m%d_%H%M_%Q_%*.gz"
+
+redshift-ds: dwh_db
+dest-table: $test_schema.streaming_load
+work-table: $test_schema.streaming_load_wk
+log-table: $test_schema.streaming_load_l
+load-options:
+  json: "auto"
+  gzip: true
+  maxerror: 0
+  timeformat: "auto"
+  truncatecolumns: true
+  acceptinvchars: " "
+skip-work: true

data/test/home/subsys/streaming_load_wk.ct

@@ -0,0 +1,4 @@
+create table streaming_load_wk (
+    col1 int
+    , col2 varchar(32)
+)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.24.2
+  version: 5.24.3
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-12 00:00:00.000000000 Z
+date: 2017-08-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -200,90 +200,15 @@ files:
 - libexec/sqldump.Linux
 - test/all.rb
 - test/home/Gemfile
-- test/home/Gemfile.lock
-- test/home/config/bricolage.yml
 - test/home/config/development/database.yml
 - test/home/config/development/password.yml
 - test/home/config/development/variable.yml
-- test/home/config/streamingload.yml
 - test/home/data/20141002-1355_00.txt
 - test/home/data/20141002-1355_01.txt
 - test/home/data/20141002-1355_02.txt
 - test/home/data/test.txt
 - test/home/jobnet-test.rb
-- test/home/log/20170117/subsys::job1/20170117_202014082/subsys-job1.log
-- test/home/log/20170117/subsys::job1/20170117_202334852/subsys-job1.log
-- test/home/log/20170117/subsys::job1/20170117_202405056/subsys-job1.log
-- test/home/log/20170117/subsys::migrate/20170117_202548409/subsys-migrate.log
-- test/home/log/20170117/subsys::migrate/20170117_202636115/subsys-migrate.log
-- test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job1.log
-- test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job2.log
-- test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job3.log
-- test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job4.log
-- test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job1.log
-- test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job2.log
-- test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job3.log
-- test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job4.log
-- test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job1.log
-- test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job2.log
-- test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job3.log
-- test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job4.log
-- test/home/log/20170629/subsys::job1/20170629_155543985/subsys-job1.log
-- test/home/log/20170629/subsys::net1/20170629_154308811/subsys-job1.log
-- test/home/log/20170629/subsys::net1/20170629_154308811/subsys-job2.log
-- test/home/log/20170629/subsys::net1/20170629_154308811/subsys-job3.log
-- test/home/log/20170629/subsys::net1/20170629_154308811/subsys-job4.log
-- test/home/log/20170629/subsys::net1/20170629_155330208/subsys-job1.log
-- test/home/log/20170629/subsys::net1/20170629_155330208/subsys-job2.log
-- test/home/log/20170629/subsys::net1/20170629_155330208/subsys-job3.log
-- test/home/log/20170629/subsys::net1/20170629_155330208/subsys-job4.log
-- test/home/log/20170630/subsys::net1/20170630_145523511/subsys-job1.log
-- test/home/log/20170630/subsys::net1/20170630_145523511/subsys-job2.log
-- test/home/log/20170630/subsys::net1/20170630_145523511/subsys-job3.log
-- test/home/log/20170630/subsys::net1/20170630_145523511/subsys-job4.log
-- test/home/log/20170630/subsys::net1/20170630_145545097/subsys-job1.log
-- test/home/log/20170630/subsys::net1/20170630_145545097/subsys-job2.log
-- test/home/log/20170630/subsys::net1/20170630_145545097/subsys-job3.log
-- test/home/log/20170630/subsys::net1/20170630_145545097/subsys-job4.log
-- test/home/log/20170630/subsys::net1/20170630_145615702/subsys-job1.log
-- test/home/log/20170630/subsys::net1/20170630_145615702/subsys-job2.log
-- test/home/log/20170630/subsys::net1/20170630_145615702/subsys-job3.log
-- test/home/log/20170630/subsys::net1/20170630_145615702/subsys-job4.log
-- test/home/log/20170630/subsys::net1/20170630_153437221/subsys-job1.log
-- test/home/log/20170630/subsys::net1/20170630_153507519/subsys-job1.log
-- test/home/log/20170630/subsys::net1/20170630_153507519/subsys-job2.log
-- test/home/log/20170630/subsys::net1/20170630_153507519/subsys-job3.log
-- test/home/log/20170630/subsys::net1/20170630_153507519/subsys-job4.log
-- test/home/log/20170704/subsys::insert.sql.job/20170704_163033119/subsys-insert.log
-- test/home/log/20170704/subsys::insert.sql.job/20170704_172410576/subsys-insert.log
-- test/home/log/20170704/subsys::insert.sql.job/20170704_173130175/subsys-insert.log
-- test/home/log/20170704/subsys::insert.sql.job/20170704_173201376/subsys-insert.log
-- test/home/log/20170704/subsys::insert.sql/20170704_164143661/subsys-insert.sql.log
-- test/home/log/20170704/subsys::insert/20170704_164335210/subsys-insert.log
-- test/home/log/20170704/subsys::insert/20170704_164344251/subsys-insert.log
-- test/home/log/20170704/subsys::insert/20170704_164723299/subsys-insert.log
-- test/home/log/20170704/subsys::net1/20170704_162457694/subsys-job1.log
-- test/home/log/20170704/subsys::net1/20170704_162544282/subsys-job1.log
-- test/home/log/20170704/subsys::net1/20170704_162544282/subsys-job2.log
-- test/home/log/20170704/subsys::net1/20170704_162544282/subsys-job3.log
-- test/home/log/20170704/subsys::net1/20170704_162544282/subsys-job4.log
-- test/home/log/20170706/subsys::net1/20170706_201157129/subsys-job1.log
-- test/home/log/20170706/subsys::net1/20170706_201157129/subsys-job2.log
-- test/home/log/20170706/subsys::net1/20170706_201157129/subsys-job3.log
-- test/home/log/20170706/subsys::net1/20170706_201157129/subsys-job4.log
-- test/home/log/20170707/subsys::insert/20170707_020050817/subsys-insert.log
-- test/home/log/20170707/subsys::insert/20170707_020050817/subsys-insert.log.status
-- test/home/log/20170707/subsys::job1/20170707_020542902/subsys-job1.log
-- test/home/log/20170707/subsys::net1/20170707_012252058/subsys-job1.log
-- test/home/log/20170707/subsys::net1/20170707_012252058/subsys-job2.log
-- test/home/log/20170707/subsys::net1/20170707_012252058/subsys-job3.log
-- test/home/log/20170707/subsys::net1/20170707_012252058/subsys-job4.log
-- test/home/log/20170707/subsys::net1/20170707_020039222/subsys-job1.log
-- test/home/log/20170707/subsys::net1/20170707_020039222/subsys-job2.log
-- test/home/log/20170707/subsys::net1/20170707_020039222/subsys-job3.log
-- test/home/log/20170707/subsys::net1/20170707_020039222/subsys-job4.log
-- test/home/revert.sh
-- test/home/s
+- test/home/subsys/20170214_1225_0_strload.data.json.gz
 - test/home/subsys/d.ct
 - test/home/subsys/insert.sql.job
 - test/home/subsys/item_pv_acc.ct
@@ -312,6 +237,9 @@ files:
 - test/home/subsys/separated.job
 - test/home/subsys/separated.sql
 - test/home/subsys/some_view-cv.sql.job
+- test/home/subsys/streaming_load.ct
+- test/home/subsys/streaming_load.job
+- test/home/subsys/streaming_load_wk.ct
 - test/home/subsys/unified.jobnet
 - test/home/subsys/unified.sql.job
 - test/home/subsys/users.ct

data/test/home/Gemfile.lock

@@ -1,41 +0,0 @@
-PATH
-  remote: ../..
-  specs:
-    bricolage (5.24.2)
-      aws-sdk (~> 2)
-      mysql2
-      pg (~> 0.18.0)
-      redis (>= 3.0.0)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    aws-sdk (2.10.9)
-      aws-sdk-resources (= 2.10.9)
-    aws-sdk-core (2.10.9)
-      aws-sigv4 (~> 1.0)
-      jmespath (~> 1.0)
-    aws-sdk-resources (2.10.9)
-      aws-sdk-core (= 2.10.9)
-    aws-sigv4 (1.0.0)
-    coderay (1.1.1)
-    jmespath (1.3.1)
-    method_source (0.8.2)
-    mysql2 (0.4.8)
-    pg (0.18.4)
-    pry (0.10.4)
-      coderay (~> 1.1.0)
-      method_source (~> 0.8.1)
-      slop (~> 3.4)
-    redis (3.3.3)
-    slop (3.6.0)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bricolage!
-  pry
-
-BUNDLED WITH
-   1.14.3

data/test/home/config/bricolage.yml

@@ -1 +0,0 @@
-enable-queue: true

data/test/home/config/streamingload.yml

@@ -1,8 +0,0 @@
-redshift-ds: sql
-event-queue-ds: sqs_strload_event
-task-queue-ds: sqs_strload_task
-url_patterns:
-  -
-    url: "s3://redshift-copy-buffer/development/log_stream/\\d{8}_\\d{4}_\\d+.*\\.gz"
-    schema: "aamine"
-    table: "pv_log"

data/test/home/log/20170117/subsys::job1/20170117_202014082/subsys-job1.log

@@ -1,3 +0,0 @@
-2017-01-17 20:20:14 +0900: INFO: development environment
-2017-01-17 20:20:14 +0900: INFO: 0.00 secs
-2017-01-17 20:20:14 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::job1/20170117_202334852/subsys-job1.log

@@ -1,3 +0,0 @@
-2017-01-17 20:23:34 +0900: INFO: development environment
-2017-01-17 20:23:34 +0900: INFO: 0.00 secs
-2017-01-17 20:23:34 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::job1/20170117_202405056/subsys-job1.log

@@ -1,3 +0,0 @@
-2017-01-17 20:24:05 +0900: INFO: development environment
-2017-01-17 20:24:05 +0900: INFO: 0.00 secs
-2017-01-17 20:24:05 +0900: INFO: SUCCESS

data/test/home/log/20170117/subsys::migrate/20170117_202548409/subsys-migrate.log

@@ -1,80 +0,0 @@
-2017-01-17 20:25:48 +0900: INFO: development environment
-2017-01-17 20:25:48 +0900: INFO: [CMD] "/Users/minero-aoki/c/bricolage/libexec/sqldump" "--json" "127.0.0.1" "9011" "work_readonly" "main" "select * from main.search_backends;" | "gzip" "{:out=>"/tmp/bricolagedev/search_backends.json.gz"}"
-2017-01-17 20:25:48.540819891 +0900 JST: [SQL] select * from main.search_backends;
-2017-01-17 20:25:48.597928759 +0900 JST: query returned
-2017-01-17 20:25:48.598038993 +0900 JST: Total 11 records
-2017-01-17 20:25:48 +0900: INFO: aws s3 cp /tmp/bricolagedev/search_backends.json.gz s3://tabemiru-data.ap-northeast-1/dev/search_backends/search_backends.json.gz
-2017-01-17 20:25:51 +0900: INFO: 3.28 secs
-2017-01-17 20:25:51 +0900: ERROR: Aws::Errors::MissingCredentialsError: unable to sign request without credentials set
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/request_signer.rb:104:in `require_credentials'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_request_signer.rb:14:in `call'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/xml/error_handler.rb:8:in `call'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/helpful_socket_errors.rb:10:in `call'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_request_signer.rb:65:in `call'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_redirects.rb:15:in `call'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/retry_errors.rb:87:in `call'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_dualstack.rb:32:in `call'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_accelerate.rb:49:in `call'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_md5s.rb:31:in `call'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_expect_100_continue.rb:21:in `call'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_bucket_name_restrictions.rb:12:in `call'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_bucket_dns.rb:31:in `call'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/rest/handler.rb:7:in `call'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/user_agent.rb:12:in `call'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/seahorse/client/plugins/endpoint.rb:41:in `call'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/param_validator.rb:21:in `call'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/seahorse/client/plugins/raise_response_errors.rb:14:in `call'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_sse_cpk.rb:19:in `call'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_dualstack.rb:24:in `call'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_accelerate.rb:34:in `call'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/idempotency_token.rb:18:in `call'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/param_converter.rb:20:in `call'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/seahorse/client/plugins/response_target.rb:21:in `call'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/seahorse/client/request.rb:70:in `send_request'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/seahorse/client/base.rb:207:in `block (2 levels) in define_operation_methods'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-resources-2.6.36/lib/aws-sdk-resources/services/s3/file_uploader.rb:42:in `block in put_object'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-resources-2.6.36/lib/aws-sdk-resources/services/s3/file_uploader.rb:49:in `open_file'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-resources-2.6.36/lib/aws-sdk-resources/services/s3/file_uploader.rb:41:in `put_object'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-resources-2.6.36/lib/aws-sdk-resources/services/s3/file_uploader.rb:34:in `upload'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-resources-2.6.36/lib/aws-sdk-resources/services/s3/object.rb:251:in `upload_file'
-	/Users/minero-aoki/c/bricolage/lib/bricolage/s3datasource.rb:188:in `block in run'
-	/Users/minero-aoki/c/bricolage/lib/bricolage/s3datasource.rb:165:in `block in each_src_dest'
-	/Users/minero-aoki/c/bricolage/lib/bricolage/s3datasource.rb:163:in `each'
-	/Users/minero-aoki/c/bricolage/lib/bricolage/s3datasource.rb:163:in `each_src_dest'
-	/Users/minero-aoki/c/bricolage/lib/bricolage/s3datasource.rb:186:in `run'
-	/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:82:in `block (2 levels) in run'
-	/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:81:in `each'
-	/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:81:in `block in run'
-	/Users/minero-aoki/c/bricolage/lib/bricolage/datasource.rb:136:in `open'
-	/Users/minero-aoki/c/bricolage/lib/bricolage/datasource.rb:140:in `open_for_batch'
-	/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:80:in `run'
-	/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:47:in `block in run'
-	/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:46:in `each'
-	/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:46:in `run'
-	/Users/minero-aoki/c/bricolage/lib/bricolage/job.rb:143:in `block in execute'
-	/Users/minero-aoki/c/bricolage/lib/bricolage/logger.rb:44:in `with_elapsed_time'
-	/Users/minero-aoki/c/bricolage/lib/bricolage/job.rb:142:in `execute'
-	/Users/minero-aoki/c/bricolage/lib/bricolage/application.rb:68:in `block in main'
-	/Users/minero-aoki/c/bricolage/lib/bricolage/application.rb:97:in `redirect_log_to_file'
-	/Users/minero-aoki/c/bricolage/lib/bricolage/application.rb:67:in `main'
-	/Users/minero-aoki/c/bricolage/lib/bricolage/application.rb:28:in `main'
-	/Users/minero-aoki/c/bricolage/bin/bricolage:6:in `<top (required)>'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/bin/bricolage:23:in `load'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/bin/bricolage:23:in `<top (required)>'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli/exec.rb:74:in `load'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli/exec.rb:74:in `kernel_load'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli/exec.rb:27:in `run'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli.rb:332:in `exec'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/vendor/thor/lib/thor/command.rb:27:in `run'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/vendor/thor/lib/thor/invocation.rb:126:in `invoke_command'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/vendor/thor/lib/thor.rb:359:in `dispatch'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli.rb:20:in `dispatch'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/vendor/thor/lib/thor/base.rb:440:in `start'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli.rb:11:in `start'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/exe/bundle:34:in `block in <top (required)>'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/friendly_errors.rb:100:in `with_friendly_errors'
-	/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/exe/bundle:26:in `<top (required)>'
-	/Users/minero-aoki/.rbenv/versions/2.3/bin/bundle:23:in `load'
-	/Users/minero-aoki/.rbenv/versions/2.3/bin/bundle:23:in `<main>'
-
-2017-01-17 20:25:51 +0900: ERROR: error: Aws::Errors::MissingCredentialsError: unable to sign request without credentials set