bricolage 5.23.3 → 5.24.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/bricolage/application.rb +160 -37
- data/lib/bricolage/commandlineapplication.rb +0 -0
- data/lib/bricolage/commandutils.rb +0 -20
- data/lib/bricolage/context.rb +19 -4
- data/lib/bricolage/job.rb +38 -38
- data/lib/bricolage/jobfile.rb +2 -1
- data/lib/bricolage/jobnet.rb +21 -3
- data/lib/bricolage/jobnetrunner.rb +102 -62
- data/lib/bricolage/logfilepath.rb +2 -2
- data/lib/bricolage/loglocator.rb +71 -0
- data/lib/bricolage/loglocatorbuilder.rb +29 -0
- data/lib/bricolage/psqldatasource.rb +1 -1
- data/lib/bricolage/s3writer.rb +22 -0
- data/lib/bricolage/version.rb +1 -1
- data/test/home/Gemfile.lock +41 -0
- data/test/home/config/bricolage.yml +3 -0
- data/test/home/config/development/database.yml +113 -0
- data/test/home/config/development/password.yml +6 -0
- data/test/home/config/development/variable.yml +7 -0
- data/test/home/config/streamingload.yml +8 -0
- data/test/home/log/20170117/subsys::job1/20170117_202014082/subsys-job1.log +3 -0
- data/test/home/log/20170117/subsys::job1/20170117_202334852/subsys-job1.log +3 -0
- data/test/home/log/20170117/subsys::job1/20170117_202405056/subsys-job1.log +3 -0
- data/test/home/log/20170117/subsys::migrate/20170117_202548409/subsys-migrate.log +80 -0
- data/test/home/log/20170117/subsys::migrate/20170117_202636115/subsys-migrate.log +79 -0
- data/test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job1.log +3 -0
- data/test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job2.log +3 -0
- data/test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job3.log +3 -0
- data/test/home/log/20170117/subsys::net1/20170117_203022461/subsys-job4.log +3 -0
- data/test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job1.log +3 -0
- data/test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job2.log +3 -0
- data/test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job3.log +3 -0
- data/test/home/log/20170117/subsys::net1/20170117_203102130/subsys-job4.log +3 -0
- data/test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job1.log +3 -0
- data/test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job2.log +3 -0
- data/test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job3.log +3 -0
- data/test/home/log/20170117/subsys::net1/20170117_203232033/subsys-job4.log +3 -0
- data/test/home/log/20170629/subsys::job1/20170629_155543985/subsys-job1.log +3 -0
- data/test/home/log/20170629/subsys::net1/20170629_154308811/subsys-job1.log +3 -0
- data/test/home/log/20170629/subsys::net1/20170629_154308811/subsys-job2.log +3 -0
- data/test/home/log/20170629/subsys::net1/20170629_154308811/subsys-job3.log +3 -0
- data/test/home/log/20170629/subsys::net1/20170629_154308811/subsys-job4.log +3 -0
- data/test/home/log/20170629/subsys::net1/20170629_155330208/subsys-job1.log +3 -0
- data/test/home/log/20170629/subsys::net1/20170629_155330208/subsys-job2.log +3 -0
- data/test/home/log/20170629/subsys::net1/20170629_155330208/subsys-job3.log +3 -0
- data/test/home/log/20170629/subsys::net1/20170629_155330208/subsys-job4.log +3 -0
- data/test/home/log/20170630/subsys::net1/20170630_145523511/subsys-job1.log +3 -0
- data/test/home/log/20170630/subsys::net1/20170630_145523511/subsys-job2.log +3 -0
- data/test/home/log/20170630/subsys::net1/20170630_145523511/subsys-job3.log +3 -0
- data/test/home/log/20170630/subsys::net1/20170630_145523511/subsys-job4.log +3 -0
- data/test/home/log/20170630/subsys::net1/20170630_145545097/subsys-job1.log +3 -0
- data/test/home/log/20170630/subsys::net1/20170630_145545097/subsys-job2.log +3 -0
- data/test/home/log/20170630/subsys::net1/20170630_145545097/subsys-job3.log +3 -0
- data/test/home/log/20170630/subsys::net1/20170630_145545097/subsys-job4.log +3 -0
- data/test/home/log/20170630/subsys::net1/20170630_145615702/subsys-job1.log +3 -0
- data/test/home/log/20170630/subsys::net1/20170630_145615702/subsys-job2.log +3 -0
- data/test/home/log/20170630/subsys::net1/20170630_145615702/subsys-job3.log +3 -0
- data/test/home/log/20170630/subsys::net1/20170630_145615702/subsys-job4.log +3 -0
- data/test/home/log/20170630/subsys::net1/20170630_153437221/subsys-job1.log +3 -0
- data/test/home/log/20170630/subsys::net1/20170630_153507519/subsys-job1.log +3 -0
- data/test/home/log/20170630/subsys::net1/20170630_153507519/subsys-job2.log +3 -0
- data/test/home/log/20170630/subsys::net1/20170630_153507519/subsys-job3.log +3 -0
- data/test/home/log/20170630/subsys::net1/20170630_153507519/subsys-job4.log +3 -0
- data/test/home/log/20170704/subsys::insert.sql.job/20170704_163033119/subsys-insert.log +14 -0
- data/test/home/log/20170704/subsys::insert.sql.job/20170704_172410576/subsys-insert.log +14 -0
- data/test/home/log/20170704/subsys::insert.sql.job/20170704_173130175/subsys-insert.log +14 -0
- data/test/home/log/20170704/subsys::insert.sql.job/20170704_173201376/subsys-insert.log +14 -0
- data/test/home/log/20170704/subsys::insert.sql/20170704_164143661/subsys-insert.sql.log +14 -0
- data/test/home/log/20170704/subsys::insert/20170704_164335210/subsys-insert.log +14 -0
- data/test/home/log/20170704/subsys::insert/20170704_164344251/subsys-insert.log +14 -0
- data/test/home/log/20170704/subsys::insert/20170704_164723299/subsys-insert.log +14 -0
- data/test/home/log/20170704/subsys::net1/20170704_162457694/subsys-job1.log +3 -0
- data/test/home/log/20170704/subsys::net1/20170704_162544282/subsys-job1.log +3 -0
- data/test/home/log/20170704/subsys::net1/20170704_162544282/subsys-job2.log +3 -0
- data/test/home/log/20170704/subsys::net1/20170704_162544282/subsys-job3.log +3 -0
- data/test/home/log/20170704/subsys::net1/20170704_162544282/subsys-job4.log +3 -0
- data/test/home/log/20170706/subsys::net1/20170706_201157129/subsys-job1.log +3 -0
- data/test/home/log/20170706/subsys::net1/20170706_201157129/subsys-job2.log +3 -0
- data/test/home/log/20170706/subsys::net1/20170706_201157129/subsys-job3.log +3 -0
- data/test/home/log/20170706/subsys::net1/20170706_201157129/subsys-job4.log +3 -0
- data/test/home/log/20170707/subsys::insert/20170707_020050817/subsys-insert.log +51 -0
- data/test/home/log/20170707/subsys::insert/20170707_020050817/subsys-insert.log.status +1 -0
- data/test/home/log/20170707/subsys::job1/20170707_020542902/subsys-job1.log +3 -0
- data/test/home/log/20170707/subsys::net1/20170707_012252058/subsys-job1.log +3 -0
- data/test/home/log/20170707/subsys::net1/20170707_012252058/subsys-job2.log +3 -0
- data/test/home/log/20170707/subsys::net1/20170707_012252058/subsys-job3.log +3 -0
- data/test/home/log/20170707/subsys::net1/20170707_012252058/subsys-job4.log +3 -0
- data/test/home/log/20170707/subsys::net1/20170707_020039222/subsys-job1.log +3 -0
- data/test/home/log/20170707/subsys::net1/20170707_020039222/subsys-job2.log +3 -0
- data/test/home/log/20170707/subsys::net1/20170707_020039222/subsys-job3.log +3 -0
- data/test/home/log/20170707/subsys::net1/20170707_020039222/subsys-job4.log +3 -0
- data/test/home/revert.sh +7 -0
- data/test/home/s +1 -0
- metadata +125 -43
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
module Bricolage
|
|
2
|
+
class S3Writer
|
|
3
|
+
def initialize(ds, key)
|
|
4
|
+
@ds = ds
|
|
5
|
+
@key = key
|
|
6
|
+
end
|
|
7
|
+
|
|
8
|
+
def url
|
|
9
|
+
@url ||= @ds.url(@key)
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
attr_reader :key
|
|
13
|
+
|
|
14
|
+
def object
|
|
15
|
+
@object ||= @ds.object(@key)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def upload(path)
|
|
19
|
+
object.upload_file(path)
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
data/lib/bricolage/version.rb
CHANGED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
PATH
|
|
2
|
+
remote: ../..
|
|
3
|
+
specs:
|
|
4
|
+
bricolage (5.24.0)
|
|
5
|
+
aws-sdk (~> 2)
|
|
6
|
+
mysql2
|
|
7
|
+
pg (~> 0.18.0)
|
|
8
|
+
redis (>= 3.0.0)
|
|
9
|
+
|
|
10
|
+
GEM
|
|
11
|
+
remote: https://rubygems.org/
|
|
12
|
+
specs:
|
|
13
|
+
aws-sdk (2.10.2)
|
|
14
|
+
aws-sdk-resources (= 2.10.2)
|
|
15
|
+
aws-sdk-core (2.10.2)
|
|
16
|
+
aws-sigv4 (~> 1.0)
|
|
17
|
+
jmespath (~> 1.0)
|
|
18
|
+
aws-sdk-resources (2.10.2)
|
|
19
|
+
aws-sdk-core (= 2.10.2)
|
|
20
|
+
aws-sigv4 (1.0.0)
|
|
21
|
+
coderay (1.1.1)
|
|
22
|
+
jmespath (1.3.1)
|
|
23
|
+
method_source (0.8.2)
|
|
24
|
+
mysql2 (0.4.6)
|
|
25
|
+
pg (0.18.4)
|
|
26
|
+
pry (0.10.4)
|
|
27
|
+
coderay (~> 1.1.0)
|
|
28
|
+
method_source (~> 0.8.1)
|
|
29
|
+
slop (~> 3.4)
|
|
30
|
+
redis (3.3.3)
|
|
31
|
+
slop (3.6.0)
|
|
32
|
+
|
|
33
|
+
PLATFORMS
|
|
34
|
+
ruby
|
|
35
|
+
|
|
36
|
+
DEPENDENCIES
|
|
37
|
+
bricolage!
|
|
38
|
+
pry
|
|
39
|
+
|
|
40
|
+
BUNDLED WITH
|
|
41
|
+
1.14.3
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
sql: &redshift_parameters
|
|
2
|
+
type: psql
|
|
3
|
+
host: <%= ENV['RSHOST'] || 'dwh.ckpd.co' %>
|
|
4
|
+
port: <%= ENV['RSPORT'] || 5439 %>
|
|
5
|
+
database: production
|
|
6
|
+
username: <%= ENV['RSUSER'] || 'tabemirudev' %>
|
|
7
|
+
pgpass: <%= user_home_relative_path '.pgpass' %>
|
|
8
|
+
encoding: utf8
|
|
9
|
+
|
|
10
|
+
#sql_dev:
|
|
11
|
+
# <<: *redshift_parameters
|
|
12
|
+
#
|
|
13
|
+
#sql_prod:
|
|
14
|
+
# <<: *redshift_parameters
|
|
15
|
+
# username: tabemiru_batch
|
|
16
|
+
|
|
17
|
+
#sql_app:
|
|
18
|
+
# type: psql
|
|
19
|
+
# host: localhost #cookpad-dwh-401.cbzwjkf60uat.ap-northeast-1.redshift.amazonaws.com
|
|
20
|
+
# port: 5445
|
|
21
|
+
# database: production
|
|
22
|
+
# username: tabemiru_batch
|
|
23
|
+
# pgpass: <%= user_home_relative_path '.pgpass' %>
|
|
24
|
+
# encoding: utf8
|
|
25
|
+
|
|
26
|
+
sql_app2:
|
|
27
|
+
type: psql
|
|
28
|
+
host: localhost
|
|
29
|
+
port: 5446
|
|
30
|
+
database: production
|
|
31
|
+
username: tabemiru_batch
|
|
32
|
+
pgpass: <%= user_home_relative_path '.pgpass' %>
|
|
33
|
+
encoding: utf8
|
|
34
|
+
|
|
35
|
+
postgres:
|
|
36
|
+
type: psql
|
|
37
|
+
host: localhost
|
|
38
|
+
port: 5432
|
|
39
|
+
database: bricolage
|
|
40
|
+
username: bricolage
|
|
41
|
+
password: bricolage
|
|
42
|
+
encoding: utf8
|
|
43
|
+
sql_log_level: DEBUG
|
|
44
|
+
|
|
45
|
+
s3:
|
|
46
|
+
type: s3
|
|
47
|
+
endpoint: "s3-ap-northeast-1.amazonaws.com"
|
|
48
|
+
region: "ap-northeast-1"
|
|
49
|
+
bucket: tabemiru-data.ap-northeast-1
|
|
50
|
+
prefix: "dev"
|
|
51
|
+
#s3cfg: <%= user_home_relative_path '.s3cfg' %>
|
|
52
|
+
#access_key_id: "<%= password 'aws_access_key_id' %>"
|
|
53
|
+
#secret_access_key: "<%= password 'aws_secret_access_key' %>"
|
|
54
|
+
#master_symmetric_key: "12345678901234567890123456789012"
|
|
55
|
+
iam_role: "arn:aws:iam::789035092620:role/RedshiftDWHTeamS3Access"
|
|
56
|
+
|
|
57
|
+
s3_log:
|
|
58
|
+
type: s3
|
|
59
|
+
endpoint: "s3-ap-northeast-1.amazonaws.com"
|
|
60
|
+
region: "ap-northeast-1"
|
|
61
|
+
bucket: "redshift-copy-buffer"
|
|
62
|
+
prefix: "development/bricolage/log"
|
|
63
|
+
|
|
64
|
+
mysql:
|
|
65
|
+
type: mysql
|
|
66
|
+
host: 127.0.0.1
|
|
67
|
+
port: 9011
|
|
68
|
+
database: main
|
|
69
|
+
username: work_readonly
|
|
70
|
+
password: <%= password 'mysql_shared_work_readonly' %>
|
|
71
|
+
encoding: utf8
|
|
72
|
+
|
|
73
|
+
mysql_summary:
|
|
74
|
+
type: mysql
|
|
75
|
+
host: 127.0.0.1
|
|
76
|
+
port: 40201
|
|
77
|
+
database: summarized_data
|
|
78
|
+
username: work_readonly
|
|
79
|
+
password: <%= password 'mysql_shared_work_readonly' %>
|
|
80
|
+
encoding: utf8
|
|
81
|
+
|
|
82
|
+
sqs_preproc:
|
|
83
|
+
type: sqs
|
|
84
|
+
url: "https://sqs.ap-northeast-1.amazonaws.com/789035092620/bricolage-preproc-dev"
|
|
85
|
+
max_number_of_messages: 10
|
|
86
|
+
visibility_timeout: 60
|
|
87
|
+
|
|
88
|
+
sqs_dispatch:
|
|
89
|
+
type: sqs
|
|
90
|
+
url: "https://sqs.ap-northeast-1.amazonaws.com/789035092620/log-stream-dev"
|
|
91
|
+
max_number_of_messages: 10
|
|
92
|
+
visibility_timeout: 60
|
|
93
|
+
|
|
94
|
+
sqs_task:
|
|
95
|
+
type: sqs
|
|
96
|
+
url: "https://sqs.ap-northeast-1.amazonaws.com/789035092620/bricolage-load-tasks-dev"
|
|
97
|
+
max_number_of_messages: 1
|
|
98
|
+
visibility_timeout: 180
|
|
99
|
+
|
|
100
|
+
sns:
|
|
101
|
+
type: sns
|
|
102
|
+
region: "ap-northeast-1"
|
|
103
|
+
topic_arn: "arn:aws:sns:ap-northeast-1:789035092620:dwh-service-notification"
|
|
104
|
+
|
|
105
|
+
td:
|
|
106
|
+
type: td
|
|
107
|
+
database: logs
|
|
108
|
+
apikey: <%= password 'td_tabemiru' %>
|
|
109
|
+
|
|
110
|
+
td_search_log:
|
|
111
|
+
type: td
|
|
112
|
+
database: search_log
|
|
113
|
+
apikey: <%= password 'td_tabemiru' %>
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
td_tabemiru: "121/c1376bb5e3972fc4a4efc942c1df100f7bc156e0"
|
|
2
|
+
mysql_shared_work_readonly: "s0uL/M@9r1"
|
|
3
|
+
mysql_shared_ckpd_readonly: "no3lnar;"
|
|
4
|
+
mysql_local_work_readonly: "ckpdtest"
|
|
5
|
+
aws_access_key_id: "AKIAI5R5BL2W73PR3FXQ"
|
|
6
|
+
aws_secret_access_key: "Mj9/cwWuruUO0qHsYFI2911G31PQaIj3+eJTBxyO"
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
2017-01-17 20:25:48 +0900: INFO: development environment
|
|
2
|
+
2017-01-17 20:25:48 +0900: INFO: [CMD] "/Users/minero-aoki/c/bricolage/libexec/sqldump" "--json" "127.0.0.1" "9011" "work_readonly" "main" "select * from main.search_backends;" | "gzip" "{:out=>"/tmp/bricolagedev/search_backends.json.gz"}"
|
|
3
|
+
2017-01-17 20:25:48.540819891 +0900 JST: [SQL] select * from main.search_backends;
|
|
4
|
+
2017-01-17 20:25:48.597928759 +0900 JST: query returned
|
|
5
|
+
2017-01-17 20:25:48.598038993 +0900 JST: Total 11 records
|
|
6
|
+
2017-01-17 20:25:48 +0900: INFO: aws s3 cp /tmp/bricolagedev/search_backends.json.gz s3://tabemiru-data.ap-northeast-1/dev/search_backends/search_backends.json.gz
|
|
7
|
+
2017-01-17 20:25:51 +0900: INFO: 3.28 secs
|
|
8
|
+
2017-01-17 20:25:51 +0900: ERROR: Aws::Errors::MissingCredentialsError: unable to sign request without credentials set
|
|
9
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/request_signer.rb:104:in `require_credentials'
|
|
10
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_request_signer.rb:14:in `call'
|
|
11
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/xml/error_handler.rb:8:in `call'
|
|
12
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/helpful_socket_errors.rb:10:in `call'
|
|
13
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_request_signer.rb:65:in `call'
|
|
14
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_redirects.rb:15:in `call'
|
|
15
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/retry_errors.rb:87:in `call'
|
|
16
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_dualstack.rb:32:in `call'
|
|
17
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_accelerate.rb:49:in `call'
|
|
18
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_md5s.rb:31:in `call'
|
|
19
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_expect_100_continue.rb:21:in `call'
|
|
20
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_bucket_name_restrictions.rb:12:in `call'
|
|
21
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_bucket_dns.rb:31:in `call'
|
|
22
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/rest/handler.rb:7:in `call'
|
|
23
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/user_agent.rb:12:in `call'
|
|
24
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/seahorse/client/plugins/endpoint.rb:41:in `call'
|
|
25
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/param_validator.rb:21:in `call'
|
|
26
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/seahorse/client/plugins/raise_response_errors.rb:14:in `call'
|
|
27
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_sse_cpk.rb:19:in `call'
|
|
28
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_dualstack.rb:24:in `call'
|
|
29
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/s3_accelerate.rb:34:in `call'
|
|
30
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/idempotency_token.rb:18:in `call'
|
|
31
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/aws-sdk-core/plugins/param_converter.rb:20:in `call'
|
|
32
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/seahorse/client/plugins/response_target.rb:21:in `call'
|
|
33
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/seahorse/client/request.rb:70:in `send_request'
|
|
34
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-core-2.6.36/lib/seahorse/client/base.rb:207:in `block (2 levels) in define_operation_methods'
|
|
35
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-resources-2.6.36/lib/aws-sdk-resources/services/s3/file_uploader.rb:42:in `block in put_object'
|
|
36
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-resources-2.6.36/lib/aws-sdk-resources/services/s3/file_uploader.rb:49:in `open_file'
|
|
37
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-resources-2.6.36/lib/aws-sdk-resources/services/s3/file_uploader.rb:41:in `put_object'
|
|
38
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-resources-2.6.36/lib/aws-sdk-resources/services/s3/file_uploader.rb:34:in `upload'
|
|
39
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/aws-sdk-resources-2.6.36/lib/aws-sdk-resources/services/s3/object.rb:251:in `upload_file'
|
|
40
|
+
/Users/minero-aoki/c/bricolage/lib/bricolage/s3datasource.rb:188:in `block in run'
|
|
41
|
+
/Users/minero-aoki/c/bricolage/lib/bricolage/s3datasource.rb:165:in `block in each_src_dest'
|
|
42
|
+
/Users/minero-aoki/c/bricolage/lib/bricolage/s3datasource.rb:163:in `each'
|
|
43
|
+
/Users/minero-aoki/c/bricolage/lib/bricolage/s3datasource.rb:163:in `each_src_dest'
|
|
44
|
+
/Users/minero-aoki/c/bricolage/lib/bricolage/s3datasource.rb:186:in `run'
|
|
45
|
+
/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:82:in `block (2 levels) in run'
|
|
46
|
+
/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:81:in `each'
|
|
47
|
+
/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:81:in `block in run'
|
|
48
|
+
/Users/minero-aoki/c/bricolage/lib/bricolage/datasource.rb:136:in `open'
|
|
49
|
+
/Users/minero-aoki/c/bricolage/lib/bricolage/datasource.rb:140:in `open_for_batch'
|
|
50
|
+
/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:80:in `run'
|
|
51
|
+
/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:47:in `block in run'
|
|
52
|
+
/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:46:in `each'
|
|
53
|
+
/Users/minero-aoki/c/bricolage/lib/bricolage/script.rb:46:in `run'
|
|
54
|
+
/Users/minero-aoki/c/bricolage/lib/bricolage/job.rb:143:in `block in execute'
|
|
55
|
+
/Users/minero-aoki/c/bricolage/lib/bricolage/logger.rb:44:in `with_elapsed_time'
|
|
56
|
+
/Users/minero-aoki/c/bricolage/lib/bricolage/job.rb:142:in `execute'
|
|
57
|
+
/Users/minero-aoki/c/bricolage/lib/bricolage/application.rb:68:in `block in main'
|
|
58
|
+
/Users/minero-aoki/c/bricolage/lib/bricolage/application.rb:97:in `redirect_log_to_file'
|
|
59
|
+
/Users/minero-aoki/c/bricolage/lib/bricolage/application.rb:67:in `main'
|
|
60
|
+
/Users/minero-aoki/c/bricolage/lib/bricolage/application.rb:28:in `main'
|
|
61
|
+
/Users/minero-aoki/c/bricolage/bin/bricolage:6:in `<top (required)>'
|
|
62
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/bin/bricolage:23:in `load'
|
|
63
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/bin/bricolage:23:in `<top (required)>'
|
|
64
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli/exec.rb:74:in `load'
|
|
65
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli/exec.rb:74:in `kernel_load'
|
|
66
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli/exec.rb:27:in `run'
|
|
67
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli.rb:332:in `exec'
|
|
68
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/vendor/thor/lib/thor/command.rb:27:in `run'
|
|
69
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/vendor/thor/lib/thor/invocation.rb:126:in `invoke_command'
|
|
70
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/vendor/thor/lib/thor.rb:359:in `dispatch'
|
|
71
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli.rb:20:in `dispatch'
|
|
72
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/vendor/thor/lib/thor/base.rb:440:in `start'
|
|
73
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/cli.rb:11:in `start'
|
|
74
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/exe/bundle:34:in `block in <top (required)>'
|
|
75
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/lib/bundler/friendly_errors.rb:100:in `with_friendly_errors'
|
|
76
|
+
/Users/minero-aoki/.rbenv/versions/2.3.1/lib/ruby/gems/2.3.0/gems/bundler-1.13.6/exe/bundle:26:in `<top (required)>'
|
|
77
|
+
/Users/minero-aoki/.rbenv/versions/2.3/bin/bundle:23:in `load'
|
|
78
|
+
/Users/minero-aoki/.rbenv/versions/2.3/bin/bundle:23:in `<main>'
|
|
79
|
+
|
|
80
|
+
2017-01-17 20:25:51 +0900: ERROR: error: Aws::Errors::MissingCredentialsError: unable to sign request without credentials set
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
2017-01-17 20:26:36 +0900: INFO: development environment
|
|
2
|
+
2017-01-17 20:26:36 +0900: INFO: [CMD] "/Users/minero-aoki/c/bricolage/libexec/sqldump" "--json" "127.0.0.1" "9011" "work_readonly" "main" "select * from main.search_backends;" | "gzip" "{:out=>"/tmp/bricolagedev/search_backends.json.gz"}"
|
|
3
|
+
2017-01-17 20:26:36.214888001 +0900 JST: [SQL] select * from main.search_backends;
|
|
4
|
+
2017-01-17 20:26:36.27827597 +0900 JST: query returned
|
|
5
|
+
2017-01-17 20:26:36.278393544 +0900 JST: Total 11 records
|
|
6
|
+
2017-01-17 20:26:36 +0900: INFO: aws s3 cp /tmp/bricolagedev/search_backends.json.gz s3://tabemiru-data.ap-northeast-1/dev/search_backends/search_backends.json.gz
|
|
7
|
+
rm -f /tmp/bricolagedev/search_backends.json.gz
|
|
8
|
+
2017-01-17 20:26:36 +0900: INFO: command: psql --no-psqlrc --host=dwh.ckpd.co --port=5439 --username=aamine production --echo-all -v ON_ERROR_STOP=true -f /var/folders/_t/f8rv7p995rq8m3j3tf45w4rm0000gp/T/1484652396_69460_3fd54d43f9d8_14594 --no-password
|
|
9
|
+
\timing on
|
|
10
|
+
Timing is on.
|
|
11
|
+
begin transaction;
|
|
12
|
+
BEGIN
|
|
13
|
+
Time: 8.269 ms
|
|
14
|
+
drop table if exists aamine.search_backends_old cascade;
|
|
15
|
+
psql:/var/folders/_t/f8rv7p995rq8m3j3tf45w4rm0000gp/T/1484652396_69460_3fd54d43f9d8_14594:5: INFO: Table "search_backends_old" does not exist and will be skipped
|
|
16
|
+
DROP TABLE
|
|
17
|
+
Time: 8.647 ms
|
|
18
|
+
drop table if exists aamine.search_backends_wk cascade;
|
|
19
|
+
psql:/var/folders/_t/f8rv7p995rq8m3j3tf45w4rm0000gp/T/1484652396_69460_3fd54d43f9d8_14594:7: INFO: Table "search_backends_wk" does not exist and will be skipped
|
|
20
|
+
DROP TABLE
|
|
21
|
+
Time: 8.031 ms
|
|
22
|
+
/* /Users/minero-aoki/c/bricolage/test/home/subsys/search_backends.ct */
|
|
23
|
+
create table aamine.search_backends_wk
|
|
24
|
+
( id int
|
|
25
|
+
, role varchar(10)
|
|
26
|
+
, group_id smallint
|
|
27
|
+
, index_type varchar(32)
|
|
28
|
+
, config_key varchar(32)
|
|
29
|
+
, updated_at timestamp
|
|
30
|
+
)
|
|
31
|
+
distkey (id)
|
|
32
|
+
sortkey (id)
|
|
33
|
+
;
|
|
34
|
+
CREATE TABLE
|
|
35
|
+
Time: 521.167 ms
|
|
36
|
+
copy aamine.search_backends_wk
|
|
37
|
+
from 's3://tabemiru-data.ap-northeast-1/dev/search_backends/search_backends.json.gz'
|
|
38
|
+
credentials 'aws_iam_role=arn:aws:iam::789035092620:role/RedshiftDWHTeamS3Access'
|
|
39
|
+
json 'auto'
|
|
40
|
+
statupdate false
|
|
41
|
+
compupdate false
|
|
42
|
+
emptyasnull
|
|
43
|
+
gzip
|
|
44
|
+
;
|
|
45
|
+
psql:/var/folders/_t/f8rv7p995rq8m3j3tf45w4rm0000gp/T/1484652396_69460_3fd54d43f9d8_14594:30: INFO: Load into table 'search_backends_wk' completed, 11 record(s) loaded successfully.
|
|
46
|
+
COPY
|
|
47
|
+
Time: 1235.569 ms
|
|
48
|
+
grant select on aamine.search_backends_wk to aamine;
|
|
49
|
+
GRANT
|
|
50
|
+
Time: 8.580 ms
|
|
51
|
+
analyze aamine.search_backends_wk;
|
|
52
|
+
ANALYZE
|
|
53
|
+
Time: 43.441 ms
|
|
54
|
+
create table if not exists aamine.search_backends (x int);
|
|
55
|
+
psql:/var/folders/_t/f8rv7p995rq8m3j3tf45w4rm0000gp/T/1484652396_69460_3fd54d43f9d8_14594:36: INFO: Relation "search_backends" already exists and will be skipped
|
|
56
|
+
CREATE TABLE
|
|
57
|
+
Time: 7.768 ms
|
|
58
|
+
alter table aamine.search_backends rename to search_backends_old;
|
|
59
|
+
ALTER TABLE
|
|
60
|
+
Time: 10.037 ms
|
|
61
|
+
alter table aamine.search_backends_wk rename to search_backends;
|
|
62
|
+
ALTER TABLE
|
|
63
|
+
Time: 11.686 ms
|
|
64
|
+
commit;
|
|
65
|
+
COMMIT
|
|
66
|
+
Time: 8482.452 ms
|
|
67
|
+
drop table if exists aamine.search_backends_old cascade;
|
|
68
|
+
DROP TABLE
|
|
69
|
+
Time: 4801.919 ms
|
|
70
|
+
;
|
|
71
|
+
Time: 8.088 ms
|
|
72
|
+
vacuum sort only aamine.search_backends;
|
|
73
|
+
VACUUM
|
|
74
|
+
Time: 36970.832 ms
|
|
75
|
+
;
|
|
76
|
+
Time: 8.746 ms
|
|
77
|
+
2017-01-17 20:27:29 +0900: INFO: status: 0 (pid 69478 exit 0)
|
|
78
|
+
2017-01-17 20:27:29 +0900: INFO: 53.05 secs
|
|
79
|
+
2017-01-17 20:27:29 +0900: INFO: SUCCESS
|