magellan-gcs-proxy 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.rubocop.yml +20 -0
- data/Gemfile +2 -0
- data/README.md +32 -12
- data/Rakefile +7 -3
- data/bin/console +3 -3
- data/example/Dockerfile +19 -0
- data/example/Gemfile +11 -0
- data/example/Gemfile.lock +129 -0
- data/example/README.md +73 -0
- data/example/VERSION +1 -0
- data/example/app.sh +19 -0
- data/example/config.yml +25 -0
- data/exe/magellan-gcs-proxy +1 -1
- data/exe/magellan-gcs-proxy-dev-progress-listener +14 -0
- data/exe/magellan-gcs-proxy-dev-setup +53 -0
- data/lib/magellan/gcs/proxy/cli.rb +24 -39
- data/lib/magellan/gcs/proxy/composite_logger.rb +37 -0
- data/lib/magellan/gcs/proxy/config.rb +44 -0
- data/lib/magellan/gcs/proxy/context.rb +33 -32
- data/lib/magellan/gcs/proxy/expand_variable.rb +20 -29
- data/lib/magellan/gcs/proxy/gcp.rb +10 -4
- data/lib/magellan/gcs/proxy/log.rb +67 -1
- data/lib/magellan/gcs/proxy/message_wrapper.rb +34 -5
- data/lib/magellan/gcs/proxy/progress_notification.rb +73 -0
- data/lib/magellan/gcs/proxy/progress_notifier_adapter.rb +24 -0
- data/lib/magellan/gcs/proxy/pubsub_progress_notifier.rb +25 -0
- data/lib/magellan/gcs/proxy/pubsub_sustainer.rb +54 -0
- data/lib/magellan/gcs/proxy/version.rb +1 -1
- data/lib/magellan/gcs/proxy.rb +20 -4
- data/magellan-gcs-proxy.gemspec +17 -15
- metadata +48 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 1232e616e80b9b1c977f0ac7f082d093a3502799
|
|
4
|
+
data.tar.gz: 8b23997772e69ac5685fc5f65b7526953ea343e2
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3e48f5f6203aac1ac89968c708bed4028390b49ad97666dd70654059bf9d55125bed1cdaa08a24a8ffd1afdbf1b4b1f34d48fa7d0973d1d9e2ff05b2e94574c2
|
|
7
|
+
data.tar.gz: 4d915466137763f3c37c59d2c3621c054b064ea3f2f9ef0cf8327f8a70a9a028b2ad9ce00dce8b3a4cd8bb3411cdc102ba316d1a87599e6066f44c6ed506ad58
|
data/.gitignore
CHANGED
data/.rubocop.yml
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
Style/Documentation:
|
|
2
|
+
Enabled: false
|
|
3
|
+
|
|
4
|
+
LineLength:
|
|
5
|
+
Max: 130
|
|
6
|
+
|
|
7
|
+
Metrics/AbcSize:
|
|
8
|
+
Max: 20
|
|
9
|
+
|
|
10
|
+
Metrics/CyclomaticComplexity:
|
|
11
|
+
Max: 8
|
|
12
|
+
|
|
13
|
+
Metrics/MethodLength:
|
|
14
|
+
Max: 15
|
|
15
|
+
|
|
16
|
+
Lint/AssignmentInCondition:
|
|
17
|
+
Enabled: false
|
|
18
|
+
|
|
19
|
+
Style/TrailingCommaInLiteral:
|
|
20
|
+
EnforcedStyleForMultiline: comma
|
data/Gemfile
CHANGED
data/README.md
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
#
|
|
1
|
+
# magellan-gcs-proxy
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
[](https://rubygems.org/gems/magellan-gcs-proxy) [](https://travis-ci.org/groovenauts/magellan-gcs-proxy)
|
|
4
|
+
|
|
5
|
+
magellan-gcs-proxy is a gem for MAGELLAN BLOCKS "Batch type" IoT board.
|
|
4
6
|
|
|
5
7
|
|
|
6
8
|
## Installation
|
|
@@ -35,28 +37,47 @@ magellan-gcs-proxy COMMAND ARGS...
|
|
|
35
37
|
### Setup
|
|
36
38
|
|
|
37
39
|
```
|
|
38
|
-
|
|
39
|
-
export TOPIC=projects/$PROJECT_ID/topics/[Topic name]
|
|
40
|
-
export SUB=projects/$PROJECT_ID/subscriptions/[Subscription name of topic]
|
|
41
|
-
$ gcloud beta pubsub topics create projects/$PROJECT_ID/topics/$TOPIC
|
|
42
|
-
$ gcloud beta pubsub topics list
|
|
43
|
-
$ gcloud beta pubsub subscriptions create $SUB --topic $TOPIC
|
|
44
|
-
$ gcloud beta pubsub subscriptions list
|
|
40
|
+
magellan-gcs-proxy-dev-setup [Project ID]
|
|
45
41
|
```
|
|
46
42
|
|
|
47
|
-
|
|
43
|
+
|
|
44
|
+
### Listen to progress subscription
|
|
48
45
|
|
|
49
46
|
```
|
|
50
|
-
$
|
|
47
|
+
$ export BLOCKS_BATCH_PROJECT_ID=[Project ID]
|
|
48
|
+
$ magellan-gcs-proxy-dev-progress-listener [Progress subscription name]
|
|
51
49
|
```
|
|
52
50
|
|
|
51
|
+
`Progress subscription name` is the name of `Progress subscription`.
|
|
52
|
+
It created by `magellan-gcs-proxy-dev-setup`.
|
|
53
|
+
You can see it by `gcloud beta pubsub subscriptions list`.
|
|
54
|
+
It starts with `test-progress-` and ends with '-sub'.
|
|
53
55
|
|
|
54
56
|
### Run application
|
|
55
57
|
|
|
56
58
|
```
|
|
59
|
+
$ export BLOCKS_BATCH_PROJECT_ID=[Project ID]
|
|
60
|
+
$ export BLOCKS_BATCH_PUBSUB_SUBSCRIPTION=[Job subscription name]
|
|
57
61
|
$ magellan-gcs-proxy COMMAND ARGS...
|
|
58
62
|
```
|
|
59
63
|
|
|
64
|
+
`Job subscription name` is the name of `Job subscription`.
|
|
65
|
+
It created by `magellan-gcs-proxy-dev-setup`.
|
|
66
|
+
You can see it by `gcloud beta pubsub subscriptions list`.
|
|
67
|
+
It starts with `test-job-` and ends with '-sub'.
|
|
68
|
+
|
|
69
|
+
### Publish message
|
|
70
|
+
|
|
71
|
+
```
|
|
72
|
+
$ export JOB_TOPIC=[Job topic name]
|
|
73
|
+
$ gcloud beta pubsub topics publish $JOB_TOPIC "" --attribute='download_files=["gs://bucket1/path/to/file"]'
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
`Job topic name` is the name of `Job topic`.
|
|
77
|
+
It created by `magellan-gcs-proxy-dev-setup`.
|
|
78
|
+
You can see it by `gcloud beta pubsub topics list`.
|
|
79
|
+
It starts with `test-job-` and ends with your user name.
|
|
80
|
+
|
|
60
81
|
|
|
61
82
|
## Development
|
|
62
83
|
|
|
@@ -72,4 +93,3 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/[USERN
|
|
|
72
93
|
## License
|
|
73
94
|
|
|
74
95
|
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
|
75
|
-
|
data/Rakefile
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
|
-
require
|
|
2
|
-
require
|
|
1
|
+
require 'bundler/gem_tasks'
|
|
2
|
+
require 'rspec/core/rake_task'
|
|
3
3
|
|
|
4
4
|
RSpec::Core::RakeTask.new(:spec)
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
require 'rubocop/rake_task'
|
|
7
|
+
RuboCop::RakeTask.new
|
|
8
|
+
|
|
9
|
+
task(:default).clear
|
|
10
|
+
task default: [:rubocop, :spec]
|
data/bin/console
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
2
|
|
|
3
|
-
require
|
|
4
|
-
require
|
|
3
|
+
require 'bundler/setup'
|
|
4
|
+
require 'magellan/gcs/proxy'
|
|
5
5
|
|
|
6
6
|
# You can add fixtures and/or initialization code here to make experimenting
|
|
7
7
|
# with your gem easier. You can also use a different console, if you like.
|
|
@@ -10,5 +10,5 @@ require "magellan/gcs/proxy"
|
|
|
10
10
|
# require "pry"
|
|
11
11
|
# Pry.start
|
|
12
12
|
|
|
13
|
-
require
|
|
13
|
+
require 'irb'
|
|
14
14
|
IRB.start
|
data/example/Dockerfile
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# [config] IMAGE_NAME: "groovenauts/batch_type_iot_example"
|
|
2
|
+
# [config]
|
|
3
|
+
# [config] WORKING_DIR: "."
|
|
4
|
+
# [config] VERSION_FILE: VERSION
|
|
5
|
+
# [config] GIT_TAG_PREFIX: 'example/'
|
|
6
|
+
# [config]
|
|
7
|
+
|
|
8
|
+
FROM ruby:2.3
|
|
9
|
+
|
|
10
|
+
COPY Gemfile* /tmp/
|
|
11
|
+
WORKDIR /tmp
|
|
12
|
+
RUN bundle install
|
|
13
|
+
|
|
14
|
+
ENV APP_HOME /usr/app/batch_type_example
|
|
15
|
+
COPY . $APP_HOME
|
|
16
|
+
WORKDIR $APP_HOME
|
|
17
|
+
RUN bundle install --without development
|
|
18
|
+
|
|
19
|
+
CMD ["bundle", "exec", "magellan-gcs-proxy", "./app.sh", "%{download_files.0}", "%{uploads_dir}"]
|
data/example/Gemfile
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
GEM
|
|
2
|
+
remote: https://rubygems.org/
|
|
3
|
+
specs:
|
|
4
|
+
addressable (2.5.0)
|
|
5
|
+
public_suffix (~> 2.0, >= 2.0.2)
|
|
6
|
+
binding_of_caller (0.7.2)
|
|
7
|
+
debug_inspector (>= 0.0.1)
|
|
8
|
+
brocket (0.2.2)
|
|
9
|
+
logger_pipe
|
|
10
|
+
thor
|
|
11
|
+
byebug (9.0.6)
|
|
12
|
+
coderay (1.1.1)
|
|
13
|
+
debug_inspector (0.0.2)
|
|
14
|
+
digest-crc (0.4.1)
|
|
15
|
+
dotenv (2.1.1)
|
|
16
|
+
faraday (0.10.0)
|
|
17
|
+
multipart-post (>= 1.2, < 3)
|
|
18
|
+
google-api-client (0.9.20)
|
|
19
|
+
addressable (~> 2.3)
|
|
20
|
+
googleauth (~> 0.5)
|
|
21
|
+
httpclient (~> 2.7)
|
|
22
|
+
hurley (~> 0.1)
|
|
23
|
+
memoist (~> 0.11)
|
|
24
|
+
mime-types (>= 1.6)
|
|
25
|
+
representable (~> 2.3.0)
|
|
26
|
+
retriable (~> 2.0)
|
|
27
|
+
google-cloud-core (0.21.1)
|
|
28
|
+
googleauth (~> 0.5.1)
|
|
29
|
+
google-cloud-logging (0.21.1)
|
|
30
|
+
google-cloud-core (~> 0.21.0)
|
|
31
|
+
google-gax (~> 0.6.0)
|
|
32
|
+
google-protobuf (~> 3.0)
|
|
33
|
+
googleapis-common-protos (~> 1.3)
|
|
34
|
+
grpc (~> 1.0)
|
|
35
|
+
orderedhash (= 0.0.6)
|
|
36
|
+
google-cloud-pubsub (0.21.0)
|
|
37
|
+
google-cloud-core (~> 0.21.0)
|
|
38
|
+
google-gax (~> 0.6.0)
|
|
39
|
+
google-protobuf (~> 3.0)
|
|
40
|
+
googleapis-common-protos (~> 1.3)
|
|
41
|
+
grpc (~> 1.0)
|
|
42
|
+
grpc-google-iam-v1 (~> 0.6.8)
|
|
43
|
+
google-cloud-storage (0.21.0)
|
|
44
|
+
digest-crc (~> 0.4)
|
|
45
|
+
google-api-client (~> 0.9.11)
|
|
46
|
+
google-cloud-core (~> 0.21.0)
|
|
47
|
+
google-gax (0.6.0)
|
|
48
|
+
googleapis-common-protos (~> 1.3.1)
|
|
49
|
+
googleauth (~> 0.5.1)
|
|
50
|
+
grpc (~> 1.0)
|
|
51
|
+
rly (~> 0.2.3)
|
|
52
|
+
google-protobuf (3.0.2)
|
|
53
|
+
googleapis-common-protos (1.3.4)
|
|
54
|
+
google-protobuf (~> 3.0)
|
|
55
|
+
grpc (~> 1.0)
|
|
56
|
+
googleauth (0.5.1)
|
|
57
|
+
faraday (~> 0.9)
|
|
58
|
+
jwt (~> 1.4)
|
|
59
|
+
logging (~> 2.0)
|
|
60
|
+
memoist (~> 0.12)
|
|
61
|
+
multi_json (~> 1.11)
|
|
62
|
+
os (~> 0.9)
|
|
63
|
+
signet (~> 0.7)
|
|
64
|
+
grpc (1.0.1)
|
|
65
|
+
google-protobuf (~> 3.0.2)
|
|
66
|
+
googleauth (~> 0.5.1)
|
|
67
|
+
grpc-google-iam-v1 (0.6.8)
|
|
68
|
+
googleapis-common-protos (~> 1.3.1)
|
|
69
|
+
googleauth (~> 0.5.1)
|
|
70
|
+
grpc (~> 1.0)
|
|
71
|
+
httpclient (2.8.2.4)
|
|
72
|
+
hurley (0.2)
|
|
73
|
+
jwt (1.5.6)
|
|
74
|
+
little-plugger (1.1.4)
|
|
75
|
+
logger_pipe (0.3.1)
|
|
76
|
+
logging (2.1.0)
|
|
77
|
+
little-plugger (~> 1.1)
|
|
78
|
+
multi_json (~> 1.10)
|
|
79
|
+
magellan-gcs-proxy (0.1.1)
|
|
80
|
+
dotenv
|
|
81
|
+
google-cloud-logging
|
|
82
|
+
google-cloud-pubsub
|
|
83
|
+
google-cloud-storage
|
|
84
|
+
logger_pipe
|
|
85
|
+
memoist (0.15.0)
|
|
86
|
+
method_source (0.8.2)
|
|
87
|
+
mime-types (3.1)
|
|
88
|
+
mime-types-data (~> 3.2015)
|
|
89
|
+
mime-types-data (3.2016.0521)
|
|
90
|
+
multi_json (1.12.1)
|
|
91
|
+
multipart-post (2.0.0)
|
|
92
|
+
orderedhash (0.0.6)
|
|
93
|
+
os (0.9.6)
|
|
94
|
+
pry (0.10.4)
|
|
95
|
+
coderay (~> 1.1.0)
|
|
96
|
+
method_source (~> 0.8.1)
|
|
97
|
+
slop (~> 3.4)
|
|
98
|
+
pry-byebug (3.4.1)
|
|
99
|
+
byebug (~> 9.0)
|
|
100
|
+
pry (~> 0.10)
|
|
101
|
+
pry-stack_explorer (0.4.9.2)
|
|
102
|
+
binding_of_caller (>= 0.7)
|
|
103
|
+
pry (>= 0.9.11)
|
|
104
|
+
public_suffix (2.0.4)
|
|
105
|
+
representable (2.3.0)
|
|
106
|
+
uber (~> 0.0.7)
|
|
107
|
+
retriable (2.1.0)
|
|
108
|
+
rly (0.2.3)
|
|
109
|
+
signet (0.7.3)
|
|
110
|
+
addressable (~> 2.3)
|
|
111
|
+
faraday (~> 0.9)
|
|
112
|
+
jwt (~> 1.5)
|
|
113
|
+
multi_json (~> 1.10)
|
|
114
|
+
slop (3.6.0)
|
|
115
|
+
thor (0.19.4)
|
|
116
|
+
uber (0.0.15)
|
|
117
|
+
|
|
118
|
+
PLATFORMS
|
|
119
|
+
ruby
|
|
120
|
+
|
|
121
|
+
DEPENDENCIES
|
|
122
|
+
brocket
|
|
123
|
+
magellan-gcs-proxy
|
|
124
|
+
pry
|
|
125
|
+
pry-byebug
|
|
126
|
+
pry-stack_explorer
|
|
127
|
+
|
|
128
|
+
BUNDLED WITH
|
|
129
|
+
1.13.6
|
data/example/README.md
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# magellan-gcs-proxy example
|
|
2
|
+
|
|
3
|
+
## How to run application locally
|
|
4
|
+
|
|
5
|
+
```
|
|
6
|
+
$ cd path/to/somewhere
|
|
7
|
+
$ git clone https://github.com/groovenauts/magellan-gcs-proxy.git
|
|
8
|
+
$ cd magellan-gcs-proxy/example
|
|
9
|
+
$ bundle
|
|
10
|
+
$ export BLOCKS_BATCH_PROJECT_ID=[Your project ID]
|
|
11
|
+
$ magellan-gcs-proxy-dev-setup $BLOCKS_BATCH_PROJECT_ID
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
Follow the steps `magellan-gcs-proxy-dev-setup` shows.
|
|
15
|
+
|
|
16
|
+
## Result
|
|
17
|
+
|
|
18
|
+
### Terminal 1
|
|
19
|
+
|
|
20
|
+
```
|
|
21
|
+
$ magellan-gcs-proxy-dev-progress-listener $BLOCKS_BATCH_PROGRESS_SUBSCRIPTION
|
|
22
|
+
|
|
23
|
+
Listening to projects/scenic-doodad-617/subscriptions/test-progress-topic-akm-sub
|
|
24
|
+
total:12 job_message_id:49719039110847 progress:1 level:info Processing message: #<Google::Cloud::Pubsub::ReceivedMessage:0x007f9c6b2127d0 @subscription=#<Google::Cloud::Pubsub::Subscription:0x007f9c6b386f58 @service=Google::Cloud::Pubsub::Service(scenic-doodad-617), @grpc=<Google::Pubsub::V1::Subscription: name: "projects/scenic-doodad-617/subscriptions/test-job-topic-akm-sub", topic: "projects/scenic-doodad-617/topics/test-job-topic-akm", push_config: <Google::Pubsub::V1::PushConfig: push_endpoint: "", attributes: {}>, ack_deadline_seconds: 10>, @name=nil, @exists=nil>, @grpc=<Google::Pubsub::V1::ReceivedMessage: ack_id: "OUVBXkASTDYMRElTK0MLKlgRTgQhIT4wPkVTRFAGFixdRkhRNxkIaFEOT14jPzUgKEUSAAgUBXx9clNcdV8zdQdRDRlzeGkmP1lGU1ARB3ReURsfWVx-SwNYChh7fWJ8a1sTCQZCe4m8xtU_Zhs9XxJLLD5-PQ", message: <Google::Pubsub::V1::PubsubMessage: data: "", attributes: {"download_files"=>"[\"gs://bucket1/dir1/database.yml\"]"}, message_id: "49719039110847", publish_time: <Google::Protobuf::Timestamp: seconds: 1480929118, nanos: 504000000>>>>
|
|
25
|
+
total:12 job_message_id:49719039110847 progress:2 level:info Download starting
|
|
26
|
+
total:12 job_message_id:49719039110847 progress:3 level:info Download completed
|
|
27
|
+
total:12 job_message_id:49719039110847 progress:5 level:info Command starting
|
|
28
|
+
total:12 job_message_id:49719039110847 progress:8 level:info Upload starting
|
|
29
|
+
total:12 job_message_id:49719039110847 progress:6 level:info Command completed
|
|
30
|
+
total:12 job_message_id:49719039110847 progress:9 level:info Upload completed
|
|
31
|
+
total:12 job_message_id:49719039110847 progress:11 level:info Acknowledged
|
|
32
|
+
total:12 job_message_id:49719039110847 progress:12 level:info Cleanup
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
### Terminal 2
|
|
36
|
+
|
|
37
|
+
```
|
|
38
|
+
$ bundle exec magellan-gcs-proxy ./app.sh %{download_files.0} %{uploads_dir}
|
|
39
|
+
|
|
40
|
+
====================================================================================================
|
|
41
|
+
progress_notification:
|
|
42
|
+
topic: projects/scenic-doodad-617/topics/test-progress-topic-akm
|
|
43
|
+
|
|
44
|
+
loggers:
|
|
45
|
+
- type: stdout
|
|
46
|
+
- type: cloud_logging
|
|
47
|
+
log_name: magellan-gcs-proxy-example-logging
|
|
48
|
+
----------------------------------------------------------------------------------------------------
|
|
49
|
+
I, [2016-12-05T18:12:32.892158 #22285] INFO -- : Start listening
|
|
50
|
+
I, [2016-12-05T18:12:34.077139 #22285] INFO -- : subscription: #<Google::Cloud::Pubsub::Subscription:0x007f9c6b386f58 @service=Google::Cloud::Pubsub::Service(scenic-doodad-617), @grpc=<Google::Pubsub::V1::Subscription: name: "projects/scenic-doodad-617/subscriptions/test-job-topic-akm-sub", topic: "projects/scenic-doodad-617/topics/test-job-topic-akm", push_config: <Google::Pubsub::V1::PushConfig: push_endpoint: "", attributes: {}>, ack_deadline_seconds: 10>, @name=nil, @exists=nil>
|
|
51
|
+
I, [2016-12-05T18:12:35.572740 #22285] INFO -- : job_message_id:49719039110847 progress:1 total:12 data:Processing message: #<Google::Cloud::Pubsub::ReceivedMessage:0x007f9c6b2127d0 @subscription=#<Google::Cloud::Pubsub::Subscription:0x007f9c6b386f58 @service=Google::Cloud::Pubsub::Service(scenic-doodad-617), @grpc=<Google::Pubsub::V1::Subscription: name: "projects/scenic-doodad-617/subscriptions/test-job-topic-akm-sub", topic: "projects/scenic-doodad-617/topics/test-job-topic-akm", push_config: <Google::Pubsub::V1::PushConfig: push_endpoint: "", attributes: {}>, ack_deadline_seconds: 10>, @name=nil, @exists=nil>, @grpc=<Google::Pubsub::V1::ReceivedMessage: ack_id: "OUVBXkASTDYMRElTK0MLKlgRTgQhIT4wPkVTRFAGFixdRkhRNxkIaFEOT14jPzUgKEUSAAgUBXx9clNcdV8zdQdRDRlzeGkmP1lGU1ARB3ReURsfWVx-SwNYChh7fWJ8a1sTCQZCe4m8xtU_Zhs9XxJLLD5-PQ", message: <Google::Pubsub::V1::PubsubMessage: data: "", attributes: {"download_files"=>"[\"gs://bucket1/dir1/database.yml\"]"}, message_id: "49719039110847", publish_time: <Google::Protobuf::Timestamp: seconds: 1480929118, nanos: 504000000>>>>
|
|
52
|
+
I, [2016-12-05T18:12:35.785083 #22285] INFO -- : job_message_id:49719039110847 progress:2 total:12 data:Download starting
|
|
53
|
+
D, [2016-12-05T18:12:35.942864 #22285] DEBUG -- : Downloading: gs://bucket1/dir1/database.yml to /var/folders/70/5yxt0jys7ss_m71y5pz_l_qc0000gn/T/workspace20161205-22285-18l5pmo/downloads/dir1/database.yml
|
|
54
|
+
I, [2016-12-05T18:12:37.559498 #22285] INFO -- : Download OK: gs://bucket1/dir1/database.yml to /var/folders/70/5yxt0jys7ss_m71y5pz_l_qc0000gn/T/workspace20161205-22285-18l5pmo/downloads/dir1/database.yml
|
|
55
|
+
I, [2016-12-05T18:12:37.796599 #22285] INFO -- : job_message_id:49719039110847 progress:3 total:12 data:Download completed
|
|
56
|
+
I, [2016-12-05T18:12:38.072803 #22285] INFO -- : job_message_id:49719039110847 progress:5 total:12 data:Command starting
|
|
57
|
+
I, [2016-12-05T18:12:38.223959 #22285] INFO -- : executing: ./app.sh /var/folders/70/5yxt0jys7ss_m71y5pz_l_qc0000gn/T/workspace20161205-22285-18l5pmo/downloads/dir1/database.yml /var/folders/70/5yxt0jys7ss_m71y5pz_l_qc0000gn/T/workspace20161205-22285-18l5pmo/uploads 2>&1
|
|
58
|
+
I, [2016-12-05T18:12:38.465296 #22285] INFO -- : SUCCESS: ./app.sh /var/folders/70/5yxt0jys7ss_m71y5pz_l_qc0000gn/T/workspace20161205-22285-18l5pmo/downloads/dir1/database.yml /var/folders/70/5yxt0jys7ss_m71y5pz_l_qc0000gn/T/workspace20161205-22285-18l5pmo/uploads 2>&1
|
|
59
|
+
I, [2016-12-05T18:12:38.859797 #22285] INFO -- : job_message_id:49719039110847 progress:6 total:12 data:Command completed
|
|
60
|
+
I, [2016-12-05T18:12:39.081097 #22285] INFO -- : job_message_id:49719039110847 progress:8 total:12 data:Upload starting
|
|
61
|
+
I, [2016-12-05T18:12:39.205161 #22285] INFO -- : Uploading: database-20161205-181238.yml to gs://bucket1/database-20161205-181238.yml
|
|
62
|
+
I, [2016-12-05T18:12:41.102121 #22285] INFO -- : Upload OK: database-20161205-181238.yml to gs://bucket1/database-20161205-181238.yml
|
|
63
|
+
I, [2016-12-05T18:12:41.346263 #22285] INFO -- : job_message_id:49719039110847 progress:9 total:12 data:Upload completed
|
|
64
|
+
I, [2016-12-05T18:12:41.662179 #22285] INFO -- : job_message_id:49719039110847 progress:11 total:12 data:Acknowledged
|
|
65
|
+
I, [2016-12-05T18:12:41.881730 #22285] INFO -- : job_message_id:49719039110847 progress:12 total:12 data:Cleanup
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
### Terminal 3 Publish message
|
|
69
|
+
|
|
70
|
+
```
|
|
71
|
+
$ gcloud beta pubsub topics publish $BLOCKS_BATCH_PUBSUB_TOPIC "" --attribute='download_files=["gs://bucket1/dir1/database.yml"]'
|
|
72
|
+
messageIds: '49718447408725'
|
|
73
|
+
```
|
data/example/VERSION
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
0.0.1
|
data/example/app.sh
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
|
|
3
|
+
usage() {
|
|
4
|
+
echo usage: `basename $0` src_path dest_path
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
if [ $# -ne 2 ]; then
|
|
8
|
+
echo `basename $0`: missing operand 1>&2
|
|
9
|
+
usage
|
|
10
|
+
exit 1
|
|
11
|
+
fi
|
|
12
|
+
|
|
13
|
+
fpath=$1
|
|
14
|
+
fname="${fpath##*/}"
|
|
15
|
+
fbase="${fname%.*}"
|
|
16
|
+
fext="${fpath##*.}"
|
|
17
|
+
mkdir -p $2
|
|
18
|
+
|
|
19
|
+
cp $1 $2/${fbase}-$(date '+%Y%m%d-%T' | tr -d :).${fext}
|
data/example/config.yml
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
<%- if progress_topic = ENV['BLOCKS_BATCH_PROGRESS_TOPIC'] -%>
|
|
2
|
+
progress_notification:
|
|
3
|
+
topic: <%= progress_topic %>
|
|
4
|
+
<%- end -%>
|
|
5
|
+
|
|
6
|
+
loggers:
|
|
7
|
+
- type: stdout
|
|
8
|
+
<%- if log_name = ENV['BLOCKS_BATCH_CLOUD_LOGGING_LOG_NAME'] -%>
|
|
9
|
+
- type: cloud_logging
|
|
10
|
+
log_name: <%= log_name %>
|
|
11
|
+
<%- end -%>
|
|
12
|
+
|
|
13
|
+
sustainer:
|
|
14
|
+
# delay:
|
|
15
|
+
# Delay message deadline for log jobs.
|
|
16
|
+
# It must be less than or equal to ACK_DEADLINE of subscription
|
|
17
|
+
# specified by BLOCKS_BATCH_PUBSUB_SUBSCRIPTION
|
|
18
|
+
# You can check ACK_DEADLINE by `gcloud beta pubsub subscriptions list`
|
|
19
|
+
# ACK_DEADLINE is set by `--ack-deadline` option for `gcloud beta pubsub subscriptions create`.
|
|
20
|
+
delay: 600
|
|
21
|
+
|
|
22
|
+
# interval
|
|
23
|
+
# The interval to send delay message. It must be lower than delay
|
|
24
|
+
# Default: 90% of delay
|
|
25
|
+
interval: 540
|
data/exe/magellan-gcs-proxy
CHANGED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
require 'magellan/gcs/proxy'
|
|
4
|
+
require 'json'
|
|
5
|
+
|
|
6
|
+
subname = ARGV.first
|
|
7
|
+
puts "Listening to #{subname}"
|
|
8
|
+
Magellan::Gcs::Proxy::GCP.pubsub.subscription(subname).tap do |sub|
|
|
9
|
+
sub.listen do |msg|
|
|
10
|
+
attrs = { 'data' => msg.data }.update(msg.attributes)
|
|
11
|
+
puts attrs.map { |k, v| "#{k}:#{v}" }.join("\t")
|
|
12
|
+
msg.ack!
|
|
13
|
+
end
|
|
14
|
+
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
|
|
3
|
+
usage() {
|
|
4
|
+
echo usage: `basename $0` GCP_PROJECT_ID
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
if [ $# -ne 1 ]; then
|
|
8
|
+
echo `basename $0`: missing operand 1>&2
|
|
9
|
+
usage
|
|
10
|
+
exit 1
|
|
11
|
+
fi
|
|
12
|
+
|
|
13
|
+
project_id=$1
|
|
14
|
+
export job_topic=projects/${project_id}/topics/test-job-topic-${USER}
|
|
15
|
+
export job_sub=projects/${project_id}/subscriptions/test-job-topic-${USER}-sub
|
|
16
|
+
|
|
17
|
+
export progress_topic=projects/${project_id}/topics/test-progress-topic-${USER}
|
|
18
|
+
export progress_sub=projects/${project_id}/subscriptions/test-progress-topic-${USER}-sub
|
|
19
|
+
|
|
20
|
+
gcloud beta pubsub topics create ${job_topic}
|
|
21
|
+
gcloud beta pubsub subscriptions create ${job_sub} --topic ${job_topic}
|
|
22
|
+
|
|
23
|
+
gcloud beta pubsub topics create ${progress_topic}
|
|
24
|
+
gcloud beta pubsub subscriptions create ${progress_sub} --topic ${progress_topic}
|
|
25
|
+
|
|
26
|
+
gcloud beta pubsub topics list >/dev/null 2>&1
|
|
27
|
+
gcloud beta pubsub subscriptions list >/dev/null 2>&1
|
|
28
|
+
|
|
29
|
+
echo "# Steps to run example"
|
|
30
|
+
echo "#"
|
|
31
|
+
echo "# At Terminal 1: Listen to progress subscription"
|
|
32
|
+
echo "#"
|
|
33
|
+
echo "cd $PWD"
|
|
34
|
+
echo "export BLOCKS_BATCH_PROJECT_ID=${project_id}"
|
|
35
|
+
echo "export BLOCKS_BATCH_PROGRESS_SUBSCRIPTION=${progress_sub}"
|
|
36
|
+
echo "magellan-gcs-proxy-dev-progress-listener \$BLOCKS_BATCH_PROGRESS_SUBSCRIPTION"
|
|
37
|
+
echo ""
|
|
38
|
+
echo "#"
|
|
39
|
+
echo "# At Terminal 2: Run application"
|
|
40
|
+
echo "#"
|
|
41
|
+
echo "cd $PWD"
|
|
42
|
+
echo "export BLOCKS_BATCH_PROJECT_ID=${project_id}"
|
|
43
|
+
echo "export BLOCKS_BATCH_PUBSUB_SUBSCRIPTION=${job_sub}"
|
|
44
|
+
echo "export BLOCKS_BATCH_PROGRESS_TOPIC=${progress_topic}"
|
|
45
|
+
echo "export BLOCKS_BATCH_CLOUD_LOGGING_LOG_NAME=magellan-gcs-proxy-example-logging"
|
|
46
|
+
echo "bundle exec magellan-gcs-proxy ./app.sh %{download_files.0} %{uploads_dir}"
|
|
47
|
+
echo ""
|
|
48
|
+
echo "#"
|
|
49
|
+
echo "# At Terminal 3: Publish message"
|
|
50
|
+
echo "#"
|
|
51
|
+
echo "cd $PWD"
|
|
52
|
+
echo "export BLOCKS_BATCH_PUBSUB_TOPIC=${job_topic}"
|
|
53
|
+
echo "gcloud beta pubsub topics publish \$BLOCKS_BATCH_PUBSUB_TOPIC \"\" --attribute='download_files=[\"gs://bucket1/path/to/file\"]'"
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
require
|
|
2
|
-
require
|
|
1
|
+
require 'magellan/gcs/proxy'
|
|
2
|
+
require 'magellan/gcs/proxy/log'
|
|
3
3
|
|
|
4
4
|
require 'json'
|
|
5
5
|
require 'logger'
|
|
@@ -18,59 +18,44 @@ module Magellan
|
|
|
18
18
|
end
|
|
19
19
|
|
|
20
20
|
def run
|
|
21
|
-
logger.info(
|
|
21
|
+
logger.info('Start listening')
|
|
22
22
|
GCP.subscription.listen do |msg|
|
|
23
|
-
|
|
24
|
-
process(msg)
|
|
25
|
-
rescue => e
|
|
26
|
-
logger.error("[#{e.class.name}] #{e.message}")
|
|
27
|
-
if ENV['VERBOSE'] =~ /true|yes|on|1/i
|
|
28
|
-
logger.debug("Backtrace\n " << e.backtrace.join("\n "))
|
|
29
|
-
end
|
|
30
|
-
end
|
|
23
|
+
process_with_error_handling(msg)
|
|
31
24
|
end
|
|
32
25
|
rescue => e
|
|
33
26
|
logger.error("[#{e.class.name}] #{e.message}")
|
|
34
27
|
raise e
|
|
35
28
|
end
|
|
36
29
|
|
|
37
|
-
def
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
30
|
+
def process_with_error_handling(msg)
|
|
31
|
+
process(msg)
|
|
32
|
+
rescue => e
|
|
33
|
+
logger.error("[#{e.class.name}] #{e.message}")
|
|
34
|
+
verbose("Backtrace\n " << e.backtrace.join("\n "))
|
|
35
|
+
end
|
|
42
36
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
37
|
+
TOTAL = 12
|
|
38
|
+
def process(msg)
|
|
39
|
+
context = Context.new(msg)
|
|
40
|
+
context.notify(1, TOTAL, "Processing message: #{msg.inspect}")
|
|
41
|
+
context.setup do
|
|
42
|
+
context.process_with_notification([2, 3, 4], TOTAL, 'Download', &:download)
|
|
46
43
|
|
|
47
|
-
context
|
|
48
|
-
logger.info("context.download done.")
|
|
49
|
-
logger.info("msg: #{msg}")
|
|
50
|
-
logger.info("context: #{context}")
|
|
44
|
+
cmd = build_command(context)
|
|
51
45
|
|
|
52
|
-
|
|
46
|
+
exec = ->(*) { LoggerPipe.run(logger, cmd, returns: :none, logging: :both) }
|
|
47
|
+
context.process_with_notification([5, 6, 7], TOTAL, 'Command', exec) do
|
|
48
|
+
context.process_with_notification([8, 9, 10], TOTAL, 'Upload', &:upload)
|
|
53
49
|
|
|
54
|
-
begin
|
|
55
|
-
LoggerPipe.run(logger, cmd, returns: :none, logging: :both)
|
|
56
|
-
rescue => e
|
|
57
|
-
logger.error("Error: #{cmd.inspect}")
|
|
58
|
-
else
|
|
59
|
-
context.upload
|
|
60
50
|
msg.acknowledge!
|
|
61
|
-
|
|
51
|
+
context.notify(11, TOTAL, 'Acknowledged')
|
|
62
52
|
end
|
|
63
53
|
end
|
|
54
|
+
context.notify(12, TOTAL, 'Cleanup')
|
|
64
55
|
end
|
|
65
56
|
|
|
66
|
-
def
|
|
67
|
-
|
|
68
|
-
JSON.parse(str)
|
|
69
|
-
end
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
def build_command(msg, context)
|
|
73
|
-
msg_wrapper = MessageWrapper.new(msg, context)
|
|
57
|
+
def build_command(context)
|
|
58
|
+
msg_wrapper = MessageWrapper.new(context)
|
|
74
59
|
ExpandVariable.expand_variables(cmd_template, msg_wrapper)
|
|
75
60
|
end
|
|
76
61
|
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
require 'magellan/gcs/proxy'
|
|
2
|
+
|
|
3
|
+
module Magellan
|
|
4
|
+
module Gcs
|
|
5
|
+
module Proxy
|
|
6
|
+
class CompositeLogger
|
|
7
|
+
attr_reader :loggers
|
|
8
|
+
def initialize(loggers)
|
|
9
|
+
@loggers = loggers
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def add(severity, message = nil, &block)
|
|
13
|
+
loggers.each do |logger|
|
|
14
|
+
begin
|
|
15
|
+
logger.add(severity, message, &block)
|
|
16
|
+
rescue => e
|
|
17
|
+
$stderr.puts("[#{e.class}] #{e.message}")
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
Logger::Severity.constants.each do |level|
|
|
23
|
+
level_dc = level.downcase
|
|
24
|
+
module_eval <<-INSTANCE_METHODS
|
|
25
|
+
def #{level_dc}(message = nil, &block)
|
|
26
|
+
add(Logger::Severity::#{level}, message, &block)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def #{level_dc}?
|
|
30
|
+
loggers.any?(&:#{level_dc}?)
|
|
31
|
+
end
|
|
32
|
+
INSTANCE_METHODS
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|