magellan-gcs-proxy 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.rubocop.yml +20 -0
- data/Gemfile +2 -0
- data/README.md +32 -12
- data/Rakefile +7 -3
- data/bin/console +3 -3
- data/example/Dockerfile +19 -0
- data/example/Gemfile +11 -0
- data/example/Gemfile.lock +129 -0
- data/example/README.md +73 -0
- data/example/VERSION +1 -0
- data/example/app.sh +19 -0
- data/example/config.yml +25 -0
- data/exe/magellan-gcs-proxy +1 -1
- data/exe/magellan-gcs-proxy-dev-progress-listener +14 -0
- data/exe/magellan-gcs-proxy-dev-setup +53 -0
- data/lib/magellan/gcs/proxy/cli.rb +24 -39
- data/lib/magellan/gcs/proxy/composite_logger.rb +37 -0
- data/lib/magellan/gcs/proxy/config.rb +44 -0
- data/lib/magellan/gcs/proxy/context.rb +33 -32
- data/lib/magellan/gcs/proxy/expand_variable.rb +20 -29
- data/lib/magellan/gcs/proxy/gcp.rb +10 -4
- data/lib/magellan/gcs/proxy/log.rb +67 -1
- data/lib/magellan/gcs/proxy/message_wrapper.rb +34 -5
- data/lib/magellan/gcs/proxy/progress_notification.rb +73 -0
- data/lib/magellan/gcs/proxy/progress_notifier_adapter.rb +24 -0
- data/lib/magellan/gcs/proxy/pubsub_progress_notifier.rb +25 -0
- data/lib/magellan/gcs/proxy/pubsub_sustainer.rb +54 -0
- data/lib/magellan/gcs/proxy/version.rb +1 -1
- data/lib/magellan/gcs/proxy.rb +20 -4
- data/magellan-gcs-proxy.gemspec +17 -15
- metadata +48 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 1232e616e80b9b1c977f0ac7f082d093a3502799
|
|
4
|
+
data.tar.gz: 8b23997772e69ac5685fc5f65b7526953ea343e2
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3e48f5f6203aac1ac89968c708bed4028390b49ad97666dd70654059bf9d55125bed1cdaa08a24a8ffd1afdbf1b4b1f34d48fa7d0973d1d9e2ff05b2e94574c2
|
|
7
|
+
data.tar.gz: 4d915466137763f3c37c59d2c3621c054b064ea3f2f9ef0cf8327f8a70a9a028b2ad9ce00dce8b3a4cd8bb3411cdc102ba316d1a87599e6066f44c6ed506ad58
|
data/.gitignore
CHANGED
data/.rubocop.yml
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
Style/Documentation:
|
|
2
|
+
Enabled: false
|
|
3
|
+
|
|
4
|
+
LineLength:
|
|
5
|
+
Max: 130
|
|
6
|
+
|
|
7
|
+
Metrics/AbcSize:
|
|
8
|
+
Max: 20
|
|
9
|
+
|
|
10
|
+
Metrics/CyclomaticComplexity:
|
|
11
|
+
Max: 8
|
|
12
|
+
|
|
13
|
+
Metrics/MethodLength:
|
|
14
|
+
Max: 15
|
|
15
|
+
|
|
16
|
+
Lint/AssignmentInCondition:
|
|
17
|
+
Enabled: false
|
|
18
|
+
|
|
19
|
+
Style/TrailingCommaInLiteral:
|
|
20
|
+
EnforcedStyleForMultiline: comma
|
data/Gemfile
CHANGED
data/README.md
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
#
|
|
1
|
+
# magellan-gcs-proxy
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
[](https://rubygems.org/gems/magellan-gcs-proxy) [](https://travis-ci.org/groovenauts/magellan-gcs-proxy)
|
|
4
|
+
|
|
5
|
+
magellan-gcs-proxy is a gem for MAGELLAN BLOCKS "Batch type" IoT board.
|
|
4
6
|
|
|
5
7
|
|
|
6
8
|
## Installation
|
|
@@ -35,28 +37,47 @@ magellan-gcs-proxy COMMAND ARGS...
|
|
|
35
37
|
### Setup
|
|
36
38
|
|
|
37
39
|
```
|
|
38
|
-
|
|
39
|
-
export TOPIC=projects/$PROJECT_ID/topics/[Topic name]
|
|
40
|
-
export SUB=projects/$PROJECT_ID/subscriptions/[Subscription name of topic]
|
|
41
|
-
$ gcloud beta pubsub topics create projects/$PROJECT_ID/topics/$TOPIC
|
|
42
|
-
$ gcloud beta pubsub topics list
|
|
43
|
-
$ gcloud beta pubsub subscriptions create $SUB --topic $TOPIC
|
|
44
|
-
$ gcloud beta pubsub subscriptions list
|
|
40
|
+
magellan-gcs-proxy-dev-setup [Project ID]
|
|
45
41
|
```
|
|
46
42
|
|
|
47
|
-
|
|
43
|
+
|
|
44
|
+
### Listen to progress subscription
|
|
48
45
|
|
|
49
46
|
```
|
|
50
|
-
$
|
|
47
|
+
$ export BLOCKS_BATCH_PROJECT_ID=[Project ID]
|
|
48
|
+
$ magellan-gcs-proxy-dev-progress-listener [Progress subscription name]
|
|
51
49
|
```
|
|
52
50
|
|
|
51
|
+
`Progress subscription name` is the name of `Progress subscription`.
|
|
52
|
+
It created by `magellan-gcs-proxy-dev-setup`.
|
|
53
|
+
You can see it by `gcloud beta pubsub subscriptions list`.
|
|
54
|
+
It starts with `test-progress-` and ends with '-sub'.
|
|
53
55
|
|
|
54
56
|
### Run application
|
|
55
57
|
|
|
56
58
|
```
|
|
59
|
+
$ export BLOCKS_BATCH_PROJECT_ID=[Project ID]
|
|
60
|
+
$ export BLOCKS_BATCH_PUBSUB_SUBSCRIPTION=[Job subscription name]
|
|
57
61
|
$ magellan-gcs-proxy COMMAND ARGS...
|
|
58
62
|
```
|
|
59
63
|
|
|
64
|
+
`Job subscription name` is the name of `Job subscription`.
|
|
65
|
+
It created by `magellan-gcs-proxy-dev-setup`.
|
|
66
|
+
You can see it by `gcloud beta pubsub subscriptions list`.
|
|
67
|
+
It starts with `test-job-` and ends with '-sub'.
|
|
68
|
+
|
|
69
|
+
### Publish message
|
|
70
|
+
|
|
71
|
+
```
|
|
72
|
+
$ export JOB_TOPIC=[Job topic name]
|
|
73
|
+
$ gcloud beta pubsub topics publish $JOB_TOPIC "" --attribute='download_files=["gs://bucket1/path/to/file"]'
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
`Job topic name` is the name of `Job topic`.
|
|
77
|
+
It created by `magellan-gcs-proxy-dev-setup`.
|
|
78
|
+
You can see it by `gcloud beta pubsub topics list`.
|
|
79
|
+
It starts with `test-job-` and ends with your user name.
|
|
80
|
+
|
|
60
81
|
|
|
61
82
|
## Development
|
|
62
83
|
|
|
@@ -72,4 +93,3 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/[USERN
|
|
|
72
93
|
## License
|
|
73
94
|
|
|
74
95
|
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
|
75
|
-
|
data/Rakefile
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
|
-
require
|
|
2
|
-
require
|
|
1
|
+
require 'bundler/gem_tasks'
|
|
2
|
+
require 'rspec/core/rake_task'
|
|
3
3
|
|
|
4
4
|
RSpec::Core::RakeTask.new(:spec)
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
require 'rubocop/rake_task'
|
|
7
|
+
RuboCop::RakeTask.new
|
|
8
|
+
|
|
9
|
+
task(:default).clear
|
|
10
|
+
task default: [:rubocop, :spec]
|
data/bin/console
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
2
|
|
|
3
|
-
require
|
|
4
|
-
require
|
|
3
|
+
require 'bundler/setup'
|
|
4
|
+
require 'magellan/gcs/proxy'
|
|
5
5
|
|
|
6
6
|
# You can add fixtures and/or initialization code here to make experimenting
|
|
7
7
|
# with your gem easier. You can also use a different console, if you like.
|
|
@@ -10,5 +10,5 @@ require "magellan/gcs/proxy"
|
|
|
10
10
|
# require "pry"
|
|
11
11
|
# Pry.start
|
|
12
12
|
|
|
13
|
-
require
|
|
13
|
+
require 'irb'
|
|
14
14
|
IRB.start
|
data/example/Dockerfile
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# [config] IMAGE_NAME: "groovenauts/batch_type_iot_example"
|
|
2
|
+
# [config]
|
|
3
|
+
# [config] WORKING_DIR: "."
|
|
4
|
+
# [config] VERSION_FILE: VERSION
|
|
5
|
+
# [config] GIT_TAG_PREFIX: 'example/'
|
|
6
|
+
# [config]
|
|
7
|
+
|
|
8
|
+
FROM ruby:2.3
|
|
9
|
+
|
|
10
|
+
COPY Gemfile* /tmp/
|
|
11
|
+
WORKDIR /tmp
|
|
12
|
+
RUN bundle install
|
|
13
|
+
|
|
14
|
+
ENV APP_HOME /usr/app/batch_type_example
|
|
15
|
+
COPY . $APP_HOME
|
|
16
|
+
WORKDIR $APP_HOME
|
|
17
|
+
RUN bundle install --without development
|
|
18
|
+
|
|
19
|
+
CMD ["bundle", "exec", "magellan-gcs-proxy", "./app.sh", "%{download_files.0}", "%{uploads_dir}"]
|
data/example/Gemfile
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
GEM
|
|
2
|
+
remote: https://rubygems.org/
|
|
3
|
+
specs:
|
|
4
|
+
addressable (2.5.0)
|
|
5
|
+
public_suffix (~> 2.0, >= 2.0.2)
|
|
6
|
+
binding_of_caller (0.7.2)
|
|
7
|
+
debug_inspector (>= 0.0.1)
|
|
8
|
+
brocket (0.2.2)
|
|
9
|
+
logger_pipe
|
|
10
|
+
thor
|
|
11
|
+
byebug (9.0.6)
|
|
12
|
+
coderay (1.1.1)
|
|
13
|
+
debug_inspector (0.0.2)
|
|
14
|
+
digest-crc (0.4.1)
|
|
15
|
+
dotenv (2.1.1)
|
|
16
|
+
faraday (0.10.0)
|
|
17
|
+
multipart-post (>= 1.2, < 3)
|
|
18
|
+
google-api-client (0.9.20)
|
|
19
|
+
addressable (~> 2.3)
|
|
20
|
+
googleauth (~> 0.5)
|
|
21
|
+
httpclient (~> 2.7)
|
|
22
|
+
hurley (~> 0.1)
|
|
23
|
+
memoist (~> 0.11)
|
|
24
|
+
mime-types (>= 1.6)
|
|
25
|
+
representable (~> 2.3.0)
|
|
26
|
+
retriable (~> 2.0)
|
|
27
|
+
google-cloud-core (0.21.1)
|
|
28
|
+
googleauth (~> 0.5.1)
|
|
29
|
+
google-cloud-logging (0.21.1)
|
|
30
|
+
google-cloud-core (~> 0.21.0)
|
|
31
|
+
google-gax (~> 0.6.0)
|
|
32
|
+
google-protobuf (~> 3.0)
|
|
33
|
+
googleapis-common-protos (~> 1.3)
|
|
34
|
+
grpc (~> 1.0)
|
|
35
|
+
orderedhash (= 0.0.6)
|
|
36
|
+
google-cloud-pubsub (0.21.0)
|
|
37
|
+
google-cloud-core (~> 0.21.0)
|
|
38
|
+
google-gax (~> 0.6.0)
|
|
39
|
+
google-protobuf (~> 3.0)
|
|
40
|
+
googleapis-common-protos (~> 1.3)
|
|
41
|
+
grpc (~> 1.0)
|
|
42
|
+
grpc-google-iam-v1 (~> 0.6.8)
|
|
43
|
+
google-cloud-storage (0.21.0)
|
|
44
|
+
digest-crc (~> 0.4)
|
|
45
|
+
google-api-client (~> 0.9.11)
|
|
46
|
+
google-cloud-core (~> 0.21.0)
|
|
47
|
+
google-gax (0.6.0)
|
|
48
|
+
googleapis-common-protos (~> 1.3.1)
|
|
49
|
+
googleauth (~> 0.5.1)
|
|
50
|
+
grpc (~> 1.0)
|
|
51
|
+
rly (~> 0.2.3)
|
|
52
|
+
google-protobuf (3.0.2)
|
|
53
|
+
googleapis-common-protos (1.3.4)
|
|
54
|
+
google-protobuf (~> 3.0)
|
|
55
|
+
grpc (~> 1.0)
|
|
56
|
+
googleauth (0.5.1)
|
|
57
|
+
faraday (~> 0.9)
|
|
58
|
+
jwt (~> 1.4)
|
|
59
|
+
logging (~> 2.0)
|
|
60
|
+
memoist (~> 0.12)
|
|
61
|
+
multi_json (~> 1.11)
|
|
62
|
+
os (~> 0.9)
|
|
63
|
+
signet (~> 0.7)
|
|
64
|
+
grpc (1.0.1)
|
|
65
|
+
google-protobuf (~> 3.0.2)
|
|
66
|
+
googleauth (~> 0.5.1)
|
|
67
|
+
grpc-google-iam-v1 (0.6.8)
|
|
68
|
+
googleapis-common-protos (~> 1.3.1)
|
|
69
|
+
googleauth (~> 0.5.1)
|
|
70
|
+
grpc (~> 1.0)
|
|
71
|
+
httpclient (2.8.2.4)
|
|
72
|
+
hurley (0.2)
|
|
73
|
+
jwt (1.5.6)
|
|
74
|
+
little-plugger (1.1.4)
|
|
75
|
+
logger_pipe (0.3.1)
|
|
76
|
+
logging (2.1.0)
|
|
77
|
+
little-plugger (~> 1.1)
|
|
78
|
+
multi_json (~> 1.10)
|
|
79
|
+
magellan-gcs-proxy (0.1.1)
|
|
80
|
+
dotenv
|
|
81
|
+
google-cloud-logging
|
|
82
|
+
google-cloud-pubsub
|
|
83
|
+
google-cloud-storage
|
|
84
|
+
logger_pipe
|
|
85
|
+
memoist (0.15.0)
|
|
86
|
+
method_source (0.8.2)
|
|
87
|
+
mime-types (3.1)
|
|
88
|
+
mime-types-data (~> 3.2015)
|
|
89
|
+
mime-types-data (3.2016.0521)
|
|
90
|
+
multi_json (1.12.1)
|
|
91
|
+
multipart-post (2.0.0)
|
|
92
|
+
orderedhash (0.0.6)
|
|
93
|
+
os (0.9.6)
|
|
94
|
+
pry (0.10.4)
|
|
95
|
+
coderay (~> 1.1.0)
|
|
96
|
+
method_source (~> 0.8.1)
|
|
97
|
+
slop (~> 3.4)
|
|
98
|
+
pry-byebug (3.4.1)
|
|
99
|
+
byebug (~> 9.0)
|
|
100
|
+
pry (~> 0.10)
|
|
101
|
+
pry-stack_explorer (0.4.9.2)
|
|
102
|
+
binding_of_caller (>= 0.7)
|
|
103
|
+
pry (>= 0.9.11)
|
|
104
|
+
public_suffix (2.0.4)
|
|
105
|
+
representable (2.3.0)
|
|
106
|
+
uber (~> 0.0.7)
|
|
107
|
+
retriable (2.1.0)
|
|
108
|
+
rly (0.2.3)
|
|
109
|
+
signet (0.7.3)
|
|
110
|
+
addressable (~> 2.3)
|
|
111
|
+
faraday (~> 0.9)
|
|
112
|
+
jwt (~> 1.5)
|
|
113
|
+
multi_json (~> 1.10)
|
|
114
|
+
slop (3.6.0)
|
|
115
|
+
thor (0.19.4)
|
|
116
|
+
uber (0.0.15)
|
|
117
|
+
|
|
118
|
+
PLATFORMS
|
|
119
|
+
ruby
|
|
120
|
+
|
|
121
|
+
DEPENDENCIES
|
|
122
|
+
brocket
|
|
123
|
+
magellan-gcs-proxy
|
|
124
|
+
pry
|
|
125
|
+
pry-byebug
|
|
126
|
+
pry-stack_explorer
|
|
127
|
+
|
|
128
|
+
BUNDLED WITH
|
|
129
|
+
1.13.6
|
data/example/README.md
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# magellan-gcs-proxy example
|
|
2
|
+
|
|
3
|
+
## How to run application locally
|
|
4
|
+
|
|
5
|
+
```
|
|
6
|
+
$ cd path/to/somewhere
|
|
7
|
+
$ git clone https://github.com/groovenauts/magellan-gcs-proxy.git
|
|
8
|
+
$ cd magellan-gcs-proxy/example
|
|
9
|
+
$ bundle
|
|
10
|
+
$ export BLOCKS_BATCH_PROJECT_ID=[Your project ID]
|
|
11
|
+
$ magellan-gcs-proxy-dev-setup $BLOCKS_BATCH_PROJECT_ID
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
Follow the steps `magellan-gcs-proxy-dev-setup` shows.
|
|
15
|
+
|
|
16
|
+
## Result
|
|
17
|
+
|
|
18
|
+
### Terminal 1
|
|
19
|
+
|
|
20
|
+
```
|
|
21
|
+
$ magellan-gcs-proxy-dev-progress-listener $BLOCKS_BATCH_PROGRESS_SUBSCRIPTION
|
|
22
|
+
|
|
23
|
+
Listening to projects/scenic-doodad-617/subscriptions/test-progress-topic-akm-sub
|
|
24
|
+
total:12 job_message_id:49719039110847 progress:1 level:info Processing message: #<Google::Cloud::Pubsub::ReceivedMessage:0x007f9c6b2127d0 @subscription=#<Google::Cloud::Pubsub::Subscription:0x007f9c6b386f58 @service=Google::Cloud::Pubsub::Service(scenic-doodad-617), @grpc=<Google::Pubsub::V1::Subscription: name: "projects/scenic-doodad-617/subscriptions/test-job-topic-akm-sub", topic: "projects/scenic-doodad-617/topics/test-job-topic-akm", push_config: <Google::Pubsub::V1::PushConfig: push_endpoint: "", attributes: {}>, ack_deadline_seconds: 10>, @name=nil, @exists=nil>, @grpc=<Google::Pubsub::V1::ReceivedMessage: ack_id: "OUVBXkASTDYMRElTK0MLKlgRTgQhIT4wPkVTRFAGFixdRkhRNxkIaFEOT14jPzUgKEUSAAgUBXx9clNcdV8zdQdRDRlzeGkmP1lGU1ARB3ReURsfWVx-SwNYChh7fWJ8a1sTCQZCe4m8xtU_Zhs9XxJLLD5-PQ", message: <Google::Pubsub::V1::PubsubMessage: data: "", attributes: {"download_files"=>"[\"gs://bucket1/dir1/database.yml\"]"}, message_id: "49719039110847", publish_time: <Google::Protobuf::Timestamp: seconds: 1480929118, nanos: 504000000>>>>
|
|
25
|
+
total:12 job_message_id:49719039110847 progress:2 level:info Download starting
|
|
26
|
+
total:12 job_message_id:49719039110847 progress:3 level:info Download completed
|
|
27
|
+
total:12 job_message_id:49719039110847 progress:5 level:info Command starting
|
|
28
|
+
total:12 job_message_id:49719039110847 progress:8 level:info Upload starting
|
|
29
|
+
total:12 job_message_id:49719039110847 progress:6 level:info Command completed
|
|
30
|
+
total:12 job_message_id:49719039110847 progress:9 level:info Upload completed
|
|
31
|
+
total:12 job_message_id:49719039110847 progress:11 level:info Acknowledged
|
|
32
|
+
total:12 job_message_id:49719039110847 progress:12 level:info Cleanup
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
### Terminal 2
|
|
36
|
+
|
|
37
|
+
```
|
|
38
|
+
$ bundle exec magellan-gcs-proxy ./app.sh %{download_files.0} %{uploads_dir}
|
|
39
|
+
|
|
40
|
+
====================================================================================================
|
|
41
|
+
progress_notification:
|
|
42
|
+
topic: projects/scenic-doodad-617/topics/test-progress-topic-akm
|
|
43
|
+
|
|
44
|
+
loggers:
|
|
45
|
+
- type: stdout
|
|
46
|
+
- type: cloud_logging
|
|
47
|
+
log_name: magellan-gcs-proxy-example-logging
|
|
48
|
+
----------------------------------------------------------------------------------------------------
|
|
49
|
+
I, [2016-12-05T18:12:32.892158 #22285] INFO -- : Start listening
|
|
50
|
+
I, [2016-12-05T18:12:34.077139 #22285] INFO -- : subscription: #<Google::Cloud::Pubsub::Subscription:0x007f9c6b386f58 @service=Google::Cloud::Pubsub::Service(scenic-doodad-617), @grpc=<Google::Pubsub::V1::Subscription: name: "projects/scenic-doodad-617/subscriptions/test-job-topic-akm-sub", topic: "projects/scenic-doodad-617/topics/test-job-topic-akm", push_config: <Google::Pubsub::V1::PushConfig: push_endpoint: "", attributes: {}>, ack_deadline_seconds: 10>, @name=nil, @exists=nil>
|
|
51
|
+
I, [2016-12-05T18:12:35.572740 #22285] INFO -- : job_message_id:49719039110847 progress:1 total:12 data:Processing message: #<Google::Cloud::Pubsub::ReceivedMessage:0x007f9c6b2127d0 @subscription=#<Google::Cloud::Pubsub::Subscription:0x007f9c6b386f58 @service=Google::Cloud::Pubsub::Service(scenic-doodad-617), @grpc=<Google::Pubsub::V1::Subscription: name: "projects/scenic-doodad-617/subscriptions/test-job-topic-akm-sub", topic: "projects/scenic-doodad-617/topics/test-job-topic-akm", push_config: <Google::Pubsub::V1::PushConfig: push_endpoint: "", attributes: {}>, ack_deadline_seconds: 10>, @name=nil, @exists=nil>, @grpc=<Google::Pubsub::V1::ReceivedMessage: ack_id: "OUVBXkASTDYMRElTK0MLKlgRTgQhIT4wPkVTRFAGFixdRkhRNxkIaFEOT14jPzUgKEUSAAgUBXx9clNcdV8zdQdRDRlzeGkmP1lGU1ARB3ReURsfWVx-SwNYChh7fWJ8a1sTCQZCe4m8xtU_Zhs9XxJLLD5-PQ", message: <Google::Pubsub::V1::PubsubMessage: data: "", attributes: {"download_files"=>"[\"gs://bucket1/dir1/database.yml\"]"}, message_id: "49719039110847", publish_time: <Google::Protobuf::Timestamp: seconds: 1480929118, nanos: 504000000>>>>
|
|
52
|
+
I, [2016-12-05T18:12:35.785083 #22285] INFO -- : job_message_id:49719039110847 progress:2 total:12 data:Download starting
|
|
53
|
+
D, [2016-12-05T18:12:35.942864 #22285] DEBUG -- : Downloading: gs://bucket1/dir1/database.yml to /var/folders/70/5yxt0jys7ss_m71y5pz_l_qc0000gn/T/workspace20161205-22285-18l5pmo/downloads/dir1/database.yml
|
|
54
|
+
I, [2016-12-05T18:12:37.559498 #22285] INFO -- : Download OK: gs://bucket1/dir1/database.yml to /var/folders/70/5yxt0jys7ss_m71y5pz_l_qc0000gn/T/workspace20161205-22285-18l5pmo/downloads/dir1/database.yml
|
|
55
|
+
I, [2016-12-05T18:12:37.796599 #22285] INFO -- : job_message_id:49719039110847 progress:3 total:12 data:Download completed
|
|
56
|
+
I, [2016-12-05T18:12:38.072803 #22285] INFO -- : job_message_id:49719039110847 progress:5 total:12 data:Command starting
|
|
57
|
+
I, [2016-12-05T18:12:38.223959 #22285] INFO -- : executing: ./app.sh /var/folders/70/5yxt0jys7ss_m71y5pz_l_qc0000gn/T/workspace20161205-22285-18l5pmo/downloads/dir1/database.yml /var/folders/70/5yxt0jys7ss_m71y5pz_l_qc0000gn/T/workspace20161205-22285-18l5pmo/uploads 2>&1
|
|
58
|
+
I, [2016-12-05T18:12:38.465296 #22285] INFO -- : SUCCESS: ./app.sh /var/folders/70/5yxt0jys7ss_m71y5pz_l_qc0000gn/T/workspace20161205-22285-18l5pmo/downloads/dir1/database.yml /var/folders/70/5yxt0jys7ss_m71y5pz_l_qc0000gn/T/workspace20161205-22285-18l5pmo/uploads 2>&1
|
|
59
|
+
I, [2016-12-05T18:12:38.859797 #22285] INFO -- : job_message_id:49719039110847 progress:6 total:12 data:Command completed
|
|
60
|
+
I, [2016-12-05T18:12:39.081097 #22285] INFO -- : job_message_id:49719039110847 progress:8 total:12 data:Upload starting
|
|
61
|
+
I, [2016-12-05T18:12:39.205161 #22285] INFO -- : Uploading: database-20161205-181238.yml to gs://bucket1/database-20161205-181238.yml
|
|
62
|
+
I, [2016-12-05T18:12:41.102121 #22285] INFO -- : Upload OK: database-20161205-181238.yml to gs://bucket1/database-20161205-181238.yml
|
|
63
|
+
I, [2016-12-05T18:12:41.346263 #22285] INFO -- : job_message_id:49719039110847 progress:9 total:12 data:Upload completed
|
|
64
|
+
I, [2016-12-05T18:12:41.662179 #22285] INFO -- : job_message_id:49719039110847 progress:11 total:12 data:Acknowledged
|
|
65
|
+
I, [2016-12-05T18:12:41.881730 #22285] INFO -- : job_message_id:49719039110847 progress:12 total:12 data:Cleanup
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
### Terminal 3 Publish message
|
|
69
|
+
|
|
70
|
+
```
|
|
71
|
+
$ gcloud beta pubsub topics publish $BLOCKS_BATCH_PUBSUB_TOPIC "" --attribute='download_files=["gs://bucket1/dir1/database.yml"]'
|
|
72
|
+
messageIds: '49718447408725'
|
|
73
|
+
```
|
data/example/VERSION
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
0.0.1
|
data/example/app.sh
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
|
|
3
|
+
usage() {
|
|
4
|
+
echo usage: `basename $0` src_path dest_path
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
if [ $# -ne 2 ]; then
|
|
8
|
+
echo `basename $0`: missing operand 1>&2
|
|
9
|
+
usage
|
|
10
|
+
exit 1
|
|
11
|
+
fi
|
|
12
|
+
|
|
13
|
+
fpath=$1
|
|
14
|
+
fname="${fpath##*/}"
|
|
15
|
+
fbase="${fname%.*}"
|
|
16
|
+
fext="${fpath##*.}"
|
|
17
|
+
mkdir -p $2
|
|
18
|
+
|
|
19
|
+
cp $1 $2/${fbase}-$(date '+%Y%m%d-%T' | tr -d :).${fext}
|
data/example/config.yml
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
<%- if progress_topic = ENV['BLOCKS_BATCH_PROGRESS_TOPIC'] -%>
|
|
2
|
+
progress_notification:
|
|
3
|
+
topic: <%= progress_topic %>
|
|
4
|
+
<%- end -%>
|
|
5
|
+
|
|
6
|
+
loggers:
|
|
7
|
+
- type: stdout
|
|
8
|
+
<%- if log_name = ENV['BLOCKS_BATCH_CLOUD_LOGGING_LOG_NAME'] -%>
|
|
9
|
+
- type: cloud_logging
|
|
10
|
+
log_name: <%= log_name %>
|
|
11
|
+
<%- end -%>
|
|
12
|
+
|
|
13
|
+
sustainer:
|
|
14
|
+
# delay:
|
|
15
|
+
# Delay message deadline for log jobs.
|
|
16
|
+
# It must be less than or equal to ACK_DEADLINE of subscription
|
|
17
|
+
# specified by BLOCKS_BATCH_PUBSUB_SUBSCRIPTION
|
|
18
|
+
# You can check ACK_DEADLINE by `gcloud beta pubsub subscriptions list`
|
|
19
|
+
# ACK_DEADLINE is set by `--ack-deadline` option for `gcloud beta pubsub subscriptions create`.
|
|
20
|
+
delay: 600
|
|
21
|
+
|
|
22
|
+
# interval
|
|
23
|
+
# The interval to send delay message. It must be lower than delay
|
|
24
|
+
# Default: 90% of delay
|
|
25
|
+
interval: 540
|
data/exe/magellan-gcs-proxy
CHANGED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
require 'magellan/gcs/proxy'
|
|
4
|
+
require 'json'
|
|
5
|
+
|
|
6
|
+
subname = ARGV.first
|
|
7
|
+
puts "Listening to #{subname}"
|
|
8
|
+
Magellan::Gcs::Proxy::GCP.pubsub.subscription(subname).tap do |sub|
|
|
9
|
+
sub.listen do |msg|
|
|
10
|
+
attrs = { 'data' => msg.data }.update(msg.attributes)
|
|
11
|
+
puts attrs.map { |k, v| "#{k}:#{v}" }.join("\t")
|
|
12
|
+
msg.ack!
|
|
13
|
+
end
|
|
14
|
+
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
|
|
3
|
+
usage() {
|
|
4
|
+
echo usage: `basename $0` GCP_PROJECT_ID
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
if [ $# -ne 1 ]; then
|
|
8
|
+
echo `basename $0`: missing operand 1>&2
|
|
9
|
+
usage
|
|
10
|
+
exit 1
|
|
11
|
+
fi
|
|
12
|
+
|
|
13
|
+
project_id=$1
|
|
14
|
+
export job_topic=projects/${project_id}/topics/test-job-topic-${USER}
|
|
15
|
+
export job_sub=projects/${project_id}/subscriptions/test-job-topic-${USER}-sub
|
|
16
|
+
|
|
17
|
+
export progress_topic=projects/${project_id}/topics/test-progress-topic-${USER}
|
|
18
|
+
export progress_sub=projects/${project_id}/subscriptions/test-progress-topic-${USER}-sub
|
|
19
|
+
|
|
20
|
+
gcloud beta pubsub topics create ${job_topic}
|
|
21
|
+
gcloud beta pubsub subscriptions create ${job_sub} --topic ${job_topic}
|
|
22
|
+
|
|
23
|
+
gcloud beta pubsub topics create ${progress_topic}
|
|
24
|
+
gcloud beta pubsub subscriptions create ${progress_sub} --topic ${progress_topic}
|
|
25
|
+
|
|
26
|
+
gcloud beta pubsub topics list >/dev/null 2>&1
|
|
27
|
+
gcloud beta pubsub subscriptions list >/dev/null 2>&1
|
|
28
|
+
|
|
29
|
+
echo "# Steps to run example"
|
|
30
|
+
echo "#"
|
|
31
|
+
echo "# At Terminal 1: Listen to progress subscription"
|
|
32
|
+
echo "#"
|
|
33
|
+
echo "cd $PWD"
|
|
34
|
+
echo "export BLOCKS_BATCH_PROJECT_ID=${project_id}"
|
|
35
|
+
echo "export BLOCKS_BATCH_PROGRESS_SUBSCRIPTION=${progress_sub}"
|
|
36
|
+
echo "magellan-gcs-proxy-dev-progress-listener \$BLOCKS_BATCH_PROGRESS_SUBSCRIPTION"
|
|
37
|
+
echo ""
|
|
38
|
+
echo "#"
|
|
39
|
+
echo "# At Terminal 2: Run application"
|
|
40
|
+
echo "#"
|
|
41
|
+
echo "cd $PWD"
|
|
42
|
+
echo "export BLOCKS_BATCH_PROJECT_ID=${project_id}"
|
|
43
|
+
echo "export BLOCKS_BATCH_PUBSUB_SUBSCRIPTION=${job_sub}"
|
|
44
|
+
echo "export BLOCKS_BATCH_PROGRESS_TOPIC=${progress_topic}"
|
|
45
|
+
echo "export BLOCKS_BATCH_CLOUD_LOGGING_LOG_NAME=magellan-gcs-proxy-example-logging"
|
|
46
|
+
echo "bundle exec magellan-gcs-proxy ./app.sh %{download_files.0} %{uploads_dir}"
|
|
47
|
+
echo ""
|
|
48
|
+
echo "#"
|
|
49
|
+
echo "# At Terminal 3: Publish message"
|
|
50
|
+
echo "#"
|
|
51
|
+
echo "cd $PWD"
|
|
52
|
+
echo "export BLOCKS_BATCH_PUBSUB_TOPIC=${job_topic}"
|
|
53
|
+
echo "gcloud beta pubsub topics publish \$BLOCKS_BATCH_PUBSUB_TOPIC \"\" --attribute='download_files=[\"gs://bucket1/path/to/file\"]'"
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
require
|
|
2
|
-
require
|
|
1
|
+
require 'magellan/gcs/proxy'
|
|
2
|
+
require 'magellan/gcs/proxy/log'
|
|
3
3
|
|
|
4
4
|
require 'json'
|
|
5
5
|
require 'logger'
|
|
@@ -18,59 +18,44 @@ module Magellan
|
|
|
18
18
|
end
|
|
19
19
|
|
|
20
20
|
def run
|
|
21
|
-
logger.info(
|
|
21
|
+
logger.info('Start listening')
|
|
22
22
|
GCP.subscription.listen do |msg|
|
|
23
|
-
|
|
24
|
-
process(msg)
|
|
25
|
-
rescue => e
|
|
26
|
-
logger.error("[#{e.class.name}] #{e.message}")
|
|
27
|
-
if ENV['VERBOSE'] =~ /true|yes|on|1/i
|
|
28
|
-
logger.debug("Backtrace\n " << e.backtrace.join("\n "))
|
|
29
|
-
end
|
|
30
|
-
end
|
|
23
|
+
process_with_error_handling(msg)
|
|
31
24
|
end
|
|
32
25
|
rescue => e
|
|
33
26
|
logger.error("[#{e.class.name}] #{e.message}")
|
|
34
27
|
raise e
|
|
35
28
|
end
|
|
36
29
|
|
|
37
|
-
def
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
30
|
+
def process_with_error_handling(msg)
|
|
31
|
+
process(msg)
|
|
32
|
+
rescue => e
|
|
33
|
+
logger.error("[#{e.class.name}] #{e.message}")
|
|
34
|
+
verbose("Backtrace\n " << e.backtrace.join("\n "))
|
|
35
|
+
end
|
|
42
36
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
37
|
+
TOTAL = 12
|
|
38
|
+
def process(msg)
|
|
39
|
+
context = Context.new(msg)
|
|
40
|
+
context.notify(1, TOTAL, "Processing message: #{msg.inspect}")
|
|
41
|
+
context.setup do
|
|
42
|
+
context.process_with_notification([2, 3, 4], TOTAL, 'Download', &:download)
|
|
46
43
|
|
|
47
|
-
context
|
|
48
|
-
logger.info("context.download done.")
|
|
49
|
-
logger.info("msg: #{msg}")
|
|
50
|
-
logger.info("context: #{context}")
|
|
44
|
+
cmd = build_command(context)
|
|
51
45
|
|
|
52
|
-
|
|
46
|
+
exec = ->(*) { LoggerPipe.run(logger, cmd, returns: :none, logging: :both) }
|
|
47
|
+
context.process_with_notification([5, 6, 7], TOTAL, 'Command', exec) do
|
|
48
|
+
context.process_with_notification([8, 9, 10], TOTAL, 'Upload', &:upload)
|
|
53
49
|
|
|
54
|
-
begin
|
|
55
|
-
LoggerPipe.run(logger, cmd, returns: :none, logging: :both)
|
|
56
|
-
rescue => e
|
|
57
|
-
logger.error("Error: #{cmd.inspect}")
|
|
58
|
-
else
|
|
59
|
-
context.upload
|
|
60
50
|
msg.acknowledge!
|
|
61
|
-
|
|
51
|
+
context.notify(11, TOTAL, 'Acknowledged')
|
|
62
52
|
end
|
|
63
53
|
end
|
|
54
|
+
context.notify(12, TOTAL, 'Cleanup')
|
|
64
55
|
end
|
|
65
56
|
|
|
66
|
-
def
|
|
67
|
-
|
|
68
|
-
JSON.parse(str)
|
|
69
|
-
end
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
def build_command(msg, context)
|
|
73
|
-
msg_wrapper = MessageWrapper.new(msg, context)
|
|
57
|
+
def build_command(context)
|
|
58
|
+
msg_wrapper = MessageWrapper.new(context)
|
|
74
59
|
ExpandVariable.expand_variables(cmd_template, msg_wrapper)
|
|
75
60
|
end
|
|
76
61
|
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
require 'magellan/gcs/proxy'
|
|
2
|
+
|
|
3
|
+
module Magellan
|
|
4
|
+
module Gcs
|
|
5
|
+
module Proxy
|
|
6
|
+
class CompositeLogger
|
|
7
|
+
attr_reader :loggers
|
|
8
|
+
def initialize(loggers)
|
|
9
|
+
@loggers = loggers
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def add(severity, message = nil, &block)
|
|
13
|
+
loggers.each do |logger|
|
|
14
|
+
begin
|
|
15
|
+
logger.add(severity, message, &block)
|
|
16
|
+
rescue => e
|
|
17
|
+
$stderr.puts("[#{e.class}] #{e.message}")
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
Logger::Severity.constants.each do |level|
|
|
23
|
+
level_dc = level.downcase
|
|
24
|
+
module_eval <<-INSTANCE_METHODS
|
|
25
|
+
def #{level_dc}(message = nil, &block)
|
|
26
|
+
add(Logger::Severity::#{level}, message, &block)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def #{level_dc}?
|
|
30
|
+
loggers.any?(&:#{level_dc}?)
|
|
31
|
+
end
|
|
32
|
+
INSTANCE_METHODS
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|