magellan-gcs-proxy 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: fbfdc826b779db7cec5c24192988e35247c56e33
|
|
4
|
+
data.tar.gz: 56879281961eebe651ac777a034505bb8cb060b7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 67dccc48b4ab362d2090cf1f82d2a2caa16201fbdf053e15e97ea89b94f03c6bd7deaeaec741321dfc92ce877aa2535a47dce84ba08e67cf2a252db94cbb0a5b
|
|
7
|
+
data.tar.gz: b5c10a2f63c1ed87498d92ad40d7a1882c49e8dae503f61b34d98c317b1abd6909cdea858dd48e13c105ad660c20674c33be8b8575d2ebe03fcf2cdd371ab84c
|
data/example/Gemfile.lock
CHANGED
|
@@ -12,8 +12,8 @@ GEM
|
|
|
12
12
|
coderay (1.1.1)
|
|
13
13
|
debug_inspector (0.0.2)
|
|
14
14
|
digest-crc (0.4.1)
|
|
15
|
-
dotenv (2.1.
|
|
16
|
-
faraday (0.
|
|
15
|
+
dotenv (2.1.2)
|
|
16
|
+
faraday (0.11.0)
|
|
17
17
|
multipart-post (>= 1.2, < 3)
|
|
18
18
|
google-api-client (0.9.20)
|
|
19
19
|
addressable (~> 2.3)
|
|
@@ -26,13 +26,14 @@ GEM
|
|
|
26
26
|
retriable (~> 2.0)
|
|
27
27
|
google-cloud-core (0.21.1)
|
|
28
28
|
googleauth (~> 0.5.1)
|
|
29
|
-
google-cloud-logging (0.23.
|
|
30
|
-
google-cloud-core (~> 0.21.
|
|
29
|
+
google-cloud-logging (0.23.2)
|
|
30
|
+
google-cloud-core (~> 0.21.1)
|
|
31
31
|
google-gax (~> 0.6.0)
|
|
32
32
|
google-protobuf (~> 3.0)
|
|
33
33
|
googleapis-common-protos (~> 1.3)
|
|
34
34
|
grpc (~> 1.0)
|
|
35
35
|
orderedhash (= 0.0.6)
|
|
36
|
+
stackdriver-core (~> 0.21.0)
|
|
36
37
|
google-cloud-storage (0.23.1)
|
|
37
38
|
digest-crc (~> 0.4)
|
|
38
39
|
google-api-client (~> 0.9.11)
|
|
@@ -65,7 +66,7 @@ GEM
|
|
|
65
66
|
logging (2.1.0)
|
|
66
67
|
little-plugger (~> 1.1)
|
|
67
68
|
multi_json (~> 1.10)
|
|
68
|
-
magellan-gcs-proxy (0.3.
|
|
69
|
+
magellan-gcs-proxy (0.3.1)
|
|
69
70
|
dotenv
|
|
70
71
|
google-cloud-logging
|
|
71
72
|
google-cloud-storage
|
|
@@ -89,7 +90,7 @@ GEM
|
|
|
89
90
|
pry-stack_explorer (0.4.9.2)
|
|
90
91
|
binding_of_caller (>= 0.7)
|
|
91
92
|
pry (>= 0.9.11)
|
|
92
|
-
public_suffix (2.0.
|
|
93
|
+
public_suffix (2.0.5)
|
|
93
94
|
representable (2.3.0)
|
|
94
95
|
uber (~> 0.0.7)
|
|
95
96
|
retriable (2.1.0)
|
|
@@ -100,6 +101,7 @@ GEM
|
|
|
100
101
|
jwt (~> 1.5)
|
|
101
102
|
multi_json (~> 1.10)
|
|
102
103
|
slop (3.6.0)
|
|
104
|
+
stackdriver-core (0.21.0)
|
|
103
105
|
thor (0.19.4)
|
|
104
106
|
uber (0.0.15)
|
|
105
107
|
|
data/example/config.yml
CHANGED
|
@@ -44,7 +44,7 @@ module Magellan
|
|
|
44
44
|
|
|
45
45
|
cmd = build_command(context)
|
|
46
46
|
|
|
47
|
-
exec = ->(*) { LoggerPipe.run(logger, cmd, returns: :none, logging: :both) }
|
|
47
|
+
exec = ->(*) { LoggerPipe.run(logger, cmd, returns: :none, logging: :both, dry_run: Proxy.config[:dryrun]) }
|
|
48
48
|
context.process_with_notification([5, 6, 7], TOTAL, 'Command', exec) do
|
|
49
49
|
context.process_with_notification([8, 9, 10], TOTAL, 'Upload', &:upload)
|
|
50
50
|
|
|
@@ -54,6 +54,7 @@ module Magellan
|
|
|
54
54
|
download_mapping.each do |url, path|
|
|
55
55
|
FileUtils.mkdir_p File.dirname(path)
|
|
56
56
|
logger.debug("Downloading: #{url} to #{path}")
|
|
57
|
+
next if Proxy.config[:dryrun]
|
|
57
58
|
uri = parse_uri(url)
|
|
58
59
|
@last_bucket_name = uri.host
|
|
59
60
|
bucket = GCP.storage.bucket(@last_bucket_name)
|
|
@@ -71,6 +72,7 @@ module Magellan
|
|
|
71
72
|
next if directory?(path)
|
|
72
73
|
url = "gs://#{bucket_name}/#{path}"
|
|
73
74
|
logger.info("Uploading: #{path} to #{url}")
|
|
75
|
+
next if Proxy.config[:dryrun]
|
|
74
76
|
bucket = GCP.storage.bucket(bucket_name)
|
|
75
77
|
bucket.create_file path, path
|
|
76
78
|
logger.info("Upload OK: #{path} to #{url}")
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
require 'magellan/gcs/proxy'
|
|
2
|
+
require 'magellan/gcs/proxy/log'
|
|
2
3
|
|
|
3
4
|
require 'logger'
|
|
4
5
|
require 'json'
|
|
@@ -7,6 +8,8 @@ module Magellan
|
|
|
7
8
|
module Gcs
|
|
8
9
|
module Proxy
|
|
9
10
|
class PubsubProgressNotifier
|
|
11
|
+
include Log
|
|
12
|
+
|
|
10
13
|
attr_reader :topic_name
|
|
11
14
|
def initialize(topic_name)
|
|
12
15
|
@topic_name = topic_name
|
|
@@ -17,7 +20,13 @@ module Magellan
|
|
|
17
20
|
end
|
|
18
21
|
|
|
19
22
|
def notify(severity, job_message, data, attrs)
|
|
20
|
-
|
|
23
|
+
attrs = { level: severity, job_message_id: job_message.message_id }.merge(attrs)
|
|
24
|
+
# attrs must be an [Hash<String,String>]
|
|
25
|
+
attrs = attrs.each_with_object({}) { |(k, v), d| d[k.to_s] = v.to_s }
|
|
26
|
+
logger.debug("Publishing progress: #{attrs.inspect}")
|
|
27
|
+
msg = Google::Apis::PubsubV1::Message.new(data: data, attributes: attrs)
|
|
28
|
+
req = Google::Apis::PubsubV1::PublishRequest.new(messages: [msg])
|
|
29
|
+
GCP.pubsub.publish_topic(topic_name, req)
|
|
21
30
|
end
|
|
22
31
|
end
|
|
23
32
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: magellan-gcs-proxy
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- akm
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2017-01-24 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: dotenv
|