magellan-gcs-proxy 0.1.4 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 50724872f81d7d6de7c7803628cd766428150f5d
4
- data.tar.gz: bc1b1ed15f3239206268f2596b9769aa22cf683f
3
+ metadata.gz: 97082e16910f8eb448e9dd90a579dab3f5d2645b
4
+ data.tar.gz: b8ef0a1084a6da1b9287a9806c91ebd1e73b0ba8
5
5
  SHA512:
6
- metadata.gz: ae86f4f6bd8b441cbc11fdd21ffa580cacbb81676d8f092b196ff961ad3924798c7d767d2c7a31e993d6b0ce67fbb6ebde99a5597bc13990c547f23158dbb40e
7
- data.tar.gz: da279b3209b1b16931a870e260c74de94a2ecc6315ecf111b7774cdc5b3d101c73a97704e9d77a6f888bda517b591ac84050610bc87fd76f64ad64a92e315a8f
6
+ metadata.gz: 412d7f881615ba6bb467cd655d69a60701297f1bcd0a32be6bea745cf41a517f88dc0d3631c628a545e2a9d5c4acb586b02bbd1fa0f596e7d471f5d7968780ec
7
+ data.tar.gz: cda4e394851a8f65fc5edd6564faae8dda6ad74c9f1fe2c52a4b1953ddca4f48775be4138703516f8245cb9dec25b13d99cc6980d4002197c7fe91a1ace7ed79
data/example/Dockerfile CHANGED
@@ -1,9 +1,8 @@
1
1
  # [config] IMAGE_NAME: "groovenauts/batch_type_iot_example"
2
2
  # [config]
3
3
  # [config] WORKING_DIR: "."
4
- # [config] VERSION_FILE: VERSION
4
+ # [config] VERSION_SCRIPT: "ruby -r ../lib/magellan/gcs/proxy/version.rb -e 'puts Magellan::Gcs::Proxy::VERSION'"
5
5
  # [config] GIT_TAG_PREFIX: 'example/'
6
- # [config]
7
6
 
8
7
  FROM ruby:2.3
9
8
 
data/example/Gemfile.lock CHANGED
@@ -33,13 +33,6 @@ GEM
33
33
  googleapis-common-protos (~> 1.3)
34
34
  grpc (~> 1.0)
35
35
  orderedhash (= 0.0.6)
36
- google-cloud-pubsub (0.21.0)
37
- google-cloud-core (~> 0.21.0)
38
- google-gax (~> 0.6.0)
39
- google-protobuf (~> 3.0)
40
- googleapis-common-protos (~> 1.3)
41
- grpc (~> 1.0)
42
- grpc-google-iam-v1 (~> 0.6.8)
43
36
  google-cloud-storage (0.23.1)
44
37
  digest-crc (~> 0.4)
45
38
  google-api-client (~> 0.9.11)
@@ -64,10 +57,6 @@ GEM
64
57
  grpc (1.0.1)
65
58
  google-protobuf (~> 3.0.2)
66
59
  googleauth (~> 0.5.1)
67
- grpc-google-iam-v1 (0.6.8)
68
- googleapis-common-protos (~> 1.3.1)
69
- googleauth (~> 0.5.1)
70
- grpc (~> 1.0)
71
60
  httpclient (2.8.3)
72
61
  hurley (0.2)
73
62
  jwt (1.5.6)
@@ -76,10 +65,9 @@ GEM
76
65
  logging (2.1.0)
77
66
  little-plugger (~> 1.1)
78
67
  multi_json (~> 1.10)
79
- magellan-gcs-proxy (0.1.4)
68
+ magellan-gcs-proxy (0.2.0)
80
69
  dotenv
81
70
  google-cloud-logging
82
- google-cloud-pubsub
83
71
  google-cloud-storage
84
72
  logger_pipe
85
73
  memoist (0.15.0)
@@ -5,7 +5,7 @@ require 'json'
5
5
 
6
6
  subname = ARGV.first
7
7
  puts "Listening to #{subname}"
8
- Magellan::Gcs::Proxy::GCP.pubsub.subscription(subname).tap do |sub|
8
+ Magellan::Gcs::Proxy::PubsubSubscription.new(subname).tap do |sub|
9
9
  sub.listen do |msg|
10
10
  attrs = { 'data' => msg.data }.update(msg.attributes)
11
11
  puts attrs.map { |k, v| "#{k}:#{v}" }.join("\t")
@@ -10,6 +10,7 @@ require 'magellan/gcs/proxy/gcp'
10
10
  require 'magellan/gcs/proxy/composite_logger'
11
11
  require 'magellan/gcs/proxy/progress_notifier_adapter'
12
12
  require 'magellan/gcs/proxy/pubsub_progress_notifier'
13
+ require 'magellan/gcs/proxy/pubsub_subscription'
13
14
  require 'magellan/gcs/proxy/pubsub_sustainer'
14
15
  require 'magellan/gcs/proxy/progress_notification'
15
16
 
@@ -35,7 +35,7 @@ module Magellan
35
35
  verbose("Backtrace\n " << e.backtrace.join("\n "))
36
36
  end
37
37
 
38
- TOTAL = 12
38
+ TOTAL = 14
39
39
  def process(msg)
40
40
  context = Context.new(msg)
41
41
  context.notify(1, TOTAL, "Processing message: #{msg.inspect}")
@@ -48,11 +48,12 @@ module Magellan
48
48
  context.process_with_notification([5, 6, 7], TOTAL, 'Command', exec) do
49
49
  context.process_with_notification([8, 9, 10], TOTAL, 'Upload', &:upload)
50
50
 
51
- msg.acknowledge!
52
- context.notify(11, TOTAL, 'Acknowledged')
51
+ context.process_with_notification([11, 12, 13], TOTAL, 'Acknowledge') do
52
+ msg.acknowledge!
53
+ end
53
54
  end
54
55
  end
55
- context.notify(12, TOTAL, 'Cleanup')
56
+ context.notify(14, TOTAL, 'Cleanup')
56
57
  end
57
58
 
58
59
  def build_command(context)
@@ -3,8 +3,8 @@ require 'magellan/gcs/proxy'
3
3
 
4
4
  require 'google/cloud/logging'
5
5
  require 'google/cloud/logging/version'
6
- require 'google/cloud/pubsub'
7
6
  require 'google/cloud/storage'
7
+ require 'google/apis/pubsub_v1'
8
8
  require 'net/http'
9
9
 
10
10
  module Magellan
@@ -24,6 +24,20 @@ module Magellan
24
24
  ENV['BLOCKS_BATCH_PROJECT_ID'] || retrieve_metadata('project/project-id')
25
25
  end
26
26
 
27
+ SCOPES = [
28
+ 'https://www.googleapis.com/auth/devstorage.full_control',
29
+ 'https://www.googleapis.com/auth/pubsub',
30
+ ].freeze
31
+
32
+ def auth
33
+ @auth ||= new_auth
34
+ end
35
+
36
+ def new_auth
37
+ logger.debug("#{self.class.name}#new_auth")
38
+ Google::Auth.get_application_default(SCOPES).tap(&:fetch_access_token!)
39
+ end
40
+
27
41
  METADATA_HOST = 'metadata.google.internal'.freeze
28
42
  METADATA_PATH_BASE = '/computeMetadata/v1/'.freeze
29
43
  METADATA_HEADER = { 'Metadata-Flavor' => 'Google' }.freeze
@@ -42,12 +56,12 @@ module Magellan
42
56
  end
43
57
 
44
58
  def pubsub
45
- @pubsub ||= Google::Cloud::Pubsub.new(project: project_id)
59
+ @pubsub ||= Google::Apis::PubsubV1::PubsubService.new.tap { |api| api.authorization = auth }
46
60
  end
47
61
 
48
62
  def subscription
49
63
  unless @subscription
50
- @subscription = pubsub.subscription(ENV['BLOCKS_BATCH_PUBSUB_SUBSCRIPTION'] || 'test-subscription')
64
+ @subscription = PubsubSubscription.new(ENV['BLOCKS_BATCH_PUBSUB_SUBSCRIPTION'] || 'test-subscription')
51
65
  logger.info("subscription: #{@subscription.inspect}")
52
66
  end
53
67
  @subscription
@@ -0,0 +1,86 @@
1
+ require 'magellan/gcs/proxy'
2
+ require 'magellan/gcs/proxy/log'
3
+
4
+ module Magellan
5
+ module Gcs
6
+ module Proxy
7
+ class PubsubSubscription
8
+ attr_reader :name, :delay
9
+ def initialize(name, delay: 1)
10
+ @name = name
11
+ @delay = delay
12
+ end
13
+
14
+ def listen
15
+ loop do
16
+ if msg = wait_for_message
17
+ yield msg
18
+ else
19
+ sleep delay
20
+ end
21
+ end
22
+ end
23
+
24
+ def pull_req
25
+ @pull_req ||= Google::Apis::PubsubV1::PullRequest.new(max_messages: 1, return_immediately: true)
26
+ end
27
+
28
+ def wait_for_message
29
+ # #<Google::Apis::PubsubV1::ReceivedMessage:0x007fdc440b58d8
30
+ # @ack_id="...",
31
+ # @message=#<Google::Apis::PubsubV1::Message:0x007fdc440be140
32
+ # @attributes={"download_files"=>"[\"gs://bucket1/path/to/file1\"]"},
33
+ # @message_id="50414480536440",
34
+ # @publish_time="2016-12-17T08:08:35.599Z">>
35
+ res = GCP.pubsub.pull_subscription(name, pull_req)
36
+ msg = (res.received_messages || []).first
37
+ msg.nil? ? nil : MessageWrapper.new(self, msg)
38
+ end
39
+
40
+ class MessageWrapper
41
+ attr_reader :subscription, :original
42
+
43
+ # @param [Magellan::Gcs::Proxy::PubsubSubscription] subscription
44
+ # @param [Google::Apis::PubsubV1::ReceivedMessage] original
45
+ def initialize(subscription, original)
46
+ @subscription = subscription
47
+ @original = original
48
+ end
49
+
50
+ def method_missing(name, *args, &block)
51
+ receiver =
52
+ case name
53
+ when :message_id, :attributes, :publish_time, :data then original.message
54
+ when :ack_id then original
55
+ end
56
+ receiver ? receiver.send(name, *args, &block) : super
57
+ end
58
+
59
+ def respond_to_missing?(name)
60
+ case name
61
+ when :message_id, :attributes, :publish_time, :data then true
62
+ when :ack_id then true
63
+ else false
64
+ end
65
+ end
66
+
67
+ def ack_options
68
+ # https://github.com/google/google-api-ruby-client/blob/master/lib/google/apis/options.rb#L48
69
+ @ack_options ||= { authorization: GCP.auth, retries: 10 }
70
+ end
71
+
72
+ def acknowledge!
73
+ req = Google::Apis::PubsubV1::AcknowledgeRequest.new(ack_ids: [ack_id])
74
+ GCP.pubsub.acknowledge_subscription(subscription.name, req, options: ack_options)
75
+ end
76
+ alias ack! acknowledge!
77
+
78
+ def delay!(new_deadline)
79
+ req = Google::Apis::PubsubV1::ModifyAckDeadlineRequest.new(ack_deadline_seconds: new_deadline, ack_ids: [ack_id])
80
+ GCP.pubsub.modify_subscription_ack_deadline(subscription.name, req)
81
+ end
82
+ end
83
+ end
84
+ end
85
+ end
86
+ end
@@ -63,7 +63,7 @@ module Magellan
63
63
  debug("is sending delay!(#{delay})")
64
64
  message.delay! delay
65
65
  debug("sent delay!(#{delay}) successfully")
66
- rescue Google::Cloud::UnavailableError => e
66
+ rescue Google::Apis::ServerError => e
67
67
  if Time.now.to_f < next_deadline
68
68
  sleep(1) # retry interval
69
69
  debug("is retrying to send delay! cause of [#{e.class.name}] #{e.message}")
@@ -1,7 +1,7 @@
1
1
  module Magellan
2
2
  module Gcs
3
3
  module Proxy
4
- VERSION = '0.1.4'.freeze
4
+ VERSION = '0.2.0'.freeze
5
5
  end
6
6
  end
7
7
  end
@@ -23,7 +23,6 @@ Gem::Specification.new do |spec|
23
23
 
24
24
  spec.add_runtime_dependency 'dotenv'
25
25
  spec.add_runtime_dependency 'google-cloud-logging'
26
- spec.add_runtime_dependency 'google-cloud-pubsub'
27
26
  spec.add_runtime_dependency 'google-cloud-storage'
28
27
  spec.add_runtime_dependency 'logger_pipe'
29
28
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: magellan-gcs-proxy
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.4
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - akm
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2016-12-16 00:00:00.000000000 Z
11
+ date: 2016-12-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: dotenv
@@ -38,20 +38,6 @@ dependencies:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
- - !ruby/object:Gem::Dependency
42
- name: google-cloud-pubsub
43
- requirement: !ruby/object:Gem::Requirement
44
- requirements:
45
- - - ">="
46
- - !ruby/object:Gem::Version
47
- version: '0'
48
- type: :runtime
49
- prerelease: false
50
- version_requirements: !ruby/object:Gem::Requirement
51
- requirements:
52
- - - ">="
53
- - !ruby/object:Gem::Version
54
- version: '0'
55
41
  - !ruby/object:Gem::Dependency
56
42
  name: google-cloud-storage
57
43
  requirement: !ruby/object:Gem::Requirement
@@ -148,7 +134,6 @@ files:
148
134
  - example/Gemfile
149
135
  - example/Gemfile.lock
150
136
  - example/README.md
151
- - example/VERSION
152
137
  - example/app.sh
153
138
  - example/config.yml
154
139
  - exe/magellan-gcs-proxy
@@ -166,6 +151,7 @@ files:
166
151
  - lib/magellan/gcs/proxy/progress_notification.rb
167
152
  - lib/magellan/gcs/proxy/progress_notifier_adapter.rb
168
153
  - lib/magellan/gcs/proxy/pubsub_progress_notifier.rb
154
+ - lib/magellan/gcs/proxy/pubsub_subscription.rb
169
155
  - lib/magellan/gcs/proxy/pubsub_sustainer.rb
170
156
  - lib/magellan/gcs/proxy/version.rb
171
157
  - magellan-gcs-proxy.gemspec
data/example/VERSION DELETED
@@ -1 +0,0 @@
1
- 0.0.1