magellan-gcs-proxy 0.1.4 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 50724872f81d7d6de7c7803628cd766428150f5d
-  data.tar.gz: bc1b1ed15f3239206268f2596b9769aa22cf683f
+  metadata.gz: 97082e16910f8eb448e9dd90a579dab3f5d2645b
+  data.tar.gz: b8ef0a1084a6da1b9287a9806c91ebd1e73b0ba8
 SHA512:
-  metadata.gz: ae86f4f6bd8b441cbc11fdd21ffa580cacbb81676d8f092b196ff961ad3924798c7d767d2c7a31e993d6b0ce67fbb6ebde99a5597bc13990c547f23158dbb40e
-  data.tar.gz: da279b3209b1b16931a870e260c74de94a2ecc6315ecf111b7774cdc5b3d101c73a97704e9d77a6f888bda517b591ac84050610bc87fd76f64ad64a92e315a8f
+  metadata.gz: 412d7f881615ba6bb467cd655d69a60701297f1bcd0a32be6bea745cf41a517f88dc0d3631c628a545e2a9d5c4acb586b02bbd1fa0f596e7d471f5d7968780ec
+  data.tar.gz: cda4e394851a8f65fc5edd6564faae8dda6ad74c9f1fe2c52a4b1953ddca4f48775be4138703516f8245cb9dec25b13d99cc6980d4002197c7fe91a1ace7ed79

data/example/Dockerfile

@@ -1,9 +1,8 @@
 # [config] IMAGE_NAME: "groovenauts/batch_type_iot_example"
 # [config]
 # [config] WORKING_DIR: "."
-# [config] VERSION_FILE: VERSION
+# [config] VERSION_SCRIPT: "ruby -r ../lib/magellan/gcs/proxy/version.rb -e 'puts Magellan::Gcs::Proxy::VERSION'"
 # [config] GIT_TAG_PREFIX: 'example/'
-# [config]
 
 FROM ruby:2.3
 

data/example/Gemfile.lock

@@ -33,13 +33,6 @@ GEM
       googleapis-common-protos (~> 1.3)
       grpc (~> 1.0)
       orderedhash (= 0.0.6)
-    google-cloud-pubsub (0.21.0)
-      google-cloud-core (~> 0.21.0)
-      google-gax (~> 0.6.0)
-      google-protobuf (~> 3.0)
-      googleapis-common-protos (~> 1.3)
-      grpc (~> 1.0)
-      grpc-google-iam-v1 (~> 0.6.8)
     google-cloud-storage (0.23.1)
       digest-crc (~> 0.4)
       google-api-client (~> 0.9.11)
@@ -64,10 +57,6 @@ GEM
     grpc (1.0.1)
       google-protobuf (~> 3.0.2)
       googleauth (~> 0.5.1)
-    grpc-google-iam-v1 (0.6.8)
-      googleapis-common-protos (~> 1.3.1)
-      googleauth (~> 0.5.1)
-      grpc (~> 1.0)
     httpclient (2.8.3)
     hurley (0.2)
     jwt (1.5.6)
@@ -76,10 +65,9 @@ GEM
     logging (2.1.0)
       little-plugger (~> 1.1)
       multi_json (~> 1.10)
-    magellan-gcs-proxy (0.1.4)
+    magellan-gcs-proxy (0.2.0)
       dotenv
       google-cloud-logging
-      google-cloud-pubsub
       google-cloud-storage
       logger_pipe
     memoist (0.15.0)

data/exe/magellan-gcs-proxy-dev-progress-listener

@@ -5,7 +5,7 @@ require 'json'
 
 subname = ARGV.first
 puts "Listening to #{subname}"
-Magellan::Gcs::Proxy::GCP.pubsub.subscription(subname).tap do |sub|
+Magellan::Gcs::Proxy::PubsubSubscription.new(subname).tap do |sub|
   sub.listen do |msg|
     attrs = { 'data' => msg.data }.update(msg.attributes)
     puts attrs.map { |k, v| "#{k}:#{v}" }.join("\t")

data/lib/magellan/gcs/proxy.rb

@@ -10,6 +10,7 @@ require 'magellan/gcs/proxy/gcp'
 require 'magellan/gcs/proxy/composite_logger'
 require 'magellan/gcs/proxy/progress_notifier_adapter'
 require 'magellan/gcs/proxy/pubsub_progress_notifier'
+require 'magellan/gcs/proxy/pubsub_subscription'
 require 'magellan/gcs/proxy/pubsub_sustainer'
 require 'magellan/gcs/proxy/progress_notification'
 

data/lib/magellan/gcs/proxy/cli.rb

@@ -35,7 +35,7 @@ module Magellan
           verbose("Backtrace\n  " << e.backtrace.join("\n  "))
         end
 
-        TOTAL = 12
+        TOTAL = 14
         def process(msg)
           context = Context.new(msg)
           context.notify(1, TOTAL, "Processing message: #{msg.inspect}")
@@ -48,11 +48,12 @@ module Magellan
             context.process_with_notification([5, 6, 7], TOTAL, 'Command', exec) do
               context.process_with_notification([8, 9, 10], TOTAL, 'Upload', &:upload)
 
-              msg.acknowledge!
-              context.notify(11, TOTAL, 'Acknowledged')
+              context.process_with_notification([11, 12, 13], TOTAL, 'Acknowledge') do
+                msg.acknowledge!
+              end
             end
           end
-          context.notify(12, TOTAL, 'Cleanup')
+          context.notify(14, TOTAL, 'Cleanup')
         end
 
         def build_command(context)

data/lib/magellan/gcs/proxy/gcp.rb

@@ -3,8 +3,8 @@ require 'magellan/gcs/proxy'
 
 require 'google/cloud/logging'
 require 'google/cloud/logging/version'
-require 'google/cloud/pubsub'
 require 'google/cloud/storage'
+require 'google/apis/pubsub_v1'
 require 'net/http'
 
 module Magellan
@@ -24,6 +24,20 @@ module Magellan
           ENV['BLOCKS_BATCH_PROJECT_ID'] || retrieve_metadata('project/project-id')
         end
 
+        SCOPES = [
+          'https://www.googleapis.com/auth/devstorage.full_control',
+          'https://www.googleapis.com/auth/pubsub',
+        ].freeze
+
+        def auth
+          @auth ||= new_auth
+        end
+
+        def new_auth
+          logger.debug("#{self.class.name}#new_auth")
+          Google::Auth.get_application_default(SCOPES).tap(&:fetch_access_token!)
+        end
+
         METADATA_HOST = 'metadata.google.internal'.freeze
         METADATA_PATH_BASE = '/computeMetadata/v1/'.freeze
         METADATA_HEADER = { 'Metadata-Flavor' => 'Google' }.freeze
@@ -42,12 +56,12 @@ module Magellan
         end
 
         def pubsub
-          @pubsub ||= Google::Cloud::Pubsub.new(project: project_id)
+          @pubsub ||= Google::Apis::PubsubV1::PubsubService.new.tap { |api| api.authorization = auth }
         end
 
         def subscription
           unless @subscription
-            @subscription = pubsub.subscription(ENV['BLOCKS_BATCH_PUBSUB_SUBSCRIPTION'] || 'test-subscription')
+            @subscription = PubsubSubscription.new(ENV['BLOCKS_BATCH_PUBSUB_SUBSCRIPTION'] || 'test-subscription')
             logger.info("subscription: #{@subscription.inspect}")
           end
           @subscription

data/lib/magellan/gcs/proxy/pubsub_subscription.rb

@@ -0,0 +1,86 @@
+require 'magellan/gcs/proxy'
+require 'magellan/gcs/proxy/log'
+
+module Magellan
+  module Gcs
+    module Proxy
+      class PubsubSubscription
+        attr_reader :name, :delay
+        def initialize(name, delay: 1)
+          @name = name
+          @delay = delay
+        end
+
+        def listen
+          loop do
+            if msg = wait_for_message
+              yield msg
+            else
+              sleep delay
+            end
+          end
+        end
+
+        def pull_req
+          @pull_req ||= Google::Apis::PubsubV1::PullRequest.new(max_messages: 1, return_immediately: true)
+        end
+
+        def wait_for_message
+          # #<Google::Apis::PubsubV1::ReceivedMessage:0x007fdc440b58d8
+          #   @ack_id="...",
+          #   @message=#<Google::Apis::PubsubV1::Message:0x007fdc440be140
+          #     @attributes={"download_files"=>"[\"gs://bucket1/path/to/file1\"]"},
+          #     @message_id="50414480536440",
+          #     @publish_time="2016-12-17T08:08:35.599Z">>
+          res = GCP.pubsub.pull_subscription(name, pull_req)
+          msg = (res.received_messages || []).first
+          msg.nil? ? nil : MessageWrapper.new(self, msg)
+        end
+
+        class MessageWrapper
+          attr_reader :subscription, :original
+
+          # @param [Magellan::Gcs::Proxy::PubsubSubscription] subscription
+          # @param [Google::Apis::PubsubV1::ReceivedMessage] original
+          def initialize(subscription, original)
+            @subscription = subscription
+            @original = original
+          end
+
+          def method_missing(name, *args, &block)
+            receiver =
+              case name
+              when :message_id, :attributes, :publish_time, :data then original.message
+              when :ack_id then original
+              end
+            receiver ? receiver.send(name, *args, &block) : super
+          end
+
+          def respond_to_missing?(name)
+            case name
+            when :message_id, :attributes, :publish_time, :data then true
+            when :ack_id then true
+            else false
+            end
+          end
+
+          def ack_options
+            # https://github.com/google/google-api-ruby-client/blob/master/lib/google/apis/options.rb#L48
+            @ack_options ||= { authorization: GCP.auth, retries: 10 }
+          end
+
+          def acknowledge!
+            req = Google::Apis::PubsubV1::AcknowledgeRequest.new(ack_ids: [ack_id])
+            GCP.pubsub.acknowledge_subscription(subscription.name, req, options: ack_options)
+          end
+          alias ack! acknowledge!
+
+          def delay!(new_deadline)
+            req = Google::Apis::PubsubV1::ModifyAckDeadlineRequest.new(ack_deadline_seconds: new_deadline, ack_ids: [ack_id])
+            GCP.pubsub.modify_subscription_ack_deadline(subscription.name, req)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/magellan/gcs/proxy/pubsub_sustainer.rb

@@ -63,7 +63,7 @@ module Magellan
           debug("is sending delay!(#{delay})")
           message.delay! delay
           debug("sent delay!(#{delay}) successfully")
-        rescue Google::Cloud::UnavailableError => e
+        rescue Google::Apis::ServerError => e
           if Time.now.to_f < next_deadline
             sleep(1) # retry interval
             debug("is retrying to send delay! cause of [#{e.class.name}] #{e.message}")

data/lib/magellan/gcs/proxy/version.rb

@@ -1,7 +1,7 @@
 module Magellan
   module Gcs
     module Proxy
-      VERSION = '0.1.4'.freeze
+      VERSION = '0.2.0'.freeze
     end
   end
 end

data/magellan-gcs-proxy.gemspec

@@ -23,7 +23,6 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency 'dotenv'
   spec.add_runtime_dependency 'google-cloud-logging'
-  spec.add_runtime_dependency 'google-cloud-pubsub'
   spec.add_runtime_dependency 'google-cloud-storage'
   spec.add_runtime_dependency 'logger_pipe'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: magellan-gcs-proxy
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.0
 platform: ruby
 authors:
 - akm
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-12-16 00:00:00.000000000 Z
+date: 2016-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dotenv
@@ -38,20 +38,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: google-cloud-pubsub
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: google-cloud-storage
   requirement: !ruby/object:Gem::Requirement
@@ -148,7 +134,6 @@ files:
 - example/Gemfile
 - example/Gemfile.lock
 - example/README.md
-- example/VERSION
 - example/app.sh
 - example/config.yml
 - exe/magellan-gcs-proxy
@@ -166,6 +151,7 @@ files:
 - lib/magellan/gcs/proxy/progress_notification.rb
 - lib/magellan/gcs/proxy/progress_notifier_adapter.rb
 - lib/magellan/gcs/proxy/pubsub_progress_notifier.rb
+- lib/magellan/gcs/proxy/pubsub_subscription.rb
 - lib/magellan/gcs/proxy/pubsub_sustainer.rb
 - lib/magellan/gcs/proxy/version.rb
 - magellan-gcs-proxy.gemspec

data/example/VERSION

@@ -1 +0,0 @@
-0.0.1