magellan-gcs-proxy 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ea3b9c9c14c544a0856ccc2fc7e5923058ea2842
-  data.tar.gz: 7097911f386a69da3f54f01ceb4ad37c8e3cfc88
+  metadata.gz: fbfdc826b779db7cec5c24192988e35247c56e33
+  data.tar.gz: 56879281961eebe651ac777a034505bb8cb060b7
 SHA512:
-  metadata.gz: 5afd3e0bc9f9510715d9b509c46b38f43f552c1006e324931c13ae12fde72db44bfd48074b864e26c48f0071c8baa4ed11945ffd7d530406235fd0b8238aa5f6
-  data.tar.gz: 35e8d6f94d1bc21e2a8080bf0244bbf52ec41323d273f81d330f5d42e85a5d39d645c3b96e20561dd4b31528f283a890b900a0b1caf5826db3a377ff32a1e25a
+  metadata.gz: 67dccc48b4ab362d2090cf1f82d2a2caa16201fbdf053e15e97ea89b94f03c6bd7deaeaec741321dfc92ce877aa2535a47dce84ba08e67cf2a252db94cbb0a5b
+  data.tar.gz: b5c10a2f63c1ed87498d92ad40d7a1882c49e8dae503f61b34d98c317b1abd6909cdea858dd48e13c105ad660c20674c33be8b8575d2ebe03fcf2cdd371ab84c

data/example/Gemfile.lock

@@ -12,8 +12,8 @@ GEM
     coderay (1.1.1)
     debug_inspector (0.0.2)
     digest-crc (0.4.1)
-    dotenv (2.1.1)
-    faraday (0.10.0)
+    dotenv (2.1.2)
+    faraday (0.11.0)
       multipart-post (>= 1.2, < 3)
     google-api-client (0.9.20)
       addressable (~> 2.3)
@@ -26,13 +26,14 @@ GEM
       retriable (~> 2.0)
     google-cloud-core (0.21.1)
       googleauth (~> 0.5.1)
-    google-cloud-logging (0.23.0)
-      google-cloud-core (~> 0.21.0)
+    google-cloud-logging (0.23.2)
+      google-cloud-core (~> 0.21.1)
       google-gax (~> 0.6.0)
       google-protobuf (~> 3.0)
       googleapis-common-protos (~> 1.3)
       grpc (~> 1.0)
       orderedhash (= 0.0.6)
+      stackdriver-core (~> 0.21.0)
     google-cloud-storage (0.23.1)
       digest-crc (~> 0.4)
       google-api-client (~> 0.9.11)
@@ -65,7 +66,7 @@ GEM
     logging (2.1.0)
       little-plugger (~> 1.1)
       multi_json (~> 1.10)
-    magellan-gcs-proxy (0.3.0)
+    magellan-gcs-proxy (0.3.1)
       dotenv
       google-cloud-logging
       google-cloud-storage
@@ -89,7 +90,7 @@ GEM
     pry-stack_explorer (0.4.9.2)
       binding_of_caller (>= 0.7)
       pry (>= 0.9.11)
-    public_suffix (2.0.4)
+    public_suffix (2.0.5)
     representable (2.3.0)
       uber (~> 0.0.7)
     retriable (2.1.0)
@@ -100,6 +101,7 @@ GEM
       jwt (~> 1.5)
       multi_json (~> 1.10)
     slop (3.6.0)
+    stackdriver-core (0.21.0)
     thor (0.19.4)
     uber (0.0.15)
 

data/example/config.yml

@@ -23,3 +23,5 @@ sustainer:
   #   The interval to send delay message. It must be lower than delay
   #   Default: 90% of delay
   interval: <%= ENV['BLOCKS_BATCH_SUSTAINER_INTERVAL'] || 540 %>
+
+dryrun: <%= ENV['BLOCKS_BATCH_DRYRUN'] || false %>

data/lib/magellan/gcs/proxy/cli.rb

@@ -44,7 +44,7 @@ module Magellan
 
             cmd = build_command(context)
 
-            exec = ->(*) { LoggerPipe.run(logger, cmd, returns: :none, logging: :both) }
+            exec = ->(*) { LoggerPipe.run(logger, cmd, returns: :none, logging: :both, dry_run: Proxy.config[:dryrun]) }
             context.process_with_notification([5, 6, 7], TOTAL, 'Command', exec) do
               context.process_with_notification([8, 9, 10], TOTAL, 'Upload', &:upload)
 

data/lib/magellan/gcs/proxy/context.rb

@@ -54,6 +54,7 @@ module Magellan
           download_mapping.each do |url, path|
             FileUtils.mkdir_p File.dirname(path)
             logger.debug("Downloading: #{url} to #{path}")
+            next if Proxy.config[:dryrun]
             uri = parse_uri(url)
             @last_bucket_name = uri.host
             bucket = GCP.storage.bucket(@last_bucket_name)
@@ -71,6 +72,7 @@ module Magellan
                   next if directory?(path)
                   url = "gs://#{bucket_name}/#{path}"
                   logger.info("Uploading: #{path} to #{url}")
+                  next if Proxy.config[:dryrun]
                   bucket = GCP.storage.bucket(bucket_name)
                   bucket.create_file path, path
                   logger.info("Upload OK: #{path} to #{url}")

data/lib/magellan/gcs/proxy/pubsub_progress_notifier.rb

@@ -1,4 +1,5 @@
 require 'magellan/gcs/proxy'
+require 'magellan/gcs/proxy/log'
 
 require 'logger'
 require 'json'
@@ -7,6 +8,8 @@ module Magellan
   module Gcs
     module Proxy
       class PubsubProgressNotifier
+        include Log
+
         attr_reader :topic_name
         def initialize(topic_name)
           @topic_name = topic_name
@@ -17,7 +20,13 @@ module Magellan
         end
 
         def notify(severity, job_message, data, attrs)
-          topic.publish data, { level: severity, job_message_id: job_message.message_id }.merge(attrs)
+          attrs = { level: severity, job_message_id: job_message.message_id }.merge(attrs)
+          # attrs must be an [Hash<String,String>]
+          attrs = attrs.each_with_object({}) { |(k, v), d| d[k.to_s] = v.to_s }
+          logger.debug("Publishing progress: #{attrs.inspect}")
+          msg = Google::Apis::PubsubV1::Message.new(data: data, attributes: attrs)
+          req = Google::Apis::PubsubV1::PublishRequest.new(messages: [msg])
+          GCP.pubsub.publish_topic(topic_name, req)
         end
       end
     end

data/lib/magellan/gcs/proxy/version.rb

@@ -1,7 +1,7 @@
 module Magellan
   module Gcs
     module Proxy
-      VERSION = '0.3.0'.freeze
+      VERSION = '0.3.1'.freeze
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: magellan-gcs-proxy
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - akm
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-12-20 00:00:00.000000000 Z
+date: 2017-01-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dotenv