magellan-gcs-proxy 0.1.1 → 0.1.2

data/lib/magellan/gcs/proxy/config.rb

@@ -0,0 +1,44 @@
+# coding: utf-8
+require 'magellan/gcs/proxy'
+
+require 'yaml'
+require 'erb'
+
+module Magellan
+  module Gcs
+    module Proxy
+      class Config
+        attr_reader :path
+        def initialize(path = './config.yml')
+          @path = path
+        end
+
+        def data
+          @data ||= load_file
+        end
+
+        def reset
+          @data = nil
+        end
+
+        def load_file
+          erb = ERB.new(File.read(path), nil, '-')
+          erb.filename = path
+          t = erb.result
+          puts '=' * 100
+          puts t
+          puts '-' * 100
+          YAML.load(t)
+        end
+
+        def [](key)
+          data[key.to_s]
+        end
+
+        def verbose?
+          ENV['VERBOSE'] =~ /true|yes|on|1/i
+        end
+      end
+    end
+  end
+end

data/lib/magellan/gcs/proxy/context.rb

@@ -1,6 +1,6 @@
 # coding: utf-8
-require "magellan/gcs/proxy"
-require "magellan/gcs/proxy/log"
+require 'magellan/gcs/proxy'
+require 'magellan/gcs/proxy/log'
 
 require 'fileutils'
 require 'uri'
@@ -10,30 +10,27 @@ module Magellan
     module Proxy
       class Context
         include Log
+        include Proxy::ProgressNotification
 
-        attr_reader :workspace, :remote_download_files
-        def initialize(workspace, remote_download_files)
-          @workspace = workspace
-          @remote_download_files = remote_download_files
+        attr_reader :message, :workspace, :remote_download_files
+        def initialize(message)
+          @message = message
+          @remote_download_files = parse_json(message.attributes['download_files'])
+          @workspace = nil
         end
 
-        KEYS = [
-          :workspace,
-          :downloads_dir, :uploads_dir,
-          :download_files,
-          :local_download_files,
-          :remote_download_files,
-        ].freeze
-
-        def [](key)
-          case key.to_sym
-          when *KEYS then send(key)
-          else nil
+        def setup
+          Dir.mktmpdir 'workspace' do |dir|
+            @workspace = dir
+            setup_dirs
+            PubsubSustainer.run(message) do
+              yield
+            end
           end
         end
 
-        def include?(key)
-          KEYS.include?(key)
+        def ltsv(hash)
+          hash.map { |k, v| "#{k}:#{v}" }.join("\t")
         end
 
         def downloads_dir
@@ -47,16 +44,12 @@ module Magellan
         def local_download_files
           @local_download_files ||= build_local_files_obj(remote_download_files, download_mapping)
         end
-        alias_method :download_files, :local_download_files
+        alias download_files local_download_files
 
         def uploads_dir
           File.join(workspace, 'uploads')
         end
 
-        def setup
-          setup_dirs
-        end
-
         def download
           download_mapping.each do |url, path|
             FileUtils.mkdir_p File.dirname(path)
@@ -64,7 +57,7 @@ module Magellan
             uri = parse_uri(url)
             @last_bucket_name = uri.host
             bucket = GCP.storage.bucket(@last_bucket_name)
-            file = bucket.file uri.path.sub(/\A\//, '')
+            file = bucket.file uri.path.sub(%r{\A/}, '')
             file.download(path)
             logger.info("Download OK: #{url} to #{path}")
           end
@@ -73,7 +66,7 @@ module Magellan
         def upload
           Dir.chdir(uploads_dir) do
             Dir.glob('**/*') do |path|
-              next if File.directory?(path)
+              next if directory?(path)
               url = "gs://#{@last_bucket_name}/#{path}"
               logger.info("Uploading: #{path} to #{url}")
               bucket = GCP.storage.bucket(@last_bucket_name)
@@ -83,8 +76,12 @@ module Magellan
           end
         end
 
+        def directory?(path)
+          File.directory?(path)
+        end
+
         def setup_dirs
-          [:downloads_dir, :uploads_dir].each{|k| Dir.mkdir(send(k))}
+          [:downloads_dir, :uploads_dir].each { |k| FileUtils.mkdir_p(send(k)) }
         end
 
         def build_mapping(base_dir, obj)
@@ -98,11 +95,16 @@ module Magellan
           case obj
           when nil then []
           when Hash then flatten_values(obj.values)
-          when Array then obj.map{|i| flatten_values(i) }
+          when Array then obj.map { |i| flatten_values(i) }
           else obj
           end
         end
 
+        def parse_json(str)
+          return nil if str.nil? || str.empty?
+          JSON.parse(str)
+        end
+
         def parse_uri(str)
           uri = URI.parse(str)
           raise "Unsupported scheme #{uri.scheme.inspect} of #{str}" unless uri.scheme == 'gs'
@@ -111,13 +113,12 @@ module Magellan
 
         def build_local_files_obj(obj, mapping)
           case obj
-          when Hash then obj.each_with_object({}){|(k,v), d| d[k] = build_local_files_obj(v, mapping)}
-          when Array then obj.map{|i| build_local_files_obj(i, mapping)}
+          when Hash then obj.each_with_object({}) { |(k, v), d| d[k] = build_local_files_obj(v, mapping) }
+          when Array then obj.map { |i| build_local_files_obj(i, mapping) }
           when String then mapping[obj]
           else obj
           end
         end
-
       end
     end
   end

data/lib/magellan/gcs/proxy/expand_variable.rb

@@ -1,53 +1,45 @@
-require "magellan/gcs/proxy"
+require 'magellan/gcs/proxy'
 
 module Magellan
   module Gcs
     module Proxy
       module ExpandVariable
-
         class InvalidReferenceError < StandardError
         end
 
         module_function
 
         def dig_variables(variable_ref, data)
-          vars = variable_ref.split(".").map{|i| (/\A\d+\z/.match(i)) ? i.to_i : i }
-          value = vars.inject(data) do |tmp, v|
-            case v
-            when String
-              if tmp.respond_to?(:[]) && tmp.respond_to?(:include?)
-                if tmp.include?(v)
-                  tmp[v]
-                else
-                  raise InvalidReferenceError, variable_ref
-                end
-              else
-                raise InvalidReferenceError, variable_ref
-              end
-            when Integer
-              case tmp
-              when Array
-                if tmp.size > v
-                  tmp[v]
-                else
-                  raise InvalidReferenceError, variable_ref
-                end
-              else
-                raise InvalidReferenceError, variable_ref
-              end
+          vars = variable_ref.split('.').map { |i| /\A\d+\z/ =~ i ? i.to_i : i }
+          vars.inject(data) do |tmp, v|
+            dig_variable(tmp, v, variable_ref)
+          end
+        end
+
+        def dig_variable(tmp, v, variable_ref)
+          case v
+          when String
+            if tmp.respond_to?(:[]) && tmp.respond_to?(:include?)
+              return tmp[v] if tmp.include?(v)
+            end
+          when Integer
+            case tmp
+            when Array
+              return tmp[v] if tmp.size > v
             end
           end
+          raise InvalidReferenceError, variable_ref
         end
 
         def expand_variables(str, data, quote_string: false)
           data ||= {}
-          str.gsub(/\%\{\s*([\w.]+)\s*\}/) do |m|
+          str.gsub(/\%\{\s*([\w.]+)\s*\}/) do |_m|
             var = Regexp.last_match(1)
             value =
               begin
                 dig_variables(var, data)
               rescue InvalidReferenceError
-                ""
+                ''
               end
 
             case value
@@ -58,7 +50,6 @@ module Magellan
           end
         end
       end
-
     end
   end
 end

data/lib/magellan/gcs/proxy/gcp.rb

@@ -1,8 +1,10 @@
 # coding: utf-8
 require 'magellan/gcs/proxy'
 
-require "google/cloud/pubsub"
-require "google/cloud/storage"
+require 'google/cloud/logging'
+require 'google/cloud/logging/version'
+require 'google/cloud/pubsub'
+require 'google/cloud/storage'
 require 'net/http'
 
 module Magellan
@@ -24,7 +26,7 @@ module Magellan
 
         METADATA_HOST = 'metadata.google.internal'.freeze
         METADATA_PATH_BASE = '/computeMetadata/v1/'.freeze
-        METADATA_HEADER = {"Metadata-Flavor" => "Google"}.freeze
+        METADATA_HEADER = { 'Metadata-Flavor' => 'Google' }.freeze
 
         def retrieve_metadata(key)
           http = Net::HTTP.new(METADATA_HOST)
@@ -51,8 +53,12 @@ module Magellan
           @subscription
         end
 
+        def logging
+          @logging ||= Google::Cloud::Logging.new(project: project_id)
+        end
+
         def reset
-          instance_variables.each {|ivar| instance_variable_set(ivar, nil)}
+          instance_variables.each { |ivar| instance_variable_set(ivar, nil) }
         end
       end
     end

data/lib/magellan/gcs/proxy/log.rb

@@ -1,10 +1,76 @@
+# coding: utf-8
 require 'logger'
+
 module Magellan
   module Gcs
     module Proxy
       module Log
+        module_function
+
+        def verbose(msg)
+          logger.debug(msg) if Proxy.config.verbose?
+        end
+
         def logger
-          @logger ||= Logger.new($stdout)
+          @logger ||= build_logger(loggers)
+        end
+
+        def build_logger(loggers)
+          case loggers.length
+          when 0 then Logger.new('/dev/null')
+          when 1 then loggers.first
+          else CompositeLogger.new(loggers)
+          end
+        end
+
+        def loggers
+          @loggers ||= build_loggers
+        end
+
+        def build_loggers
+          (Proxy.config[:loggers] || []).map do |logger_def|
+            config = logger_def.dup
+            type = config.delete('type')
+            case type
+            when 'stdout' then Logger.new($stdout)
+            when 'stderr' then Logger.new($stderr)
+            when 'cloud_logging' then build_cloud_logging_logger(config)
+            else raise "Unsupported logger type: #{type} with #{config.inspect}"
+            end
+          end
+        end
+
+        CLOUD_LOGGING_RESOURCE_KEYS = [
+          :project_id,
+          :cluster_name,
+          :namespace_id,
+          :instance_id,
+          :pod_id,
+          :container_name,
+          :zone,
+        ].freeze
+
+        def build_cloud_logging_logger(config)
+          log_name = config['log_name']
+          return nil unless log_name
+          # container
+          # GKE Container	A Google Container Engine (GKE) container instance.
+          #   project_id: The identifier of the GCP project associated with this resource (e.g., my-project).
+          #   cluster_name: An immutable name for the cluster the container is running in.
+          #   namespace_id: Immutable ID of the cluster namespace the container is running in.
+          #   instance_id: Immutable ID of the GCE instance the container is running in.
+          #   pod_id: Immutable ID of the pod the container is running in.
+          #   container_name: Immutable name of the container.
+          #   zone: The GCE zone in which the instance is running.
+          # See https://cloud.google.com/logging/docs/api/v2/resource-list
+          options = CLOUD_LOGGING_RESOURCE_KEYS.each_with_object({}) do |key, d|
+            if v = ENV["BLOCKS_BATCH_CLOUD_LOGGING_#{key.to_s.upcase}"]
+              d[key] = v
+            end
+          end
+          resource = GCP.logging.resource 'container', options
+          Google::Cloud::Logging::Logger.new GCP.logging, log_name, resource,
+                                             magellan_gcs_proxy: Magellan::Gcs::Proxy::VERSION
         end
       end
     end

data/lib/magellan/gcs/proxy/message_wrapper.rb

@@ -1,12 +1,13 @@
-require "magellan/gcs/proxy"
+require 'magellan/gcs/proxy'
 
 module Magellan
   module Gcs
     module Proxy
       class MessageWrapper
         attr_reader :msg, :context
-        def initialize(msg, context)
-          @msg, @context = msg, context
+        def initialize(context)
+          @msg = context.message
+          @context = ContextAccessor.new(context)
         end
 
         def [](key)
@@ -26,6 +27,31 @@ module Magellan
           Attrs.new(msg.attributes)
         end
 
+        class ContextAccessor
+          attr_accessor :context
+          def initialize(context)
+            @context = context
+          end
+
+          KEYS = [
+            :workspace,
+            :downloads_dir, :uploads_dir,
+            :download_files,
+            :local_download_files,
+            :remote_download_files
+          ].freeze
+
+          def [](key)
+            case key.to_sym
+            when *KEYS then context.send(key)
+            end
+          end
+
+          def include?(key)
+            KEYS.include?(key)
+          end
+        end
+
         class Attrs
           attr_reader :data
           def initialize(data)
@@ -35,7 +61,11 @@ module Magellan
           def [](key)
             value = data[key]
             if value.is_a?(String) && value =~ /\A\[.*\]\z|\A\{.*\}\z/
-              JSON.parse(value) rescue value
+              begin
+                JSON.parse(value)
+              rescue
+                value
+              end
             else
               value
             end
@@ -44,7 +74,6 @@ module Magellan
           def include?(key)
             data.include?(key) || data.include?(key.to_sym)
           end
-
         end
       end
     end

data/lib/magellan/gcs/proxy/progress_notification.rb

@@ -0,0 +1,73 @@
+require 'magellan/gcs/proxy'
+
+module Magellan
+  module Gcs
+    module Proxy
+      module ProgressNotification
+        include Log
+
+        def process_with_notification(numbers, total, base_message, main = nil)
+          start_no, complete_no, error_no = *numbers
+          notify(start_no, total, "#{base_message} starting")
+          begin
+            main ? main.call(self) : yield(self)
+          rescue => e
+            notify(error_no, total, "#{base_message} error: [#{e.class}] #{e.message}", severity: :error)
+            raise e unless main
+          else
+            notify(complete_no, total, "#{base_message} completed")
+            yield(self) if main
+          end
+        end
+
+        def notify(progress, total, data, severity: :info)
+          notifier.notify(severity, message, data, progress: progress, total: total)
+        end
+
+        def notifier
+          @notifier ||= build_notifier
+        end
+
+        # Build the Notifier object like these...
+        #
+        # CompositeNotifier
+        #   @notifiers:
+        #     PubsubProgressNotifier
+        #     ProgressNotifierAdapter
+        #       @logger:
+        #         CompositeLogger
+        #           @loggers:
+        #             Logger
+        #             Google::Cloud::Logging::Logger
+        def build_notifier
+          notifiers = []
+          if c = Proxy.config[:progress_notification]
+            notifiers << PubsubProgressNotifier.new(c['topic'])
+          end
+          notifiers << ProgressNotifierAdapter.new(logger)
+          case notifiers.length
+          when 1 then notifiers.first
+          else CompositeNotifier.new(notifiers)
+          end
+        end
+
+        class CompositeNotifier
+          attr_reader :notifiers
+          def initialize(notifiers)
+            @notifiers = notifiers
+          end
+
+          def notify(*args, &block)
+            notifiers.each do |notifier|
+              begin
+                notifier.notify(*args, &block)
+              rescue => e
+                $stderr.puts("[#{e.class}] #{e.message}")
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/magellan/gcs/proxy/progress_notifier_adapter.rb

@@ -0,0 +1,24 @@
+require 'magellan/gcs/proxy'
+
+module Magellan
+  module Gcs
+    module Proxy
+      class ProgressNotifierAdapter
+        attr_reader :logger
+        def initialize(logger)
+          @logger = logger
+        end
+
+        def ltsv(hash)
+          hash.map { |k, v| "#{k}:#{v}" }.join("\t")
+        end
+
+        def notify(severity, job_message, data, attrs)
+          d = { job_message_id: job_message.message_id }.merge(attrs)
+          d[:data] = data # Show data at the end of string
+          logger.send(severity, ltsv(d))
+        end
+      end
+    end
+  end
+end

data/lib/magellan/gcs/proxy/pubsub_progress_notifier.rb

@@ -0,0 +1,25 @@
+require 'magellan/gcs/proxy'
+
+require 'logger'
+require 'json'
+
+module Magellan
+  module Gcs
+    module Proxy
+      class PubsubProgressNotifier
+        attr_reader :topic_name
+        def initialize(topic_name)
+          @topic_name = topic_name
+        end
+
+        def topic
+          @topic ||= GCP.pubsub.topic(topic_name)
+        end
+
+        def notify(severity, job_message, data, attrs)
+          topic.publish data, { level: severity, job_message_id: job_message.message_id }.merge(attrs)
+        end
+      end
+    end
+  end
+end

data/lib/magellan/gcs/proxy/pubsub_sustainer.rb

@@ -0,0 +1,54 @@
+require 'magellan/gcs/proxy'
+
+require 'logger'
+require 'json'
+
+module Magellan
+  module Gcs
+    module Proxy
+      class PubsubSustainer
+        include Log
+
+        class << self
+          def run(message)
+            raise "#{name}.run requires block" unless block_given?
+            if c = Proxy.config[:sustainer]
+              t = Thread.new(message, c['delay'], c['interval']) do |msg, delay, interval|
+                Thread.current[:processing_message] = true
+                new(msg, delay: delay, interval: interval).run
+              end
+              begin
+                yield
+              ensure
+                t[:processing_message] = false
+                t.join
+              end
+            else
+              yield
+            end
+          end
+        end
+
+        attr_reader :message, :delay, :interval
+        def initialize(message, delay: 10, interval: nil)
+          @message = message
+          @delay = delay.to_i
+          @interval = (interval || @delay * 0.9).to_f
+        end
+
+        def run
+          loop do
+            sleep(interval)
+            break unless Thread.current[:processing_message]
+            begin
+              message.delay! delay
+            rescue => e
+              logger.error(e)
+              break
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/magellan/gcs/proxy/version.rb

@@ -1,7 +1,7 @@
 module Magellan
   module Gcs
     module Proxy
-      VERSION = "0.1.1"
+      VERSION = '0.1.2'.freeze
     end
   end
 end

data/lib/magellan/gcs/proxy.rb

@@ -1,14 +1,30 @@
-require "magellan/gcs/proxy/version"
-require 'magellan/gcs/proxy/cli'
-require 'magellan/gcs/proxy/context'
+require 'dotenv'
+Dotenv.load
+
+require 'magellan/gcs/proxy/version'
 require 'magellan/gcs/proxy/expand_variable'
-require 'magellan/gcs/proxy/gcp'
+require 'magellan/gcs/proxy/config'
 require 'magellan/gcs/proxy/log'
+require 'magellan/gcs/proxy/gcp'
+
+require 'magellan/gcs/proxy/composite_logger'
+require 'magellan/gcs/proxy/progress_notifier_adapter'
+require 'magellan/gcs/proxy/pubsub_progress_notifier'
+require 'magellan/gcs/proxy/pubsub_sustainer'
+require 'magellan/gcs/proxy/progress_notification'
+
 require 'magellan/gcs/proxy/message_wrapper'
+require 'magellan/gcs/proxy/context'
+require 'magellan/gcs/proxy/cli'
 
 module Magellan
   module Gcs
     module Proxy
+      class << self
+        def config
+          @config ||= Config.new
+        end
+      end
     end
   end
 end