magellan-gcs-proxy 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bca7eb4b15232e1190e59339f68cca01ef9929eb
-  data.tar.gz: c1d15432f65404b91aabae522b18ce2cac1fa172
+  metadata.gz: e2d9990ba10bfc49119dc24969dac164773e526b
+  data.tar.gz: 135838363859a7755d687ec3f9ec5f6d17639383
 SHA512:
-  metadata.gz: 59fa94585e7f984de73fd5d1cd36f5f499dcfa252e9def885effb9881fca87c4e7dd84d703d22d500c3358918216a0108c152e132ba93528507cce11f4b3736f
-  data.tar.gz: e27613b66bee86d93148171851cc19e0ec0a7a7a316d3600f457f6a9150a8660db8956bd064dade9c259bcf3d0bcbf6aa18f9fc8ad0d712ace7082e6d16e310c
+  metadata.gz: 3c6176dc804d2ddb6c913973296c6c343e8faaf11ceb9dedc4ce17875b7f0353c1f166cd68602ee2444390b99eb61f3372d3439444fea0cd5decd607577867da
+  data.tar.gz: 51772e727ad2ea9a7c813d4f62efa041a5c770f7f4713b4bd613d81943a4f8dadd7061ac346eaef9a8f1d079fd9e8e351d7e4b6e2ad57e7bf28c3500cb551518

data/README.md

@@ -1,8 +1,7 @@
-# Magellan::Gcs::Proxy
+# magellang-gcs-proxy
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/magellan/gcs/proxy`. To experiment with that code, run `bin/console` for an interactive prompt.
+magellang-gcs-proxy is a gem for MAGELLAN BLOCKS "Batch type" IoT board.
 
-TODO: Delete this and the text above, and describe your gem
 
 ## Installation
 
@@ -22,7 +21,42 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+```
+magellan-gcs-proxy COMMAND ARGS...
+```
+
+### TODO How it works
+
+### TODO Expanding variables
+
+
+## Debugging with gcloud
+
+### Setup
+
+```
+export PROJECT_ID=[Project ID]
+export TOPIC=projects/$PROJECT_ID/topics/[Topic name]
+export SUB=projects/$PROJECT_ID/subscriptions/[Subscription name of topic]
+$ gcloud beta pubsub topics create projects/$PROJECT_ID/topics/$TOPIC
+$ gcloud beta pubsub topics list
+$ gcloud beta pubsub subscriptions create $SUB --topic $TOPIC
+$ gcloud beta pubsub subscriptions list
+```
+
+### Publish message
+
+```
+$ gcloud beta pubsub topics publish $TOPIC "" --attribute='download_files=["gs://bucket1/path/to/file"]'
+```
+
+
+### Run application
+
+```
+$ magellan-gcs-proxy COMMAND ARGS...
+```
+
 
 ## Development
 

data/lib/magellan/gcs/proxy/cli.rb

@@ -1,15 +1,16 @@
 require "magellan/gcs/proxy"
-require "magellan/gcs/proxy/pubsub_operation"
+require "magellan/gcs/proxy/log"
 
 require 'json'
 require 'logger'
 require 'tmpdir'
+require 'logger_pipe'
 
 module Magellan
   module Gcs
     module Proxy
       class Cli
-        include PubsubOperation
+        include Log
 
         attr_reader :cmd_template
         def initialize(*args)
@@ -18,11 +19,14 @@ module Magellan
 
         def run
           logger.info("Start listening")
-          sub.listen do |msg|
+          GCP.subscription.listen do |msg|
             begin
               process(msg)
             rescue => e
               logger.error("[#{e.class.name}] #{e.message}")
+              if ENV['VERBOSE'] =~ /true|yes|on|1/i
+                logger.debug("Backtrace\n  " << e.backtrace.join("\n  "))
+              end
             end
           end
         rescue => e
@@ -34,32 +38,31 @@ module Magellan
           logger.info("Processing message: #{msg.inspect}")
           Dir.mktmpdir 'workspace' do |dir|
             dfiles = parse(msg.attributes['download_files'])
-            ufiles =  parse(msg.attributes['upload_files'])
+            logger.info("dfiles: #{dfiles}")
 
-            context = Context.new(dir, dfiles, ufiles)
+            context = Context.new(dir, dfiles)
             context.setup
+            logger.info("context.setup done.")
 
             context.download
+            logger.info("context.download done.")
+            logger.info("msg: #{msg}")
+            logger.info("context: #{context}")
 
             cmd = build_command(msg, context)
 
-            logger.info("Executing command: #{cmd.inspect}")
-
-            if system(cmd)
+            begin
+              LoggerPipe.run(logger, cmd, returns: :none, logging: :both)
+            rescue => e
+              logger.error("Error: #{cmd.inspect}")
+            else
               context.upload
-
-              sub.acknowledge msg
+              msg.acknowledge!
               logger.info("Complete processing and acknowledged")
-            else
-              logger.error("Error: #{cmd.inspect}")
             end
           end
         end
 
-        def logger
-          @logger ||= Logger.new($stdout)
-        end
-
         def parse(str)
           return nil if str.nil? || str.empty?
           JSON.parse(str)

data/lib/magellan/gcs/proxy/context.rb

@@ -1,27 +1,28 @@
+# coding: utf-8
 require "magellan/gcs/proxy"
-require "magellan/gcs/proxy/file_operation"
+require "magellan/gcs/proxy/log"
 
+require 'fileutils'
 require 'uri'
 
 module Magellan
   module Gcs
     module Proxy
       class Context
-        include FileOperation
-        
-        attr_reader :workspace, :remote_download_files, :remote_upload_files
-        def initialize(workspace, remote_download_files, remote_upload_files)
+        include Log
+
+        attr_reader :workspace, :remote_download_files
+        def initialize(workspace, remote_download_files)
           @workspace = workspace
           @remote_download_files = remote_download_files
-          @remote_upload_files = remote_upload_files
         end
 
         KEYS = [
           :workspace,
           :downloads_dir, :uploads_dir,
-          :download_files, :upload_files,
-          :local_download_files, :local_upload_files,
-          :remote_download_files, :remote_upload_files
+          :download_files,
+          :local_download_files,
+          :remote_download_files,
         ].freeze
 
         def [](key)
@@ -52,35 +53,33 @@ module Magellan
           File.join(workspace, 'uploads')
         end
 
-        def upload_mapping
-          @upload_mapping ||= build_mapping(uploads_dir, remote_upload_files)
-        end
-
-        def local_upload_files
-          @local_upload_files ||= build_local_files_obj(remote_upload_files, upload_mapping)
-        end
-        alias_method :upload_files, :local_upload_files
-
         def setup
           setup_dirs
         end
 
         def download
           download_mapping.each do |url, path|
-            logger.info("Downloading: #{url}")
+            FileUtils.mkdir_p File.dirname(path)
+            logger.debug("Downloading: #{url} to #{path}")
             uri = parse_uri(url)
-            bucket = storage.bucket(uri.host)
+            @last_bucket_name = uri.host
+            bucket = GCP.storage.bucket(@last_bucket_name)
             file = bucket.file uri.path.sub(/\A\//, '')
             file.download(path)
+            logger.info("Download OK: #{url} to #{path}")
           end
         end
 
         def upload
-          upload_mapping.each do |url, path|
-            logger.info("Uploading: #{url}")
-            uri = parse_uri(url)
-            bucket = storage.bucket(uri.host)
-            bucket.create_file path, uri.path.sub(/\A\//, '')
+          Dir.chdir(uploads_dir) do
+            Dir.glob('**/*') do |path|
+              next if File.directory?(path)
+              url = "gs://#{@last_bucket_name}/#{path}"
+              logger.info("Uploading: #{path} to #{url}")
+              bucket = GCP.storage.bucket(@last_bucket_name)
+              bucket.create_file path, path
+              logger.info("Upload OK: #{path} to #{url}")
+            end
           end
         end
 
@@ -97,6 +96,7 @@ module Magellan
 
         def flatten_values(obj)
           case obj
+          when nil then []
           when Hash then flatten_values(obj.values)
           when Array then obj.map{|i| flatten_values(i) }
           else obj

data/lib/magellan/gcs/proxy/gcp.rb

@@ -0,0 +1,60 @@
+# coding: utf-8
+require 'magellan/gcs/proxy'
+
+require "google/cloud/pubsub"
+require "google/cloud/storage"
+require 'net/http'
+
+module Magellan
+  module Gcs
+    module Proxy
+      module GCP
+        extend Log
+        include Log
+
+        module_function
+
+        def project_id
+          @project_id ||= retrieve_project_id
+        end
+
+        def retrieve_project_id
+          ENV['BLOCKS_BATCH_PROJECT_ID'] || retrieve_metadata('project/project-id')
+        end
+
+        METADATA_HOST = 'metadata.google.internal'.freeze
+        METADATA_PATH_BASE = '/computeMetadata/v1/'.freeze
+        METADATA_HEADER = {"Metadata-Flavor" => "Google"}.freeze
+
+        def retrieve_metadata(key)
+          http = Net::HTTP.new(METADATA_HOST)
+          res = http.get(METADATA_PATH_BASE + key, METADATA_HEADER)
+          case res.code
+          when /\A2\d{2}\z/ then res.body
+          else raise "[#{res.code}] #{res.body}"
+          end
+        end
+
+        def storage
+          @storage ||= Google::Cloud::Storage.new(project: project_id)
+        end
+
+        def pubsub
+          @pubsub ||= Google::Cloud::Pubsub.new(project: project_id)
+        end
+
+        def subscription
+          unless @subscription
+            @subscription = pubsub.subscription(ENV['BLOCKS_BATCH_PUBSUB_SUBSCRIPTION'] || 'test-subscription')
+            logger.info("subscription: #{@subscription.inspect}")
+          end
+          @subscription
+        end
+
+        def reset
+          instance_variables.each {|ivar| instance_variable_set(ivar, nil)}
+        end
+      end
+    end
+  end
+end

data/lib/magellan/gcs/proxy/log.rb

@@ -0,0 +1,12 @@
+require 'logger'
+module Magellan
+  module Gcs
+    module Proxy
+      module Log
+        def logger
+          @logger ||= Logger.new($stdout)
+        end
+      end
+    end
+  end
+end

data/lib/magellan/gcs/proxy/version.rb

@@ -1,7 +1,7 @@
 module Magellan
   module Gcs
     module Proxy
-      VERSION = "0.1.0"
+      VERSION = "0.1.1"
     end
   end
 end

data/lib/magellan/gcs/proxy.rb

@@ -2,9 +2,9 @@ require "magellan/gcs/proxy/version"
 require 'magellan/gcs/proxy/cli'
 require 'magellan/gcs/proxy/context'
 require 'magellan/gcs/proxy/expand_variable'
-require 'magellan/gcs/proxy/file_operation'
+require 'magellan/gcs/proxy/gcp'
+require 'magellan/gcs/proxy/log'
 require 'magellan/gcs/proxy/message_wrapper'
-require 'magellan/gcs/proxy/pubsub_operation'
 
 module Magellan
   module Gcs

data/magellan-gcs-proxy.gemspec

@@ -23,6 +23,7 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency "google-cloud-pubsub"
   spec.add_runtime_dependency "google-cloud-storage"
+  spec.add_runtime_dependency "logger_pipe"
 
   spec.add_development_dependency "bundler", "~> 1.13"
   spec.add_development_dependency "rake", "~> 10.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: magellan-gcs-proxy
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - akm
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-11-21 00:00:00.000000000 Z
+date: 2016-11-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-cloud-pubsub
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: logger_pipe
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -104,9 +118,9 @@ files:
 - lib/magellan/gcs/proxy/cli.rb
 - lib/magellan/gcs/proxy/context.rb
 - lib/magellan/gcs/proxy/expand_variable.rb
-- lib/magellan/gcs/proxy/file_operation.rb
+- lib/magellan/gcs/proxy/gcp.rb
+- lib/magellan/gcs/proxy/log.rb
 - lib/magellan/gcs/proxy/message_wrapper.rb
-- lib/magellan/gcs/proxy/pubsub_operation.rb
 - lib/magellan/gcs/proxy/version.rb
 - magellan-gcs-proxy.gemspec
 homepage: https://github.com/groovenauts/magellan-gcs-proxy

data/lib/magellan/gcs/proxy/file_operation.rb

@@ -1,46 +0,0 @@
-require "magellan/gcs/proxy"
-
-require 'uri'
-require "google/cloud/storage"
-
-module Magellan
-  module Gcs
-    module Proxy
-      module FileOperation
-        def storage
-          @storage ||= Google::Cloud::Storage.new(
-            # default credential を利用するため、プロジェクトの指定はしない
-            # project: ENV['GOOGLE_PROJECT'] || 'dummy-project-id',
-            # keyfile: ENV['GOOGLE_KEY_JSON_FILE'],
-          )
-        end
-
-        def download(base_dir, urls)
-          (urls || []).each do |url|
-            logger.info("Downloading: #{url}")
-            uri = parse_uri(url)
-            bucket = storage.bucket(uri.host)
-            file = bucket.file uri.path.sub(/\A\//, '')
-            file.download File.join(base_dir, uri.path)
-          end
-        end
-
-        def upload(base_dir, urls)
-          (urls || []).each do |url|
-            logger.info("Uploading: #{url}")
-            uri = parse_uri(url)
-            bucket = storage.bucket(uri.host)
-            bucket.create_file File.join(base_dir, uri.path), uri.path.sub(/\A\//, '')
-          end
-        end
-
-        def parse_uri(str)
-          uri = URI.parse(str)
-          raise "Unsupported scheme #{uri.scheme.inspect} of #{str}" unless uri.scheme == 'gs'
-          uri
-        end
-
-      end
-    end
-  end
-end

data/lib/magellan/gcs/proxy/pubsub_operation.rb

@@ -1,38 +0,0 @@
-require "magellan/gcs/proxy"
-
-require "google/cloud/pubsub"
-
-module Magellan
-  module Gcs
-    module Proxy
-      module PubsubOperation
-        def pubsub
-          @pubsub ||= Google::Cloud::Pubsub.new(
-            # default credential を利用するため、プロジェクトの指定はしない
-            # project: ENV['GOOGLE_PROJECT'] || 'dummy-project-id',
-            # keyfile: ENV['GOOGLE_KEY_JSON_FILE'],
-          )
-        end
-
-        def topic
-          unless @topic
-            topic_name = ENV['BATCH_TOPIC_NAME'] || 'test-topic'
-            @topic = pubsub.topic(topic_name) || pubsub.create_topic(topic_name)
-            logger.info("topic: #{@topic.inspect}")
-          end
-          @topic
-        end
-
-        def sub
-          unless @sub
-            sub_name = ENV['BATCH_SUBSCRIPTION_NAME'] || 'test-subscription'
-            @sub = topic.subscription(sub_name) || topic.subscribe(sub_name)
-            logger.info("subscription: #{@sub.inspect}")
-          end
-          @sub
-        end
-
-      end
-    end
-  end
-end