magellan-gcs-proxy 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 97082e16910f8eb448e9dd90a579dab3f5d2645b
-  data.tar.gz: b8ef0a1084a6da1b9287a9806c91ebd1e73b0ba8
+  metadata.gz: ea3b9c9c14c544a0856ccc2fc7e5923058ea2842
+  data.tar.gz: 7097911f386a69da3f54f01ceb4ad37c8e3cfc88
 SHA512:
-  metadata.gz: 412d7f881615ba6bb467cd655d69a60701297f1bcd0a32be6bea745cf41a517f88dc0d3631c628a545e2a9d5c4acb586b02bbd1fa0f596e7d471f5d7968780ec
-  data.tar.gz: cda4e394851a8f65fc5edd6564faae8dda6ad74c9f1fe2c52a4b1953ddca4f48775be4138703516f8245cb9dec25b13d99cc6980d4002197c7fe91a1ace7ed79
+  metadata.gz: 5afd3e0bc9f9510715d9b509c46b38f43f552c1006e324931c13ae12fde72db44bfd48074b864e26c48f0071c8baa4ed11945ffd7d530406235fd0b8238aa5f6
+  data.tar.gz: 35e8d6f94d1bc21e2a8080bf0244bbf52ec41323d273f81d330f5d42e85a5d39d645c3b96e20561dd4b31528f283a890b900a0b1caf5826db3a377ff32a1e25a

data/example/Dockerfile

@@ -15,4 +15,4 @@ COPY . $APP_HOME
 WORKDIR $APP_HOME
 RUN bundle install --without development
 
-CMD ["bundle", "exec", "magellan-gcs-proxy", "./app.sh", "%{download_files.0}", "%{uploads_dir}"]
+CMD ["bundle", "exec", "magellan-gcs-proxy", "./app.sh", "%{download_files.0}", "%{downloads_dir}", "%{uploads_dir}", "test"]

data/example/Gemfile.lock

@@ -65,7 +65,7 @@ GEM
     logging (2.1.0)
       little-plugger (~> 1.1)
       multi_json (~> 1.10)
-    magellan-gcs-proxy (0.2.0)
+    magellan-gcs-proxy (0.3.0)
       dotenv
       google-cloud-logging
       google-cloud-storage

data/example/app.sh

@@ -1,19 +1,47 @@
 #!/bin/bash
 
 usage() {
-  echo usage: `basename $0` src_path dest_path
+  echo usage: `basename $0` download_filepath downloads_dir uploads_dir suffix
 }
 
-if [ $# -ne 2 ]; then
+if [ $# -ne 4 ]; then
   echo `basename $0`: missing operand 1>&2
   usage
   exit 1
 fi
 
 fpath=$1
+downloads_dir=$2
+uploads_dir=$3
+suffix=$4
+
+dl_localpath=${fpath##$downloads_dir/}
+dl_bucket=$(echo $dl_localpath | cut -d '/' -f1)
+dl_relpath=${dl_localpath##$dl_bucket/}
+dl_fname=${dl_relpath##*/}
+dl_dir=${dl_relpath%$dl_fname}
+
+echo "dl_localpath=${dl_localpath}"
+echo "dl_bucket=${dl_bucket}"
+echo "dl_relpath=${dl_relpath}"
+echo "dl_fname=${dl_fname}"
+echo "dl_dir=${dl_dir}"
+
+
 fname="${fpath##*/}"
 fbase="${fname%.*}"
 fext="${fpath##*.}"
-mkdir -p $2
 
-cp $1 $2/${fbase}-$(date '+%Y%m%d-%T' | tr -d :).${fext}
+echo "fname=${fname}"
+echo "fbase=${fbase}"
+echo "fext=${fext}"
+
+
+ul_dir="$3/${dl_bucket}/${dl_dir}"
+ul_path="${ul_dir}${fbase}-${suffix}.${fext}"
+
+echo "ul_dir=${ul_dir}"
+echo "ul_path=${ul_path}"
+
+mkdir -p $ul_dir
+cp $fpath $ul_path

data/exe/magellan-gcs-proxy-dev-setup

@@ -43,7 +43,7 @@ echo "export BLOCKS_BATCH_PROJECT_ID=${project_id}"
 echo "export BLOCKS_BATCH_PUBSUB_SUBSCRIPTION=${job_sub}"
 echo "export BLOCKS_BATCH_PROGRESS_TOPIC=${progress_topic}"
 echo "export BLOCKS_BATCH_CLOUD_LOGGING_LOG_NAME=magellan-gcs-proxy-example-logging"
-echo "bundle exec magellan-gcs-proxy ./app.sh %{download_files.0} %{uploads_dir}"
+echo "bundle exec magellan-gcs-proxy ./app.sh %{download_files.0} %{downloads_dir} %{uploads_dir} test"
 echo ""
 echo "#"
 echo "# At Terminal 3: Publish message"

data/lib/magellan/gcs/proxy/context.rb

@@ -65,13 +65,17 @@ module Magellan
 
         def upload
           Dir.chdir(uploads_dir) do
-            Dir.glob('**/*') do |path|
-              next if directory?(path)
-              url = "gs://#{@last_bucket_name}/#{path}"
-              logger.info("Uploading: #{path} to #{url}")
-              bucket = GCP.storage.bucket(@last_bucket_name)
-              bucket.create_file path, path
-              logger.info("Upload OK: #{path} to #{url}")
+            Dir.glob('*') do |bucket_name|
+              Dir.chdir(bucket_name) do
+                Dir.glob('**/*') do |path|
+                  next if directory?(path)
+                  url = "gs://#{bucket_name}/#{path}"
+                  logger.info("Uploading: #{path} to #{url}")
+                  bucket = GCP.storage.bucket(bucket_name)
+                  bucket.create_file path, path
+                  logger.info("Upload OK: #{path} to #{url}")
+                end
+              end
             end
           end
         end
@@ -87,7 +91,7 @@ module Magellan
         def build_mapping(base_dir, obj)
           flatten_values(obj).flatten.each_with_object({}) do |url, d|
             uri = parse_uri(url)
-            d[url] = File.join(base_dir, uri.path)
+            d[url] = File.join(base_dir, uri.host, uri.path)
           end
         end
 

data/lib/magellan/gcs/proxy/version.rb

@@ -1,7 +1,7 @@
 module Magellan
   module Gcs
     module Proxy
-      VERSION = '0.2.0'.freeze
+      VERSION = '0.3.0'.freeze
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: magellan-gcs-proxy
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - akm
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-12-19 00:00:00.000000000 Z
+date: 2016-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dotenv