gcs 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cf9d720d07e17816a3153871d3270c62cedc6636
+  data.tar.gz: 14d98afb77a2604726d50af1495f8567001c8387
+SHA512:
+  metadata.gz: a6c852593cfdbebe19960b7e2a98cefaa688c849587e58d64ab90aad7ceefff7dc587d2397672bf3dba8eb64f6f69fbebe4c78b98f0e2bc0c7662337c4311054
+  data.tar.gz: d70fa373f3fd6e57fa2560befab7f1e70fd57253f58ba3087820227381fb93ba6a47b4eb7078036db1029a32255a88be1ce0c6cb9bfa0036f9c306ed7c6e7ca0

data/.gitignore

@@ -0,0 +1,16 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+/config/*.json
+/vendor/bundle

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,8 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.1
+  - 2.3.4
+before_install:
+  - if [ "${TRAVIS_SECURE_ENV_VARS}" = true ]; then openssl aes-256-cbc -K $encrypted_22290c477736_key -iv $encrypted_22290c477736_iv -in config/service_account.json.enc -out config/service_account.json -d; fi
+

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in gcs.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2017 Groovenauts, Inc.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,69 @@
+# gcs-ruby
+
+[![Build Status](https://travis-ci.org/groovenauts/gcs-ruby.svg?branch=master)](https://travis-ci.org/groovenauts/gcs-ruby)
+
+Groovenauts' wrapper library for Google Cloud Storage with [google-api-ruby-client](https://github.com/google/google-api-ruby-client).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'gcs'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install gcs
+
+## Usage
+
+```
+gcs = Gcs.new(email_address, private_key)
+
+# get bucket list
+gcs.bucket(project_id)
+
+# get object list
+gcs.list_objects("myBucket", prefix: "path/to/subdir", max_results: 1000)
+
+# get object metadata
+get_object "myBucket", "myObject"
+get_object "gs://myBucket/myObject"
+
+# download object content
+io = StringIO.new
+get_object "myBucket", "myObject", download_dest: io
+get_object "gs://myBucket/myObject", download_dest: io
+
+# delete object
+gcs.delete_object("myBucket", "myObject")
+gcs.delete_object("gs://myBucket/myObject")
+
+# create new object
+io = StringIO.new("Hello, World!\")
+gcs.insert_object("myBucket", "myObject", io)
+
+# copy recursively
+gcs.copy_tree("gs://myBucket1/src", "gs://myBucket2/dest")
+
+# delete recursively
+gcs.remove_tree("gs://myBucket/dir")
+
+# read object content with size limitation
+gcs.read_partial("gs://myBucket/myObject", limit: 1024) # => read first part of object at least 1024 bytes.
+gcs.read_partial("myBucket", "myObjet", limit: 1024)
+gcs.read_partial("gs://myBucket/myObject", limit: 1024, trim_after_last_delimiter: "\n") #=> remove substr after last "\n"
+
+# initiate resumable upload
+# return session URL to upload object content by PUT method requests.
+# see https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload
+# origin_domain keyword arg was for CORS setting.
+gcs.initiate_resumable_upload("myBucket", "myObject", content_type: "text/plain", origin_domain: "http://example.com")
+gcs.initiate_resumable_upload("gs://myBucket/myObject", content_type: "text/plain", origin_domain: "http://example.com")
+```
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "gcs"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+service_account_json = File.expand_path("../../config/service_account.json", __FILE__)
+if File.readable?(service_account_json)
+  json = JSON.parse(File.read(service_account_json))
+  @client = Gcs.new(json["client_email"], OpenSSL::PKey::RSA.new(json["private_key"]))
+end
+
+require "pry"
+if @client
+  Pry.start(@client)
+else
+  Pry.start
+end

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/gcs.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'gcs/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "gcs"
+  spec.version       = Gcs::VERSION
+  spec.authors       = ["Groovenauts, Inc."]
+  spec.email         = ["tech@groovenauts.jp"]
+
+  spec.summary       = %q{Groovenauts' wrapper library for Google Cloud Storage with google-api-ruby-client}
+  spec.description   = %q{Groovenauts' wrapper library for Google Cloud Storage with google-api-ruby-client}
+  spec.homepage      = "https://github.com/groovenauts/gcs-ruby"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.13"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake-compiler"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_dependency "google-api-client"
+end

data/lib/gcs.rb

@@ -0,0 +1,234 @@
+# coding: utf-8
+
+require "net/http"
+require "cgi"
+require "json"
+
+require "gcs/version"
+require "google/apis/storage_v1"
+
+class Gcs
+  include Google::Apis::StorageV1
+  def initialize(email_address = nil, private_key = nil, scope: "cloud-platform")
+    @api = Google::Apis::StorageV1::StorageService.new
+    scope_url = "https://www.googleapis.com/auth/#{scope}"
+    if email_address and private_key
+      auth = Signet::OAuth2::Client.new(
+        token_credential_uri: "https://accounts.google.com/o/oauth2/token",
+        audience: "https://accounts.google.com/o/oauth2/token",
+        scope: scope_url,
+        issuer: email_address,
+        signing_key: private_key)
+    else
+      auth = Google::Auth.get_application_default([scope_url])
+    end
+    auth.fetch_access_token!
+    @api.authorization = auth
+  end
+
+  def buckets(project_id)
+    @api.list_buckets(project_id, max_results: 1000).items || []
+  end
+
+  def bucket(name)
+    @api.get_bucket(name)
+  rescue Google::Apis::ClientError
+    if $!.status_code == 404
+      return nil
+    else
+      raise
+    end
+  end
+
+  def insert_bucket(project_id, name, storage_class: "STANDARD", acl: nil, default_object_acl: nil, location: nil)
+    b = Bucket.new(
+      name: name,
+      storage_class: storage_class
+    )
+    b.location = location if location
+    b.acl = acl if acl
+    b.default_object_acl = default_object_acl if default_object_acl
+    @api.insert_bucket(project_id, b)
+  end
+
+  def delete_bucket(name)
+    @api.delete_bucket(name)
+  rescue Google::Apis::ClientError
+    if $!.status_code == 404
+      return nil
+    else
+      raise
+    end
+  end
+
+  def self.ensure_bucket_object(bucket, object=nil)
+    if object.nil? and bucket.start_with?("gs://")
+      bucket = bucket.sub(%r{\Ags://}, "")
+      bucket, object = bucket.split("/", 2)
+    end
+    return [bucket, object]
+  end
+
+  def _ensure_bucket_object(bucket, object=nil)
+    self.class.ensure_bucket_object(bucket, object)
+  end
+
+  def get_object(bucket, object=nil, download_dest: nil)
+    bucket, object = _ensure_bucket_object(bucket, object)
+    begin
+      obj = @api.get_object(bucket, object)
+      if download_dest
+        @api.get_object(bucket, object, generation: obj.generation, download_dest: download_dest)
+      end
+      obj
+    rescue Google::Apis::ClientError
+      if $!.status_code == 404
+        return nil
+      else
+        raise
+      end
+    end
+  end
+
+  def read_partial(bucket, object=nil, limit: 1024*1024, trim_after_last_delimiter: nil, &blk)
+    bucket, object = _ensure_bucket_object(bucket, object)
+    uri = URI("https://www.googleapis.com/download/storage/v1/b/#{CGI.escape(bucket)}/o/#{CGI.escape(object).gsub("+", "%20")}?alt=media")
+    Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
+      req = Net::HTTP::Get.new(uri.request_uri)
+      req["Authorization"] = "Bearer #{@api.authorization.access_token}"
+      http.request(req) do |res|
+        case res
+        when Net::HTTPSuccess
+          if blk
+            res.read_body(&blk)
+            return res
+          else
+            total = "".force_encoding(Encoding::ASCII_8BIT)
+            res.read_body do |part|
+              total << part
+              if total.bytesize > limit
+                break
+              end
+            end
+            if trim_after_last_delimiter
+              i = total.rindex(trim_after_last_delimiter.force_encoding(Encoding::ASCII_8BIT))
+              if i.nil?
+                # If no delimiter was found, return empty string.
+                # This is because caller expect not to incomplete line. (ex: Newline Delimited JSON)
+                i = -1
+              end
+              total[(i+1)..-1] = ""
+            end
+            return total
+          end
+        when Net::HTTPNotFound
+          return nil
+        else
+          raise "Gcs.read_partial failed with HTTP status #{res.code}: #{res.body}"
+        end
+      end
+    end
+  end
+
+  def list_objects(bucket, delimiter: "/", prefix: "", page_token: nil, max_results: nil)
+    @api.list_objects(bucket, delimiter: delimiter, prefix: prefix, page_token: page_token, max_results: max_results)
+  end
+
+  def delete_object(bucket, object=nil, if_generation_match: nil)
+    bucket, object = _ensure_bucket_object(bucket, object)
+    @api.delete_object(bucket, object, if_generation_match: if_generation_match)
+  end
+
+  # @param [String] bucket
+  # @param [String] object name
+  # @param [String|IO] source
+  # @param [String] content_type
+  # @param [String] content_encoding
+  #
+  # @return [Google::Apis::StorageV1::Object]
+  def insert_object(bucket, name, source, content_type: nil, content_encoding: nil, if_generation_match: nil)
+    bucket, name = _ensure_bucket_object(bucket, name)
+    obj = Google::Apis::StorageV1::Object.new(name: name)
+    @api.insert_object(bucket, obj, content_encoding: content_encoding, upload_source: source, content_type: content_type,
+                       if_generation_match: if_generation_match)
+  end
+
+  def rewrite(src_bucket, src_object, dest_bucket, dest_object, if_generation_match: nil)
+    r = @api.rewrite_object(src_bucket, src_object, dest_bucket, dest_object, if_generation_match: if_generation_match)
+    until r.done
+      r = @api.rewrite_object(src_bucket, src_object, dest_bucket, dest_object, rewrite_token: r.rewrite_token, if_generation_match: if_generation_match)
+    end
+    r
+  end
+
+  def copy_tree(src, dest)
+    src_bucket, src_path = self.class.ensure_bucket_object(src)
+    dest_bucket, dest_path = self.class.ensure_bucket_object(dest)
+    src_path = src_path + "/" unless src_path[-1] == "/"
+    dest_path = dest_path + "/" unless dest_path[-1] == "/"
+    res = list_objects(src_bucket, prefix: src_path)
+    (res.items || []).each do |o|
+      next if o.name[-1] == "/"
+      dest_obj_name = dest_path + o.name.sub(/\A#{Regexp.escape(src_path)}/, "")
+      self.rewrite(src_bucket, o.name, dest_bucket, dest_obj_name)
+    end
+    (res.prefixes || []).each do |p|
+      copy_tree("gs://#{src_bucket}/#{p}", "gs://#{dest_bucket}/#{dest_path}#{p.sub(/\A#{Regexp.escape(src_path)}/, "")}")
+    end
+  end
+
+  def copy_object(src, dest)
+    src_bucket, src_path = self.class.ensure_bucket_object(src)
+    dest_bucket, dest_path = self.class.ensure_bucket_object(dest)
+    self.rewrite(src_bucket, src_path, dest_bucket, dest_path)
+  end
+
+  def remove_tree(gcs_url)
+    bucket, path = self.class.ensure_bucket_object(gcs_url)
+    if path.size > 0 and path[-1] != "/"
+      path = path + "/"
+    end
+    next_page_token = nil
+    loop do
+      begin
+        res = list_objects(bucket, prefix: path, delimiter: nil, page_token: next_page_token)
+      rescue Google::Apis::ClientError
+        if $!.status_code == 404
+          return nil
+        else
+          raise
+        end
+      end
+
+      # batch request あたりの API 呼び出しの量は API の種類によって異なり
+      # Cloud Storage JSON API のドキュメントでは 100 となってるけど1000でもいけたので1000に変更
+      # ref. https://cloud.google.com/storage/docs/json_api/v1/how-tos/batch
+      (res.items || []).each_slice(1000) do |objs|
+        @api.batch do
+          objs.each do |o|
+            @api.delete_object(bucket, o.name) {|_, err| raise err if err and (not(err.respond_to?(:status_code)) or (err.status_code != 404))}
+          end
+        end
+      end
+      break unless res.next_page_token
+      next_page_token = res.next_page_token
+    end
+  end
+
+  def initiate_resumable_upload(bucket, object=nil, content_type: "application/octet-stream", origin_domain: nil)
+    bucket, object = self.class.ensure_bucket_object(bucket, object)
+    uri = URI("https://www.googleapis.com/upload/storage/v1/b/#{CGI.escape(bucket)}/o?uploadType=resumable")
+    http = Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
+      req = Net::HTTP::Post.new(uri.request_uri)
+      req["content-type"] = "application/json; charset=UTF-8"
+      req["Authorization"] = "Bearer #{@api.authorization.access_token}"
+      req["X-Upload-Content-Type"] = content_type
+      if origin_domain
+        req["Origin"] = origin_domain
+      end
+      req.body = JSON.generate({ "name" => object })
+      res = http.request(req)
+      return res["location"]
+    end
+  end
+end

data/lib/gcs/version.rb

@@ -0,0 +1,3 @@
+class Gcs
+  VERSION = "0.1.1"
+end

metadata

@@ -0,0 +1,140 @@
+--- !ruby/object:Gem::Specification
+name: gcs
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Groovenauts, Inc.
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-12-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: google-api-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Groovenauts' wrapper library for Google Cloud Storage with google-api-ruby-client
+email:
+- tech@groovenauts.jp
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- config/service_account.json.enc
+- gcs.gemspec
+- lib/gcs.rb
+- lib/gcs/version.rb
+homepage: https://github.com/groovenauts/gcs-ruby
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.13
+signing_key: 
+specification_version: 4
+summary: Groovenauts' wrapper library for Google Cloud Storage with google-api-ruby-client
+test_files: []