blobby-gcs 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f09547aa636cb68c076fa0ad2f2253b1f357a1af97cee53993f5ad5f9ed8df37
+  data.tar.gz: b8941452f69592af379bcee4c87fc38f95f1d94600a5022044417c3a3fa8b322
+SHA512:
+  metadata.gz: 11c590018bfa1ed10f1a6b452e2f88fc6915171e872210fd1e9dcfcb46aaa98c5aad7fd55edaea07e174d91805fc89e523a3f342c3665774468ee47c9725d4b5
+  data.tar.gz: 5ff02e4d413bffd69b0a06abbc5a81f4c9287407841869b89b18a0d2eacf878c372642c50bc608ba990b09dad78f75e8be596d3ade2c1e6bc6d5d692b32bee47

data/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in blobby-gcs.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,121 @@
+PATH
+  remote: .
+  specs:
+    blobby-gcs (0.1)
+      blobby (~> 1.1.0)
+      google-cloud-storage (~> 1.25)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    ast (2.4.0)
+    blobby (1.1.1)
+    bundler-audit (0.6.1)
+      bundler (>= 1.2.0, < 3)
+      thor (~> 0.18)
+    byebug (11.0.1)
+    coderay (1.1.2)
+    declarative (0.0.10)
+    declarative-option (0.1.0)
+    diff-lcs (1.3)
+    digest-crc (0.4.1)
+    faraday (0.17.3)
+      multipart-post (>= 1.2, < 3)
+    google-api-client (0.36.3)
+      addressable (~> 2.5, >= 2.5.1)
+      googleauth (~> 0.9)
+      httpclient (>= 2.8.1, < 3.0)
+      mini_mime (~> 1.0)
+      representable (~> 3.0)
+      retriable (>= 2.0, < 4.0)
+      signet (~> 0.12)
+    google-cloud-core (1.4.1)
+      google-cloud-env (~> 1.0)
+    google-cloud-env (1.3.0)
+      faraday (~> 0.11)
+    google-cloud-storage (1.25.0)
+      addressable (~> 2.5)
+      digest-crc (~> 0.4)
+      google-api-client (~> 0.33)
+      google-cloud-core (~> 1.2)
+      googleauth (~> 0.9)
+      mini_mime (~> 1.0)
+    googleauth (0.10.0)
+      faraday (~> 0.12)
+      jwt (>= 1.4, < 3.0)
+      memoist (~> 0.16)
+      multi_json (~> 1.11)
+      os (>= 0.9, < 2.0)
+      signet (~> 0.12)
+    httpclient (2.8.3)
+    jaro_winkler (1.5.4)
+    jwt (2.2.1)
+    memoist (0.16.2)
+    method_source (0.9.2)
+    mini_mime (1.0.2)
+    multi_json (1.14.1)
+    multipart-post (2.1.1)
+    os (1.0.1)
+    parallel (1.19.1)
+    parser (2.7.0.1)
+      ast (~> 2.4.0)
+    pry (0.12.2)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    pry-byebug (3.7.0)
+      byebug (~> 11.0)
+      pry (~> 0.10)
+    public_suffix (4.0.2)
+    rainbow (3.0.0)
+    rake (10.5.0)
+    representable (3.0.4)
+      declarative (< 0.1.0)
+      declarative-option (< 0.2.0)
+      uber (< 0.2.0)
+    retriable (3.1.2)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.1)
+      rspec-support (~> 3.9.1)
+    rspec-expectations (3.9.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.2)
+    rubocop (0.78.0)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.6)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 1.7)
+    ruby-progressbar (1.10.1)
+    signet (0.12.0)
+      addressable (~> 2.3)
+      faraday (~> 0.9)
+      jwt (>= 1.5, < 3.0)
+      multi_json (~> 1.10)
+    thor (0.20.3)
+    uber (0.1.0)
+    unicode-display_width (1.6.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  blobby-gcs!
+  bundler (~> 2.1)
+  bundler-audit (~> 0.6.1)
+  pry-byebug (~> 3.7)
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  rubocop (~> 0.78.0)
+
+BUNDLED WITH
+   2.1.0

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Jerome Paul
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,68 @@
+# Blobby::GCSStore [![Build status](https://badge.buildkite.com/3a10d87beff89aab18d733ccd5fb3080e76020079c00ebdaab.svg)](https://buildkite.com/gs/blobby-gcs)
+
+This gem provides a Google Cloud Storage (GCS) implementation of the "store" interface defined by the [`blobby`](https://github.com/realestate-com-au/blobby) gem. It's been packaged separately to avoid adding dependencies to the core gem.
+
+The simplest use-case is writing to a single bucket:
+
+```ruby
+gcs_store = Blobby.store("gs://<bucket_name>/<directory_path_inside_bucket>")
+gcs_store["key"].write("IO or string file content")
+```
+
+Configuration is provided using the [same environment variables](https://googleapis.dev/ruby/google-cloud-storage/latest/file.AUTHENTICATION.html) as the `google-cloud-storage` gem (which this one is wrapping).
+
+```ruby
+require 'blobby-gcs'
+
+ENV["GOOGLE_CLOUD_PROJECT"] = "my-project-id"
+ENV["GOOGLE_APPLICATION_CREDENTIALS"] = "path/to/keyfile.json"
+
+gcs_store = Blobby.store("gs://<bucket_name>/<directory_path_inside_bucket>")
+```
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'blobby-gcs'
+```
+
+And then execute:
+
+```bash
+$ bundle
+```
+
+Or install it yourself as:
+
+```bash
+$ gem install blobby-gcs
+```
+
+## Development
+
+You can run the tests within Docker via `auto/test` or directly within your shell using `scripts/test`.
+
+There are integration tests within this gem which point to a real **Google Cloud Storage** bucket, so credentials are required. If you do not already have `~/.config/gcloud/application_default_credentials.json` from authenticating with [`gcloud`](https://cloud.google.com/sdk/gcloud/), run:
+
+```bash
+$ gcloud auth application-default login
+```
+
+You will also need to provide the name of a GCS bucket you have `storage.objectAdmin` access to (and unfortunately, also `storage.legacyBucketReader`):
+
+```bash
+export GOOGLE_CLOUD_PROJECT=unused
+export BLOBBY_GCS_TEST_BUCKET=...name of bucket...
+```
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and publish the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/greensync/blobby-gcs.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/blobby-gcs.gemspec

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = 'blobby-gcs'
+  spec.version       = 0.1
+  spec.authors       = ['Jerome Paul']
+  spec.email         = ['jerome.paul@greensync.com.au']
+
+  spec.summary       = 'Store BLOBs in Google Cloud Storage'
+  spec.homepage      = 'https://github.com/greensync/blobby-gcs'
+  spec.license       = 'MIT'
+
+  spec.files += Dir['lib/**/*']
+  spec.files += Dir['LICENSE.txt']
+  spec.files += Dir['blobby-gcs.gemspec']
+  spec.files += Dir['Gemfile']
+  spec.files += Dir['Gemfile.lock']
+  spec.files += Dir['README.md']
+
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 2.1'
+  spec.add_development_dependency 'bundler-audit', '~> 0.6.1'
+  spec.add_development_dependency 'pry-byebug', '~> 3.7'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop', '~> 0.78.0'
+
+  spec.add_runtime_dependency('blobby', '~> 1.1.0')
+  spec.add_runtime_dependency('google-cloud-storage', '~> 1.25')
+end

data/lib/blobby-gcs.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require 'blobby'
+require 'blobby/gcs_store'

data/lib/blobby/gcs_store.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require 'blobby'
+require 'blobby/key_transforming_store'
+require 'google/cloud/storage'
+require 'uri'
+
+module Blobby
+  # A BLOB store backed by Google Cloud Storage.
+  #
+  class GCSStore
+
+    def self.from_uri(uri)
+      uri = URI(uri)
+      raise ArgumentError, "invalid GCS address: #{uri}" unless uri.scheme == 'gs'
+
+      bucket_name = uri.host
+      prefix = uri.path.sub(%r{\A/}, '').sub(%r{/\Z}, '')
+      raise ArgumentError, 'no bucket specified' if bucket_name.nil?
+
+      store = new(bucket_name)
+      store = KeyTransformingStore.new(store) { |key| prefix + '/' + key } unless prefix.empty?
+      store
+    end
+
+    def initialize(bucket_name, gcs_options: {})
+      @bucket_name = bucket_name
+      @gcs_options = gcs_options
+    end
+
+    def available?
+      !!bucket&.exists?
+    end
+
+    def [](key)
+      KeyConstraint.must_allow!(key)
+      StoredObject.new(bucket, key)
+    end
+
+    class StoredObject
+
+      def initialize(bucket, key)
+        @bucket = bucket
+        @key = key
+      end
+
+      # https://googleapis.dev/ruby/google-cloud-storage/latest/Google/Cloud/Storage/File.html#exists%3F-instance_method
+      # bucket.file(key) returns nil, if the file does not exist!
+      def exists?
+        !!gcs_file&.exists?
+      end
+
+      # https://googleapis.dev/ruby/google-cloud-storage/latest/Google/Cloud/Storage/File.html#download-instance_method
+      def read
+        return nil unless exists?
+
+        io = gcs_file.download
+        if block_given?
+          while (chunk = io.read(512))
+            yield chunk
+          end
+          nil
+        else
+          io.read
+        end
+      end
+
+      # https://googleapis.dev/ruby/google-cloud-storage/latest/Google/Cloud/Storage/Bucket.html#create_file-instance_method
+      def write(content)
+        content = StringIO.new(content) unless content.respond_to?(:read)
+        bucket.create_file(content, key)
+        nil
+      end
+
+      # https://googleapis.dev/ruby/google-cloud-storage/latest/Google/Cloud/Storage/File.html#delete-instance_method
+      def delete
+        return false unless exists?
+
+        gcs_file.delete
+      end
+
+      private
+
+      attr_reader :bucket, :key
+
+      def gcs_file
+        bucket.file(key)
+      end
+
+    end
+
+    private
+
+    attr_reader :bucket_name, :gcs_options
+
+    def bucket
+      @bucket ||= gcs_client.bucket(bucket_name, skip_lookup: true)
+    end
+
+    def gcs_client
+      @gcs_client ||= Google::Cloud::Storage.new(gcs_options)
+    end
+
+  end
+
+  register_store_factory 'gs', GCSStore
+end

metadata

@@ -0,0 +1,162 @@
+--- !ruby/object:Gem::Specification
+name: blobby-gcs
+version: !ruby/object:Gem::Version
+  version: '0.1'
+platform: ruby
+authors:
+- Jerome Paul
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-01-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: bundler-audit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.1
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.78.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.78.0
+- !ruby/object:Gem::Dependency
+  name: blobby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+- !ruby/object:Gem::Dependency
+  name: google-cloud-storage
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.25'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.25'
+description: 
+email:
+- jerome.paul@greensync.com.au
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- blobby-gcs.gemspec
+- lib/blobby-gcs.rb
+- lib/blobby/gcs_store.rb
+homepage: https://github.com/greensync/blobby-gcs
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Store BLOBs in Google Cloud Storage
+test_files: []