bfs-s3 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 99fd7cbae95d18698976a0e6bc061a8c89f4767ee58186903dceb321fefccbf4
+  data.tar.gz: 66abcf654b856ca9da91319352bceec93d806635f6587676e7eee16a6fab7450
+SHA512:
+  metadata.gz: 11a36864299e8b1b549cbda57dcf1fdf3cba4e2a2eb5bd6b3a18f5617adc5bcb8a267dbfa5522659b083e4d7d349a5717c52641f4cd987d175624297df7f6b68
+  data.tar.gz: 890d3fec3c4653f23b7c85f98f2ab89ad219728633858e5558161951c7572c98fe879850ab0b349a485f8a42ffd85e65f432e4b3f418099475a4e5866a884edf

data/bfs-s3.gemspec

@@ -0,0 +1,22 @@
+Gem::Specification.new do |s|
+  s.name        = 'bfs-s3'
+  s.version     = File.read(File.expand_path('../.version', __dir__)).strip
+  s.platform    = Gem::Platform::RUBY
+
+  s.licenses    = ['Apache-2.0']
+  s.summary     = 'S3 bucket adapter for bfs'
+  s.description = 'https://github.com/bsm/bfs.rb'
+
+  s.authors     = ['Dimitrij Denissenko']
+  s.email       = 'dimitrij@blacksquaremedia.com'
+  s.homepage    = 'https://github.com/bsm/bfs.rb'
+
+  s.executables   = []
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- spec/*`.split("\n")
+  s.require_paths = ['lib']
+  s.required_ruby_version = '>= 2.2.0'
+
+  s.add_dependency 'aws-sdk-s3'
+  s.add_dependency 'bfs'
+end

data/lib/bfs/bucket/s3.rb

@@ -0,0 +1,141 @@
+require 'bfs'
+require 'aws-sdk-s3'
+require 'cgi'
+
+module BFS
+  module Bucket
+    # S3 buckets are operating on s3
+    class S3 < Abstract
+      attr_reader :name, :sse, :acl, :storage_class
+
+      # Initializes a new S3 bucket
+      # @param [String] name the bucket name
+      # @param [Hash] opts options
+      # @option opts [String] :region default region
+      # @option opts [String] :sse default server-side-encryption setting
+      # @option opts [Aws::Credentials] :credentials credentials object
+      # @option opts [String] :access_key_id custom AWS access key ID
+      # @option opts [String] :secret_access_key custom AWS secret access key
+      # @option opts [Symbol] :acl canned ACL
+      # @option opts [String] :storage_class storage class
+      # @option opts [Aws::S3::Client] :client custom client, uses default_client by default
+      def initialize(name, opts={})
+        opts = opts.dup
+        opts.keys.each do |key|
+          opts[key.to_s] = opts.delete(key)
+        end
+
+        @name = name
+        @sse = opts['sse'] || opts['server_side_encryption']
+        @credentials = opts['credentials']
+        @credentials ||= Aws::Credentials.new(opts['access_key_id'].to_s, opts['secret_access_key'].to_s) if opts['access_key_id']
+        @acl = opts['acl'].to_sym if opts.key?('acl')
+        @storage_class = opts['storage_class']
+        @client = opts['client'] || Aws::S3::Client.new(region: opts['region'])
+      end
+
+      # Lists the contents of a bucket using a glob pattern
+      def ls(pattern='**/*', opts={})
+        @client.list_objects_v2(opts.merge(bucket: name)).contents.select do |obj|
+          File.fnmatch?(pattern, obj.key, File::FNM_PATHNAME)
+        end.map(&:key)
+      end
+
+      # Info returns the object info
+      def info(path, opts={})
+        path = norm_path(path)
+        opts = opts.merge(
+          bucket: name,
+          max_keys: 1,
+          prefix: opts[:prefix] ? File.join(opts[:prefix], path) : path,
+        )
+        object = @client.list_objects_v2(opts).contents.first
+        raise BFS::FileNotFound, path unless object
+
+        BFS::FileInfo.new(path, object.size, object.last_modified)
+      rescue Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::NoSuchBucket
+        raise BFS::FileNotFound, path
+      end
+
+      # Creates a new file and opens it for writing
+      def create(path, opts={}, &block)
+        path = norm_path(path)
+        opts = opts.merge(
+          bucket: name,
+          key: path,
+        )
+        opts[:acl] ||= @acl if @acl
+        opts[:server_side_encryption] ||= @sse if @sse
+        opts[:storage_class] ||= @storage_class if @storage_class
+
+        temp = BFS::TempWriter.new(path) do |t|
+          File.open(t) do |file|
+            @client.put_object(opts.merge(body: file))
+          end
+        end
+        return temp unless block
+
+        begin
+          yield temp
+        ensure
+          temp.close
+        end
+      end
+
+      # Opens an existing file for reading
+      def open(path, opts={}, &block)
+        path = norm_path(path)
+        temp = Tempfile.new(File.basename(path))
+        temp.close
+
+        opts = opts.merge(
+          response_target: temp.path,
+          bucket: name,
+          key: path,
+        )
+        @client.get_object(opts)
+
+        File.open(temp.path, &block)
+      rescue Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::NoSuchBucket
+        raise BFS::FileNotFound, path
+      end
+
+      # Deletes a file.
+      def rm(path, opts={})
+        path = norm_path(path)
+        opts = opts.merge(
+          bucket: name,
+          key: path,
+        )
+        @client.delete_object(opts)
+      rescue Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::NoSuchBucket # rubocop:disable Lint/HandleExceptions
+      end
+
+      # Copies a file.
+      def cp(src, dst, opts={})
+        src = norm_path(src)
+        dst = norm_path(dst)
+        opts = opts.merge(
+          bucket: name,
+          copy_source: "/#{name}/#{src}",
+          key: dst,
+        )
+        @client.copy_object(opts)
+      rescue Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::NoSuchBucket
+        raise BFS::FileNotFound, src
+      end
+    end
+  end
+end
+
+BFS.register('s3') do |url|
+  params = CGI.parse(url.query)
+
+  BFS::Bucket::S3.new url.host,
+    region: params.key?('region') ? params['region'].first : nil,
+    sse: params.key?('sse') ? params['sse'].first : nil,
+    access_key_id: params.key?('access_key_id') ? params['access_key_id'].first : nil,
+    secret_access_key: params.key?('secret_access_key') ? params['secret_access_key'].first : nil,
+    acl: params.key?('acl') ? params['acl'].first : nil,
+    storage_class: params.key?('storage_class') ? params['storage_class'].first : nil
+end

data/lib/bfs/s3.rb

@@ -0,0 +1 @@
+require 'bfs/bucket/s3'

data/spec/bfs/bucket/s3_spec.rb

@@ -0,0 +1,74 @@
+require 'spec_helper'
+
+RSpec.describe BFS::Bucket::S3 do
+  let(:client)  { double('Aws::S3::Client') }
+  let(:files)   { {} }
+  subject { described_class.new('mock-bucket', client: client) }
+
+  # stub put_object calls and store file data
+  before do
+    allow(client).to receive(:put_object).with(hash_including(bucket: 'mock-bucket')) do |opts|
+      files[opts[:key]] = opts[:body].read
+      nil
+    end
+  end
+
+  # stub get_object calls
+  before do
+    allow(client).to receive(:get_object).with(hash_including(bucket: 'mock-bucket')) do |opts|
+      data = files[opts[:key]]
+      raise Aws::S3::Errors::NoSuchKey.new(nil, nil) unless data
+
+      File.open(opts[:response_target], 'w') {|f| f.write(data) }
+      nil
+    end
+  end
+
+  before do
+    allow(client).to receive(:delete_object).with(hash_including(bucket: 'mock-bucket')) do |opts|
+      raise Aws::S3::Errors::NoSuchKey.new(nil, nil) unless files.key?(opts[:key])
+
+      files.delete(opts[:key])
+      nil
+    end
+  end
+
+  # stub list_objects_v2 calls
+  before do
+    allow(client).to receive(:list_objects_v2).with(bucket: 'mock-bucket') do |*|
+      contents = files.keys.map {|key| Aws::S3::Types::Object.new(key: key) }
+      double 'ListObjectsV2Response', contents: contents
+    end
+  end
+
+  # stub list_objects_v2, single object calls
+  before do
+    match = double 'ListObjectsV2Response', contents: [
+      Aws::S3::Types::Object.new(key: 'a/b/c.txt', size: 10, last_modified: Time.now),
+    ]
+    no_match = double 'ListObjectsV2Response', contents: []
+
+    allow(client).to receive(:list_objects_v2).with(bucket: 'mock-bucket', max_keys: 1, prefix: 'a/b/c.txt').and_return(match)
+    allow(client).to receive(:list_objects_v2).with(bucket: 'mock-bucket', max_keys: 1, prefix: 'missing.txt').and_return(no_match)
+  end
+
+  # stub copy_object calls
+  before do
+    allow(client).to receive(:copy_object).with(hash_including(bucket: 'mock-bucket')) do |opts|
+      src = opts[:copy_source].sub('/mock-bucket/', '')
+      raise Aws::S3::Errors::NoSuchKey.new(nil, nil) unless files.key?(src)
+
+      files[opts[:key]] = files[src]
+      nil
+    end
+  end
+
+  it_behaves_like 'a bucket'
+
+  it 'should resolve from URL' do
+    bucket = BFS.resolve('s3://mock-bucket?acl=private&region=eu-west-2')
+    expect(bucket).to be_instance_of(described_class)
+    expect(bucket.name).to eq('mock-bucket')
+    expect(bucket.acl).to eq(:private)
+  end
+end

metadata

@@ -0,0 +1,76 @@
+--- !ruby/object:Gem::Specification
+name: bfs-s3
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Dimitrij Denissenko
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-10-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bfs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: https://github.com/bsm/bfs.rb
+email: dimitrij@blacksquaremedia.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- bfs-s3.gemspec
+- lib/bfs/bucket/s3.rb
+- lib/bfs/s3.rb
+- spec/bfs/bucket/s3_spec.rb
+homepage: https://github.com/bsm/bfs.rb
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: S3 bucket adapter for bfs
+test_files:
+- spec/bfs/bucket/s3_spec.rb