s3-toolkit 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: d1adcaadc316e76b88696d1e2c5122182032dfc4f8a428da3d72ec33fe98629c
+  data.tar.gz: b19c8833cce6cdd0b0df6ddf9ba3f6d8ded64a365337f7452841c7086f8f49ba
+SHA512:
+  metadata.gz: d21272f300b14ba67d0c7f0311313100a6dbeb13b64c1e946e718d696a69672aab838660dab2a1d8e0935f10d0abb7e169e40b379c6e5ffd9e70039743e8ed73
+  data.tar.gz: 633dfd04dcd269e84b2880469b62d751812ba733d2b1f1eab6c292f7dae174d462edb83274f59618c107108e7e9fd3842f208dcd6ed30874cc1f01fd79487ac3

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+# Change Log
+
+## 0.1.0
+
+- [TT-9472] Initial Project with fetch support

data/README.md

@@ -0,0 +1,42 @@
+[![Ruby Style Guide](https://img.shields.io/badge/code_style-standard-brightgreen.svg)](https://github.com/testdouble/standard)
+[![Gem Version](https://badge.fury.io/rb/s3-toolkit.svg)](http://badge.fury.io/rb/s3-toolkit)
+[![Build Status](https://github.com/sealink/s3-toolkit/workflows/Build%20and%20Test/badge.svg?branch=master)](https://github.com/sealink/s3-toolkit/actions)
+[![Coverage Status](https://coveralls.io/repos/sealink/s3-toolkit/badge.svg)](https://coveralls.io/r/sealink/s3-toolkit)
+
+## Purpose
+
+This provides a simple way to fetch files from an s3 bucket using the AWS Ruby SDK.
+
+Once this gem is installed you can use it to download the contents of a remote s3 bucket.
+
+For example
+
+```ruby
+bundle exec s3-toolkit fetch s3://remote-bucket/config/ config/
+```
+
+To show all available options
+
+```ruby
+bundle exec s3-toolkit --help
+```
+
+## Release
+
+To publish a new version of this gem the following steps must be taken.
+
+* Update the version in the following files
+  ```
+    CHANGELOG.md
+    lib/s3-toolkit/version.rb
+  ````
+* Create a tag using the format v0.1.0
+* Follow build progress in GitHub actions
+
+## Contributing
+
+1. Fork it ( https://github.com/sealink/s3-toolkit/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/exe/s3-toolkit

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "s3_toolkit"
+
+Dry::CLI.new(S3Toolkit::Commands).call

data/lib/s3-toolkit/commands.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module S3Toolkit
+  module Commands
+    extend Dry::CLI::Registry
+
+    class Fetch < Dry::CLI::Command
+      desc "Fetch the contents of a remote URL into a local folder"
+
+      argument :s3_url, type: :string, required: true, desc: "S3 Bucket / Prefix to fetch"
+      argument :dest_dir, type: :string, required: true, desc: "Destination to copy into"
+
+      option :overwrite, type: :boolean, required: false, default: false, desc: "Overwrite existing files"
+      option :request_concurrency, type: :integer, required: false, default: 10, desc: "Request concurrency"
+      option :region, type: :string, required: false, default: "ap-southeast-2", desc: "S3 client region"
+
+      def call(s3_url:, dest_dir:, **options)
+        S3Toolkit::Fetcher.call(
+          s3_url: s3_url,
+          dest_dir: dest_dir,
+          request_concurrency: options.fetch(:request_concurrency),
+          region: options.fetch(:region),
+          overwrite: options.fetch(:overwrite)
+        )
+      end
+    end
+
+    register "fetch", Fetch
+  end
+end

data/lib/s3-toolkit/fetcher.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module S3Toolkit
+  class Fetcher
+    def self.call(region:, s3_url:, dest_dir:, request_concurrency:, overwrite:, client: nil)
+      files = 0
+      FileUtils.mkdir_p(dest_dir)
+      queue = Queue.new
+
+      bucket, key = S3Toolkit::UrlParser.call(s3_url)
+
+      s3 = client || Aws::S3::Client.new(region: region)
+      resp = s3.list_objects(bucket: bucket, prefix: key)
+      resp.contents.each { |content| queue.push(content.key) }
+
+      puts "Found #{queue.size} files"
+      request_concurrency.times.map do
+        Thread.new do
+          until queue.empty?
+            download = queue.pop
+            base_name = File.basename(download)
+            destination = "#{dest_dir}/#{base_name}"
+            next unless !File.exist?(destination) || overwrite
+
+            puts "download #{s3_url}/#{base_name} to #{destination}"
+            File.open(destination, "wb") do |file|
+              resp = s3.get_object({bucket: bucket, key: download}, target: file)
+              files += 1
+            end
+          end
+          Thread.exit
+        end
+      end.each(&:join)
+      files
+    end
+  end
+end

data/lib/s3-toolkit/url_parser.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module S3Toolkit
+  class UrlParser
+    def self.call(s3_url)
+      raise URI::InvalidURIError, "Unexpected format, expected: s3://<bucket>/<key>" unless s3_url&.start_with?("s3://")
+
+      uri = URI.parse(s3_url)
+      bucket = uri.host
+      key = uri.path[1..] # Strip the leading /
+
+      [bucket, key]
+    end
+  end
+end

data/lib/s3-toolkit/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module S3Toolkit
+  VERSION = "0.1.0"
+end

data/lib/s3_toolkit.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module S3Toolkit
+end
+
+require "dry/cli"
+require "aws-sdk-s3"
+
+require "s3-toolkit/version"
+require "s3-toolkit/fetcher"
+require "s3-toolkit/url_parser"
+require "s3-toolkit/commands"

data/s3-toolkit.gemspec

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+lib = File.expand_path("lib", __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "s3-toolkit/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "s3-toolkit"
+  spec.version = S3Toolkit::VERSION
+  spec.authors = ["John Mortlock"]
+  spec.email = ["john.mortlock@sealink.com.au"]
+
+  spec.summary = "Simple tools for managing s3 buckets in ruby"
+  spec.description = "Simple tools to download contents of s3 buckets when the aws cli is to big a dependency to use"
+  spec.homepage = "https://github.com/sealink/s3-toolkit"
+  spec.license = "MIT"
+
+  spec.files = Dir["CHANGELOG.md", "README.md", "s3-toolkit.gemspec", "lib/**/*"]
+  spec.bindir = "exe"
+  spec.executables = ["s3-toolkit"]
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = ">= 2.6"
+
+  spec.add_dependency "aws-sdk-s3", "~> 1"
+  spec.add_dependency "dry-cli"
+
+  spec.add_development_dependency "coverage-kit"
+  spec.add_development_dependency "coveralls_reborn"
+  spec.add_development_dependency "nokogiri"
+  spec.add_development_dependency "pry-byebug"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "standard"
+  spec.add_development_dependency "simplecov-rcov"
+end

metadata

@@ -0,0 +1,180 @@
+--- !ruby/object:Gem::Specification
+name: s3-toolkit
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- John Mortlock
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2021-05-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: dry-cli
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: coverage-kit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: coveralls_reborn
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: standard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov-rcov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Simple tools to download contents of s3 buckets when the aws cli is to
+  big a dependency to use
+email:
+- john.mortlock@sealink.com.au
+executables:
+- s3-toolkit
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- README.md
+- exe/s3-toolkit
+- lib/s3-toolkit/commands.rb
+- lib/s3-toolkit/fetcher.rb
+- lib/s3-toolkit/url_parser.rb
+- lib/s3-toolkit/version.rb
+- lib/s3_toolkit.rb
+- s3-toolkit.gemspec
+homepage: https://github.com/sealink/s3-toolkit
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.6'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.15
+signing_key: 
+specification_version: 4
+summary: Simple tools for managing s3 buckets in ruby
+test_files: []