s3-toolkit 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/CHANGELOG.md +5 -0
- data/README.md +42 -0
- data/exe/s3-toolkit +7 -0
- data/lib/s3-toolkit/commands.rb +30 -0
- data/lib/s3-toolkit/fetcher.rb +37 -0
- data/lib/s3-toolkit/url_parser.rb +15 -0
- data/lib/s3-toolkit/version.rb +5 -0
- data/lib/s3_toolkit.rb +12 -0
- data/s3-toolkit.gemspec +35 -0
- metadata +180 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: d1adcaadc316e76b88696d1e2c5122182032dfc4f8a428da3d72ec33fe98629c
|
4
|
+
data.tar.gz: b19c8833cce6cdd0b0df6ddf9ba3f6d8ded64a365337f7452841c7086f8f49ba
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: d21272f300b14ba67d0c7f0311313100a6dbeb13b64c1e946e718d696a69672aab838660dab2a1d8e0935f10d0abb7e169e40b379c6e5ffd9e70039743e8ed73
|
7
|
+
data.tar.gz: 633dfd04dcd269e84b2880469b62d751812ba733d2b1f1eab6c292f7dae174d462edb83274f59618c107108e7e9fd3842f208dcd6ed30874cc1f01fd79487ac3
|
data/CHANGELOG.md
ADDED
data/README.md
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
[![Ruby Style Guide](https://img.shields.io/badge/code_style-standard-brightgreen.svg)](https://github.com/testdouble/standard)
|
2
|
+
[![Gem Version](https://badge.fury.io/rb/s3-toolkit.svg)](http://badge.fury.io/rb/s3-toolkit)
|
3
|
+
[![Build Status](https://github.com/sealink/s3-toolkit/workflows/Build%20and%20Test/badge.svg?branch=master)](https://github.com/sealink/s3-toolkit/actions)
|
4
|
+
[![Coverage Status](https://coveralls.io/repos/sealink/s3-toolkit/badge.svg)](https://coveralls.io/r/sealink/s3-toolkit)
|
5
|
+
|
6
|
+
## Purpose
|
7
|
+
|
8
|
+
This provides a simple way to fetch files from an s3 bucket using the AWS Ruby SDK.
|
9
|
+
|
10
|
+
Once this gem is installed you can use it to download the contents of a remote s3 bucket.
|
11
|
+
|
12
|
+
For example
|
13
|
+
|
14
|
+
```ruby
|
15
|
+
bundle exec s3-toolkit fetch s3://remote-bucket/config/ config/
|
16
|
+
```
|
17
|
+
|
18
|
+
To show all available options
|
19
|
+
|
20
|
+
```ruby
|
21
|
+
bundle exec s3-toolkit --help
|
22
|
+
```
|
23
|
+
|
24
|
+
## Release
|
25
|
+
|
26
|
+
To publish a new version of this gem the following steps must be taken.
|
27
|
+
|
28
|
+
* Update the version in the following files
|
29
|
+
```
|
30
|
+
CHANGELOG.md
|
31
|
+
lib/s3-toolkit/version.rb
|
32
|
+
````
|
33
|
+
* Create a tag using the format v0.1.0
|
34
|
+
* Follow build progress in GitHub actions
|
35
|
+
|
36
|
+
## Contributing
|
37
|
+
|
38
|
+
1. Fork it ( https://github.com/sealink/s3-toolkit/fork )
|
39
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
40
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
41
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
42
|
+
5. Create a new Pull Request
|
data/exe/s3-toolkit
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module S3Toolkit
|
4
|
+
module Commands
|
5
|
+
extend Dry::CLI::Registry
|
6
|
+
|
7
|
+
class Fetch < Dry::CLI::Command
|
8
|
+
desc "Fetch the contents of a remote URL into a local folder"
|
9
|
+
|
10
|
+
argument :s3_url, type: :string, required: true, desc: "S3 Bucket / Prefix to fetch"
|
11
|
+
argument :dest_dir, type: :string, required: true, desc: "Destination to copy into"
|
12
|
+
|
13
|
+
option :overwrite, type: :boolean, required: false, default: false, desc: "Overwrite existing files"
|
14
|
+
option :request_concurrency, type: :integer, required: false, default: 10, desc: "Request concurrency"
|
15
|
+
option :region, type: :string, required: false, default: "ap-southeast-2", desc: "S3 client region"
|
16
|
+
|
17
|
+
def call(s3_url:, dest_dir:, **options)
|
18
|
+
S3Toolkit::Fetcher.call(
|
19
|
+
s3_url: s3_url,
|
20
|
+
dest_dir: dest_dir,
|
21
|
+
request_concurrency: options.fetch(:request_concurrency),
|
22
|
+
region: options.fetch(:region),
|
23
|
+
overwrite: options.fetch(:overwrite)
|
24
|
+
)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
register "fetch", Fetch
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module S3Toolkit
|
4
|
+
class Fetcher
|
5
|
+
def self.call(region:, s3_url:, dest_dir:, request_concurrency:, overwrite:, client: nil)
|
6
|
+
files = 0
|
7
|
+
FileUtils.mkdir_p(dest_dir)
|
8
|
+
queue = Queue.new
|
9
|
+
|
10
|
+
bucket, key = S3Toolkit::UrlParser.call(s3_url)
|
11
|
+
|
12
|
+
s3 = client || Aws::S3::Client.new(region: region)
|
13
|
+
resp = s3.list_objects(bucket: bucket, prefix: key)
|
14
|
+
resp.contents.each { |content| queue.push(content.key) }
|
15
|
+
|
16
|
+
puts "Found #{queue.size} files"
|
17
|
+
request_concurrency.times.map do
|
18
|
+
Thread.new do
|
19
|
+
until queue.empty?
|
20
|
+
download = queue.pop
|
21
|
+
base_name = File.basename(download)
|
22
|
+
destination = "#{dest_dir}/#{base_name}"
|
23
|
+
next unless !File.exist?(destination) || overwrite
|
24
|
+
|
25
|
+
puts "download #{s3_url}/#{base_name} to #{destination}"
|
26
|
+
File.open(destination, "wb") do |file|
|
27
|
+
resp = s3.get_object({bucket: bucket, key: download}, target: file)
|
28
|
+
files += 1
|
29
|
+
end
|
30
|
+
end
|
31
|
+
Thread.exit
|
32
|
+
end
|
33
|
+
end.each(&:join)
|
34
|
+
files
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module S3Toolkit
|
4
|
+
class UrlParser
|
5
|
+
def self.call(s3_url)
|
6
|
+
raise URI::InvalidURIError, "Unexpected format, expected: s3://<bucket>/<key>" unless s3_url&.start_with?("s3://")
|
7
|
+
|
8
|
+
uri = URI.parse(s3_url)
|
9
|
+
bucket = uri.host
|
10
|
+
key = uri.path[1..] # Strip the leading /
|
11
|
+
|
12
|
+
[bucket, key]
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
data/lib/s3_toolkit.rb
ADDED
data/s3-toolkit.gemspec
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
lib = File.expand_path("lib", __dir__)
|
4
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
|
+
require "s3-toolkit/version"
|
6
|
+
|
7
|
+
Gem::Specification.new do |spec|
|
8
|
+
spec.name = "s3-toolkit"
|
9
|
+
spec.version = S3Toolkit::VERSION
|
10
|
+
spec.authors = ["John Mortlock"]
|
11
|
+
spec.email = ["john.mortlock@sealink.com.au"]
|
12
|
+
|
13
|
+
spec.summary = "Simple tools for managing s3 buckets in ruby"
|
14
|
+
spec.description = "Simple tools to download contents of s3 buckets when the aws cli is to big a dependency to use"
|
15
|
+
spec.homepage = "https://github.com/sealink/s3-toolkit"
|
16
|
+
spec.license = "MIT"
|
17
|
+
|
18
|
+
spec.files = Dir["CHANGELOG.md", "README.md", "s3-toolkit.gemspec", "lib/**/*"]
|
19
|
+
spec.bindir = "exe"
|
20
|
+
spec.executables = ["s3-toolkit"]
|
21
|
+
spec.require_paths = ["lib"]
|
22
|
+
|
23
|
+
spec.required_ruby_version = ">= 2.6"
|
24
|
+
|
25
|
+
spec.add_dependency "aws-sdk-s3", "~> 1"
|
26
|
+
spec.add_dependency "dry-cli"
|
27
|
+
|
28
|
+
spec.add_development_dependency "coverage-kit"
|
29
|
+
spec.add_development_dependency "coveralls_reborn"
|
30
|
+
spec.add_development_dependency "nokogiri"
|
31
|
+
spec.add_development_dependency "pry-byebug"
|
32
|
+
spec.add_development_dependency "rspec"
|
33
|
+
spec.add_development_dependency "standard"
|
34
|
+
spec.add_development_dependency "simplecov-rcov"
|
35
|
+
end
|
metadata
ADDED
@@ -0,0 +1,180 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: s3-toolkit
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- John Mortlock
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2021-05-31 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: aws-sdk-s3
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: dry-cli
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: coverage-kit
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: coveralls_reborn
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: nokogiri
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: pry-byebug
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: rspec
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: standard
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: simplecov-rcov
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ">="
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :development
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ">="
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0'
|
139
|
+
description: Simple tools to download contents of s3 buckets when the aws cli is to
|
140
|
+
big a dependency to use
|
141
|
+
email:
|
142
|
+
- john.mortlock@sealink.com.au
|
143
|
+
executables:
|
144
|
+
- s3-toolkit
|
145
|
+
extensions: []
|
146
|
+
extra_rdoc_files: []
|
147
|
+
files:
|
148
|
+
- CHANGELOG.md
|
149
|
+
- README.md
|
150
|
+
- exe/s3-toolkit
|
151
|
+
- lib/s3-toolkit/commands.rb
|
152
|
+
- lib/s3-toolkit/fetcher.rb
|
153
|
+
- lib/s3-toolkit/url_parser.rb
|
154
|
+
- lib/s3-toolkit/version.rb
|
155
|
+
- lib/s3_toolkit.rb
|
156
|
+
- s3-toolkit.gemspec
|
157
|
+
homepage: https://github.com/sealink/s3-toolkit
|
158
|
+
licenses:
|
159
|
+
- MIT
|
160
|
+
metadata: {}
|
161
|
+
post_install_message:
|
162
|
+
rdoc_options: []
|
163
|
+
require_paths:
|
164
|
+
- lib
|
165
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
166
|
+
requirements:
|
167
|
+
- - ">="
|
168
|
+
- !ruby/object:Gem::Version
|
169
|
+
version: '2.6'
|
170
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
171
|
+
requirements:
|
172
|
+
- - ">="
|
173
|
+
- !ruby/object:Gem::Version
|
174
|
+
version: '0'
|
175
|
+
requirements: []
|
176
|
+
rubygems_version: 3.2.15
|
177
|
+
signing_key:
|
178
|
+
specification_version: 4
|
179
|
+
summary: Simple tools for managing s3 buckets in ruby
|
180
|
+
test_files: []
|