s3repo 0.3.4 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.prospectus +11 -0
- data/CHANGELOG.md +4 -0
- data/README.md +62 -3
- data/bin/s3repo +2 -2
- data/circle.yml +8 -0
- data/lib/s3repo/cache.rb +7 -2
- data/lib/s3repo/client.rb +1 -1
- data/lib/s3repo/repo.rb +11 -9
- data/lib/s3repo/version.rb +2 -1
- data/s3repo.gemspec +8 -5
- data/spec/fixtures/cassettes/cache.yml +547 -0
- data/spec/fixtures/cassettes/etag_cache.yml +590 -0
- data/spec/fixtures/cassettes/nil_cache.yml +46 -0
- data/spec/s3repo/base_spec.rb +71 -0
- data/spec/s3repo/cache_spec.rb +48 -0
- data/spec/s3repo_spec.rb +8 -0
- data/spec/spec_helper.rb +29 -6
- metadata +69 -15
- data/.travis.yml +0 -14
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 037f096771d38b162f01efabbd4f21a3d38d2e54
|
4
|
+
data.tar.gz: 2639022f0ac61664ed66c9bdd49b28e8b3dc4d13
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 40e4eba132b5c8374a4c5da86dc5043aca1700daf69c616ee91f018bd214e010cf64bda56b0c1187388252ffc21d95e328be2b68daf1f10d0fdf0d940c2f6978
|
7
|
+
data.tar.gz: d09e26179074b0c678e798bc814b1a2e4277bea5738d914c7af0c64c306896f4da46423dc03ec2b8af3f0917d432f707f8e8d38614598d50c7fe2dddc6cd7067
|
data/.prospectus
ADDED
data/CHANGELOG.md
ADDED
data/README.md
CHANGED
@@ -3,15 +3,74 @@ s3repo
|
|
3
3
|
|
4
4
|
[](https://rubygems.org/gems/s3repo)
|
5
5
|
[](https://gemnasium.com/amylum/s3repo)
|
6
|
-
[](https://circleci.com/gh/amylum/s3repo)
|
7
|
+
[](https://codecov.io/github/amylum/s3repo)
|
8
|
+
[](https://www.codacy.com/app/akerl/s3repo)
|
9
9
|
[](https://tldrlegal.com/license/mit-license)
|
10
10
|
|
11
11
|
Simple library for interacting with an Archlinux repo stored in S3
|
12
12
|
|
13
13
|
## Usage
|
14
14
|
|
15
|
+
### From the command line
|
16
|
+
|
17
|
+
This tool provides an `s3repo` script that can be used to manage an S3 bucket with packages. It is configured via environment variable:
|
18
|
+
|
19
|
+
* AWS_ACCESS_KEY_ID -- Used by the AWS API library to authenticate for the S3 bucket
|
20
|
+
* AWS_SECRET_ACCESS_KEY -- Used by the AWS API library to authenticate for the S3 bucket
|
21
|
+
* AWS_REGION -- Used by the AWS API library to know which region to use for the bucket
|
22
|
+
* S3_BUCKET -- Controls which S3 bucket is used for packages
|
23
|
+
* MAKEPKG_FLAGS -- Flags used for makepkg calls when building packages
|
24
|
+
* S3REPO_TMPDIR -- Sets the temp path for local cache of metadata files. Optional, otherwise looks up a path from the system
|
25
|
+
* S3REPO_SIGN_DB -- Set to sign the DB metadata
|
26
|
+
* S3REPO_SIGN_PACKAGES -- Set to sign the packages
|
27
|
+
|
28
|
+
#### Build a package
|
29
|
+
|
30
|
+
This assumes you're in the parent directory of your package's directory, which should contain its PKGBUILD and any other files it needs. It's essentially a wrapper around makepkg.
|
31
|
+
|
32
|
+
```
|
33
|
+
s3repo build bash
|
34
|
+
```
|
35
|
+
|
36
|
+
#### Upload a package
|
37
|
+
|
38
|
+
This takes an already built package and uploads its .pkg.tar.xz (and optionally .pkg.tar.xz.sig) to S3.
|
39
|
+
|
40
|
+
```
|
41
|
+
s3repo upload bash
|
42
|
+
```
|
43
|
+
|
44
|
+
#### Remove a package
|
45
|
+
|
46
|
+
This removes a named package from the repo metdata (it doesn't remove the actual package files, which you can do after this using prune).
|
47
|
+
|
48
|
+
```
|
49
|
+
s3repo remove bash-amylum-4.3p42_2-1-x86_64
|
50
|
+
```
|
51
|
+
|
52
|
+
#### Prune packages
|
53
|
+
|
54
|
+
This removes any files from S3 that aren't referenced in the metadata DB.
|
55
|
+
|
56
|
+
```
|
57
|
+
s3repo prune
|
58
|
+
```
|
59
|
+
|
60
|
+
### From Ruby code
|
61
|
+
|
62
|
+
I use this primarily for serving the packages using [amylum/server](https://github.com/amylum/server). Like the command line tool, it expects AWS API credentials via environment variables, but you can pass in the bucket using either environmnt or a parameter when creating your S3Repo object:
|
63
|
+
|
64
|
+
```
|
65
|
+
repo = S3Repo.new(bucket: 'my_bucket')
|
66
|
+
```
|
67
|
+
|
68
|
+
You can call build_packages, add_packages, remove_packages, and prune_files on this, which work almost exactly as their command line counterparts above.
|
69
|
+
|
70
|
+
The library also offers `.packages` and `.signatures`, which return arrays of packages and signatures in the bucket. You can use `.include? 'package_name'` to check for a package.
|
71
|
+
|
72
|
+
To get the contents of a file, call `.serve 'package_name'`.
|
73
|
+
|
15
74
|
## Installation
|
16
75
|
|
17
76
|
gem install s3repo
|
data/bin/s3repo
CHANGED
@@ -41,10 +41,10 @@ Mercenary.program(:s3repo) do |p|
|
|
41
41
|
|
42
42
|
p.command(:prune) do |c|
|
43
43
|
c.syntax 'prune'
|
44
|
-
c.description 'Prune
|
44
|
+
c.description 'Prune orphaned files from the repo'
|
45
45
|
|
46
46
|
c.action do
|
47
|
-
S3Repo.new.
|
47
|
+
S3Repo.new.prune_files
|
48
48
|
end
|
49
49
|
end
|
50
50
|
|
data/circle.yml
ADDED
data/lib/s3repo/cache.rb
CHANGED
@@ -6,7 +6,12 @@ module S3Repo
|
|
6
6
|
##
|
7
7
|
# Cache object, stores S3 objects on disk
|
8
8
|
class Cache < Base
|
9
|
-
TMPDIRS = [
|
9
|
+
TMPDIRS = [
|
10
|
+
ENV['S3REPO_TMPDIR'],
|
11
|
+
ENV['TMPDIR'],
|
12
|
+
Dir.tmpdir,
|
13
|
+
'/tmp/s3repo'
|
14
|
+
].freeze
|
10
15
|
|
11
16
|
def initialize(params = {})
|
12
17
|
super
|
@@ -14,7 +19,7 @@ module S3Repo
|
|
14
19
|
end
|
15
20
|
|
16
21
|
def serve(key, refresh = true)
|
17
|
-
File.open(download(key, refresh)
|
22
|
+
File.open(download(key, refresh), &:read)
|
18
23
|
rescue Aws::S3::Errors::NoSuchKey
|
19
24
|
nil
|
20
25
|
end
|
data/lib/s3repo/client.rb
CHANGED
data/lib/s3repo/repo.rb
CHANGED
@@ -26,17 +26,21 @@ module S3Repo
|
|
26
26
|
metadata.remove_packages(packages)
|
27
27
|
end
|
28
28
|
|
29
|
-
def
|
29
|
+
def prune_files
|
30
30
|
if orphans.empty?
|
31
|
-
puts 'No orphaned
|
31
|
+
puts 'No orphaned files'
|
32
32
|
return
|
33
33
|
end
|
34
|
-
puts "Pruning
|
34
|
+
puts "Pruning files: #{orphans.join(', ')}"
|
35
35
|
client.delete_objects(delete: { objects: orphans.map { |x| { key: x } } })
|
36
36
|
end
|
37
37
|
|
38
38
|
def packages
|
39
|
-
package_cache.cache {
|
39
|
+
package_cache.cache { parse_objects(/.*\.pkg\.tar\.xz$/) }
|
40
|
+
end
|
41
|
+
|
42
|
+
def signatures
|
43
|
+
parse_objects(/.*\.pkg\.tar\.xz\.sig$/)
|
40
44
|
end
|
41
45
|
|
42
46
|
def include?(key)
|
@@ -60,7 +64,7 @@ module S3Repo
|
|
60
64
|
end
|
61
65
|
|
62
66
|
def orphans
|
63
|
-
packages.map(&:key).reject do |x|
|
67
|
+
(packages + signatures).map(&:key).reject do |x|
|
64
68
|
metadata.packages.include? x.reverse.split('-', 2).last.reverse
|
65
69
|
end
|
66
70
|
end
|
@@ -73,10 +77,8 @@ module S3Repo
|
|
73
77
|
@package_cache ||= BasicCache::TimeCache.new lifetime: 60
|
74
78
|
end
|
75
79
|
|
76
|
-
def
|
77
|
-
client.list_objects(bucket: bucket).contents.select
|
78
|
-
x.key.match(/.*\.pkg\.tar\.xz$/)
|
79
|
-
end
|
80
|
+
def parse_objects(regex)
|
81
|
+
client.list_objects(bucket: bucket).contents.select { |x| x.key =~ regex }
|
80
82
|
end
|
81
83
|
end
|
82
84
|
end
|
data/lib/s3repo/version.rb
CHANGED
data/s3repo.gemspec
CHANGED
@@ -18,12 +18,15 @@ Gem::Specification.new do |s|
|
|
18
18
|
s.executables = ['s3repo']
|
19
19
|
|
20
20
|
s.add_dependency 'mercenary', '~> 0.3.4'
|
21
|
-
s.add_dependency 'aws-sdk', '~> 2.
|
21
|
+
s.add_dependency 'aws-sdk', '~> 2.2.0'
|
22
22
|
s.add_dependency 'basiccache', '~> 1.0.0'
|
23
23
|
|
24
|
-
s.add_development_dependency 'rubocop', '~> 0.
|
25
|
-
s.add_development_dependency 'rake', '~> 10.
|
26
|
-
s.add_development_dependency '
|
27
|
-
s.add_development_dependency 'rspec', '~> 3.
|
24
|
+
s.add_development_dependency 'rubocop', '~> 0.36.0'
|
25
|
+
s.add_development_dependency 'rake', '~> 10.5.0'
|
26
|
+
s.add_development_dependency 'codecov', '~> 0.1.1'
|
27
|
+
s.add_development_dependency 'rspec', '~> 3.4.0'
|
28
28
|
s.add_development_dependency 'fuubar', '~> 2.0.0'
|
29
|
+
s.add_development_dependency 'webmock', '~> 1.22.0'
|
30
|
+
s.add_development_dependency 'vcr', '~> 3.0.0'
|
31
|
+
s.add_development_dependency 'climate_control', '~> 0.0.3'
|
29
32
|
end
|