deb-s3 0.7.1 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +87 -27
  3. data/lib/deb/s3.rb +1 -1
  4. data/lib/deb/s3/cli.rb +149 -73
  5. data/lib/deb/s3/lock.rb +58 -0
  6. data/lib/deb/s3/manifest.rb +10 -4
  7. data/lib/deb/s3/package.rb +1 -1
  8. data/lib/deb/s3/release.rb +2 -2
  9. data/lib/deb/s3/utils.rb +3 -1
  10. metadata +31 -29
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 904679e411b5fa26a8eee92c48460b742e6c1418
4
- data.tar.gz: eef06f5681d0871f579ef3beb24c249f467bf2ac
3
+ metadata.gz: 7e1bc9874f303cb3ec4a83a5dd1edf8948bf62bd
4
+ data.tar.gz: d2a16561f73b2dfd0c324f99a4336a6082f5a0e3
5
5
  SHA512:
6
- metadata.gz: a8d5260d87f9bd394f82eb5c45f44361152bacff274b3cb95127b44da0a6561c185b2891c904d45658acc1da4b52a33a9317e1de0b73131bff9c999a0305dcf5
7
- data.tar.gz: 1df2e5e2a552a983b71a17e76679a1b25a4f0415f77983fb1c6af5a6b0956eb6faffd9744a8128d76fb575a03e94af582f46e4a30da945a5958da910cadfdd19
6
+ metadata.gz: 54c58ff8308b6c8a1f219df3fbb21d6a9693917f2a853b1be44269443f84f5ffc4ff66e6270d23c98651adb8c430dddc9351eccca317b60279d7977112e2c272
7
+ data.tar.gz: c1a01cae8a4b91996e8d919cf41a8a2c8a4e6abe4ceb2c90496081fa5f9f77430472bb39067f65feaa5bf40305bf84043aa6f74bff0b9db234af868708e929c1
data/README.md CHANGED
@@ -5,7 +5,7 @@
5
5
  `deb-s3` is a simple utility to make creating and managing APT repositories on
6
6
  S3.
7
7
 
8
- Most existing existing guides on using S3 to host an APT repository have you
8
+ Most existing guides on using S3 to host an APT repository have you
9
9
  using something like [reprepro](http://mirrorer.alioth.debian.org/) to generate
10
10
  the repository file structure, and then [s3cmd](http://s3tools.org/s3cmd) to
11
11
  sync the files to S3.
@@ -61,18 +61,33 @@ Usage:
61
61
  deb-s3 upload FILES
62
62
 
63
63
  Options:
64
- -a, [--arch=ARCH] # The architecture of the package in the APT repository.
65
- [--sign=SIGN] # Sign the Release file. Use --sign with your key ID to use a specific key.
66
- -p, [--preserve-versions] # Whether to preserve other versions of a package in the repository when uploading one.
67
- -b, [--bucket=BUCKET] # The name of the S3 bucket to upload to.
68
- -c, [--codename=CODENAME] # The codename of the APT repository.
69
- # Default: stable
70
- -m, [--component=COMPONENT] # The component of the APT repository.
71
- # Default: main
72
- [--access-key-id=ACCESS_KEY] # The access key for connecting to S3.
73
- [--secret-access-key=SECRET_KEY] # The secret key for connecting to S3.
74
- -v, [--visibility=VISIBILITY] # The access policy for the uploaded files. Can be public, private, or authenticated.
75
- # Default: public
64
+ -a, [--arch=ARCH] # The architecture of the package in the APT repository.
65
+ -p, [--preserve-versions], [--no-preserve-versions] # Whether to preserve other versions of a package in the repository when uploading one.
66
+ -l, [--lock], [--no-lock] # Whether to check for an existing lock on the repository to prevent simultaneous updates
67
+ [--fail-if-exists], [--no-fail-if-exists] # Whether to overwrite any existing package that has the same filename in the pool or the same name and version in the manifest.
68
+ -b, [--bucket=BUCKET] # The name of the S3 bucket to upload to.
69
+ [--prefix=PREFIX] # The path prefix to use when storing on S3.
70
+ -o, [--origin=ORIGIN] # The origin to use in the repository Release file.
71
+ [--suite=SUITE] # The suite to use in the repository Release file.
72
+ -c, [--codename=CODENAME] # The codename of the APT repository.
73
+ # Default: stable
74
+ -m, [--component=COMPONENT] # The component of the APT repository.
75
+ # Default: main
76
+ [--access-key-id=ACCESS_KEY_ID] # The access key for connecting to S3.
77
+ [--secret-access-key=SECRET_ACCESS_KEY] # The secret key for connecting to S3.
78
+ [--endpoint=ENDPOINT] # The region endpoint for connecting to S3.
79
+ # Default: s3.amazonaws.com
80
+ [--force-path-style], [--no-force-path-style] # Use S3 path style instead of subdomains.
81
+ [--proxy-uri=PROXY_URI] # The URI of the proxy to send service requests through.
82
+ [--use-ssl], [--no-use-ssl] # Whether to use HTTP or HTTPS for request transport.
83
+ # Default: true
84
+ -v, [--visibility=VISIBILITY] # The access policy for the uploaded files. Can be public, private, or authenticated.
85
+ # Default: public
86
+ [--sign=SIGN] # Sign the Release file when uploading a package, or when verifying it after removing a package. Use --sign with your key ID to use a specific key.
87
+ [--gpg-options=GPG_OPTIONS] # Additional command line options to pass to GPG when signing.
88
+ -e, [--encryption], [--no-encryption] # Use S3 server side encryption.
89
+ -q, [--quiet], [--no-quiet] # Doesn't output information, just returns status appropriately.
90
+ -C, [--cache-control=CACHE_CONTROL] # Add cache-control headers to S3 objects.
76
91
 
77
92
  Uploads the given files to a S3 bucket as an APT repository.
78
93
  ```
@@ -83,7 +98,7 @@ packages in the specified component, codename and architecture.
83
98
 
84
99
  Now to delete the package:
85
100
  ```console
86
- $ deb-s3 delete --arch amd64 --bucket my-bucket --versions 1.0.0 my-deb-package
101
+ $ deb-s3 delete my-deb-package --arch amd64 --bucket my-bucket --versions 1.0.0
87
102
  >> Retrieving existing manifests
88
103
  -- Deleting my-deb-package version 1.0.0
89
104
  >> Uploading new manifests to S3
@@ -91,8 +106,41 @@ $ deb-s3 delete --arch amd64 --bucket my-bucket --versions 1.0.0 my-deb-package
91
106
  -- Transferring dists/stable/main/binary-amd64/Packages.gz
92
107
  -- Transferring dists/stable/Release
93
108
  >> Update complete.
109
+ ```
94
110
 
95
- ````
111
+ ```
112
+ Usage:
113
+ deb-s3 delete PACKAGE
114
+
115
+ Options:
116
+ -a, [--arch=ARCH] # The architecture of the package in the APT repository.
117
+ [--versions=one two three] # The space-delimited versions of PACKAGE to delete. If not specified, ALL VERSIONS will be deleted. Fair warning. E.g. --versions "0.1 0.2 0.3"
118
+ -b, [--bucket=BUCKET] # The name of the S3 bucket to upload to.
119
+ [--prefix=PREFIX] # The path prefix to use when storing on S3.
120
+ -o, [--origin=ORIGIN] # The origin to use in the repository Release file.
121
+ [--suite=SUITE] # The suite to use in the repository Release file.
122
+ -c, [--codename=CODENAME] # The codename of the APT repository.
123
+ # Default: stable
124
+ -m, [--component=COMPONENT] # The component of the APT repository.
125
+ # Default: main
126
+ [--access-key-id=ACCESS_KEY_ID] # The access key for connecting to S3.
127
+ [--secret-access-key=SECRET_ACCESS_KEY] # The secret key for connecting to S3.
128
+ [--endpoint=ENDPOINT] # The region endpoint for connecting to S3.
129
+ # Default: s3.amazonaws.com
130
+ [--force-path-style], [--no-force-path-style] # Use S3 path style instead of subdomains.
131
+ [--proxy-uri=PROXY_URI] # The URI of the proxy to send service requests through.
132
+ [--use-ssl], [--no-use-ssl] # Whether to use HTTP or HTTPS for request transport.
133
+ # Default: true
134
+ -v, [--visibility=VISIBILITY] # The access policy for the uploaded files. Can be public, private, or authenticated.
135
+ # Default: public
136
+ [--sign=SIGN] # Sign the Release file when uploading a package, or when verifying it after removing a package. Use --sign with your key ID to use a specific key.
137
+ [--gpg-options=GPG_OPTIONS] # Additional command line options to pass to GPG when signing.
138
+ -e, [--encryption], [--no-encryption] # Use S3 server side encryption.
139
+ -q, [--quiet], [--no-quiet] # Doesn't output information, just returns status appropriately.
140
+ -C, [--cache-control=CACHE_CONTROL] # Add cache-control headers to S3 objects.
141
+
142
+ Remove the package named PACKAGE. If --versions is not specified, deleteall versions of PACKAGE. Otherwise, only the specified versions will be deleted.
143
+ ```
96
144
 
97
145
  You can also verify an existing APT repository on S3 using the `verify` command:
98
146
 
@@ -109,18 +157,30 @@ Usage:
109
157
  deb-s3 verify
110
158
 
111
159
  Options:
112
- -f, [--fix-manifests] # Whether to fix problems in manifests when verifying.
113
- [--sign=SIGN] # Sign the Release file. Use --sign with your key ID to use a specific key.
114
- -b, [--bucket=BUCKET] # The name of the S3 bucket to upload to.
115
- -c, [--codename=CODENAME] # The codename of the APT repository.
116
- # Default: stable
117
- -m, [--component=COMPONENT] # The component of the APT repository.
118
- # Default: main
119
- [--access-key-id=ACCESS_KEY] # The access key for connecting to S3.
120
- [--secret-access-key=SECRET_KEY] # The secret key for connecting to S3.
121
- -v, [--visibility=VISIBILITY] # The access policy for the uploaded files. Can be public, private, or authenticated.
122
- # Default: public
160
+ -f, [--fix-manifests], [--no-fix-manifests] # Whether to fix problems in manifests when verifying.
161
+ -b, [--bucket=BUCKET] # The name of the S3 bucket to upload to.
162
+ [--prefix=PREFIX] # The path prefix to use when storing on S3.
163
+ -o, [--origin=ORIGIN] # The origin to use in the repository Release file.
164
+ [--suite=SUITE] # The suite to use in the repository Release file.
165
+ -c, [--codename=CODENAME] # The codename of the APT repository.
166
+ # Default: stable
167
+ -m, [--component=COMPONENT] # The component of the APT repository.
168
+ # Default: main
169
+ [--access-key-id=ACCESS_KEY_ID] # The access key for connecting to S3.
170
+ [--secret-access-key=SECRET_ACCESS_KEY] # The secret key for connecting to S3.
171
+ [--endpoint=ENDPOINT] # The region endpoint for connecting to S3.
172
+ # Default: s3.amazonaws.com
173
+ [--force-path-style], [--no-force-path-style] # Use S3 path style instead of subdomains.
174
+ [--proxy-uri=PROXY_URI] # The URI of the proxy to send service requests through.
175
+ [--use-ssl], [--no-use-ssl] # Whether to use HTTP or HTTPS for request transport.
176
+ # Default: true
177
+ -v, [--visibility=VISIBILITY] # The access policy for the uploaded files. Can be public, private, or authenticated.
178
+ # Default: public
179
+ [--sign=SIGN] # Sign the Release file when uploading a package, or when verifying it after removing a package. Use --sign with your key ID to use a specific key.
180
+ [--gpg-options=GPG_OPTIONS] # Additional command line options to pass to GPG when signing.
181
+ -e, [--encryption], [--no-encryption] # Use S3 server side encryption.
182
+ -q, [--quiet], [--no-quiet] # Doesn't output information, just returns status appropriately.
183
+ -C, [--cache-control=CACHE_CONTROL] # Add cache-control headers to S3 objects.
123
184
 
124
185
  Verifies that the files in the package manifests exist
125
186
  ```
126
-
data/lib/deb/s3.rb CHANGED
@@ -1,6 +1,6 @@
1
1
  # -*- encoding : utf-8 -*-
2
2
  module Deb
3
3
  module S3
4
- VERSION = "0.7.1"
4
+ VERSION = "0.8.0"
5
5
  end
6
6
  end
data/lib/deb/s3/cli.rb CHANGED
@@ -10,6 +10,7 @@ require "deb/s3/utils"
10
10
  require "deb/s3/manifest"
11
11
  require "deb/s3/package"
12
12
  require "deb/s3/release"
13
+ require "deb/s3/lock"
13
14
 
14
15
  class Deb::S3::CLI < Thor
15
16
  class_option :bucket,
@@ -60,6 +61,11 @@ class Deb::S3::CLI < Thor
60
61
  :desc => "The region endpoint for connecting to S3.",
61
62
  :default => "s3.amazonaws.com"
62
63
 
64
+ class_option :force_path_style,
65
+ :default => false,
66
+ :type => :boolean,
67
+ :desc => "Use S3 path style instead of subdomains."
68
+
63
69
  class_option :proxy_uri,
64
70
  :type => :string,
65
71
  :desc => "The URI of the proxy to send service requests through."
@@ -78,20 +84,20 @@ class Deb::S3::CLI < Thor
78
84
 
79
85
  class_option :sign,
80
86
  :type => :string,
81
- :desc => "Sign the Release file when uploading a package," +
82
- "or when verifying it after removing a package." +
87
+ :desc => "Sign the Release file when uploading a package, " +
88
+ "or when verifying it after removing a package. " +
83
89
  "Use --sign with your key ID to use a specific key."
84
90
 
85
91
  class_option :gpg_options,
86
92
  :default => "",
87
93
  :type => :string,
88
- :desc => "Additional command line options to pass to GPG when signing"
94
+ :desc => "Additional command line options to pass to GPG when signing."
89
95
 
90
96
  class_option :encryption,
91
97
  :default => false,
92
98
  :type => :boolean,
93
99
  :aliases => "-e",
94
- :desc => "Use S3 server side encryption"
100
+ :desc => "Use S3 server side encryption."
95
101
 
96
102
  class_option :quiet,
97
103
  :type => :boolean,
@@ -101,7 +107,7 @@ class Deb::S3::CLI < Thor
101
107
  class_option :cache_control,
102
108
  :type => :string,
103
109
  :aliases => "-C",
104
- :desc => "Add cache-control headers to S3 objects"
110
+ :desc => "Add cache-control headers to S3 objects."
105
111
 
106
112
  desc "upload FILES",
107
113
  "Uploads the given files to a S3 bucket as an APT repository."
@@ -118,6 +124,19 @@ class Deb::S3::CLI < Thor
118
124
  :desc => "Whether to preserve other versions of a package " +
119
125
  "in the repository when uploading one."
120
126
 
127
+ option :lock,
128
+ :default => false,
129
+ :type => :boolean,
130
+ :aliases => "-l",
131
+ :desc => "Whether to check for an existing lock on the repository " +
132
+ "to prevent simultaneous updates "
133
+
134
+ option :fail_if_exists,
135
+ :default => false,
136
+ :type => :boolean,
137
+ :desc => "Whether to overwrite any existing package that has the same " +
138
+ "filename in the pool or the same name and version in the manifest."
139
+
121
140
  def upload(*files)
122
141
  if files.nil? || files.empty?
123
142
  error("You must specify at least one file to upload")
@@ -131,67 +150,104 @@ class Deb::S3::CLI < Thor
131
150
  # configure AWS::S3
132
151
  configure_s3_client
133
152
 
134
- # retrieve the existing manifests
135
- log("Retrieving existing manifests")
136
- release = Deb::S3::Release.retrieve(options[:codename], options[:origin], options[:suite], options[:cache_control])
137
- manifests = {}
138
- release.architectures.each do |arch|
139
- manifests[arch] = Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control])
140
- end
153
+ begin
154
+ if options[:lock]
155
+ log("Checking for existing lock file")
156
+ if Deb::S3::Lock.locked?(options[:codename], component, options[:arch], options[:cache_control])
157
+ lock = Deb::S3::Lock.current(options[:codename], component, options[:arch], options[:cache_control])
158
+ log("Repository is locked by another user: #{lock.user} at host #{lock.host}")
159
+ log("Attempting to obtain a lock")
160
+ Deb::S3::Lock.wait_for_lock(options[:codename], component, options[:arch], options[:cache_control])
161
+ end
162
+ log("Locking repository for updates")
163
+ Deb::S3::Lock.lock(options[:codename], component, options[:arch], options[:cache_control])
164
+ @lock_acquired = true
165
+ end
141
166
 
142
- packages_arch_all = []
143
-
144
- # examine all the files
145
- files.collect { |f| Dir.glob(f) }.flatten.each do |file|
146
- log("Examining package file #{File.basename(file)}")
147
- pkg = Deb::S3::Package.parse_file(file)
148
-
149
- # copy over some options if they weren't given
150
- arch = options[:arch] || pkg.architecture
151
-
152
- # validate we have them
153
- error("No architcture given and unable to determine one for #{file}. " +
154
- "Please specify one with --arch [i386|amd64].") unless arch
155
-
156
- # If the arch is all and the list of existing manifests is none, then
157
- # throw an error. This is mainly the case when initializing a brand new
158
- # repository. With "all", we won't know which architectures they're using.
159
- if arch == "all" && manifests.count == 0
160
- error("Package #{File.basename(file)} had architecture \"all\", " +
161
- "however noexisting package lists exist. This can often happen " +
162
- "if the first package you are add to a new repository is an " +
163
- "\"all\" architecture file. Please use --arch [i386|amd64] or " +
164
- "another platform type to upload the file.")
167
+ # retrieve the existing manifests
168
+ log("Retrieving existing manifests")
169
+ release = Deb::S3::Release.retrieve(options[:codename], options[:origin], options[:suite], options[:cache_control])
170
+ manifests = {}
171
+ release.architectures.each do |arch|
172
+ manifests[arch] = Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control], options[:fail_if_exists])
165
173
  end
166
174
 
167
- # retrieve the manifest for the arch if we don't have it already
168
- manifests[arch] ||= Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control])
175
+ packages_arch_all = []
176
+
177
+ # examine all the files
178
+ files.collect { |f| Dir.glob(f) }.flatten.each do |file|
179
+ log("Examining package file #{File.basename(file)}")
180
+ pkg = Deb::S3::Package.parse_file(file)
181
+
182
+ # copy over some options if they weren't given
183
+ arch = options[:arch] || pkg.architecture
184
+
185
+ # If they've specified an arch type that doesn't match the package let them know
186
+ if options.key?("arch") && options[:arch] != pkg.architecture
187
+ warn("You specified architecture #{options[:arch]} but package #{pkg.name} has architecture type of #{pkg.architecture}")
188
+ end
189
+
190
+ # validate we have them
191
+ error("No architcture given and unable to determine one for #{file}. " +
192
+ "Please specify one with --arch [i386|amd64|armhf].") unless arch
193
+
194
+ # If the arch is all and the list of existing manifests is none, then
195
+ # throw an error. This is mainly the case when initializing a brand new
196
+ # repository. With "all", we won't know which architectures they're using.
197
+ if arch == "all" && manifests.count == 0
198
+ error("Package #{File.basename(file)} had architecture \"all\", " +
199
+ "however noexisting package lists exist. This can often happen " +
200
+ "if the first package you are add to a new repository is an " +
201
+ "\"all\" architecture file. Please use --arch [i386|amd64|armhf] or " +
202
+ "another platform type to upload the file.")
203
+ end
204
+
205
+ # retrieve the manifest for the arch if we don't have it already
206
+ manifests[arch] ||= Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control], options[:fail_if_exists])
169
207
 
170
- # add package in manifests
171
- manifests[arch].add(pkg, options[:preserve_versions])
208
+ # add package in manifests
209
+ begin
210
+ manifests[arch].add(pkg, options[:preserve_versions])
211
+ rescue Deb::S3::Utils::AlreadyExistsError => e
212
+ error("Preparing manifest failed because: #{e}")
213
+ end
172
214
 
173
- # If arch is all, we must add this package in all arch available
174
- if arch == 'all'
175
- packages_arch_all << pkg
215
+ # If arch is all, we must add this package in all arch available
216
+ if arch == 'all'
217
+ packages_arch_all << pkg
218
+ end
176
219
  end
177
- end
178
220
 
179
- manifests.each do |arch, manifest|
180
- next if arch == 'all'
181
- packages_arch_all.each do |pkg|
182
- manifest.add(pkg, options[:preserve_versions], false)
221
+ manifests.each do |arch, manifest|
222
+ next if arch == 'all'
223
+ packages_arch_all.each do |pkg|
224
+ begin
225
+ manifest.add(pkg, options[:preserve_versions], false)
226
+ rescue Deb::S3::Utils::AlreadyExistsError => e
227
+ error("Preparing manifest failed because: #{e}")
228
+ end
229
+ end
183
230
  end
184
- end
185
231
 
186
- # upload the manifest
187
- log("Uploading packages and new manifests to S3")
188
- manifests.each_value do |manifest|
189
- manifest.write_to_s3 { |f| sublog("Transferring #{f}") }
190
- release.update_manifest(manifest)
191
- end
192
- release.write_to_s3 { |f| sublog("Transferring #{f}") }
232
+ # upload the manifest
233
+ log("Uploading packages and new manifests to S3")
234
+ manifests.each_value do |manifest|
235
+ begin
236
+ manifest.write_to_s3 { |f| sublog("Transferring #{f}") }
237
+ rescue Deb::S3::Utils::AlreadyExistsError => e
238
+ error("Uploading manifest failed because: #{e}")
239
+ end
240
+ release.update_manifest(manifest)
241
+ end
242
+ release.write_to_s3 { |f| sublog("Transferring #{f}") }
193
243
 
194
- log("Update complete.")
244
+ log("Update complete.")
245
+ ensure
246
+ if options[:lock] && @lock_acquired
247
+ Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
248
+ log("Lock released.")
249
+ end
250
+ end
195
251
  end
196
252
 
197
253
  desc "list", "Lists packages in given codename, component, and optionally architecture"
@@ -199,7 +255,7 @@ class Deb::S3::CLI < Thor
199
255
  option :long,
200
256
  :type => :boolean,
201
257
  :aliases => '-l',
202
- :desc => "Shows all package information in original format",
258
+ :desc => "Shows all package information in original format.",
203
259
  :default => false
204
260
 
205
261
  option :arch,
@@ -216,7 +272,8 @@ class Deb::S3::CLI < Thor
216
272
  widths = [0, 0]
217
273
  rows = archs.map { |arch|
218
274
  manifest = Deb::S3::Manifest.retrieve(options[:codename], component,
219
- arch, options[:cache_control])
275
+ arch, options[:cache_control],
276
+ false)
220
277
  manifest.packages.map do |package|
221
278
  if options[:long]
222
279
  package.generate
@@ -256,7 +313,7 @@ class Deb::S3::CLI < Thor
256
313
 
257
314
  # retrieve the existing manifests
258
315
  manifest = Deb::S3::Manifest.retrieve(options[:codename], component, arch,
259
- options[:cache_control])
316
+ options[:cache_control], false)
260
317
  package = manifest.packages.detect { |p|
261
318
  p.name == package_name && p.full_version == version
262
319
  }
@@ -273,7 +330,7 @@ class Deb::S3::CLI < Thor
273
330
  option :cache_control,
274
331
  :type => :string,
275
332
  :aliases => "-C",
276
- :desc => "Add cache-control headers to S3 objects"
333
+ :desc => "Add cache-control headers to S3 objects."
277
334
 
278
335
  option :arch,
279
336
  :type => :string,
@@ -283,8 +340,8 @@ class Deb::S3::CLI < Thor
283
340
  option :versions,
284
341
  :default => nil,
285
342
  :type => :array,
286
- :desc => "The space-delimited versions of PACKAGE to delete. If not" +
287
- "specified, ALL VERSIONS will be deleted. Fair warning." +
343
+ :desc => "The space-delimited versions of PACKAGE to delete. If not " +
344
+ "specified, ALL VERSIONS will be deleted. Fair warning. " +
288
345
  "E.g. --versions \"0.1 0.2 0.3\""
289
346
 
290
347
  option :preserve_versions,
@@ -294,6 +351,12 @@ class Deb::S3::CLI < Thor
294
351
  :desc => "Whether to preserve other versions of a package " +
295
352
  "in the repository when uploading one."
296
353
 
354
+ option :fail_if_exists,
355
+ :default => true,
356
+ :type => :boolean,
357
+ :desc => "Whether to overwrite any existing package that has the same " +
358
+ "filename in the pool or the same name and version in the manifest."
359
+
297
360
  def copy(package_name, to_codename, to_component)
298
361
  if package_name.nil?
299
362
  error "You must specify a package name."
@@ -323,10 +386,12 @@ class Deb::S3::CLI < Thor
323
386
  log "Retrieving existing manifests"
324
387
  from_manifest = Deb::S3::Manifest.retrieve(options[:codename],
325
388
  component, arch,
326
- options[:cache_control])
389
+ options[:cache_control],
390
+ false)
327
391
  to_release = Deb::S3::Release.retrieve(to_codename)
328
392
  to_manifest = Deb::S3::Manifest.retrieve(to_codename, to_component, arch,
329
- options[:cache_control])
393
+ options[:cache_control],
394
+ options[:fail_if_exists])
330
395
  packages = from_manifest.packages.select { |p|
331
396
  p.name == package_name &&
332
397
  (versions.nil? || versions.include?(p.full_version))
@@ -336,10 +401,18 @@ class Deb::S3::CLI < Thor
336
401
  end
337
402
 
338
403
  packages.each do |package|
339
- to_manifest.add package, options[:preserve_versions], false
404
+ begin
405
+ to_manifest.add package, options[:preserve_versions], false
406
+ rescue Deb::S3::Utils::AlreadyExistsError => e
407
+ error("Preparing manifest failed because: #{e}")
408
+ end
340
409
  end
341
410
 
342
- to_manifest.write_to_s3 { |f| sublog("Transferring #{f}") }
411
+ begin
412
+ to_manifest.write_to_s3 { |f| sublog("Transferring #{f}") }
413
+ rescue Deb::S3::Utils::AlreadyExistsError => e
414
+ error("Copying manifest failed because: #{e}")
415
+ end
343
416
  to_release.update_manifest(to_manifest)
344
417
  to_release.write_to_s3 { |f| sublog("Transferring #{f}") }
345
418
 
@@ -359,8 +432,8 @@ class Deb::S3::CLI < Thor
359
432
  option :versions,
360
433
  :default => nil,
361
434
  :type => :array,
362
- :desc => "The space-delimited versions of PACKAGE to delete. If not" +
363
- "specified, ALL VERSIONS will be deleted. Fair warning." +
435
+ :desc => "The space-delimited versions of PACKAGE to delete. If not " +
436
+ "specified, ALL VERSIONS will be deleted. Fair warning. " +
364
437
  "E.g. --versions \"0.1 0.2 0.3\""
365
438
 
366
439
  def delete(package)
@@ -385,7 +458,7 @@ class Deb::S3::CLI < Thor
385
458
  # retrieve the existing manifests
386
459
  log("Retrieving existing manifests")
387
460
  release = Deb::S3::Release.retrieve(options[:codename], options[:origin], options[:suite])
388
- manifest = Deb::S3::Manifest.retrieve(options[:codename], component, options[:arch], options[:cache_control])
461
+ manifest = Deb::S3::Manifest.retrieve(options[:codename], component, options[:arch], options[:cache_control], false)
389
462
 
390
463
  deleted = manifest.delete_package(package, versions)
391
464
  if deleted.length == 0
@@ -396,7 +469,7 @@ class Deb::S3::CLI < Thor
396
469
  end
397
470
  else
398
471
  deleted.each { |p|
399
- sublog("Deleting #{p.name} version #{p.version}")
472
+ sublog("Deleting #{p.name} version #{p.full_version}")
400
473
  }
401
474
  end
402
475
 
@@ -426,7 +499,7 @@ class Deb::S3::CLI < Thor
426
499
  release.architectures.each do |arch|
427
500
  log("Checking for missing packages in: #{options[:codename]}/#{options[:component]} #{arch}")
428
501
  manifest = Deb::S3::Manifest.retrieve(options[:codename], component,
429
- arch, options[:cache_control])
502
+ arch, options[:cache_control], false)
430
503
  missing_packages = []
431
504
 
432
505
  manifest.packages.each do |p|
@@ -502,7 +575,8 @@ class Deb::S3::CLI < Thor
502
575
  settings = {
503
576
  :s3_endpoint => options[:endpoint],
504
577
  :proxy_uri => options[:proxy_uri],
505
- :use_ssl => options[:use_ssl]
578
+ :use_ssl => options[:use_ssl],
579
+ :s3_force_path_style => options[:force_path_style]
506
580
  }
507
581
  settings.merge!(provider.credentials)
508
582
 
@@ -522,8 +596,10 @@ class Deb::S3::CLI < Thor
522
596
  :private
523
597
  when "authenticated"
524
598
  :authenticated_read
599
+ when "bucket_owner"
600
+ :bucket_owner_full_control
525
601
  else
526
- error("Invalid visibility setting given. Can be public, private, or authenticated.")
602
+ error("Invalid visibility setting given. Can be public, private, authenticated, or bucket_owner.")
527
603
  end
528
604
  end
529
605
  end
data/lib/deb/s3/lock.rb ADDED
@@ -0,0 +1,58 @@
1
+ # -*- encoding : utf-8 -*-
2
+ require "tempfile"
3
+ require "socket"
4
+ require "etc"
5
+
6
+ class Deb::S3::Lock
7
+ attr_accessor :user
8
+ attr_accessor :host
9
+
10
+ def initialize
11
+ @user = nil
12
+ @host = nil
13
+ end
14
+
15
+ class << self
16
+ def locked?(codename, component = nil, architecture = nil, cache_control = nil)
17
+ Deb::S3::Utils.s3_exists?(lock_path(codename, component, architecture, cache_control))
18
+ end
19
+
20
+ def wait_for_lock(codename, component = nil, architecture = nil, cache_control = nil, max_attempts=60, wait=10)
21
+ attempts = 0
22
+ while self.locked?(codename, component, architecture, cache_control) do
23
+ attempts += 1
24
+ throw "Unable to obtain a lock after #{max_attempts}, giving up." if attempts > max_attempts
25
+ sleep(wait)
26
+ end
27
+ end
28
+
29
+ def lock(codename, component = nil, architecture = nil, cache_control = nil)
30
+ lockfile = Tempfile.new("lockfile")
31
+ lockfile.write("#{Etc.getlogin}@#{Socket.gethostname}")
32
+ lockfile.close
33
+
34
+ Deb::S3::Utils.s3_store(lockfile.path,
35
+ lock_path(codename, component, architecture, cache_control),
36
+ "text/plain",
37
+ cache_control)
38
+ end
39
+
40
+ def unlock(codename, component = nil, architecture = nil, cache_control = nil)
41
+ Deb::S3::Utils.s3_remove(lock_path(codename, component, architecture, cache_control))
42
+ end
43
+
44
+ def current(codename, component = nil, architecture = nil, cache_control = nil)
45
+ lock_content = Deb::S3::Utils.s3_read(lock_path(codename, component, architecture, cache_control))
46
+ lock_content = lock_content.split('@')
47
+ lock = Deb::S3::Lock.new
48
+ lock.user = lock_content[0]
49
+ lock.host = lock_content[1] if lock_content.size > 1
50
+ lock
51
+ end
52
+
53
+ private
54
+ def lock_path(codename, component = nil, architecture = nil, cache_control = nil)
55
+ "dists/#{codename}/#{component}/binary-#{architecture}/lockfile"
56
+ end
57
+ end
58
+ end
data/lib/deb/s3/manifest.rb CHANGED
@@ -11,6 +11,7 @@ class Deb::S3::Manifest
11
11
  attr_accessor :component
12
12
  attr_accessor :cache_control
13
13
  attr_accessor :architecture
14
+ attr_accessor :fail_if_exists
14
15
 
15
16
  attr_accessor :files
16
17
 
@@ -24,10 +25,11 @@ class Deb::S3::Manifest
24
25
  @architecture = nil
25
26
  @files = {}
26
27
  @cache_control = ""
28
+ @fail_if_exists = false
27
29
  end
28
30
 
29
31
  class << self
30
- def retrieve(codename, component, architecture, cache_control)
32
+ def retrieve(codename, component, architecture, cache_control, fail_if_exists)
31
33
  m = if s = Deb::S3::Utils.s3_read("dists/#{codename}/#{component}/binary-#{architecture}/Packages")
32
34
  self.parse_packages(s)
33
35
  else
@@ -38,6 +40,7 @@ class Deb::S3::Manifest
38
40
  m.component = component
39
41
  m.architecture = architecture
40
42
  m.cache_control = cache_control
43
+ m.fail_if_exists = fail_if_exists
41
44
  m
42
45
  end
43
46
 
@@ -52,6 +55,9 @@ class Deb::S3::Manifest
52
55
  end
53
56
 
54
57
  def add(pkg, preserve_versions, needs_uploading=true)
58
+ if self.fail_if_exists
59
+ packages.each { |p| raise AlreadyExistsError, "package #{pkg.name}_#{pkg.full_version} already exists with different filename (#{p.url_filename})" if p.name == pkg.name && p.full_version == pkg.full_version && File.basename(p.url_filename) != File.basename(pkg.filename) }
60
+ end
55
61
  if preserve_versions
56
62
  packages.delete_if { |p| p.name == pkg.name && p.full_version == pkg.full_version }
57
63
  else
@@ -69,7 +75,7 @@ class Deb::S3::Manifest
69
75
  if p.name != pkg
70
76
  p
71
77
  # Also include the packages not matching a specified version
72
- elsif (!versions.nil? and p.name == pkg and !versions.include?(p.version) and !versions.include?("#{p.version}-#{p.iteration}"))
78
+ elsif (!versions.nil? and p.name == pkg and !versions.include?(p.version) and !versions.include?("#{p.version}-#{p.iteration}") and !versions.include?(p.full_version))
73
79
  p
74
80
  end
75
81
  }
@@ -88,7 +94,7 @@ class Deb::S3::Manifest
88
94
  # store any packages that need to be stored
89
95
  @packages_to_be_upload.each do |pkg|
90
96
  yield pkg.url_filename if block_given?
91
- s3_store(pkg.filename, pkg.url_filename, 'application/octet-stream; charset=binary')
97
+ s3_store(pkg.filename, pkg.url_filename, 'application/octet-stream; charset=binary', self.cache_control, self.fail_if_exists)
92
98
  end
93
99
 
94
100
  # generate the Packages file
@@ -97,7 +103,7 @@ class Deb::S3::Manifest
97
103
  pkgs_temp.close
98
104
  f = "dists/#{@codename}/#{@component}/binary-#{@architecture}/Packages"
99
105
  yield f if block_given?
100
- s3_store(pkgs_temp.path, f, 'binary/octet-stream; charset=binary', self.cache_control)
106
+ s3_store(pkgs_temp.path, f, 'text/plain; charset=UTF-8', self.cache_control)
101
107
  @files["#{@component}/binary-#{@architecture}/Packages"] = hashfile(pkgs_temp.path)
102
108
  pkgs_temp.unlink
103
109
 
data/lib/deb/s3/package.rb CHANGED
@@ -56,7 +56,7 @@ class Deb::S3::Package
56
56
  end
57
57
 
58
58
  def extract_control(package)
59
- if system("which dpkg &> /dev/null")
59
+ if system("which dpkg > /dev/null 2>&1")
60
60
  `dpkg -f #{package}`
61
61
  else
62
62
  # ar fails to find the control.tar.gz tarball within the .deb
data/lib/deb/s3/release.rb CHANGED
@@ -98,7 +98,7 @@ class Deb::S3::Release
98
98
  release_tmp.puts self.generate
99
99
  release_tmp.close
100
100
  yield self.filename if block_given?
101
- s3_store(release_tmp.path, self.filename, 'binary/octet-stream; charset=binary', self.cache_control)
101
+ s3_store(release_tmp.path, self.filename, 'text/plain; charset=UTF-8', self.cache_control)
102
102
 
103
103
  # sign the file, if necessary
104
104
  if Deb::S3::Utils.signing_key
@@ -130,7 +130,7 @@ class Deb::S3::Release
130
130
  def validate_others
131
131
  to_apply = []
132
132
  self.components.each do |comp|
133
- %w(amd64 i386).each do |arch|
133
+ %w(amd64 i386 armhf).each do |arch|
134
134
  next if self.files.has_key?("#{comp}/binary-#{arch}/Packages")
135
135
 
136
136
  m = Deb::S3::Manifest.new
data/lib/deb/s3/utils.rb CHANGED
@@ -22,6 +22,7 @@ module Deb::S3::Utils
22
22
  def encryption= v; @encryption = v end
23
23
 
24
24
  class SafeSystemError < RuntimeError; end
25
+ class AlreadyExistsError < RuntimeError; end
25
26
 
26
27
  def safesystem(*args)
27
28
  success = system(*args)
@@ -63,7 +64,7 @@ module Deb::S3::Utils
63
64
  Deb::S3::Utils.s3.buckets[Deb::S3::Utils.bucket].objects[s3_path(path)].read
64
65
  end
65
66
 
66
- def s3_store(path, filename=nil, content_type='application/octet-stream; charset=binary', cache_control=nil)
67
+ def s3_store(path, filename=nil, content_type='application/octet-stream; charset=binary', cache_control=nil, fail_if_exists=false)
67
68
  filename = File.basename(path) unless filename
68
69
  obj = Deb::S3::Utils.s3.buckets[Deb::S3::Utils.bucket].objects[s3_path(filename)]
69
70
 
@@ -72,6 +73,7 @@ module Deb::S3::Utils
72
73
  # check if the object already exists
73
74
  if obj.exists?
74
75
  return if (file_md5.to_s == obj.etag.gsub('"', '') or file_md5.to_s == obj.metadata['md5'])
76
+ raise AlreadyExistsError, "file #{obj.public_url} already exists with different contents" if fail_if_exists
75
77
  end
76
78
 
77
79
  options = {:acl => Deb::S3::Utils.access_policy, :content_type => content_type, :metadata => {'md5' => file_md5}}
metadata CHANGED
@@ -1,71 +1,71 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: deb-s3
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.1
4
+ version: 0.8.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ken Robertson
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-03-12 00:00:00.000000000 Z
11
+ date: 2016-04-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: thor
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ~>
17
+ - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: 0.18.0
19
+ version: 0.19.0
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ~>
24
+ - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: 0.18.0
26
+ version: 0.19.0
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: aws-sdk
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - ~>
31
+ - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '1.18'
33
+ version: '1.66'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - ~>
38
+ - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: '1.18'
40
+ version: '1.66'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: minitest
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - '>='
45
+ - - "~>"
46
46
  - !ruby/object:Gem::Version
47
- version: '0'
47
+ version: '5'
48
48
  type: :development
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - '>='
52
+ - - "~>"
53
53
  - !ruby/object:Gem::Version
54
- version: '0'
54
+ version: '5'
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: rake
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - '>='
59
+ - - "~>"
60
60
  - !ruby/object:Gem::Version
61
- version: '0'
61
+ version: '11'
62
62
  type: :development
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - '>='
66
+ - - "~>"
67
67
  - !ruby/object:Gem::Version
68
- version: '0'
68
+ version: '11'
69
69
  description: Easily create and manage an APT repository on S3.
70
70
  email: ken@invalidlogic.com
71
71
  executables:
@@ -73,18 +73,20 @@ executables:
73
73
  extensions: []
74
74
  extra_rdoc_files: []
75
75
  files:
76
+ - README.md
77
+ - bin/deb-s3
78
+ - lib/deb/s3.rb
79
+ - lib/deb/s3/cli.rb
80
+ - lib/deb/s3/lock.rb
81
+ - lib/deb/s3/manifest.rb
82
+ - lib/deb/s3/package.rb
76
83
  - lib/deb/s3/release.rb
77
84
  - lib/deb/s3/templates/package.erb
78
85
  - lib/deb/s3/templates/release.erb
79
- - lib/deb/s3/package.rb
80
- - lib/deb/s3/cli.rb
81
- - lib/deb/s3/manifest.rb
82
86
  - lib/deb/s3/utils.rb
83
- - lib/deb/s3.rb
84
- - bin/deb-s3
85
- - README.md
86
87
  homepage: http://invalidlogic.com/
87
- licenses: []
88
+ licenses:
89
+ - MIT
88
90
  metadata: {}
89
91
  post_install_message:
90
92
  rdoc_options: []
@@ -92,17 +94,17 @@ require_paths:
92
94
  - lib
93
95
  required_ruby_version: !ruby/object:Gem::Requirement
94
96
  requirements:
95
- - - '>='
97
+ - - ">="
96
98
  - !ruby/object:Gem::Version
97
- version: '0'
99
+ version: 1.9.3
98
100
  required_rubygems_version: !ruby/object:Gem::Requirement
99
101
  requirements:
100
- - - '>='
102
+ - - ">="
101
103
  - !ruby/object:Gem::Version
102
104
  version: '0'
103
105
  requirements: []
104
106
  rubyforge_project:
105
- rubygems_version: 2.0.14
107
+ rubygems_version: 2.4.5.1
106
108
  signing_key:
107
109
  specification_version: 4
108
110
  summary: Easily create and manage an APT repository on S3.