frm 0.0.9 → 0.0.10
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/frm +8 -10
- data/frm.gemspec +1 -1
- data/lib/frm/arch_release.rb +180 -0
- data/lib/frm/base.rb +17 -70
- data/lib/frm/package.rb +40 -10
- data/lib/frm/release_pusher.rb +46 -19
- data/lib/frm/s3.rb +48 -51
- data/lib/frm/version.rb +1 -1
- data/lib/frm.rb +87 -2
- data/test/deb.rb +1 -1
- metadata +19 -3
- data/lib/frm/release.rb +0 -107
data/bin/frm
CHANGED
@@ -24,13 +24,13 @@ FRM allows you to easily create debian repos on S3
|
|
24
24
|
|
25
25
|
Usage: frm [options] package1 [package2] ..."
|
26
26
|
|
27
|
-
options[:access_key] =
|
27
|
+
options[:access_key] = ENV['AWS_ACCESS_KEY_ID']
|
28
28
|
opts.on( '-a', '--access-key ACCESS_KEY', 'S3 public access key' ) do |key|
|
29
29
|
options[:access_key] = key
|
30
30
|
end
|
31
31
|
|
32
32
|
|
33
|
-
options[:secret_key] =
|
33
|
+
options[:secret_key] = ENV['AWS_SECRET_ACCESS_KEY']
|
34
34
|
opts.on( '-s', '--secret-key SECRET_KEY', 'S3 private access key' ) do |key|
|
35
35
|
options[:secret_key] = key
|
36
36
|
end
|
@@ -74,7 +74,7 @@ optparse.parse!
|
|
74
74
|
|
75
75
|
|
76
76
|
ARGV.each do |package|
|
77
|
-
packages <<
|
77
|
+
packages << package
|
78
78
|
end
|
79
79
|
|
80
80
|
|
@@ -87,7 +87,8 @@ end
|
|
87
87
|
print_and_exit "you need to specify at least one package to publish!!!" if packages.empty?
|
88
88
|
|
89
89
|
packages.each do |package|
|
90
|
-
|
90
|
+
puts "package is #{package}"
|
91
|
+
print_and_exit "package '#{package}' does not exist!!!" unless File.exists?(package)
|
91
92
|
print_and_exit "package '#{package}' does not appear to be a valid deb" unless system("dpkg --field #{package} > /dev/null 2>&1 ")
|
92
93
|
end
|
93
94
|
|
@@ -95,16 +96,13 @@ end
|
|
95
96
|
# if we have gotten this far it means that all required parameters have been set, so let's begin
|
96
97
|
|
97
98
|
|
98
|
-
# chdir to the top level of the code
|
99
|
-
Dir.chdir(File.dirname(File.dirname(__FILE__)))
|
100
|
-
|
101
99
|
# require the necessary FRM libs
|
102
|
-
|
100
|
+
require_relative '../lib/frm'
|
103
101
|
|
104
102
|
puts "creating release..."
|
105
|
-
|
103
|
+
release = FRM::Release.new(options.merge(package_paths: packages))
|
106
104
|
|
107
105
|
puts "pushing release..."
|
108
|
-
|
106
|
+
release.push
|
109
107
|
|
110
108
|
puts "looks like everything ran ok!"
|
data/frm.gemspec
CHANGED
@@ -0,0 +1,180 @@
|
|
1
|
+
module FRM
|
2
|
+
|
3
|
+
class ArchRelease < Base
|
4
|
+
attr_reader :opts
|
5
|
+
|
6
|
+
def initialize(opts={})
|
7
|
+
super()
|
8
|
+
@opts = opts
|
9
|
+
@opts[:packages] = []
|
10
|
+
@opts[:package_paths] ||= []
|
11
|
+
@opts[:release] ||= run ". /etc/lsb-release && echo $DISTRIB_CODENAME"
|
12
|
+
@opts[:component] ||= 'main'
|
13
|
+
@opts[:origin] ||= 'FRM'
|
14
|
+
@opts[:lablel] ||= 'FRM'
|
15
|
+
@opts[:description] = "FRM apt repo"
|
16
|
+
@opts[:package_paths].each { |path| @opts[:packages] << Package.new(path) }
|
17
|
+
case run("uname -m")
|
18
|
+
when "x86_64"
|
19
|
+
@opts[:arch] ||= 'amd64'
|
20
|
+
else
|
21
|
+
@opts[:arch] ||= run "uname -m"
|
22
|
+
end
|
23
|
+
raise "you need to pass a remote_store object that responds to exists? and get"\
|
24
|
+
unless @opts[:remote_store] \
|
25
|
+
and @opts[:remote_store].respond_to? 'exists?' \
|
26
|
+
and @opts[:remote_store].respond_to? 'get'
|
27
|
+
|
28
|
+
@opts[:release_file_path] = "#{@opts[:component]}/binary-#{@opts[:arch]}/Release"
|
29
|
+
@opts[:package_file_path] = "#{@opts[:component]}/binary-#{@opts[:arch]}/Packages"
|
30
|
+
@opts[:gzipped_package_file_path] = "#{@opts[:component]}/binary-#{@opts[:arch]}/Packages.gz"
|
31
|
+
|
32
|
+
@opts[:package_file] = merge_package_files
|
33
|
+
@opts[:gzipped_package_file] = generate_gzip_pipe(@opts[:package_file]).read
|
34
|
+
@opts[:release_file] = release_file
|
35
|
+
|
36
|
+
@opts[:package_file_size] = @opts[:package_file].size
|
37
|
+
@opts[:gzipped_package_file_size] = @opts[:gzipped_package_file].size
|
38
|
+
@opts[:release_file_size] = @opts[:release_file].size
|
39
|
+
|
40
|
+
@opts[:package_file_md5sum] = compute_md5(@opts[:package_file])
|
41
|
+
@opts[:gzipped_package_file_md5sum] = compute_md5(@opts[:gzipped_package_file])
|
42
|
+
@opts[:release_file_md5sum] = compute_md5(@opts[:release_file])
|
43
|
+
|
44
|
+
@opts[:package_file_sha1] = compute_sha1(@opts[:package_file])
|
45
|
+
@opts[:gzipped_package_file_sha1] = compute_sha1(@opts[:gzipped_package_file])
|
46
|
+
@opts[:release_file_sha1] = compute_sha1(@opts[:release_file])
|
47
|
+
|
48
|
+
@opts[:package_file_sha256] = compute_sha2(@opts[:package_file])
|
49
|
+
@opts[:gzipped_package_file_sha256] = compute_sha2(@opts[:gzipped_package_file])
|
50
|
+
@opts[:release_file_sha256] = compute_sha2(@opts[:release_file])
|
51
|
+
end
|
52
|
+
|
53
|
+
def push
|
54
|
+
@opts[:packages].each do |p|
|
55
|
+
if @opts[:remote_store].exists?(p.info['Filename'])
|
56
|
+
STDERR.puts "package #{p.path} already exists"
|
57
|
+
unless @opts[:remote_store].etag(p.info['Filename']) == p.info['MD5sum']
|
58
|
+
raise <<EOE
|
59
|
+
trying to overwrite this package file: #{remote_path}
|
60
|
+
local md5 is #{package.info['MD5sum']}
|
61
|
+
remote md5 (etag) is #{@s3.etag(remote_path,@bucket)}
|
62
|
+
EOE
|
63
|
+
end
|
64
|
+
else
|
65
|
+
STDERR.puts "pushing package #{p.path}"
|
66
|
+
@opts[:remote_store].put(p.info['Filename'],p.content)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
STDERR.puts "pushing arch release files"
|
70
|
+
@opts[:remote_store].put(File.join("dists/#{@opts[:release]}/",@opts[:release_file_path]),@opts[:release_file])
|
71
|
+
@opts[:remote_store].put(File.join("dists/#{@opts[:release]}/",@opts[:gzipped_package_file_path]),@opts[:gzipped_package_file])
|
72
|
+
@opts[:remote_store].put(File.join("dists/#{@opts[:release]}/",@opts[:package_file_path]),@opts[:package_file])
|
73
|
+
end
|
74
|
+
|
75
|
+
private
|
76
|
+
|
77
|
+
def release_file
|
78
|
+
return <<-EOF
|
79
|
+
Component: #{@opts[:component]}
|
80
|
+
Origin: #{@opts[:origin]}
|
81
|
+
Label: #{@opts[:lablel]}
|
82
|
+
Architecture: #{@opts[:arch]}
|
83
|
+
Description: #{@opts[:description]}
|
84
|
+
EOF
|
85
|
+
end
|
86
|
+
|
87
|
+
def previous_package_file
|
88
|
+
package_file_path = "dists/#{@opts[:release]}/#{@opts[:package_file_path]}"
|
89
|
+
return @opts[:remote_store].get(package_file_path) \
|
90
|
+
if @opts[:remote_store].exists? package_file_path
|
91
|
+
return ""
|
92
|
+
end
|
93
|
+
|
94
|
+
|
95
|
+
# given a package file as a string, parse out the next package
|
96
|
+
def parse_package_stub(package_file='')
|
97
|
+
|
98
|
+
# find the first package header
|
99
|
+
stub_start = package_file.index(/^Package: /)
|
100
|
+
return nil if stub_start.nil?
|
101
|
+
|
102
|
+
# find the end of the first package section
|
103
|
+
stub_end = package_file.index(/^$/,stub_start)
|
104
|
+
raise "could not parse #{package_file}" if stub_end.nil?
|
105
|
+
|
106
|
+
# extract out the next stub
|
107
|
+
stub = package_file.slice(stub_start,stub_end) + "\n"
|
108
|
+
raise "could not parse #{package_file}" if stub.nil?
|
109
|
+
|
110
|
+
# pull out the package line from the stub
|
111
|
+
package_line = stub[/^Package: .*$/]
|
112
|
+
raise "could not parse #{package_file}" if package_line.nil?
|
113
|
+
|
114
|
+
# pull out the name of the package
|
115
|
+
package = package_line.sub('Package: ','').strip
|
116
|
+
raise "could not read package name from #{stub}" if package.empty?
|
117
|
+
|
118
|
+
return stub, stub_end, package
|
119
|
+
end
|
120
|
+
|
121
|
+
|
122
|
+
# recursive function to merge two sets of packages
|
123
|
+
def merge_package_files(opts={})
|
124
|
+
|
125
|
+
# initial settings
|
126
|
+
opts[:previous_package_file] ||= previous_package_file
|
127
|
+
opts[:new_packages] ||= @opts[:packages].collect{|p| p.to_h}.sort{|a,b| a['Package'] <=> b['Package'] }
|
128
|
+
opts[:new_package_file] ||= ""
|
129
|
+
|
130
|
+
# if we are at the end of both the old package list and the new set of packages then finish
|
131
|
+
return opts[:new_package_file] \
|
132
|
+
if opts[:previous_package_file].strip.empty? \
|
133
|
+
and opts[:new_packages].empty?
|
134
|
+
|
135
|
+
# if we don't have any new packages to add then return the current list with the old appended
|
136
|
+
return (opts[:new_package_file] << opts[:previous_package_file]) \
|
137
|
+
if opts[:new_packages].empty?
|
138
|
+
|
139
|
+
# parse next package stub
|
140
|
+
stub, stub_end, next_package = parse_package_stub(opts[:previous_package_file])
|
141
|
+
|
142
|
+
# if we don't have and more old packages to add then return the
|
143
|
+
# current list with the new appended
|
144
|
+
return (opts[:new_package_file] << opts[:new_packages].collect{|p| FRM::Package.hash_to_stub(p)}.join("\n")) if stub.nil?
|
145
|
+
|
146
|
+
# compare the next previous existing pacakge to the next new package
|
147
|
+
case next_package <=> opts[:new_packages].first['Package']
|
148
|
+
when -1
|
149
|
+
opts[:new_package_file] << stub
|
150
|
+
opts[:previous_package_file].slice!(0..stub_end)
|
151
|
+
when 1
|
152
|
+
opts[:new_package_file] << FRM::Package.hash_to_stub(opts[:new_packages].shift)
|
153
|
+
when 0
|
154
|
+
# both packages have the same name
|
155
|
+
STDERR.puts "I see two version of this pakcage: #{next_package}"
|
156
|
+
previous_version_line = stub[/^Version: .*$/]
|
157
|
+
raise "could not get version from package stub: \n#{stub}" if previous_version_line.nil?
|
158
|
+
|
159
|
+
previous_version_raw = previous_version_line.sub('Version: ','').strip
|
160
|
+
raise "could not get version from package stub: \n#{stub}" if previous_version_raw.empty?
|
161
|
+
previous_version = Gem::Version.new(previous_version_raw)
|
162
|
+
|
163
|
+
newer_version_raw = opts[:new_packages].first['Version']
|
164
|
+
newer_version = Gem::Version.new(newer_version_raw)
|
165
|
+
|
166
|
+
raise "previous version #{previous_version_raw} is equal to #{newer_version_raw}. Exiting..." \
|
167
|
+
if previous_version == newer_version
|
168
|
+
|
169
|
+
raise "previous version #{previous_version_raw} is newer than #{newer_version_raw}. Exiting..." \
|
170
|
+
if previous_version > newer_version
|
171
|
+
|
172
|
+
opts[:new_package_file] << FRM::Package.hash_to_stub(opts[:new_packages].shift)
|
173
|
+
opts[:previous_package_file].slice!(0..stub_end)
|
174
|
+
end
|
175
|
+
|
176
|
+
return merge_package_files(opts)
|
177
|
+
end
|
178
|
+
|
179
|
+
end
|
180
|
+
end
|
data/lib/frm/base.rb
CHANGED
@@ -1,5 +1,8 @@
|
|
1
1
|
module FRM
|
2
2
|
class Base
|
3
|
+
def initialize
|
4
|
+
@retries = 3
|
5
|
+
end
|
3
6
|
|
4
7
|
def compute_md5(string)
|
5
8
|
Digest::MD5.hexdigest(string)
|
@@ -58,80 +61,24 @@ module FRM
|
|
58
61
|
return unzipped_string
|
59
62
|
end
|
60
63
|
|
61
|
-
def
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
stub << line
|
75
|
-
end
|
76
|
-
package['Description'].rstrip!
|
77
|
-
return package, stub
|
64
|
+
def handle_errors(&block)
|
65
|
+
retries = 0
|
66
|
+
begin
|
67
|
+
yield
|
68
|
+
rescue Object => error_object
|
69
|
+
if retries < @retries
|
70
|
+
sleep(2**retries * 1) #exponential backoff, 1s base
|
71
|
+
retries += 1
|
72
|
+
STDERR.puts "encountered error #{error_object.inspect}, retrying attempt #{retries}."
|
73
|
+
retry
|
74
|
+
else
|
75
|
+
logger.error "encountered error #{error_object.inspect}, aborting."
|
76
|
+
raise error_object
|
78
77
|
end
|
79
78
|
end
|
80
|
-
nil
|
81
|
-
end
|
82
|
-
|
83
|
-
def merge_package_file(in_pipe,out_pipe,package_list)
|
84
|
-
sorted_list = package_list.sort { |a,b| a['Package'] <=> b['Package'] }
|
85
|
-
merge(in_pipe,out_pipe,sorted_list)
|
86
79
|
end
|
87
|
-
|
88
|
-
private
|
89
|
-
|
90
|
-
def create_package_stub(package_hash)
|
91
|
-
return_value = ''
|
92
|
-
package_hash.each do |key,value|
|
93
|
-
return_value << "#{key}: #{value}\n"
|
94
|
-
end
|
95
|
-
return_value << "\n"
|
96
|
-
end
|
97
|
-
|
98
|
-
def merge(in_pipe,out_pipe,package_list)
|
99
|
-
return if out_pipe.closed?
|
100
|
-
return if in_pipe.closed? and package_list.empty
|
101
80
|
|
102
|
-
|
103
|
-
while line = in_pipe.gets
|
104
|
-
out_pipe.puts line
|
105
|
-
end
|
106
|
-
in_pipe.close
|
107
|
-
out_pipe.close
|
108
|
-
return
|
109
|
-
end
|
110
|
-
|
111
|
-
if in_pipe.closed?
|
112
|
-
package_list.each {|package_hash| out_pipe.write(create_package_stub(package_hash)) }
|
113
|
-
out_pipe.close
|
114
|
-
return
|
115
|
-
end
|
116
|
-
|
117
|
-
current_package, stub = parse_package_stub in_pipe
|
118
|
-
|
119
|
-
if current_package['Package'] < package_list.first['Package']
|
120
|
-
out_pipe.puts stub
|
121
|
-
out_pipe.puts ""
|
122
|
-
merge(in_pipe,out_pipe,package_list)
|
123
|
-
return
|
124
|
-
elsif current_package['Package'] > package_list.first['Package']
|
125
|
-
while ( ! package_list.empty? ) and current_package['Package'] > package_list.first['Package']
|
126
|
-
out_pipe.write create_package_stub(package_list.shift)
|
127
|
-
end
|
128
|
-
elsif current_package['Package'] == package_list.first['Package']
|
129
|
-
out_pipe.write create_package_stub(package_list.shift)
|
130
|
-
end
|
131
|
-
|
132
|
-
merge(in_pipe,out_pipe,package_list)
|
133
|
-
return nil
|
134
|
-
end
|
81
|
+
private
|
135
82
|
|
136
83
|
end
|
137
84
|
end
|
data/lib/frm/package.rb
CHANGED
@@ -1,12 +1,15 @@
|
|
1
1
|
module FRM
|
2
2
|
class Package < Base
|
3
|
-
|
4
|
-
|
5
|
-
def initialize(path
|
6
|
-
raise "you need to specify a path!!!" if path.nil?
|
3
|
+
attr_reader :path, :content, :info
|
4
|
+
|
5
|
+
def initialize(path)
|
7
6
|
@path = path
|
8
|
-
|
9
|
-
|
7
|
+
|
8
|
+
raise "you need to pass a path!!!" if path.nil?
|
9
|
+
raise "Can not find file: '#{path}'" unless File.exists?(path)
|
10
|
+
raise "this system does not have the dpkg binary" unless system 'which dpkg > /dev/null'
|
11
|
+
raise "file #{path} is not a deb" unless system "dpkg --field #{path} > /dev/null"
|
12
|
+
|
10
13
|
begin
|
11
14
|
@content = File.read(path)
|
12
15
|
rescue Object => o
|
@@ -15,10 +18,37 @@ module FRM
|
|
15
18
|
STDERR.puts o.backtrace
|
16
19
|
raise "Could not open file '#{path}'. Exiting..."
|
17
20
|
end
|
18
|
-
|
19
|
-
@
|
20
|
-
@sha1 = compute_sha1(@content)
|
21
|
-
@sha2 = compute_sha2(@content)
|
21
|
+
|
22
|
+
@info = get_package_info
|
22
23
|
end
|
24
|
+
|
25
|
+
def to_stub
|
26
|
+
@info.collect{|k,v| "#{k}: #{v}"}.join("\n") + "\n\n"
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.hash_to_stub(hash)
|
30
|
+
hash.collect{|k,v| "#{k}: #{v}"}.join("\n") + "\n\n"
|
31
|
+
end
|
32
|
+
|
33
|
+
def to_h
|
34
|
+
return @info
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def get_package_info
|
40
|
+
h = {}
|
41
|
+
['Package', 'Version', 'License' ,'Vendor' ,'Architecture' ,'Maintainer', 'Installed-Size' ,'Section', 'Priority', 'Homepage', 'Description' ].each do |value|
|
42
|
+
h[value] = run "dpkg --field #{@path} #{value}"
|
43
|
+
end
|
44
|
+
|
45
|
+
h['Filename'] = "pool/main/#{h['Package'][0]}/#{h['Package']}/#{File.basename @path}"
|
46
|
+
h['Size'] = File.size(@path)
|
47
|
+
h['MD5sum'] = compute_md5(@content)
|
48
|
+
h['SHA1'] = compute_sha1(@content)
|
49
|
+
h['SHA256'] = compute_sha2(@content)
|
50
|
+
return h
|
51
|
+
end
|
52
|
+
|
23
53
|
end
|
24
54
|
end
|
data/lib/frm/release_pusher.rb
CHANGED
@@ -14,28 +14,44 @@ module FRM
|
|
14
14
|
|
15
15
|
|
16
16
|
def push_release_files(package_release)
|
17
|
-
# TODO: un-hardcode this
|
18
17
|
release_path = @prefix + "/dists/#{@release}/Release"
|
19
|
-
@s3.put(release_path,package_release.release_file,@bucket)
|
20
|
-
|
21
18
|
in_release_path = @prefix + "/dists/#{@release}/InRelease"
|
22
|
-
@s3.put(in_release_path,gpg_clearsign(package_release.release_file),@bucket)
|
23
|
-
|
24
19
|
gpg_release_path = @prefix + "/dists/#{@release}/Release.gpg"
|
25
|
-
@s3.put(gpg_release_path,gpg_detached(package_release.release_file),@bucket)
|
26
|
-
|
27
|
-
release_file_path = @prefix + "/dists/#{@release}/" + package_release.component
|
28
|
-
@s3.put(release_file_path + '/Release',package_release.short_release_file,@bucket)
|
29
|
-
@s3.put(release_file_path + '/Packages',package_release.package_file,@bucket)
|
30
|
-
@s3.put(release_file_path + '/Packages.gz',package_release.gzipped_package_file,@bucket)
|
31
20
|
|
21
|
+
unless @s3.exists?(release_path,@bucket)
|
22
|
+
STDERR.puts "pushing new release files..."
|
23
|
+
@s3.put(release_path,package_release.release_file,@bucket)
|
24
|
+
@s3.put(in_release_path,gpg_clearsign(package_release.release_file),@bucket)
|
25
|
+
@s3.put(gpg_release_path,gpg_detached(package_release.release_file),@bucket)
|
26
|
+
|
27
|
+
|
28
|
+
release_file_path = @prefix + "/dists/#{@release}/" + package_release.component
|
29
|
+
@s3.put(release_file_path + '/Release',package_release.short_release_file,@bucket)
|
30
|
+
@s3.put(release_file_path + '/Packages',package_release.package_file,@bucket)
|
31
|
+
@s3.put(release_file_path + '/Packages.gz',package_release.gzipped_package_file,@bucket)
|
32
|
+
|
33
|
+
|
34
|
+
i386_release_file_path = @prefix + "/dists/#{@release}/" + 'main/binary-i386'
|
35
|
+
@s3.put(i386_release_file_path + '/Release',package_release.i386_release_file,@bucket)
|
36
|
+
@s3.put(i386_release_file_path + '/Packages',package_release.i386_packages_file,@bucket)
|
37
|
+
@s3.put(i386_release_file_path + '/Packages.gz',package_release.gzipped_i386_packages_file,@bucket)
|
38
|
+
|
39
|
+
# push public key
|
40
|
+
@s3.put(@prefix + '/public.key',gpg_export_pubkey,@bucket)
|
41
|
+
else
|
42
|
+
STDERR.puts "updating releases file"
|
43
|
+
release_file = @s3.get(release_path,@bucket)
|
44
|
+
File.open('/tmp/in','w'){ |f|
|
45
|
+
f.write(release_file)
|
46
|
+
}
|
47
|
+
in_buffer = File.open('/tmp/in',"r")
|
48
|
+
out_buffer = File.open('/tmp/fd',"w")
|
49
|
+
STDERR.puts "package_release.packages is #{package_release.packages.inspect}"
|
50
|
+
merge_package_file(in_buffer,out_buffer,package_release.packages)
|
32
51
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
@s3.put(i386_release_file_path + '/Packages.gz',package_release.gzipped_i386_packages_file,@bucket)
|
37
|
-
# push public key
|
38
|
-
@s3.put(@prefix + '/public.key',gpg_export_pubkey,@bucket)
|
52
|
+
STDERR.puts "not yet implmented"
|
53
|
+
Kernel.exit 1
|
54
|
+
end
|
39
55
|
end
|
40
56
|
|
41
57
|
|
@@ -44,8 +60,19 @@ module FRM
|
|
44
60
|
end
|
45
61
|
|
46
62
|
def push_package(package)
|
47
|
-
remote_path = @prefix + '/' + package.
|
48
|
-
@s3.
|
63
|
+
remote_path = @prefix + '/' + package.info['Filename']
|
64
|
+
if @s3.exists?(remote_path,@bucket)
|
65
|
+
unless @s3.etag(remote_path,@bucket) == package.info['MD5sum']
|
66
|
+
error_message = <<EOE
|
67
|
+
trying to overwrite this package file: #{remote_path}
|
68
|
+
local md5 is #{package.info['MD5sum']}
|
69
|
+
remote md5 (etag) is #{@s3.etag(remote_path,@bucket)}
|
70
|
+
EOE
|
71
|
+
raise error_message
|
72
|
+
end
|
73
|
+
else
|
74
|
+
@s3.put(remote_path,package.content,@bucket)
|
75
|
+
end
|
49
76
|
end
|
50
77
|
|
51
78
|
|
data/lib/frm/s3.rb
CHANGED
@@ -3,76 +3,73 @@ require 'aws-sdk'
|
|
3
3
|
module FRM
|
4
4
|
class S3 < Base
|
5
5
|
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
@
|
10
|
-
|
11
|
-
|
6
|
+
def initialize(opts={})
|
7
|
+
super()
|
8
|
+
@opts = opts
|
9
|
+
@opts[:public_repo] ||= false
|
10
|
+
@opts[:acl] = @opts[:public_repo] ? :public_read : :private
|
11
|
+
@opts[:aws_access_key] ||= ENV['AWS_ACCESS_KEY_ID']
|
12
|
+
@opts[:aws_secret_key] ||= ENV['AWS_SECRET_ACCESS_KEY']
|
13
|
+
raise "you either need to pass an aws_access_key option or set the AWS_ACCESS_KEY environment variable" \
|
14
|
+
if @opts[:aws_access_key].nil?
|
15
|
+
raise "you either need to pass an aws_secret_key option or set the AWS_SECRET_KEY environment variable" \
|
16
|
+
if @opts[:aws_secret_key].nil?
|
17
|
+
raise "you need to pass in a bucket param" unless @opts[:bucket]
|
18
|
+
raise "you need to pass in a prefix param" unless @opts[:prefix]
|
19
|
+
@opts[:prefix] << '/' unless @opts[:prefix][-1] == '/'
|
20
|
+
AWS.config(:access_key_id => @opts[:access_key_id],
|
21
|
+
:secret_access_key => @opts[:secret_access_key])
|
12
22
|
@s3 = AWS::S3.new
|
13
|
-
@acl = public_repo ? :public_read : :private
|
14
23
|
end
|
15
|
-
|
16
24
|
|
17
|
-
def
|
18
|
-
|
25
|
+
def exists?(relative_path)
|
26
|
+
handle_errors do
|
27
|
+
return @s3.buckets[@opts[:bucket]].objects[full_path(relative_path)].exists?
|
28
|
+
end
|
19
29
|
end
|
20
30
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
@s3.buckets[bucket].objects[key].write(value,acl: @acl)
|
26
|
-
return true
|
27
|
-
rescue Object => o
|
28
|
-
print_retry(__method__,o)
|
29
|
-
end
|
30
|
-
raise "could not put object!!!" if i == (@max_retries - 1)
|
31
|
+
def etag(relative_path)
|
32
|
+
quoted_etag = ""
|
33
|
+
handle_errors do
|
34
|
+
quoted_etag = @s3.buckets[@opts[:bucket]].objects[full_path(relative_path)].etag
|
31
35
|
end
|
32
|
-
|
36
|
+
# stupid method call is actually putting quotes in the string,
|
37
|
+
# so let's remove those:
|
38
|
+
return quoted_etag.gsub(/"/,'')
|
33
39
|
end
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
begin
|
39
|
-
return @s3.buckets[bucket].objects[key].read
|
40
|
-
rescue Object => o
|
41
|
-
print_retry(__method__,o)
|
42
|
-
end
|
43
|
-
raise "could not get object!!!" if i == (@max_retries - 1)
|
40
|
+
|
41
|
+
def put(relative_path,value)
|
42
|
+
handle_errors do
|
43
|
+
@s3.buckets[@opts[:bucket]].objects[full_path(relative_path)].write(value,acl: @opts[:acl]) #
|
44
44
|
end
|
45
|
-
|
45
|
+
return true
|
46
46
|
end
|
47
47
|
|
48
|
+
def get(relative_path)
|
49
|
+
handle_errors do
|
50
|
+
return @s3.buckets[@opts[:bucket]].objects[full_path(relative_path)].read
|
51
|
+
end
|
52
|
+
end
|
48
53
|
|
49
|
-
def delete(
|
50
|
-
|
51
|
-
@s3.buckets[bucket].objects[
|
52
|
-
return true
|
53
|
-
rescue Object => o
|
54
|
-
print_retry(__method__,o)
|
54
|
+
def delete(relative_path)
|
55
|
+
handle_errors do
|
56
|
+
@s3.buckets[@opts[:bucket]].objects[full_path(relative_path)].delete
|
55
57
|
end
|
58
|
+
return true
|
56
59
|
end
|
57
60
|
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
@s3.buckets[bucket].objects[old_key].move_to(new_key)
|
62
|
-
return true
|
63
|
-
rescue Object => o
|
64
|
-
print_retry(__method__,o)
|
61
|
+
def move(old_relative_path,new_relative_path)
|
62
|
+
handle_errors do
|
63
|
+
@s3.buckets[@opts[:bucket]].objects[full_path(old_relative_path)].move_to(new_relative_path)
|
65
64
|
end
|
65
|
+
return true
|
66
66
|
end
|
67
67
|
|
68
68
|
protected
|
69
69
|
|
70
|
-
def
|
71
|
-
|
72
|
-
STDERR.puts error.inspect
|
73
|
-
STDERR.puts ", retrying..."
|
74
|
-
sleep 2
|
70
|
+
def full_path(relative_path="")
|
71
|
+
return @opts[:prefix] + relative_path
|
75
72
|
end
|
76
|
-
|
73
|
+
|
77
74
|
end
|
78
75
|
end
|
data/lib/frm/version.rb
CHANGED
data/lib/frm.rb
CHANGED
@@ -1,14 +1,99 @@
|
|
1
|
-
|
2
1
|
require 'zlib'
|
3
2
|
require 'tempfile'
|
4
3
|
require 'digest/md5'
|
5
4
|
require 'digest/sha1'
|
6
5
|
require 'digest/sha2'
|
6
|
+
require 'logger'
|
7
|
+
require 'net/ntp'
|
7
8
|
|
8
9
|
require_relative 'frm/base'
|
9
10
|
require_relative 'frm/s3'
|
10
11
|
require_relative 'frm/package'
|
11
|
-
require_relative 'frm/
|
12
|
+
require_relative 'frm/arch_release'
|
12
13
|
require_relative 'frm/release_pusher'
|
13
14
|
require_relative 'frm/deb'
|
14
15
|
|
16
|
+
|
17
|
+
module FRM
|
18
|
+
class Release < Base
|
19
|
+
|
20
|
+
def initialize(opts={})
|
21
|
+
super()
|
22
|
+
@opts = opts
|
23
|
+
@opts[:component] ||= 'main'
|
24
|
+
@opts[:origin] ||= 'FRM'
|
25
|
+
@opts[:label] ||= 'FRM'
|
26
|
+
@opts[:description] = "FRM apt repo"
|
27
|
+
@opts[:release] ||= run ". /etc/lsb-release && echo $DISTRIB_CODENAME"
|
28
|
+
handle_errors{@time = Net::NTP.get("us.pool.ntp.org").time.getutc}
|
29
|
+
|
30
|
+
case run("uname -m")
|
31
|
+
when "x86_64"
|
32
|
+
@opts[:arch] ||= 'amd64'
|
33
|
+
else
|
34
|
+
@opts[:arch] ||= run "uname -m"
|
35
|
+
end
|
36
|
+
|
37
|
+
@opts[:remote_store] = FRM::S3.new(opts)
|
38
|
+
@arch_releases = [ArchRelease.new(opts)]
|
39
|
+
@arch_releases << ArchRelease.new(arch: 'i386',remote_store: @opts[:remote_store]) \
|
40
|
+
if @opts[:arch] == 'amd64'
|
41
|
+
|
42
|
+
@release_file = release_file
|
43
|
+
@in_release_file = gpg_clearsign(release_file)
|
44
|
+
@release_gpg_file = gpg_detached(release_file)
|
45
|
+
|
46
|
+
@release_file_path = "dists/#{@opts[:release]}/Release"
|
47
|
+
@in_release_file_path = "dists/#{@opts[:release]}/InRelease"
|
48
|
+
@release_gpg_file_path = "dists/#{@opts[:release]}/Release.gpg"
|
49
|
+
|
50
|
+
end
|
51
|
+
|
52
|
+
def push
|
53
|
+
@arch_releases.each {|arch_release| arch_release.push}
|
54
|
+
@opts[:remote_store].put(@release_file_path,@release_file)
|
55
|
+
@opts[:remote_store].put(@in_release_file_path,@in_release_file)
|
56
|
+
@opts[:remote_store].put(@release_gpg_file_path,@release_gpg_file)
|
57
|
+
|
58
|
+
# push public key
|
59
|
+
@opts[:remote_store].put('public.key',gpg_export_pubkey)
|
60
|
+
end
|
61
|
+
|
62
|
+
def merge
|
63
|
+
end
|
64
|
+
|
65
|
+
private
|
66
|
+
|
67
|
+
|
68
|
+
# ubuntu precicse 64 wants i386 debs by default :\
|
69
|
+
def generate_i386_stubs()
|
70
|
+
end
|
71
|
+
|
72
|
+
def release_file()
|
73
|
+
partial_release_file = <<EOF
|
74
|
+
Origin: #{@opts[:origin]}
|
75
|
+
Label: #{@opts[:label]}
|
76
|
+
Codename: #{@opts[:release]}
|
77
|
+
Date: #{@time.getutc.strftime("%a, %d %b %Y %H:%M:%S UTC")}
|
78
|
+
Architectures: #{@arch_releases.collect{|r| r.opts[:arch]}.join(' ')}
|
79
|
+
Components: #{@opts[:component]}
|
80
|
+
Description: #{@opts[:description]}
|
81
|
+
EOF
|
82
|
+
|
83
|
+
%w{MD5Sum SHA1 SHA256}.each do |hash|
|
84
|
+
partial_release_file << "#{hash}:\n"
|
85
|
+
@arch_releases.each do |arch_release|
|
86
|
+
%w{package gzipped_package release}.each do |file|
|
87
|
+
partial_release_file << " #{arch_release.opts["#{file}_file_#{hash.downcase}".to_sym]} "
|
88
|
+
partial_release_file << arch_release.opts["#{file}_file_size".to_sym].to_s
|
89
|
+
partial_release_file << " "
|
90
|
+
partial_release_file << arch_release.opts["#{file}_file_path".to_sym]
|
91
|
+
partial_release_file << "\n"
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
return partial_release_file
|
96
|
+
end
|
97
|
+
|
98
|
+
end
|
99
|
+
end
|
data/test/deb.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: frm
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.10
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2013-01-10 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: aws-sdk
|
@@ -27,6 +27,22 @@ dependencies:
|
|
27
27
|
- - ! '>='
|
28
28
|
- !ruby/object:Gem::Version
|
29
29
|
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: net-ntp
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ~>
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: 2.1.1
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: 2.1.1
|
30
46
|
description: FRM makes it easy to build package repositories on S3
|
31
47
|
email:
|
32
48
|
- ermal14@gmail.com
|
@@ -43,10 +59,10 @@ files:
|
|
43
59
|
- bin/frm
|
44
60
|
- frm.gemspec
|
45
61
|
- lib/frm.rb
|
62
|
+
- lib/frm/arch_release.rb
|
46
63
|
- lib/frm/base.rb
|
47
64
|
- lib/frm/deb.rb
|
48
65
|
- lib/frm/package.rb
|
49
|
-
- lib/frm/release.rb
|
50
66
|
- lib/frm/release_pusher.rb
|
51
67
|
- lib/frm/s3.rb
|
52
68
|
- lib/frm/version.rb
|
data/lib/frm/release.rb
DELETED
@@ -1,107 +0,0 @@
|
|
1
|
-
module FRM
|
2
|
-
|
3
|
-
class PackageRelease < Base
|
4
|
-
attr_reader :packages, :standards_version, :priority, :package_file, :gzipped_package_file, :release_file, :short_release_file, :component, :release, :i386_release_file, :i386_packages_file, :gzipped_i386_packages_file
|
5
|
-
def initialize(packages={},release='natty',component='main/binary-amd64')
|
6
|
-
@release = release
|
7
|
-
@component = component
|
8
|
-
@standards_version = standards_version
|
9
|
-
@priority = priority
|
10
|
-
@packages = []
|
11
|
-
packages.each { |package| @packages << Package.new(package,@release) }
|
12
|
-
if component == 'main/binary-amd64'
|
13
|
-
generate_i386_stubs
|
14
|
-
end
|
15
|
-
@i386_release_file = <<EOF
|
16
|
-
Component: main
|
17
|
-
Origin: apt.cloudscaling.com
|
18
|
-
Label: apt repository #{@release}
|
19
|
-
Architecture: i386
|
20
|
-
Description: Cloudscaling APT repository
|
21
|
-
EOF
|
22
|
-
@i386_packages_file = ""
|
23
|
-
@gzipped_i386_packages_file = generate_gzip_pipe(@i386_packages_file).read
|
24
|
-
@package_file = generate_package_file
|
25
|
-
@gzipped_package_file = generate_gzip_pipe(@package_file).read
|
26
|
-
@short_release_file = generate_short_release_file
|
27
|
-
@release_file = generate_release_file
|
28
|
-
end
|
29
|
-
|
30
|
-
private
|
31
|
-
|
32
|
-
|
33
|
-
# ubuntu precicse 64 wants i386 debs by default :\
|
34
|
-
def generate_i386_stubs()
|
35
|
-
end
|
36
|
-
|
37
|
-
|
38
|
-
def generate_release_file()
|
39
|
-
partial_release_file = "Origin: apt.cloudscaling.com
|
40
|
-
Label: apt repository #{@release}
|
41
|
-
Codename: #{@release}
|
42
|
-
Date: Thu, 22 Dec 2011 00:29:55 UTC
|
43
|
-
Architectures: amd64 i386
|
44
|
-
Components: main universe multiverse
|
45
|
-
Description: Cloudscaling APT repository
|
46
|
-
MD5Sum:
|
47
|
-
#{compute_md5(@package_file)} #{@package_file.size} #{@component}/Packages
|
48
|
-
#{compute_md5(@gzipped_package_file)} #{@gzipped_package_file.size} #{@component}/Packages.gz
|
49
|
-
#{compute_md5(@short_release_file)} #{@short_release_file.size} #{@component}/Release
|
50
|
-
#{compute_md5(@i386_packages_file)} #{@i386_packages_file.size} main/binary-i386/Packages
|
51
|
-
#{compute_md5(@gzipped_i386_packages_file)} #{@gzipped_i386_packages_file.size} main/binary-i386/Packages.gz
|
52
|
-
#{compute_md5(@i386_release_file)} #{@i386_release_file.size} main/binary-i386/Release
|
53
|
-
SHA1:
|
54
|
-
#{compute_sha1(@package_file)} #{@package_file.size} #{@component}/Packages
|
55
|
-
#{compute_sha1(@gzipped_package_file)} #{@gzipped_package_file.size} #{@component}/Packages.gz
|
56
|
-
#{compute_sha1(@short_release_file)} #{@short_release_file.size} #{@component}/Release
|
57
|
-
#{compute_sha1(@i386_packages_file)} #{@i386_packages_file.size} main/binary-i386/Packages
|
58
|
-
#{compute_sha1(@gzipped_i386_packages_file)} #{@gzipped_i386_packages_file.size} main/binary-i386/Packages.gz
|
59
|
-
#{compute_sha1(@i386_release_file)} #{@i386_release_file.size} main/binary-i386/Release
|
60
|
-
SHA256:
|
61
|
-
#{compute_sha2(@package_file)} #{@package_file.size} #{@component}/Packages
|
62
|
-
#{compute_sha2(@gzipped_package_file)} #{@gzipped_package_file.size} #{@component}/Packages.gz
|
63
|
-
#{compute_sha2(@short_release_file)} #{@short_release_file.size} #{@component}/Release
|
64
|
-
#{compute_sha2(@i386_packages_file)} #{@i386_packages_file.size} main/binary-i386/Packages
|
65
|
-
#{compute_sha2(@gzipped_i386_packages_file)} #{@gzipped_i386_packages_file.size} main/binary-i386/Packages.gz
|
66
|
-
#{compute_sha2(@i386_release_file)} #{@i386_release_file.size} main/binary-i386/Release
|
67
|
-
"
|
68
|
-
return partial_release_file
|
69
|
-
end
|
70
|
-
|
71
|
-
|
72
|
-
def generate_short_release_file
|
73
|
-
"Component: main
|
74
|
-
Origin: apt.cloudscaling.com
|
75
|
-
Label: apt repository #{@release}
|
76
|
-
Architecture: amd64
|
77
|
-
Description: Cloudscaling APT repository
|
78
|
-
"
|
79
|
-
end
|
80
|
-
|
81
|
-
|
82
|
-
def filename(package)
|
83
|
-
filename = File.basename package.path
|
84
|
-
shortname = run("dpkg --field #{package.path} Package")
|
85
|
-
first_letter = shortname[0]
|
86
|
-
package.repo_filename = "pool/main/#{first_letter}/#{shortname}/#{filename}"
|
87
|
-
end
|
88
|
-
|
89
|
-
def generate_package_file()
|
90
|
-
package_file = ''
|
91
|
-
@packages.each { |package| package_file << generate_package_stub(package) }
|
92
|
-
return package_file
|
93
|
-
end
|
94
|
-
|
95
|
-
def generate_package_stub(package)
|
96
|
-
package_stub = ''
|
97
|
-
package_stub << run("dpkg --field #{package.path}") + "\n"
|
98
|
-
package_stub << "Filename: #{filename(package)}\n"
|
99
|
-
package_stub << "Size: #{package.size}\n"
|
100
|
-
package_stub << "MD5sum: #{package.md5}\n"
|
101
|
-
package_stub << "SHA1: #{package.sha1}\n"
|
102
|
-
package_stub << "SHA256: #{package.sha2}\n"
|
103
|
-
package_stub << "\n"
|
104
|
-
return package_stub
|
105
|
-
end
|
106
|
-
end
|
107
|
-
end
|