right_publish 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.rspec +4 -0
- data/Gemfile +20 -0
- data/Gemfile.lock +114 -0
- data/README.rdoc +113 -0
- data/Rakefile +48 -0
- data/VERSION +1 -0
- data/bin/right_publish +8 -0
- data/lib/right_publish/profile.rb +96 -0
- data/lib/right_publish/repo.rb +159 -0
- data/lib/right_publish/repos/apt.rb +178 -0
- data/lib/right_publish/repos/gem.rb +44 -0
- data/lib/right_publish/repos/yum.rb +167 -0
- data/lib/right_publish/storage.rb +90 -0
- data/lib/right_publish/stores/local.rb +29 -0
- data/lib/right_publish/stores/s3.rb +35 -0
- data/lib/right_publish.rb +97 -0
- data/right_publish.gemspec +101 -0
- data/spec/repo_manager_spec.rb +65 -0
- data/spec/repo_spec.rb +74 -0
- data/spec/repos/apt_spec.rb +288 -0
- data/spec/repos/gem_spec.rb +79 -0
- data/spec/repos/yum_spec.rb +282 -0
- data/spec/spec_helper.rb +30 -0
- data/spec/storage_manager_spec.rb +71 -0
- data/spec/storage_spec.rb +317 -0
- data/spec/stores/local_spec.rb +56 -0
- data/spec/stores/s3_spec.rb +44 -0
- metadata +274 -0
@@ -0,0 +1,178 @@
|
|
1
|
+
module RightPublish
|
2
|
+
class AptRepo
|
3
|
+
include RightPublish::Repo
|
4
|
+
|
5
|
+
DEFAULT_APT_AUTO = true
|
6
|
+
DEFAULT_APT_DIR = 'apt/'
|
7
|
+
DEFAULT_DESCRIPTION = "RightScale RightLink Repository"
|
8
|
+
REPO_KEY = :apt_repo
|
9
|
+
|
10
|
+
REPO_OPTIONS = {
|
11
|
+
:dists=>:addr_optional,
|
12
|
+
:description => DEFAULT_DESCRIPTION,
|
13
|
+
:auto=>DEFAULT_APT_AUTO,
|
14
|
+
:subdir=>DEFAULT_APT_DIR,
|
15
|
+
:gpg_key_id => :attr_optional,
|
16
|
+
:gpg_password => :attr_optional }
|
17
|
+
|
18
|
+
BIN_EXTENSION = 'deb'
|
19
|
+
SRC_EXTENSION = 'dsc'
|
20
|
+
BIN_ALL_ARCH = 'all'
|
21
|
+
|
22
|
+
@@supported_archs = [ 'i386', 'amd64' ]
|
23
|
+
@@all_archs = [ 'i386', 'amd64', BIN_ALL_ARCH ]
|
24
|
+
|
25
|
+
def publish(file_or_dir, target)
|
26
|
+
repo_updates = {}
|
27
|
+
|
28
|
+
# If we specified a target, let's make sure it's in our profile
|
29
|
+
if target
|
30
|
+
fail("specified distribution is not listed in this profile!") unless repo_config[:dists].include?(target)
|
31
|
+
end
|
32
|
+
|
33
|
+
pkgs = []
|
34
|
+
get_pkg_list(file_or_dir, [BIN_EXTENSION, SRC_EXTENSION]) do |path|
|
35
|
+
if path.end_with? BIN_EXTENSION and repo_config[:auto]
|
36
|
+
arch = AptRepo.pkg_parts(path)[:arch] || fail("could not determine architecture for package: #{path}")
|
37
|
+
fail("you probably want to specify a distribution target for binary packages!") unless target || arch == BIN_ALL_ARCH
|
38
|
+
end
|
39
|
+
pkgs << path
|
40
|
+
end
|
41
|
+
|
42
|
+
# Synchronize the upstream source to our local cache
|
43
|
+
fetch
|
44
|
+
|
45
|
+
repo_path = File.join(Profile.config[:local_storage][:cache_dir], repo_config[:subdir])
|
46
|
+
write_conf_file repo_path if repo_config[:auto]
|
47
|
+
|
48
|
+
pkgs.each do |pkg|
|
49
|
+
Profile.log("Adding package [#{pkg}]...")
|
50
|
+
if repo_config[:auto]
|
51
|
+
# If source package, remove existing sources so we don't get a hash clash
|
52
|
+
auto_repo_remove(repo_path, pkg, target)
|
53
|
+
auto_repo_add(repo_path, pkg, target)
|
54
|
+
else
|
55
|
+
trivial_repo_add(pkg)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
rebuild_index(repo_config[:subdir]) if not repo_config[:auto]
|
59
|
+
|
60
|
+
# Commit back to remote storage
|
61
|
+
store
|
62
|
+
end
|
63
|
+
|
64
|
+
def self.pkg_parts(pkg_name)
|
65
|
+
result = {:name=>nil, :version=>nil, :arch=>nil, :ext=>nil}
|
66
|
+
if pkg_name.end_with?(SRC_EXTENSION)
|
67
|
+
if /([A-Za-z0-9\.\-+]+)_([A-Za-z0-9\-\.:~]+)\.#{SRC_EXTENSION}\Z/.match(pkg_name)
|
68
|
+
result[:name] = $1
|
69
|
+
result[:version] = $2
|
70
|
+
result[:ext] = SRC_EXTENSION
|
71
|
+
end
|
72
|
+
else
|
73
|
+
if /([A-Za-z0-9\.\-+]+)_([A-Za-z0-9\-\.:~]+)_(#{@@all_archs.join('|')})\.#{BIN_EXTENSION}\Z/.match(pkg_name)
|
74
|
+
result[:name] = $1
|
75
|
+
result[:version] = $2
|
76
|
+
result[:arch] = $3
|
77
|
+
result[:ext] = BIN_EXTENSION
|
78
|
+
end
|
79
|
+
end
|
80
|
+
result
|
81
|
+
end
|
82
|
+
|
83
|
+
private
|
84
|
+
|
85
|
+
def auto_repo_add(repo_path, pkg, target)
|
86
|
+
targets = (target && Array(target)) || repo_config[:dists]
|
87
|
+
targets.each do |t|
|
88
|
+
sub_command = (pkg.end_with?(BIN_EXTENSION) && 'includedeb') || 'includedsc'
|
89
|
+
ask_passphrase = (repo_config[:gpg_key_id]) ? "--ask-passphrase " : ""
|
90
|
+
cmd = "reprepro #{ask_passphrase}-C main -b #{repo_path} #{sub_command} #{t} #{pkg} 2>&1 >/dev/null"
|
91
|
+
if repo_config[:gpg_key_id]
|
92
|
+
exited = shellout_with_password(cmd)
|
93
|
+
else
|
94
|
+
exited = system(cmd)
|
95
|
+
end
|
96
|
+
|
97
|
+
raise RuntimeError, "apt package installation failed; cannot continue publishing" unless exited
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
def auto_repo_remove(repo_path, pkg, target)
|
102
|
+
sub_command = (pkg.end_with?(BIN_EXTENSION) && 'remove') || 'removesrc'
|
103
|
+
pkg_name = AptRepo.pkg_parts(pkg)[:name]
|
104
|
+
Profile.log("Removing any existing files for #{pkg_name}")
|
105
|
+
|
106
|
+
targets = (target && Array(target)) || repo_config[:dists]
|
107
|
+
targets.each do |t|
|
108
|
+
ask_passphrase = (repo_config[:gpg_key_id]) ? "--ask-passphrase" : ""
|
109
|
+
cmd = "reprepro #{ask_passphrase} -b #{repo_path} #{sub_command} #{t} #{pkg_name} 2>&1 >/dev/null"
|
110
|
+
if repo_config[:gpg_key_id]
|
111
|
+
exited = shellout_with_password(cmd)
|
112
|
+
else
|
113
|
+
exited = system(cmd)
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
def rebuild_index(subdir)
|
119
|
+
Profile.log("Rebuilding repository index...")
|
120
|
+
do_in_subdir(subdir) do
|
121
|
+
indexed = system("dpkg-scanpackages binaries /dev/null 2>/dev/null | gzip -c9 > Packages.gz")
|
122
|
+
raise RuntimeError, "apt package installation failed; cannot continue publishing" unless indexed
|
123
|
+
indexed = system("dpkg-scansources sources /dev/null 2>/dev/null | gzip -c9 > Sources.gz")
|
124
|
+
raise RuntimeError, "apt package installation failed; cannot continue publishing" unless indexed
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
def trivial_repo_add(pkg)
|
129
|
+
pkg_info = AptRepo.pkg_parts(pkg)
|
130
|
+
|
131
|
+
if pkg_info[:ext].eql?(BIN_EXTENSION)
|
132
|
+
sub_dir = File.join(repo_config[:subdir], 'binaries')
|
133
|
+
else
|
134
|
+
pkg_name = pkg_info[:name]
|
135
|
+
src_file = Dir.glob("#{File.dirname(pkg)}/#{pkg_name}_*.orig.tar.gz")
|
136
|
+
raise RuntimeError, "could not find original source for #{pkg}, missing or ambiguous." unless src_file.size == 1
|
137
|
+
|
138
|
+
diff_file = pkg.sub(/#{SRC_EXTENSION}$/, 'diff.gz')
|
139
|
+
sub_dir = File.join(repo_config[:subdir], 'sources')
|
140
|
+
raise RuntimeError, "missing the debian diff file for #{pkg}." unless File.file?(diff_file)
|
141
|
+
|
142
|
+
do_in_subdir(sub_dir) { prune_all("#{pkg_info[:name]}*.orig.tar.gz") }
|
143
|
+
install_file(src_file[0], sub_dir)
|
144
|
+
|
145
|
+
do_in_subdir(sub_dir) { prune_all("#{pkg_info[:name]}*.diff.gz") }
|
146
|
+
install_file(diff_file, sub_dir)
|
147
|
+
end
|
148
|
+
|
149
|
+
do_in_subdir(sub_dir) { prune_all("#{pkg_info[:name]}_*#{pkg_info[:arch]}.#{pkg_info[:ext]}") }
|
150
|
+
install_file(pkg, sub_dir)
|
151
|
+
end
|
152
|
+
|
153
|
+
def write_conf_file(repo_path)
|
154
|
+
destination_dir = File.expand_path(File.join(repo_path, 'conf'))
|
155
|
+
|
156
|
+
FileUtils.mkdir_p destination_dir
|
157
|
+
config_file = open( File.join(destination_dir, 'distributions'), 'wb' )
|
158
|
+
|
159
|
+
repo_config[:dists].each { |dist| config_file << dist_conf(dist) }
|
160
|
+
config_file.close
|
161
|
+
end
|
162
|
+
|
163
|
+
def dist_conf(dist)
|
164
|
+
<<EOF
|
165
|
+
Origin: RightScale
|
166
|
+
Codename: #{dist}
|
167
|
+
Version: 1.0
|
168
|
+
Architectures: #{@@supported_archs.join(' ')} source
|
169
|
+
Components: main
|
170
|
+
Description: #{repo_config[:description]}
|
171
|
+
#{(repo_config[:gpg_key_id] && "SignWith: #{repo_config[:gpg_key_id]}") || nil}
|
172
|
+
|
173
|
+
EOF
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
RightPublish::RepoManager.register_repo(AptRepo)
|
178
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
module RightPublish
|
2
|
+
class GemRepo
|
3
|
+
include RightPublish::Repo
|
4
|
+
|
5
|
+
DEFAULT_GEM_DIR = 'gems/'
|
6
|
+
REPO_KEY = :gem_repo
|
7
|
+
REPO_OPTIONS = {:subdir=>DEFAULT_GEM_DIR}
|
8
|
+
|
9
|
+
GEMS_SUBDIRECTORY = 'gems'
|
10
|
+
GEM_EXT = 'gem'
|
11
|
+
|
12
|
+
def publish(file_or_dir, target)
|
13
|
+
gems_list = get_pkg_list(file_or_dir, GEM_EXT)
|
14
|
+
|
15
|
+
# Synchronize the upstream source to our local cache
|
16
|
+
fetch
|
17
|
+
|
18
|
+
# Copy gem files to the repository
|
19
|
+
destination = File.join(repo_config[:subdir], GEMS_SUBDIRECTORY)
|
20
|
+
gems_list.each do |path|
|
21
|
+
do_in_subdir(destination) { prune_all("#{pkg_parts(path)[:name]}-*.gem") }
|
22
|
+
install_file(path, destination)
|
23
|
+
end
|
24
|
+
|
25
|
+
# Rebuild the gem index
|
26
|
+
Profile.log("Rebuilding Gem Index...")
|
27
|
+
do_in_subdir(repo_config[:subdir]) do
|
28
|
+
indexed = system('gem generate_index 2>&1 >/dev/null') # requires 'builder' gem to be installed
|
29
|
+
raise Exception, "gem generate_index failed; cannot continue publishing" unless indexed
|
30
|
+
end
|
31
|
+
|
32
|
+
# Commit back to remote storage
|
33
|
+
store
|
34
|
+
end
|
35
|
+
|
36
|
+
def pkg_parts(name)
|
37
|
+
result = {:name=>nil, :version=>nil}
|
38
|
+
result[:name], result[:version] = /([A-Za-z0-9\-_]+)-([0-9\.]+)\.#{GEM_EXT}\Z/.match(name).captures
|
39
|
+
result
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
RightPublish::RepoManager.register_repo(GemRepo)
|
44
|
+
end
|
@@ -0,0 +1,167 @@
|
|
1
|
+
|
2
|
+
|
3
|
+
module RightPublish
|
4
|
+
|
5
|
+
class YumRepo
|
6
|
+
include RightPublish::Repo
|
7
|
+
|
8
|
+
DEFAULT_YUM_EPEL = 1
|
9
|
+
DEFAULT_YUM_DIR = 'yum/'
|
10
|
+
DEFAULT_DESCRIPTION = "RightScale RightLink Repository"
|
11
|
+
REPO_KEY = :yum_repo
|
12
|
+
DEFAULT_EPEL_LAYOUT = "rightlink"
|
13
|
+
REPO_OPTIONS = {
|
14
|
+
:dists => :attr_optional,
|
15
|
+
:epel => DEFAULT_YUM_EPEL,
|
16
|
+
:epel_layout => DEFAULT_EPEL_LAYOUT,
|
17
|
+
:subdir => DEFAULT_YUM_DIR,
|
18
|
+
:description => DEFAULT_DESCRIPTION,
|
19
|
+
:gpg_key_id => :attr_optional,
|
20
|
+
:gpg_password => :attr_optional }
|
21
|
+
YUM_EXT = 'rpm'
|
22
|
+
|
23
|
+
BIN_ALL_ARCH = 'noarch'
|
24
|
+
SRC_ALL_ARCH = 'src'
|
25
|
+
SRC_ALL_PATH = 'SRPMS'
|
26
|
+
|
27
|
+
ARCHITECTURES = [ 'i386', 'x86_64' ]
|
28
|
+
PKG_TYPES = [ ARCHITECTURES, BIN_ALL_ARCH, SRC_ALL_ARCH ]
|
29
|
+
|
30
|
+
def publish(file_or_dir, target)
|
31
|
+
repo_updates = {}
|
32
|
+
|
33
|
+
get_pkg_list(file_or_dir, YUM_EXT) do |path|
|
34
|
+
# Determine package architectures, and sort
|
35
|
+
appropriate_repos(path, target) do |repo_path|
|
36
|
+
repo_path = File.join(repo_config[:subdir], repo_path)
|
37
|
+
repo_updates[repo_path] ||= []
|
38
|
+
repo_updates[repo_path].push(path)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
return if repo_updates.size == 0
|
42
|
+
|
43
|
+
# Synchronize the upstream source to our local cache
|
44
|
+
fetch
|
45
|
+
|
46
|
+
full_pkg_paths = []
|
47
|
+
repo_updates.each_pair do |repo_path, pkgs|
|
48
|
+
pkgs.each do |pkg|
|
49
|
+
import_pkg( pkg, repo_path )
|
50
|
+
full_pkg_paths << File.join(repo_path, File.basename(pkg))
|
51
|
+
end
|
52
|
+
end
|
53
|
+
sign_files full_pkg_paths
|
54
|
+
|
55
|
+
# Rebuild the yum index'
|
56
|
+
repo_updates.each_key do |repo_path|
|
57
|
+
regen_metadata( repo_path )
|
58
|
+
end
|
59
|
+
|
60
|
+
# Commit back to remote storage
|
61
|
+
store
|
62
|
+
end
|
63
|
+
|
64
|
+
def import_pkg( pkg, repo_path )
|
65
|
+
# Remove any instances of this package at a different versionh
|
66
|
+
yum_prune( pkg, repo_path )
|
67
|
+
install_file( pkg, repo_path )
|
68
|
+
end
|
69
|
+
|
70
|
+
def sign_files( pkgs )
|
71
|
+
if repo_config[:gpg_key_id]
|
72
|
+
do_in_subdir('') do
|
73
|
+
|
74
|
+
cmd = "rpm --define '%_gpg_name #{repo_config[:gpg_key_id]}' --addsign #{pkgs.join(' ')} 2>&1 >/dev/null"
|
75
|
+
exited = shellout_with_password(cmd)
|
76
|
+
|
77
|
+
raise Exception, "rpm signing failed; cannot continue publishing" unless exited
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def pkg_parts(path)
|
83
|
+
@infocache ||= {}
|
84
|
+
return @infocache[path] if @infocache.has_key?(path)
|
85
|
+
|
86
|
+
result = {:name=>nil, :version=>nil, :arch=>nil, :release=>nil}
|
87
|
+
|
88
|
+
query = IO.popen("rpm --queryformat '%{NAME} %{VERSION} %{RELEASE} %{ARCH}' -qp #{path}")
|
89
|
+
result[:name], result[:version], result[:release], result[:arch] = query.readline.split(' ')
|
90
|
+
|
91
|
+
# RPM query doesn't identify package as a source package, we just
|
92
|
+
# have to test the filename.
|
93
|
+
result[:arch] = SRC_ALL_ARCH if path.end_with?('src.rpm')
|
94
|
+
|
95
|
+
@infocache[path] = result
|
96
|
+
result
|
97
|
+
end
|
98
|
+
|
99
|
+
private
|
100
|
+
|
101
|
+
def repo_dir(epel_ver, dist, dist_ver, arch)
|
102
|
+
if repo_config[:epel_layout] == "rightscale-software"
|
103
|
+
File.join(dist_ver, arch)
|
104
|
+
else
|
105
|
+
File.join(epel_ver.to_s, dist, dist_ver, arch)
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def appropriate_repos(path, target)
|
110
|
+
unless repo_config[:epel]
|
111
|
+
# This is a flat repo
|
112
|
+
yield ''
|
113
|
+
return
|
114
|
+
end
|
115
|
+
|
116
|
+
pkg_arch = pkg_parts(path)[:arch] || fail("could not determine architecture for package: #{path}")
|
117
|
+
|
118
|
+
if target
|
119
|
+
#TODO Check that the specified dist is configured in the profile (sanity)
|
120
|
+
dist, dist_ver = target.split("/")
|
121
|
+
fail("dist format needs to be in form 'dist/dist_version'!") unless dist && dist_ver
|
122
|
+
dists = {dist => [dist_ver]}
|
123
|
+
else
|
124
|
+
unless [BIN_ALL_ARCH,SRC_ALL_ARCH].include? pkg_arch
|
125
|
+
fail("need to specify a distribution with binary packages!")
|
126
|
+
end
|
127
|
+
dists = repo_config[:dists]
|
128
|
+
end
|
129
|
+
|
130
|
+
dists.each_pair do |dist,dist_vers|
|
131
|
+
dist_vers.each do |ver|
|
132
|
+
# Source RPMS stored in SRPMS architecture, noarchs go to all architectures
|
133
|
+
case pkg_arch
|
134
|
+
when SRC_ALL_ARCH
|
135
|
+
[SRC_ALL_PATH]
|
136
|
+
when BIN_ALL_ARCH
|
137
|
+
ARCHITECTURES
|
138
|
+
else
|
139
|
+
[pkg_arch]
|
140
|
+
end.each { |arch| yield repo_dir(repo_config[:epel].to_s, dist.to_s, ver.to_s, arch) }
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
def regen_metadata(repo_path)
|
146
|
+
# Rebuild the yum index
|
147
|
+
Profile.log("Rebuilding Yum Repo [#{repo_path}]...")
|
148
|
+
do_in_subdir(repo_path) do
|
149
|
+
exit_val = system('createrepo --update -o $(pwd) $(pwd) 2>&1 >/dev/null')
|
150
|
+
raise Exception, "yum regen_metadata failed; cannot continue publishing" unless exit_val
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
def yum_prune(pkg, repo_path)
|
155
|
+
our_name = pkg_parts(pkg)[:name]
|
156
|
+
our_arch = pkg_parts(pkg)[:arch]
|
157
|
+
|
158
|
+
do_in_subdir(repo_path) do
|
159
|
+
Dir.glob("*.#{YUM_EXT}") do |rpm|
|
160
|
+
File.unlink(rpm) if pkg_parts(rpm)[:name] == our_name && pkg_parts(rpm)[:arch] == our_arch
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
166
|
+
RightPublish::RepoManager.register_repo(YumRepo)
|
167
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
require 'right_publish/profile'
|
2
|
+
|
3
|
+
module RightPublish
|
4
|
+
class StorageManager
|
5
|
+
@@storage_type_regex = /\A(\w+)_storage/
|
6
|
+
@@storage_table = {}
|
7
|
+
|
8
|
+
def self.get_storage(type)
|
9
|
+
@@storage_table[type] if @@storage_table[type]
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.storage_types()
|
13
|
+
type_hash = {}
|
14
|
+
@@storage_table.each_key { |k| type_hash[@@storage_type_regex.match(k.to_s)[1]] = k }
|
15
|
+
type_hash
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.register_storage(module_type)
|
19
|
+
storage_key = module_type::STORAGE_KEY
|
20
|
+
|
21
|
+
if module_type.respond_to?(:get_directories) && @@storage_type_regex.match(storage_key.to_s)
|
22
|
+
@@storage_table[storage_key] = module_type
|
23
|
+
RightPublish::Profile.instance.register_section(@@storage_type_regex.match(storage_key.to_s)[1], module_type::STORAGE_OPTIONS)
|
24
|
+
else
|
25
|
+
raise TypeError
|
26
|
+
end
|
27
|
+
nil
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
module Storage
|
32
|
+
def self.sync_dirs(src, dest, options={})
|
33
|
+
src_dir = src.get_directories()
|
34
|
+
dest_dir = dest.get_directories()
|
35
|
+
|
36
|
+
src_map = get_etag_map(src_dir, options) {|file| file.etag}
|
37
|
+
dest_map = get_etag_map(dest_dir, options)
|
38
|
+
|
39
|
+
dest_map.each_pair do |hash, files|
|
40
|
+
files.each do |path, file|
|
41
|
+
unless src_map.include? hash and src_map[hash].include? path
|
42
|
+
Profile.log("Removing: #{path}", :debug)
|
43
|
+
file.destroy
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end if options[:sweep]
|
47
|
+
|
48
|
+
src_map.each_pair do |hash, files|
|
49
|
+
# If we already have this data, just copy it, otherwise
|
50
|
+
# sync down a copy and make copies of that.
|
51
|
+
if dest_map.include? hash
|
52
|
+
local_copy = dest_map[hash].first.last
|
53
|
+
else
|
54
|
+
remote_file = files.shift
|
55
|
+
Profile.log("Synchronizing: #{remote_file.first}", :debug)
|
56
|
+
local_copy = dest_dir.files.create(:key=>remote_file.first, :body=>remote_file.last.body, :acl=>'public-read')
|
57
|
+
end
|
58
|
+
|
59
|
+
files.each_key do |file|
|
60
|
+
unless dest_map.include? hash and dest_map[hash].include? file
|
61
|
+
Profile.log("Duplicating: #{file}", :debug)
|
62
|
+
local_copy.copy(dest_dir.key, file, 'x-amz-acl'=>'public-read')
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
private
|
69
|
+
|
70
|
+
def self.get_etag_map(dir, options={})
|
71
|
+
hash_map = {}
|
72
|
+
dir.files.each do |file|
|
73
|
+
next if file.key[-1,1] == '/'
|
74
|
+
next if file.key[0,options[:subdir].size] != options[:subdir] if options.has_key? :subdir
|
75
|
+
|
76
|
+
md5 = dir.compute_md5(file)
|
77
|
+
|
78
|
+
if hash_map.has_key? md5
|
79
|
+
hash_map[md5][file.key] = file
|
80
|
+
else
|
81
|
+
hash_map[md5] = {file.key=>file}
|
82
|
+
end
|
83
|
+
end
|
84
|
+
hash_map
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
require 'right_publish/stores/local'
|
90
|
+
require 'right_publish/stores/s3'
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'fog'
|
2
|
+
require 'digest/md5'
|
3
|
+
|
4
|
+
module RightPublish
|
5
|
+
module LocalStorage
|
6
|
+
|
7
|
+
DEFAULT_LOCAL_CACHEDIR = '~/.rp_cache'
|
8
|
+
STORAGE_KEY = :local_storage
|
9
|
+
STORAGE_OPTIONS = { :cache_dir=>DEFAULT_LOCAL_CACHEDIR}
|
10
|
+
def compute_md5(file)
|
11
|
+
Digest::MD5.file(File.join(service.path_to(self.key), file.key)).hexdigest
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.get_directories()
|
15
|
+
Profile.log("Connecting to local cache.", :debug)
|
16
|
+
conn = Fog::Storage.new(
|
17
|
+
:provider=>"Local",
|
18
|
+
:local_root=>Profile.config[STORAGE_KEY][:cache_dir] )
|
19
|
+
|
20
|
+
Profile.log("Attaching to local cache: [#{Profile.config[STORAGE_KEY][:cache_dir]}].", :debug)
|
21
|
+
conn.directories.create(:key=>'.') unless File.exists? Profile.config[STORAGE_KEY][:cache_dir]
|
22
|
+
local_dir = conn.directories.get('.')
|
23
|
+
local_dir.extend(LocalStorage)
|
24
|
+
local_dir
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
RightPublish::StorageManager.register_storage(LocalStorage)
|
29
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require 'fog'
|
2
|
+
|
3
|
+
module RightPublish
|
4
|
+
module S3Storage
|
5
|
+
|
6
|
+
STORAGE_KEY = :s3_storage
|
7
|
+
STORAGE_OPTIONS = {
|
8
|
+
:access_id => :attr_optional,
|
9
|
+
:access_key => :attr_optional,
|
10
|
+
:region => :attr_optional,
|
11
|
+
:remote_path => :attr_needed,
|
12
|
+
}
|
13
|
+
|
14
|
+
def compute_md5(file)
|
15
|
+
file.etag
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.get_directories()
|
19
|
+
Profile.log("Connecting to S3.", :debug)
|
20
|
+
conn = Fog::Storage.new(
|
21
|
+
:provider => "AWS",
|
22
|
+
:aws_access_key_id => Profile.config[:remote_storage][:access_id],
|
23
|
+
:aws_secret_access_key => Profile.config[:remote_storage][:access_key],
|
24
|
+
:region => Profile.config[:remote_storage][:region]
|
25
|
+
)
|
26
|
+
|
27
|
+
Profile.log("Attaching to bucket: [#{Profile.config[:remote_storage][:remote_path]}].", :debug)
|
28
|
+
aws_bucket = conn.directories.get(Profile.config[:remote_storage][:remote_path])
|
29
|
+
aws_bucket.extend(S3Storage)
|
30
|
+
aws_bucket
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
RightPublish::StorageManager.register_storage(S3Storage)
|
35
|
+
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
#
|
2
|
+
# === Synopsis:
|
3
|
+
# RightScale RightPublish (right_publish) - (c) 2013 RightScale Inc
|
4
|
+
#
|
5
|
+
|
6
|
+
require 'trollop'
|
7
|
+
|
8
|
+
require 'right_publish/profile'
|
9
|
+
require 'right_publish/repo'
|
10
|
+
|
11
|
+
module RightPublish
|
12
|
+
module Main
|
13
|
+
REPOSITORY_TYPES = RepoManager.repo_types()
|
14
|
+
SUB_COMMANDS = %w(publish fetch store)
|
15
|
+
|
16
|
+
def self.run()
|
17
|
+
options = parse_args
|
18
|
+
|
19
|
+
begin
|
20
|
+
profile = RightPublish::Profile.instance
|
21
|
+
profile.load(options[:profile])
|
22
|
+
rescue LoadError => e
|
23
|
+
puts e
|
24
|
+
exit -1
|
25
|
+
end
|
26
|
+
|
27
|
+
# These options override profile attributes.
|
28
|
+
profile.settings[:local_storage][:cache_dir] = options[:local_cache] if options[:local_cache]
|
29
|
+
profile.settings[:verbose] = true if options[:verbose]
|
30
|
+
|
31
|
+
# We already know this is a valid type, parse_args checked
|
32
|
+
repo = RepoManager.get_repository(REPOSITORY_TYPES[options[:repo_type]])
|
33
|
+
begin
|
34
|
+
case options[:cmd]
|
35
|
+
when "publish"
|
36
|
+
repo.publish(options[:files], options[:dist])
|
37
|
+
when "store"
|
38
|
+
repo.store
|
39
|
+
when "fetch"
|
40
|
+
repo.fetch
|
41
|
+
else
|
42
|
+
end
|
43
|
+
rescue RuntimeError => e
|
44
|
+
RightPublish::Profile.log("Fatal Error:\n\t#{e}", :error)
|
45
|
+
exit -1
|
46
|
+
end
|
47
|
+
RightPublish::Profile.log("Success!")
|
48
|
+
end
|
49
|
+
|
50
|
+
def self.parse_args()
|
51
|
+
options = Trollop.options do
|
52
|
+
version "RightPublish alpha (c) 2013 RightScale Inc"
|
53
|
+
banner <<-EOS
|
54
|
+
RightPublish can manage a YUM/APT/RubyGem repository in remote storage, e.g. S3.
|
55
|
+
|
56
|
+
Usage:
|
57
|
+
right_publish [global_options] <command> [file1, file2, ...]
|
58
|
+
|
59
|
+
commands:
|
60
|
+
fetch
|
61
|
+
publish
|
62
|
+
store
|
63
|
+
|
64
|
+
global options:
|
65
|
+
EOS
|
66
|
+
|
67
|
+
opt :local_cache, "Local cache location", :type => String
|
68
|
+
opt :profile, "Publish profile", :type => String
|
69
|
+
opt :repo_type, "Repository type: #{REPOSITORY_TYPES.keys.inspect}", :type => String
|
70
|
+
opt :dist, "Target distribution. Required for binary packages. If unspecified for noarch and source packages, will copy to all distributions specified in profile.", :type => String
|
71
|
+
opt :verbose, "Verbose output"
|
72
|
+
|
73
|
+
stop_on SUB_COMMANDS
|
74
|
+
end
|
75
|
+
|
76
|
+
options[:cmd]= ARGV.shift
|
77
|
+
options[:files] = expand_argv_globs
|
78
|
+
|
79
|
+
Trollop.die "argument --profile is required" unless options[:profile]
|
80
|
+
Trollop.die "argument --repo-type is required" unless options[:repo_type]
|
81
|
+
Trollop.die "profile does not exist: #{options[:profile]}" unless File.exist?(options[:profile])
|
82
|
+
Trollop.die "invalid repository type: #{options[:repo_type]}" unless REPOSITORY_TYPES[options[:repo_type]]
|
83
|
+
|
84
|
+
options
|
85
|
+
end
|
86
|
+
|
87
|
+
def self.expand_argv_globs
|
88
|
+
files = []
|
89
|
+
|
90
|
+
ARGV.each do |glob|
|
91
|
+
files += Dir.glob(glob)
|
92
|
+
end
|
93
|
+
|
94
|
+
files
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|