yum_s3_sync 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +14 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +114 -0
- data/Rakefile +1 -0
- data/bin/yums3sync +35 -0
- data/bin/yums3syncdir +87 -0
- data/lib/yum_s3_sync/http_downloader.rb +36 -0
- data/lib/yum_s3_sync/repo_syncer.rb +50 -0
- data/lib/yum_s3_sync/s3_deleter.rb +24 -0
- data/lib/yum_s3_sync/s3_downloader.rb +27 -0
- data/lib/yum_s3_sync/s3_file_lister.rb +23 -0
- data/lib/yum_s3_sync/s3_uploader.rb +37 -0
- data/lib/yum_s3_sync/version.rb +3 -0
- data/lib/yum_s3_sync/yum_repository.rb +139 -0
- data/lib/yum_s3_sync.rb +12 -0
- data/yum_s3_sync.gemspec +25 -0
- metadata +119 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 23ccfe9d584a3f0764dc5af307715ac8695aa64c
|
4
|
+
data.tar.gz: ca3c0fdbb706a4efd312644b00928c00db287a77
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 0e56e5c4af3f7f23beed8560af4aa849e0197752bd2251f1f3ea67b62eeb349dc31d57c8048a191785bd2a71d2eb0f497385d3e0b3cd3fef011802b34839e64c
|
7
|
+
data.tar.gz: bcb588bba36f39bf2eba56e0b240ef32c2f97b000bec75e3b77948ba13cf557c8a920ffdd859ece47aa4f289ef87e8cd78e451f6bbac997428f8cdec68dcf89a
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2014 Hein-Pieter van Braam
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,114 @@
|
|
1
|
+
# YumS3Sync
|
2
|
+
|
3
|
+
Synchronize a Yum repository over HTTP to S3. This can be useful when you want to mirror Internet Yum repositories 'locally' or want to run the synchronization from within EC2 rather than pushing it from outsede.
|
4
|
+
|
5
|
+
There are two command line tools supplied:
|
6
|
+
|
7
|
+
yums3sync - Synchronizes a single Yum repository to a bucket:/prefix
|
8
|
+
yums3syncdir - Scan a directory of links for repositories then synchronize all of them to bucket:/prefix
|
9
|
+
|
10
|
+
YumS3sync is smart enough to only copy files changes between runs, and does almost nothing if nothing changed on the source end.
|
11
|
+
|
12
|
+
Syncdir is effectively a wrapper around sync to make it a little easier to synchronize a large amount of repositories without having to know exactly which ones you're synchronizing. I use it to synchronize internal Yum repositories made by other developers to AWS without having to make configuration changes.
|
13
|
+
|
14
|
+
After the syncronization you can use the repository using Yum with an S3 plugin or configure a webserver to act as a proxy between S3 and Yum. One of these two options is required if you do not want to make your S3 bucket internet-readable.
|
15
|
+
|
16
|
+
## Installation
|
17
|
+
|
18
|
+
Clone this git repository and build a Gem. The program should work without additional dependencies on RHEL/Centos 6 with EPEL enabled. It is tested on Ruby 1.8.7 and 2.1
|
19
|
+
|
20
|
+
## Usage
|
21
|
+
|
22
|
+
### Yums3sync
|
23
|
+
|
24
|
+
```
|
25
|
+
Usage: yums3sync [options]
|
26
|
+
-s, --source SOURCE HTTP source URL
|
27
|
+
-b, --bucket BUCKET Target bucket name
|
28
|
+
-p, --prefix PREFIX Target bucket prefix
|
29
|
+
```
|
30
|
+
|
31
|
+
Example usage:
|
32
|
+
|
33
|
+
```
|
34
|
+
# yums3sync --source http://mrepo.corporate.com/mrepo/rhel6/x86_64/epel6/ --bucket my-org-repository --prefix rhel6/x86_64/epel6
|
35
|
+
```
|
36
|
+
|
37
|
+
This will make the contents of the 'epel6' repository available under ``my-org-repository::/rhel6/x86_64/epel6``. Running the command again will copy only changes, if any.
|
38
|
+
|
39
|
+
### Yums3syncdir
|
40
|
+
|
41
|
+
```
|
42
|
+
Usage: yums3syncdir [options]
|
43
|
+
-s, --source SOURCEDIR HTTP source base URL
|
44
|
+
-b, --bucket BUCKET Target bucket name
|
45
|
+
-p, --prefix PREFIX Target bucket prefix
|
46
|
+
-x, --exclude prefix1,prefix2 Exclude prefixes
|
47
|
+
```
|
48
|
+
|
49
|
+
Example usage:
|
50
|
+
|
51
|
+
```
|
52
|
+
# yums3syncdir --source http://mrepo.corporate.com/mrepo/rhel6/x86_64/ --bucket my-org-repository --prefix rhel6/x86_64 --exclude disc1,iso
|
53
|
+
```
|
54
|
+
|
55
|
+
Assuming ``http://mrepo.corporate.com/mrepo/rhel6/x86_64/`` has an ``index.html`` listing other repositories this will synchronize all of them to ``my-org-repository:/rhel6/x86_64``
|
56
|
+
|
57
|
+
|
58
|
+
### Serving the repository
|
59
|
+
|
60
|
+
Serving the repository is easy with an Apache (or other) reverse proxy. We first configure the S3 bucket to only allow access from certain referrers. We will use this as a secret to authenticate our proxies to S3.
|
61
|
+
|
62
|
+
```
|
63
|
+
{
|
64
|
+
"Version": "2012-10-17",
|
65
|
+
"Statement": [
|
66
|
+
{
|
67
|
+
"Sid": "PublicReadGetObject",
|
68
|
+
"Effect": "Allow",
|
69
|
+
"Principal": "*",
|
70
|
+
"Action": [
|
71
|
+
"s3:List*",
|
72
|
+
"s3:Get*"
|
73
|
+
],
|
74
|
+
"Resource": "arn:aws:s3:::my-org-repository/*",
|
75
|
+
"Condition": {
|
76
|
+
"StringEquals": {
|
77
|
+
"aws:Referer": "MySecretString"
|
78
|
+
}
|
79
|
+
}
|
80
|
+
}
|
81
|
+
]
|
82
|
+
}
|
83
|
+
```
|
84
|
+
|
85
|
+
Next we configure Apache to serve up the repository through mod\_proxy.
|
86
|
+
|
87
|
+
```
|
88
|
+
<IfModule proxy_module>
|
89
|
+
<IfModule proxy_http_module>
|
90
|
+
<IfModule headers_module>
|
91
|
+
ProxyRequests off
|
92
|
+
ProxyErrorOverride On
|
93
|
+
|
94
|
+
# We use '/mrepo/' to match our internal repositories.
|
95
|
+
<LocationMatch "/mrepo/">
|
96
|
+
ProxyPass http://my-org-repository.s3-eu-west-1.amazonaws.com/
|
97
|
+
|
98
|
+
Header set Cache-Control "max-age=300, public"
|
99
|
+
RequestHeader set Referer MySecretString
|
100
|
+
</LocationMatch>
|
101
|
+
|
102
|
+
|
103
|
+
ErrorLog logs/error_log
|
104
|
+
LogLevel warn
|
105
|
+
|
106
|
+
# Don't log our secret in our logfiles
|
107
|
+
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{User-Agent}i\"" proxylogger
|
108
|
+
CustomLog logs/access_log proxylogger
|
109
|
+
</IfModule>
|
110
|
+
</IfModule>
|
111
|
+
</IfModule>
|
112
|
+
```
|
113
|
+
|
114
|
+
Optionally add an ELB in front of a couple of these and serve!
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require 'bundler/gem_tasks'
|
data/bin/yums3sync
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'optparse'
|
4
|
+
require 'yum_s3_sync'
|
5
|
+
|
6
|
+
options = {}
|
7
|
+
opt_parser = OptionParser.new do |opts|
|
8
|
+
opts.banner = 'Usage: yums3sync [options]'
|
9
|
+
|
10
|
+
opts.on('-s', '--source SOURCE', 'HTTP source URL') do |s|
|
11
|
+
options[:source_base] = s
|
12
|
+
end
|
13
|
+
opts.on('-b', '--bucket BUCKET', 'Target bucket name') do |b|
|
14
|
+
options[:target_bucket] = b
|
15
|
+
end
|
16
|
+
opts.on('-p', '--prefix PREFIX', 'Target bucket prefix') do |p|
|
17
|
+
options[:target_base] = p
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
opt_parser.parse!
|
22
|
+
|
23
|
+
if !options[:source_base] || !options[:target_bucket] || !options[:target_base]
|
24
|
+
puts opt_parser
|
25
|
+
exit 1
|
26
|
+
end
|
27
|
+
|
28
|
+
repo_syncer = YumS3Sync::RepoSyncer.new(options[:source_base], options[:target_bucket], options[:target_base])
|
29
|
+
|
30
|
+
begin
|
31
|
+
repo_syncer.sync
|
32
|
+
rescue StandardError => e
|
33
|
+
puts e.message
|
34
|
+
exit 1
|
35
|
+
end
|
data/bin/yums3syncdir
ADDED
@@ -0,0 +1,87 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'optparse'
|
4
|
+
require 'yum_s3_sync'
|
5
|
+
|
6
|
+
require 'net/http'
|
7
|
+
require 'uri'
|
8
|
+
require 'nokogiri'
|
9
|
+
|
10
|
+
options = {}
|
11
|
+
opt_parser = OptionParser.new do |opts|
|
12
|
+
opts.banner = 'Usage: yums3syncdir [options]'
|
13
|
+
|
14
|
+
opts.on('-s', '--source SOURCEDIR', 'HTTP source base URL') do |s|
|
15
|
+
options[:source_base] = s
|
16
|
+
end
|
17
|
+
|
18
|
+
opts.on('-b', '--bucket BUCKET', 'Target bucket name') do |b|
|
19
|
+
options[:target_bucket] = b
|
20
|
+
end
|
21
|
+
|
22
|
+
opts.on('-p', '--prefix PREFIX', 'Target bucket prefix') do |p|
|
23
|
+
options[:target_base] = p
|
24
|
+
end
|
25
|
+
|
26
|
+
opts.on('-x', '--exclude prefix1,prefix2', Array, 'Exclude prefixes') do |x|
|
27
|
+
options[:exclude] = x
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
opt_parser.parse!
|
32
|
+
|
33
|
+
if !options[:source_base] || !options[:target_bucket] || !options[:target_base]
|
34
|
+
puts opt_parser
|
35
|
+
exit 1
|
36
|
+
end
|
37
|
+
|
38
|
+
def parent?(path)
|
39
|
+
return true if path.start_with?('/')
|
40
|
+
return true if path.include?('../')
|
41
|
+
end
|
42
|
+
|
43
|
+
uri = URI(options[:source_base])
|
44
|
+
|
45
|
+
repositories = []
|
46
|
+
|
47
|
+
Net::HTTP.start(uri.host) do |http|
|
48
|
+
http.request_get(uri.path) do|response|
|
49
|
+
unless response.code.start_with?('2')
|
50
|
+
puts "Unable to download index from #{uri} : #{response.message}"
|
51
|
+
exit 1
|
52
|
+
end
|
53
|
+
|
54
|
+
if response['content-type'].start_with?('text/html')
|
55
|
+
response.read_body do |html|
|
56
|
+
doc = Nokogiri::HTML(html)
|
57
|
+
|
58
|
+
doc.css('html a').each do |link|
|
59
|
+
begin
|
60
|
+
link_uri = URI(link['href'])
|
61
|
+
rescue URI::InvalidURIError => e
|
62
|
+
puts "Invalid URI: #{link['href']} skipping"
|
63
|
+
next
|
64
|
+
end
|
65
|
+
|
66
|
+
next if link_uri.scheme && link_uri.host != uri.host
|
67
|
+
next if link_uri.query
|
68
|
+
next if parent?(link_uri.path)
|
69
|
+
next if options[:exclude].any? { |pattern| link_uri.path.start_with?(pattern) }
|
70
|
+
|
71
|
+
repositories.push link_uri.path
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
repositories.each do |repository|
|
80
|
+
repo_syncer = YumS3Sync::RepoSyncer.new(options[:source_base] + '/' + repository, options[:target_bucket], options[:target_base] + '/' + repository)
|
81
|
+
|
82
|
+
begin
|
83
|
+
repo_syncer.sync
|
84
|
+
rescue StandardError => e
|
85
|
+
puts "Error syncing #{repository} skipping : #{e.message}"
|
86
|
+
end
|
87
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'open-uri'
|
2
|
+
|
3
|
+
module YumS3Sync
|
4
|
+
class HTTPDownloader
|
5
|
+
def initialize(baseurl)
|
6
|
+
@baseurl = baseurl
|
7
|
+
end
|
8
|
+
|
9
|
+
def download(relative_url)
|
10
|
+
retries = 0
|
11
|
+
|
12
|
+
url = "#{@baseurl}/#{relative_url}"
|
13
|
+
puts "Downloading #{url}"
|
14
|
+
|
15
|
+
begin
|
16
|
+
open("#{url}")
|
17
|
+
rescue OpenURI::HTTPError => e
|
18
|
+
if e.io.status[0] == '404'
|
19
|
+
raise "File #{url} does not exist 404"
|
20
|
+
end
|
21
|
+
|
22
|
+
raise e
|
23
|
+
rescue SocketError => e
|
24
|
+
if retries < 10
|
25
|
+
retries += 1
|
26
|
+
puts "Error downloading #{url} : #{e.message} retry ##{retries}"
|
27
|
+
sleep(1)
|
28
|
+
retry
|
29
|
+
else
|
30
|
+
puts "Error downloading #{url} : #{e.message}"
|
31
|
+
raise e
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'yum_s3_sync'
|
4
|
+
|
5
|
+
module YumS3Sync
|
6
|
+
class RepoSyncer
|
7
|
+
def initialize(source_base, target_bucket, target_base)
|
8
|
+
@source_base = source_base
|
9
|
+
@target_bucket = target_bucket
|
10
|
+
@target_base = target_base
|
11
|
+
end
|
12
|
+
|
13
|
+
def sync
|
14
|
+
http_downloader = YumS3Sync::HTTPDownloader.new(@source_base)
|
15
|
+
source_repository = YumS3Sync::YumRepository.new(http_downloader)
|
16
|
+
|
17
|
+
s3_downloader = YumS3Sync::S3Downloader.new(@target_bucket, @target_base)
|
18
|
+
dest_repository = YumS3Sync::YumRepository.new(s3_downloader)
|
19
|
+
|
20
|
+
new_packages = source_repository.compare(dest_repository)
|
21
|
+
s3_uploader = YumS3Sync::S3Uploader.new(@target_bucket, @target_base, http_downloader)
|
22
|
+
|
23
|
+
metadata = []
|
24
|
+
source_repository.metadata.each do |_type, file|
|
25
|
+
metadata.push file[:href]
|
26
|
+
end
|
27
|
+
|
28
|
+
new_packages.each do |package|
|
29
|
+
s3_uploader.upload(package)
|
30
|
+
end
|
31
|
+
|
32
|
+
if !dest_repository.exists? || !new_packages.empty?
|
33
|
+
metadata.each do |file|
|
34
|
+
s3_uploader.upload(file)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
unless new_packages.empty?
|
39
|
+
s3_file_lister = YumS3Sync::S3FileLister.new(@target_bucket, @target_base)
|
40
|
+
s3_deleter = YumS3Sync::S3Deleter.new(@target_bucket, @target_base)
|
41
|
+
|
42
|
+
s3_file_lister.list.each do |file|
|
43
|
+
if !source_repository.packages[file] && !metadata.include?(file)
|
44
|
+
s3_deleter.delete(file)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'aws-sdk'
|
2
|
+
|
3
|
+
module YumS3Sync
|
4
|
+
class S3Deleter
|
5
|
+
def initialize(bucket, prefix)
|
6
|
+
@bucket = bucket
|
7
|
+
@prefix = prefix
|
8
|
+
end
|
9
|
+
|
10
|
+
def delete(file)
|
11
|
+
s3 = AWS::S3.new
|
12
|
+
|
13
|
+
target = "#{@prefix}/#{file}"
|
14
|
+
target.gsub!(/\/+/, '/')
|
15
|
+
|
16
|
+
dest_obj = s3.buckets[@bucket].objects[target]
|
17
|
+
|
18
|
+
if dest_obj.exists?
|
19
|
+
puts "Deleting #{@bucket}::#{target}"
|
20
|
+
dest_obj.delete
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'aws-sdk'
|
2
|
+
|
3
|
+
module YumS3Sync
|
4
|
+
class S3Downloader
|
5
|
+
def initialize(bucket, prefix)
|
6
|
+
@bucket = bucket
|
7
|
+
@prefix = prefix
|
8
|
+
end
|
9
|
+
|
10
|
+
def download(relative_url)
|
11
|
+
target = "#{@prefix}/#{relative_url}"
|
12
|
+
target.gsub!(/\/+/, '/')
|
13
|
+
|
14
|
+
puts "Downloading #{@bucket}::#{target}"
|
15
|
+
|
16
|
+
s3 = AWS::S3.new
|
17
|
+
file = s3.buckets[@bucket].objects[target]
|
18
|
+
|
19
|
+
begin
|
20
|
+
return StringIO.new(file.read)
|
21
|
+
rescue AWS::S3::Errors::NoSuchKey
|
22
|
+
end
|
23
|
+
|
24
|
+
nil
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'aws-sdk'
|
2
|
+
|
3
|
+
module YumS3Sync
|
4
|
+
class S3FileLister
|
5
|
+
def initialize(bucket, prefix)
|
6
|
+
@bucket = bucket
|
7
|
+
@prefix = prefix
|
8
|
+
end
|
9
|
+
|
10
|
+
def list
|
11
|
+
files = []
|
12
|
+
|
13
|
+
s3 = AWS::S3.new
|
14
|
+
s3.buckets[@bucket].objects.with_prefix(@prefix).each do |file|
|
15
|
+
basename = file.key.sub(/#{@prefix}\/*/, '')
|
16
|
+
|
17
|
+
files.push basename
|
18
|
+
end
|
19
|
+
|
20
|
+
files
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'aws-sdk'
|
2
|
+
|
3
|
+
module YumS3Sync
|
4
|
+
class S3Uploader
|
5
|
+
def initialize(bucket, prefix, downloader)
|
6
|
+
@bucket = bucket
|
7
|
+
@prefix = prefix
|
8
|
+
@downloader = downloader
|
9
|
+
end
|
10
|
+
|
11
|
+
def upload(file)
|
12
|
+
retries = 0
|
13
|
+
s3 = AWS::S3.new
|
14
|
+
|
15
|
+
source_file = @downloader.download(file)
|
16
|
+
target = "#{@prefix}/#{file}"
|
17
|
+
target.gsub!(/\/+/, '/')
|
18
|
+
dest_obj = s3.buckets[@bucket].objects[target]
|
19
|
+
dest_obj.delete if dest_obj.exists?
|
20
|
+
|
21
|
+
puts "Uploading #{@bucket}::#{target}"
|
22
|
+
begin
|
23
|
+
dest_obj.write(:file => source_file)
|
24
|
+
rescue StandardError => e
|
25
|
+
if retries < 10
|
26
|
+
retries += 1
|
27
|
+
puts "Error uploading #{@bucket}::#{target} : #{e.message} retry ##{retries}"
|
28
|
+
sleep(1)
|
29
|
+
retry
|
30
|
+
else
|
31
|
+
puts "Error uploading #{@bucket}::#{target} : #{e.message} giving up"
|
32
|
+
raise e
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,139 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'zlib'
|
4
|
+
require 'rexml/document'
|
5
|
+
require 'rexml/streamlistener'
|
6
|
+
|
7
|
+
module YumS3Sync
|
8
|
+
class YumRepository
|
9
|
+
attr_accessor :metadata
|
10
|
+
|
11
|
+
def initialize(downloader)
|
12
|
+
@downloader = downloader
|
13
|
+
|
14
|
+
repomd_parser = RepModListener.new
|
15
|
+
repomd_file = @downloader.download('repodata/repomd.xml')
|
16
|
+
if repomd_file
|
17
|
+
REXML::Document.parse_stream(repomd_file, repomd_parser)
|
18
|
+
@metadata = repomd_parser.metadata
|
19
|
+
@metadata['repomd'] = { :href => 'repodata/repomd.xml' }
|
20
|
+
else
|
21
|
+
@metadata = { 'primary' => nil }
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def parse_packages
|
26
|
+
return {} unless @metadata['primary']
|
27
|
+
|
28
|
+
primary_file = @downloader.download(@metadata['primary'][:href])
|
29
|
+
return {} unless primary_file
|
30
|
+
|
31
|
+
gzstream = Zlib::GzipReader.new(primary_file)
|
32
|
+
package_parser = PackageListener.new
|
33
|
+
|
34
|
+
REXML::Document.parse_stream(gzstream, package_parser)
|
35
|
+
package_parser.packages
|
36
|
+
end
|
37
|
+
|
38
|
+
def packages
|
39
|
+
@parsed_packages ||= parse_packages
|
40
|
+
end
|
41
|
+
|
42
|
+
def compare(other)
|
43
|
+
diff_packages = []
|
44
|
+
|
45
|
+
if !other.metadata['primary'] || metadata['primary'][:checksum] != other.metadata['primary'][:checksum]
|
46
|
+
packages.each do |package, checksum|
|
47
|
+
if other.packages[package] != checksum
|
48
|
+
diff_packages.push package
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
diff_packages
|
54
|
+
end
|
55
|
+
|
56
|
+
def exists?
|
57
|
+
@metadata['primary']
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
class PackageListener
|
62
|
+
attr_accessor :packages
|
63
|
+
include REXML::StreamListener
|
64
|
+
|
65
|
+
def initialize
|
66
|
+
self.packages = {}
|
67
|
+
end
|
68
|
+
|
69
|
+
def tag_start(name, *attrs)
|
70
|
+
@current_tag = name
|
71
|
+
case name
|
72
|
+
when 'metadata'
|
73
|
+
puts "Parsing #{attrs[0]['packages']} packages"
|
74
|
+
when 'package'
|
75
|
+
@package = {}
|
76
|
+
when 'location'
|
77
|
+
@package['href'] = attrs[0]['href']
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def tag_end(name)
|
82
|
+
case name
|
83
|
+
when 'package'
|
84
|
+
if @package
|
85
|
+
packages[@package['href']] = @package['checksum']
|
86
|
+
@package = nil
|
87
|
+
else
|
88
|
+
fail 'Unmatched <package> tag'
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
def text(data)
|
94
|
+
return if data =~ /^\s+$/
|
95
|
+
if @current_tag == 'checksum'
|
96
|
+
@package['checksum'] = data
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
class RepModListener
|
102
|
+
attr_accessor :metadata
|
103
|
+
include REXML::StreamListener
|
104
|
+
|
105
|
+
def initialize
|
106
|
+
self.metadata = {}
|
107
|
+
end
|
108
|
+
|
109
|
+
def tag_start(name, *attrs)
|
110
|
+
@current_tag = name
|
111
|
+
case name
|
112
|
+
when 'data'
|
113
|
+
@data = {}
|
114
|
+
@data['type'] = attrs[0]['type']
|
115
|
+
when 'location'
|
116
|
+
@data['location'] = attrs[0]['href']
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
def tag_end(name)
|
121
|
+
case name
|
122
|
+
when 'data'
|
123
|
+
if @data
|
124
|
+
metadata[@data['type']] = { :href => @data['location'], :checksum => @data['checksum'] }
|
125
|
+
@data = nil
|
126
|
+
else
|
127
|
+
fail 'Unmatched <data> tag'
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
def text(data)
|
133
|
+
return if data =~ /^\s+$/
|
134
|
+
if @current_tag == 'checksum'
|
135
|
+
@data['checksum'] = data
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
data/lib/yum_s3_sync.rb
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
require 'yum_s3_sync/version'
|
2
|
+
require 'yum_s3_sync/yum_repository'
|
3
|
+
require 'yum_s3_sync/http_downloader'
|
4
|
+
require 'yum_s3_sync/repo_syncer'
|
5
|
+
require 'yum_s3_sync/s3_downloader'
|
6
|
+
require 'yum_s3_sync/s3_uploader'
|
7
|
+
require 'yum_s3_sync/s3_file_lister'
|
8
|
+
require 'yum_s3_sync/s3_deleter'
|
9
|
+
|
10
|
+
module YumS3Sync
|
11
|
+
# Your code goes here...
|
12
|
+
end
|
data/yum_s3_sync.gemspec
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'yum_s3_sync/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = 'yum_s3_sync'
|
8
|
+
spec.version = YumS3Sync::VERSION
|
9
|
+
spec.authors = ['Hein-Pieter van Braam']
|
10
|
+
spec.email = ['hp@tmm.cx']
|
11
|
+
spec.summary = 'Simple program to synchronize Yum repositories with S3 buckets'
|
12
|
+
spec.homepage = ''
|
13
|
+
spec.license = 'MIT'
|
14
|
+
|
15
|
+
spec.files = `git ls-files -z`.split("\x0")
|
16
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
17
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
18
|
+
spec.require_paths = ['lib']
|
19
|
+
|
20
|
+
spec.add_development_dependency 'bundler', '~> 1.7'
|
21
|
+
spec.add_development_dependency 'rake', '~> 10.0'
|
22
|
+
|
23
|
+
spec.add_dependency 'nokogiri', '>= 1.4.3'
|
24
|
+
spec.add_dependency 'aws-sdk'
|
25
|
+
end
|
metadata
ADDED
@@ -0,0 +1,119 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: yum_s3_sync
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.3
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Hein-Pieter van Braam
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-11-14 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.7'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.7'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '10.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: nokogiri
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 1.4.3
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 1.4.3
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: aws-sdk
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
description:
|
70
|
+
email:
|
71
|
+
- hp@tmm.cx
|
72
|
+
executables:
|
73
|
+
- yums3sync
|
74
|
+
- yums3syncdir
|
75
|
+
extensions: []
|
76
|
+
extra_rdoc_files: []
|
77
|
+
files:
|
78
|
+
- ".gitignore"
|
79
|
+
- Gemfile
|
80
|
+
- LICENSE.txt
|
81
|
+
- README.md
|
82
|
+
- Rakefile
|
83
|
+
- bin/yums3sync
|
84
|
+
- bin/yums3syncdir
|
85
|
+
- lib/yum_s3_sync.rb
|
86
|
+
- lib/yum_s3_sync/http_downloader.rb
|
87
|
+
- lib/yum_s3_sync/repo_syncer.rb
|
88
|
+
- lib/yum_s3_sync/s3_deleter.rb
|
89
|
+
- lib/yum_s3_sync/s3_downloader.rb
|
90
|
+
- lib/yum_s3_sync/s3_file_lister.rb
|
91
|
+
- lib/yum_s3_sync/s3_uploader.rb
|
92
|
+
- lib/yum_s3_sync/version.rb
|
93
|
+
- lib/yum_s3_sync/yum_repository.rb
|
94
|
+
- yum_s3_sync.gemspec
|
95
|
+
homepage: ''
|
96
|
+
licenses:
|
97
|
+
- MIT
|
98
|
+
metadata: {}
|
99
|
+
post_install_message:
|
100
|
+
rdoc_options: []
|
101
|
+
require_paths:
|
102
|
+
- lib
|
103
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
104
|
+
requirements:
|
105
|
+
- - ">="
|
106
|
+
- !ruby/object:Gem::Version
|
107
|
+
version: '0'
|
108
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
109
|
+
requirements:
|
110
|
+
- - ">="
|
111
|
+
- !ruby/object:Gem::Version
|
112
|
+
version: '0'
|
113
|
+
requirements: []
|
114
|
+
rubyforge_project:
|
115
|
+
rubygems_version: 2.2.2
|
116
|
+
signing_key:
|
117
|
+
specification_version: 4
|
118
|
+
summary: Simple program to synchronize Yum repositories with S3 buckets
|
119
|
+
test_files: []
|