yum_s3_sync 0.0.3 → 0.0.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 23ccfe9d584a3f0764dc5af307715ac8695aa64c
-  data.tar.gz: ca3c0fdbb706a4efd312644b00928c00db287a77
+  metadata.gz: eaae8aaa8330a034cd7997a9d5dee0c4034b305c
+  data.tar.gz: 1212ab5d4e1688fe2cd8f3bfb168abfb5f2e2c99
 SHA512:
-  metadata.gz: 0e56e5c4af3f7f23beed8560af4aa849e0197752bd2251f1f3ea67b62eeb349dc31d57c8048a191785bd2a71d2eb0f497385d3e0b3cd3fef011802b34839e64c
-  data.tar.gz: bcb588bba36f39bf2eba56e0b240ef32c2f97b000bec75e3b77948ba13cf557c8a920ffdd859ece47aa4f289ef87e8cd78e451f6bbac997428f8cdec68dcf89a
+  metadata.gz: 4f9c878f67a70767968e6de7bc76326b359cd33a30dbd036390e4cb1caa2e381aebfa1f3ad25fcec82c10d92b9bbc24c32fce6a8272522112b20f8334d8758a6
+  data.tar.gz: c57dc78761bd06c09ada734014e3e3aadfa483dbf13a9eabf75d17352dc6699b4be814b22ea43214ae147ef1e869284474b211a1fbd9abc3f7cd381440ee5fba

data/README.md

@@ -4,8 +4,8 @@ Synchronize a Yum repository over HTTP to S3. This can be useful when you want t
 
 There are two command line tools supplied:
 
-yums3sync - Synchronizes a single Yum repository to a bucket:/prefix
-yums3syncdir - Scan a directory of links for repositories then synchronize all of them to bucket:/prefix
+* ``yums3sync`` - Synchronizes a single Yum repository to a bucket:/prefix
+* ``yums3syncdir`` - Scan a directory of links for repositories then synchronize all of them to bucket:/prefix
 
 YumS3sync is smart enough to only copy files changes between runs, and does almost nothing if nothing changed on the source end. 
 
@@ -15,6 +15,12 @@ After the syncronization you can use the repository using Yum with an S3 plugin
 
 ## Installation
 
+### Gem
+```
+gem install yum_s3_sync
+```
+
+### Manual
 Clone this git repository and build a Gem. The program should work without additional dependencies on RHEL/Centos 6 with EPEL enabled. It is tested on Ruby 1.8.7 and 2.1
 
 ## Usage
@@ -26,6 +32,8 @@ Usage: yums3sync [options]
     -s, --source SOURCE              HTTP source URL
     -b, --bucket BUCKET              Target bucket name
     -p, --prefix PREFIX              Target bucket prefix
+    -k, --keep                       Never overwrite exitant files
+    -n, --dry-run                    Don't make any changes
 ```
 
 Example usage:
@@ -44,6 +52,8 @@ Usage: yums3syncdir [options]
     -b, --bucket BUCKET              Target bucket name
     -p, --prefix PREFIX              Target bucket prefix
     -x, --exclude prefix1,prefix2    Exclude prefixes
+    -k, --keep                       Never overwrite exitant files
+    -n, --dry-run                    Don't make any changes
 ``` 
 
 Example usage:

data/bin/yums3sync

@@ -16,6 +16,12 @@ opt_parser = OptionParser.new do |opts|
   opts.on('-p', '--prefix PREFIX', 'Target bucket prefix') do |p|
     options[:target_base] = p
   end
+  opts.on('-k', '--keep', 'Never overwrite exitant files') do |k|
+    options[:keep] = true
+  end
+  opts.on('-n', '--dry-run', 'Don\'t make any changes') do |n|
+    options[:dry_run] = true
+  end
 end
 
 opt_parser.parse!
@@ -25,7 +31,7 @@ if !options[:source_base] || !options[:target_bucket] || !options[:target_base]
   exit 1
 end
 
-repo_syncer = YumS3Sync::RepoSyncer.new(options[:source_base], options[:target_bucket], options[:target_base])
+repo_syncer = YumS3Sync::RepoSyncer.new(options[:source_base], options[:target_bucket], options[:target_base], options[:keep], options[:dry_run])
 
 begin
   repo_syncer.sync

data/bin/yums3syncdir

@@ -14,18 +14,21 @@ opt_parser = OptionParser.new do |opts|
   opts.on('-s', '--source SOURCEDIR', 'HTTP source base URL') do |s|
     options[:source_base] = s
   end
-
   opts.on('-b', '--bucket BUCKET', 'Target bucket name') do |b|
     options[:target_bucket] = b
   end
-
   opts.on('-p', '--prefix PREFIX', 'Target bucket prefix') do |p|
     options[:target_base] = p
   end
-
   opts.on('-x', '--exclude prefix1,prefix2', Array, 'Exclude prefixes') do |x|
     options[:exclude] = x
   end
+  opts.on('-k', '--keep', 'Never overwrite exitant files') do |k|
+    options[:keep] = true
+  end
+  opts.on('-n', '--dry-run', 'Don\'t make any changes') do |n|
+    options[:dry_run] = true
+  end
 end
 
 opt_parser.parse!
@@ -77,7 +80,7 @@ Net::HTTP.start(uri.host) do |http|
 end
 
 repositories.each do |repository|
-  repo_syncer = YumS3Sync::RepoSyncer.new(options[:source_base] + '/' + repository, options[:target_bucket], options[:target_base] + '/' + repository)
+  repo_syncer = YumS3Sync::RepoSyncer.new(options[:source_base] + '/' + repository, options[:target_bucket], options[:target_base] + '/' + repository, options[:keep], options[:dry_run])
 
   begin
     repo_syncer.sync

data/lib/yum_s3_sync/http_downloader.rb

@@ -1,4 +1,5 @@
 require 'open-uri'
+require 'socket'
 
 module YumS3Sync
   class HTTPDownloader

data/lib/yum_s3_sync/repo_syncer.rb

@@ -1,13 +1,16 @@
 #!/usr/bin/env ruby
 
 require 'yum_s3_sync'
+require 'parallel'
 
 module YumS3Sync
   class RepoSyncer
-    def initialize(source_base, target_bucket, target_base)
+    def initialize(source_base, target_bucket, target_base, keep = false, dry_run = false)
       @source_base = source_base
       @target_bucket = target_bucket
       @target_base = target_base
+      @keep = keep
+      @dry_run = dry_run
     end
 
     def sync
@@ -17,34 +20,49 @@ module YumS3Sync
       s3_downloader = YumS3Sync::S3Downloader.new(@target_bucket, @target_base)
       dest_repository = YumS3Sync::YumRepository.new(s3_downloader)
 
+      s3_uploader = YumS3Sync::S3Uploader.new(@target_bucket, @target_base, http_downloader, @dry_run)
+
+      s3_file_lister = YumS3Sync::S3FileLister.new(@target_bucket, @target_base)
+      s3_deleter = YumS3Sync::S3Deleter.new(@target_bucket, @target_base, @dry_run)
+
       new_packages = source_repository.compare(dest_repository)
-      s3_uploader = YumS3Sync::S3Uploader.new(@target_bucket, @target_base, http_downloader)
 
       metadata = []
-      source_repository.metadata.each do |_type, file|
+      new_metadata = false
+      source_repository.metadata.each do |type, file|
+        if !dest_repository.metadata[type] || dest_repository.metadata[type][:checksum] != file[:checksum]
+          new_metadata = true
+        end
+          
         metadata.push file[:href]
       end
 
       new_packages.each do |package|
-        s3_uploader.upload(package)
+        s3_uploader.upload(package, @keep)
       end
 
-      if !dest_repository.exists? || !new_packages.empty?
+      if !dest_repository.exists? || !new_packages.empty? || new_metadata
         metadata.each do |file|
-          s3_uploader.upload(file)
+          s3_uploader.upload(file, true)
         end
       end
 
-      unless new_packages.empty?
-        s3_file_lister = YumS3Sync::S3FileLister.new(@target_bucket, @target_base)
-        s3_deleter = YumS3Sync::S3Deleter.new(@target_bucket, @target_base)
+      file_names = s3_file_lister.list
 
-        s3_file_lister.list.each do |file|
-          if !source_repository.packages[file] && !metadata.include?(file)
-            s3_deleter.delete(file)
-          end
+      puts "Locating removed files"
+      file_names.each do |filename|
+        if !source_repository.packages[filename] && !metadata.include?(filename)
+          s3_deleter.delete(filename)
         end
       end
+
+      puts "Locating missing files"
+      source_repository.packages.each do |package, data|
+        unless file_names.include? package
+          s3_uploader.upload(package, true)
+        end
+      end
+
     end
   end
 end

data/lib/yum_s3_sync/s3_deleter.rb

@@ -2,9 +2,10 @@ require 'aws-sdk'
 
 module YumS3Sync
   class S3Deleter
-    def initialize(bucket, prefix)
+    def initialize(bucket, prefix, dry_run = false)
       @bucket = bucket
       @prefix = prefix
+      @dry_run = dry_run
     end
 
     def delete(file)
@@ -16,8 +17,12 @@ module YumS3Sync
       dest_obj = s3.buckets[@bucket].objects[target]
 
       if dest_obj.exists?
-        puts "Deleting #{@bucket}::#{target}"
-        dest_obj.delete
+        if @dry_run
+          puts "Dry-run: Deleting #{@bucket}::#{target}"
+        else
+          puts "Deleting #{@bucket}::#{target}"
+          dest_obj.delete
+        end
       end
     end
   end

data/lib/yum_s3_sync/s3_file_lister.rb

@@ -1,4 +1,5 @@
 require 'aws-sdk'
+require 'parallel'
 
 module YumS3Sync
   class S3FileLister
@@ -8,16 +9,15 @@ module YumS3Sync
     end
 
     def list
-      files = []
-
       s3 = AWS::S3.new
-      s3.buckets[@bucket].objects.with_prefix(@prefix).each do |file|
-        basename = file.key.sub(/#{@prefix}\/*/, '')
 
-        files.push basename
-      end
+      puts "Listing all files in #{@bucket}:#{@prefix}"
+      s3_objects = s3.buckets[@bucket].objects.with_prefix(@prefix)
 
-      files
+      s3_objects.map do |file|
+        basename = file.key.sub(/#{@prefix}\/*/, '')
+        basename
+      end
     end
   end
 end

data/lib/yum_s3_sync/s3_uploader.rb

@@ -2,25 +2,36 @@ require 'aws-sdk'
 
 module YumS3Sync
   class S3Uploader
-    def initialize(bucket, prefix, downloader)
+    def initialize(bucket, prefix, downloader, dry_run = false)
       @bucket = bucket
       @prefix = prefix
       @downloader = downloader
+      @dry_run = dry_run
     end
 
-    def upload(file)
+    def upload(file, overwrite = false)
       retries = 0
       s3 = AWS::S3.new
 
-      source_file = @downloader.download(file)
-      target = "#{@prefix}/#{file}"
-      target.gsub!(/\/+/, '/')
-      dest_obj = s3.buckets[@bucket].objects[target]
-      dest_obj.delete if dest_obj.exists?
-
-      puts "Uploading #{@bucket}::#{target}"
       begin
-        dest_obj.write(:file => source_file)
+        target = "#{@prefix}/#{file}"
+        target.gsub!(/\/+/, '/')
+        dest_obj = s3.buckets[@bucket].objects[target]
+
+        if dest_obj.exists? && ! overwrite
+          puts "Already exists: skipping #{@bucket}::#{target}" 
+          return
+        end
+
+        source_file = @downloader.download(file)
+
+        if @dry_run 
+          puts "Dry-run: Uploading #{@bucket}::#{target}"
+        else
+          puts "Uploading #{@bucket}::#{target}"
+          dest_obj.delete if dest_obj.exists?
+          dest_obj.write(:file => source_file)
+        end
       rescue StandardError => e
         if retries < 10
           retries += 1

data/lib/yum_s3_sync/version.rb

@@ -1,3 +1,3 @@
 module YumS3Sync
-  VERSION = '0.0.3'
+  VERSION = '0.0.13'
 end

data/lib/yum_s3_sync/yum_repository.rb

@@ -1,6 +1,7 @@
 #!/usr/bin/env ruby
 
 require 'zlib'
+require 'nokogiri'
 require 'rexml/document'
 require 'rexml/streamlistener'
 
@@ -9,13 +10,19 @@ module YumS3Sync
     attr_accessor :metadata
 
     def initialize(downloader)
+      @metadata = {}
       @downloader = downloader
 
-      repomd_parser = RepModListener.new
       repomd_file = @downloader.download('repodata/repomd.xml')
       if repomd_file
-        REXML::Document.parse_stream(repomd_file, repomd_parser)
-        @metadata = repomd_parser.metadata
+        doc = Nokogiri::XML(repomd_file)
+        doc.xpath("//xmlns:data").each do |file|
+          metadata[file['type']] = {
+            :href => file.xpath('xmlns:location')[0]['href'],
+            :checksum => file.xpath('xmlns:checksum')[0].child.to_s
+          }
+        end
+
         @metadata['repomd'] = { :href => 'repodata/repomd.xml' }
       else
         @metadata = { 'primary' => nil }
@@ -28,11 +35,16 @@ module YumS3Sync
       primary_file = @downloader.download(@metadata['primary'][:href])
       return {} unless primary_file
 
+      puts "Parsing #{@metadata['primary'][:href]}"
       gzstream = Zlib::GzipReader.new(primary_file)
-      package_parser = PackageListener.new
 
-      REXML::Document.parse_stream(gzstream, package_parser)
-      package_parser.packages
+      doc = Nokogiri::XML(gzstream)
+      packages = {}
+      doc.xpath("//xmlns:package").each do |package|
+        packages[package.xpath("xmlns:location")[0]['href']] = package.xpath("xmlns:checksum")[0].child.to_s
+      end
+
+      packages
     end
 
     def packages
@@ -58,82 +70,4 @@ module YumS3Sync
     end
   end
 
-  class PackageListener
-    attr_accessor :packages
-    include REXML::StreamListener
-
-    def initialize
-      self.packages = {}
-    end
-
-    def tag_start(name, *attrs)
-      @current_tag = name
-      case name
-      when 'metadata'
-        puts "Parsing #{attrs[0]['packages']} packages"
-      when 'package'
-        @package = {}
-      when 'location'
-        @package['href'] = attrs[0]['href']
-      end
-    end
-
-    def tag_end(name)
-      case name
-      when 'package'
-        if @package
-          packages[@package['href']] = @package['checksum']
-          @package = nil
-        else
-          fail 'Unmatched <package> tag'
-        end
-      end
-    end
-
-    def text(data)
-      return if data =~ /^\s+$/
-      if @current_tag == 'checksum'
-        @package['checksum'] = data
-      end
-    end
-  end
-
-  class RepModListener
-    attr_accessor :metadata
-    include REXML::StreamListener
-
-    def initialize
-      self.metadata = {}
-    end
-
-    def tag_start(name, *attrs)
-      @current_tag = name
-      case name
-      when 'data'
-        @data = {}
-        @data['type'] = attrs[0]['type']
-      when 'location'
-        @data['location'] = attrs[0]['href']
-      end
-    end
-
-    def tag_end(name)
-      case name
-      when 'data'
-        if @data
-          metadata[@data['type']] = { :href => @data['location'], :checksum => @data['checksum'] }
-          @data = nil
-        else
-          fail 'Unmatched <data> tag'
-        end
-      end
-    end
-
-    def text(data)
-      return if data =~ /^\s+$/
-      if @current_tag == 'checksum'
-        @data['checksum'] = data
-      end
-    end
-  end
 end

data/yum_s3_sync.gemspec

@@ -21,5 +21,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake', '~> 10.0'
 
   spec.add_dependency 'nokogiri', '>= 1.4.3'
+  spec.add_dependency 'parallel'
   spec.add_dependency 'aws-sdk'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yum_s3_sync
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.13
 platform: ruby
 authors:
 - Hein-Pieter van Braam
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-14 00:00:00.000000000 Z
+date: 2015-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.4.3
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
@@ -112,7 +126,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Simple program to synchronize Yum repositories with S3 buckets