docker-registry-sync 0.1.5 → 0.2.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MDJmMGE4YzQyMmNjMmE3ZmEzNzI1MTIyZDgyMTgyMDJhOWQ3OGMxOA==
+    MjJjMmUyZWRiNWY1NjYzMDI1MTZiMjAxZDZhMWY5MDgzZjNjOTFiOA==
   data.tar.gz: !binary |-
-    YWVhNDY4ZTYwYTcwMmRlNWEwODU5ZDYyMzg2ZmMzNDg5ZDVhNmExMQ==
+    Nzc5MWQxNzhiZmUxOGU0NTRlNzEyYTU2MmEzODE5ODhiMTE3MDllMw==
 SHA512:
   metadata.gz: !binary |-
-    NzFjNDc1MzFiNjM2ZGQ2ZWRlMDE2OGQyZTcyNjFjMGU3ZjAyNTUyZjQzODQx
-    YTUxYTQxNmU4NDhjNWFhNmM0NmQ3NDA1YjZjMGQxNDIxMDBkZTYwNDFlNzMw
-    NTdmYzBiNzViYWU0NzdjODM1NzZkMzU1ZjE3MTc0YWVjMzAwNzQ=
+    Yzc0MTExY2M3MjkxYTFlNzQxY2ExYjhiMTg5YTI0ODI1MTY4ZDdkNDA4YmQ2
+    MTA3ODNkMzU1ZWExODcyZTViNDExMjgzZGQ4ZjdjMDk1ZDQxNWE1YmFmNzI4
+    YWI2ZWVkN2VmM2NlZTYyMWExMmVjZDY3OGJiZTExY2Y3NGRkZjk=
   data.tar.gz: !binary |-
-    NGRlODNhMjljYTI1MGNiYTAwMWE5ZWQ2NDU3MGY1YTZkNTFjZjgzNTMwMjg5
-    Yzc0YmY0MGVkYTNkYmI5MDI2NzVmZjE3MjhjNDNhNTk5YmQ4YzA2ODA5Yjdk
-    MTIyYjZmNGZhYTI2YmMyNjhhNzczOTZlY2YxZWU2OTU1ZDIyYjU=
+    NzIyMzM5OWY2ZDRkYTA2MmZlMDE5Y2ViYTZkYTE1YjE3NWVlMmFhY2RjNzZk
+    YTM1YjJhNWQxNzQ2NDEwMjU5ZTY2NjhiZjJmOWMyNWJkYmVmZGQzYzQ5ODAw
+    MjJkMmY4ZjZkNmQyZGFkOGFjMTBmYzRkNWEwNWNlOGE2ZDQyODU=

data/bin/docker-registry-sync

@@ -64,6 +64,12 @@ EOC
     options[:source_sse] = true
   end
 
+  options[:pool] = 5
+  opts.on('-n POOL', '--pool POOL', "Size of worker thread pool, defaults to 5.") do |t|
+    puts "pool: #{t}"
+    options[:pool] = t.to_i
+  end
+
   options[:unset_proxy] = false
   opts.on(nil, '--unset-proxy', "Use if 'http_proxy' is set in your environment, but you don't want to use it...") do |_u|
     options[:unset_proxy] = true
@@ -96,6 +102,7 @@ unless options[:unset_proxy]
   end
 end
 
+puts options, cmd
 unless verify_opts(options, cmd)
   puts "Both '--source-bucket' and '--target-buckets' must be defined for 'sync' and 'queue-sync' commands\n"
   puts "'--queue' must be defined for 'queue-sync' and 'run-sync' commands\n\n"
@@ -108,7 +115,8 @@ CMD.configure(
   options[:target_buckets],
   options[:sqs_queue],
   options[:sse],
-  options[:source_sse]
+  options[:source_sse],
+  options[:pool]
 )
 
 ec = 1

data/lib/docker/registry/sync/cmd.rb

@@ -1,4 +1,6 @@
 require 'json'
+require 'thread'
+require 'monitor'
 require 'docker/registry/sync/s3'
 require 'docker/registry/sync/sqs'
 
@@ -9,7 +11,7 @@ module Docker
         include Docker::Registry::Sync
 
         class << self
-          def configure(source_bucket, target_buckets, sqs_queue, use_sse, source_uses_sse)
+          def configure(source_bucket, target_buckets, sqs_queue, use_sse, source_uses_sse, pool)
             unless source_bucket.nil?
               source_region, source_bucket = source_bucket.split(':')
             else
@@ -27,6 +29,7 @@ module Docker
             else
               sqs_region, sqs_uri = nil, nil
             end
+
             Docker::Registry::Sync.configure do |config|
               config.source_bucket = source_bucket
               config.source_region = source_region
@@ -34,6 +37,7 @@ module Docker
               config.source_sse = source_uses_sse
               config.sse = use_sse
               config.sqs_region = sqs_region
+              config.pool_size = pool
               config.sqs_url = "https://#{sqs_uri}"
             end
             @config = Docker::Registry::Sync.config
@@ -44,15 +48,53 @@ module Docker
 
             Signal.trap('INT') do
               @config.logger.error 'Received INT signal...'
+              @producer_finished = true
               @terminated = true
             end
             Signal.trap('TERM') do
               @config.logger.error 'Received TERM signal...'
+              @producer_finished = true
               @terminated = true
             end
           end
 
+          def start_workers
+            @threads = Array.new(@config.pool_size)
+            @work_queue = Queue.new
+            @status_queue = Queue.new
+
+            @threads.extend(MonitorMixin)
+            @threads_available = @threads.new_cond
+
+            @producer_finished = false
+            @consumer_thread = Thread.new do
+              sync_key_consumer
+            end
+          end
+
+          def finalize_workers
+            return true
+            @producer_finished = true
+            @consumer_thread.join
+            @threads.each { |t| t.join unless t.nil? }
+            @config.logger.info "Processing job results..."
+
+            success = true
+            loop do
+              begin
+                # One job filure is a run failure
+                success &&= @status_queue.pop(true)
+              rescue ThreadError
+                @config.logger.info "Finished processing job results..."
+                break
+              end
+            end
+            success && !@terminated
+          end
+
           def sync(image, tag)
+            configure_signal_handlers
+            start_workers
             success = false
             @config.target_buckets.each do |region, bucket, sse|
               if image_exists?(image, bucket, region)
@@ -61,6 +103,7 @@ module Docker
                 success = sync_repo(image, bucket, region, !sse.nil?)
               end
             end
+            success &&= finalize_workers
             success ? 0 : 1
           end
 
@@ -85,13 +128,15 @@ module Docker
           def run_sync
             ec = 1
             configure_signal_handlers
+            start_workers
             begin
               @config.logger.info 'Polling queue for images to sync...'
               sqs = Aws::SQS::Client.new(region: @config.sqs_region)
               resp = sqs.receive_message(
                 queue_url: @config.sqs_url,
                 max_number_of_messages: 1,
-                visibility_timeout: 900, # Give ourselves 15min to sync the image
+                #visibility_timeout: 900, # Give ourselves 15min to sync the image
+                visibility_timeout: 30, # Give ourselves 15min to sync the image
                 wait_time_seconds: 10, # Wait a maximum of 10s for a new message
               )
               @config.logger.info "SQS returned #{resp.messages.length} new images to sync..."
@@ -102,15 +147,20 @@ module Docker
 
                 if image_exists?(data['image'], data['target']['bucket'], data['target']['region'])
                   @config.logger.info("Syncing tag: #{data['image']}:#{data['tag']} to #{data['target']['region']}:#{data['target']['bucket']}")
-                  if sync_tag(data['image'], data['tag'], data['target']['bucket'], data['target']['region'], data['target']['sse'], data['source']['bucket'], data['source']['region'])
+                  success = sync_tag(data['image'], data['tag'], data['target']['bucket'], data['target']['region'], data['target']['sse'], data['source']['bucket'], data['source']['region'])
+                  success &&= finalize_workers
+
+                  if success
                     @config.logger.info("Finished syncing tag: #{data['image']}:#{data['tag']} to #{data['target']['region']}:#{data['target']['bucket']}")
                     finalize_message(message.receipt_handle)
                   else
                     @config.logger.info("Falied to sync tag, leaving on queue: #{data['image']}:#{data['tag']} to #{data['target']['region']}:#{data['target']['bucket']}")
                   end
                 else
+                  success = sync_repo(data['image'], data['target']['bucket'], data['target']['region'], data['target']['sse'], data['source']['bucket'], data['source']['region'])
+                  success &&= finalize_workers
                   @config.logger.info("Syncing image: #{data['image']} to #{data['target']['region']}:#{data['target']['bucket']}")
-                  if sync_repo(data['image'], data['target']['bucket'], data['target']['region'], data['target']['sse'], data['source']['bucket'], data['source']['region'])
+                  if success
                     @config.logger.info("Finished syncing image: #{data['image']} to #{data['target']['region']}:#{data['target']['bucket']}")
                     finalize_message(message.receipt_handle)
                   else

data/lib/docker/registry/sync/configuration.rb

@@ -53,7 +53,7 @@ module Docker
       end
 
       class Configuration
-        attr_accessor :source_bucket, :source_region, :target_buckets, :sqs_region, :sqs_url, :empty_queue_sleep_time, :sse, :source_sse
+        attr_accessor :source_bucket, :source_region, :target_buckets, :sqs_region, :sqs_url, :empty_queue_sleep_time, :sse, :source_sse, :pool_size
         attr_accessor :log_level, :logger
 
         def initialize
@@ -69,6 +69,7 @@ module Docker
           @sse = false
 
           @empty_queue_sleep_time = 5
+          @pool_size = 5
 
           @log_level = :debug
           @logger = Docker::Registry::Sync::STDLogger

data/lib/docker/registry/sync/s3.rb

@@ -36,6 +36,7 @@ module Docker
               sync_image(img_id, bucket, region, sse, source_bucket, source_region)
             rescue Exception => e
               @config.logger.error "An unexpected error occoured while syncing tag #{image}:#{tag}: #{e}"
+              @config.logger.error e.backtrace
               false
             else
               true
@@ -58,6 +59,7 @@ module Docker
               end
             rescue Exception => e
               @config.logger.error "An unexpected error occoured while syncing repo #{repo}: #{e}"
+              @config.logger.error e.backtrace
               false
             else
               true
@@ -99,6 +101,7 @@ module Docker
             keys.each do |key|
               @config.logger.info "Syncing key #{source_bucket}/#{key} to bucket #{target_bucket}"
               opts = {acl: 'bucket-owner-full-control',
+                      region: target_client.get_bucket_location(bucket: target_bucket).location_constraint,
                       bucket: target_bucket,
                       key: key,
                       copy_source: "#{source_bucket}/#{key}"}
@@ -108,7 +111,52 @@ module Docker
               if @config.source_sse
                 opts[:copy_source_sse_customer_algorithm] = 'AES256'
               end
-              target_client.copy_object(opts)
+              @work_queue << opts
+              sleep 0.1
+            end
+          end
+
+          def sync_key_consumer
+            @config.logger.info "Starting sync consumer..."
+            loop do
+              break if @producer_finished && @work_queue.length == 0
+              t_index = nil
+
+              begin
+                busy = @threads.select { |t| t.nil? || t.status == false  || t['finished'].nil? == false }.length == 0
+              end until !busy
+              t_index = @threads.rindex { |t| t.nil? || t.status == false || t['finished'].nil? == false }
+
+              begin
+                opts = @work_queue.pop(true)
+              rescue ThreadError
+                @config.logger.info "No work found on the queue, sleeping..."
+                sleep 1
+              else
+                if opts[:key]
+                  @threads[t_index].join unless @threads[t_index].nil?
+                  @threads[t_index] = Thread.new do
+                    @config.logger.info "Worker syncing key: #{opts[:key]}"
+                    target_client = Aws::S3::Client.new(region: opts[:region])
+                    opts.delete :region
+                    success = false
+                    begin
+                      target_client.copy_object(opts)
+                      success = true
+                    rescue Exception => e
+                      @config.logger.error "An unknown error occoured while copying object in s3: #{e}"
+                      @config.logger.error e.backtrace
+                    end
+                    Thread.current['finished'] = true
+                    @threads.synchronize do
+                      @status_queue << success
+                    end
+                    @config.logger.info "Worker finished syncing key: #{opts[:key]}"
+                  end
+                else
+                  @config.logger.info "Queued work empty: #{opts}"
+                end
+              end
             end
           end
         end

data/lib/docker/registry/sync/version.rb

@@ -1,7 +1,7 @@
 module Docker
   module Registry
     module Sync
-      VERSION = '0.1.5'
+      VERSION = '0.2.0'
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: docker-registry-sync
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.0
 platform: ruby
 authors:
 - Brian Oldfield
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-04 00:00:00.000000000 Z
+date: 2015-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler