docker-registry-sync 0.1.3 → 0.1.4

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NzUzOWU0NTYzYjIxOTU0NmE1NTg2Zjg4M2U2MDdiY2Y4MTBkYmRhNw==
+    OTY5YTA1Y2I2Y2M2Nzk0OTQ3MjE3ZGY0MjUyNmYyNjJlYmVmZDU5Yw==
   data.tar.gz: !binary |-
-    M2JiMmQ2OWExODdiMGIzZTBmMzQ2M2Q3ZGExNzAyOTdkZmJhMzY4NQ==
+    Yjg0ZGZkNzcwNjRiMWQwOWI2YzAyMTg2NGNkYjMzNWVkY2NkM2E2ZQ==
 SHA512:
   metadata.gz: !binary |-
-    Y2FlMjc2YWU5MWQwM2U2NDY2MWE2OWZkMzA4MjM2N2M3NDYzZWNkYTllNzMw
-    MDgyNDA3MjhmZjEzNTZkNjQyOWZjZDc2OWM3ZmUyOTgzZWMzZTliZjAzOTc0
-    NjU3YmU4NmQ5MzhlZWFjZDViYTczMzM5Yjc1ZTE4ODY3M2QxNWE=
+    NGVmMDkzNzgzNzI4YTQ1MDliYTJiNGNjMGUxYjA1ZDA3ODc3MmU5MzgzYjUy
+    ODk3NWFkMDJmM2FiNTM1MDY5OGNlYjVlMzcwMjNhY2FjZDA5OTM2ZDU1Mjcy
+    NmRkMGRmODAzNWM1OWI3ZDVkNmU0NGFiZTEzZDYzZDg2NTExMDk=
   data.tar.gz: !binary |-
-    MjkzODFhMzkzMWVkOTBlMWM1NGY1NDc0NDA4OTUyNzExMWE3MzAwNzUwNDNk
-    MWQ1YmZiNjdhOTVmZGZkMzQyNzg2MTY2MDE2NzgwOWJhZmMyYTM5ZGRmMTJj
-    MDk3NGRhZjNkYTEwM2Y1ZGFlOTVlNDlhZTdmZTI4NjExODI0MGM=
+    N2FiMjhiMWM4YTZlMTVhMDNiOGU3MTA0NzUyMmE5NjE0M2M3NzA2ZDYxNWI4
+    ZTk2Mzc1MTM1OGMzZTJiMTU5ZDA3Yjc5YjFkNmY2OGQ1MGYzNTM5OGU1ZWI4
+    OTllNWMxNjhiYmNkZTAxMGY2MTAzMGY2NTAwZmQwM2U2NTk2ODE=

data/lib/docker/registry/sync/cmd.rb

@@ -56,7 +56,7 @@ module Docker
               if image_exists?(image, bucket, region)
                 success = sync_tag(image, tag, bucket, region)
               else
-                success = sync_image(image, bucket, region)
+                success = sync_repo(image, bucket, region)
               end
             end
             success ? 0 : 1
@@ -107,7 +107,7 @@ module Docker
                   end
                 else
                   @config.logger.info("Syncing image: #{data['image']} to #{data['target']['region']}:#{data['target']['bucket']}")
-                  if sync_image(data['image'], data['target']['bucket'], data['target']['region'], data['source']['bucket'], data['source']['region'])
+                  if sync_repo(data['image'], data['target']['bucket'], data['target']['region'], data['source']['bucket'], data['source']['region'])
                     @config.logger.info("Finished syncing image: #{data['image']} to #{data['target']['region']}:#{data['target']['bucket']}")
                     finalize_message(message.receipt_handle)
                   else

data/lib/docker/registry/sync/configuration.rb

@@ -32,18 +32,21 @@ module Docker
           def debug(msg)
             if do_log?(:debug, Docker::Registry::Sync.config.log_level)
               STDOUT.puts "[DEBUG] #{msg}"
+              STDOUT.flush
             end
           end
 
           def info(msg)
             if do_log?(:info, Docker::Registry::Sync.config.log_level)
               STDOUT.puts "[INFO] #{msg}"
+              STDOUT.flush
             end
           end
 
           def error(msg)
             if do_log?(:error, Docker::Registry::Sync.config.log_level)
               STDERR.puts "[ERROR] #{msg}"
+              STDERR.flush
             end
           end
         end

data/lib/docker/registry/sync/s3.rb

@@ -18,34 +18,6 @@ module Docker
             end
           end
 
-          def sync_prefix(source_client, target_client, target_bucket, prefix, source_bucket = nil)
-            source_bucket ||= @config.source_bucket
-            keys = []
-            img_resp = source_client.list_objects(bucket: source_bucket, prefix: prefix)
-
-            loop do
-              img_resp.contents.each do |item|
-                keys << item.key
-              end
-              if img_resp.last_page?
-                break
-              else
-                img_resp.next_page
-              end
-            end
-            sync_keys(target_client, target_bucket, keys)
-          end
-
-          def sync_keys(target_client, target_bucket, keys, source_bucket = nil)
-            source_bucket ||= @config.source_bucket
-            keys.each do |key|
-              @config.logger.info "Syncing key #{source_bucket}/#{key} to bucket #{target_bucket}"
-              target_client.copy_object(acl: 'bucket-owner-full-control',
-                                        bucket: target_bucket,
-                                        key: key,
-                                        copy_source: "#{source_bucket}/#{key}")
-            end
-          end
 
           def sync_tag(image, tag, bucket, region, source_bucket = nil, source_region = nil)
             source_region ||= @config.source_region
@@ -58,11 +30,10 @@ module Docker
               keys = ["tag#{tag}_json", "tag_#{tag}", '_index_images'].map do |key|
                 "registry/repositories/#{image}/#{key}"
               end
-              sync_keys(s3_target, bucket, keys)
+              sync_keys(s3_target, bucket, keys, source_bucket)
 
-              img_id_resp = s3_source.get_object(bucket: source_bucket, key: "registry/repositories/#{image}/tag_#{tag}")
-              img_prefix = "registry/images/#{img_id_resp.body.read}/"
-              sync_prefix(s3_source, s3_target, bucket, img_prefix)
+              img_id = s3_source.get_object(bucket: source_bucket, key: "registry/repositories/#{image}/tag_#{tag}").body.read
+              sync_image(img_id, bucket, region, source_bucket, source_region)
             rescue Exception => e
               @config.logger.error "An unexpected error occoured while syncing tag #{image}:#{tag}: #{e}"
               false
@@ -71,28 +42,68 @@ module Docker
             end
           end
 
-          def sync_image(image, bucket, region, source_bucket = nil, source_region = nil)
+          def sync_repo(repo, bucket, region, source_bucket = nil, source_region = nil)
             source_region ||= @config.source_region
             source_bucket ||= @config.source_bucket
             s3_source = Aws::S3::Client.new(region: source_region)
             s3_target = Aws::S3::Client.new(region: region)
 
             begin
-              rep_prefix = "registry/repositories/#{image}/"
-              sync_prefix(s3_source, s3_target, bucket, rep_prefix)
+              rep_prefix = "registry/repositories/#{repo}/"
+              sync_prefix(s3_source, s3_target, bucket, rep_prefix, source_bucket)
 
-              img_index_resp = s3_source.get_object(bucket: source_bucket, key: "registry/repositories/#{image}/_index_images")
+              img_index_resp = s3_source.get_object(bucket: source_bucket, key: "registry/repositories/#{repo}/_index_images")
               JSON.load(img_index_resp.body.read).each do |image|
-                image_prefix = "registry/images/#{image['id']}/"
-                sync_prefix(s3_source, s3_target, bucket, image_prefix)
+                sync_image(image['id'], bucket, region, source_bucket, source_region)
               end
             rescue Exception => e
-              @config.logger.error "An unexpected error occoured while syncing image #{image}: #{e}"
+              @config.logger.error "An unexpected error occoured while syncing repo #{repo}: #{e}"
               false
             else
               true
             end
           end
+
+          def sync_image(image_id, bucket, region, source_bucket = nil, source_region = nil)
+            source_region ||= @config.source_region
+            source_bucket ||= @config.source_bucket
+            s3_source = Aws::S3::Client.new(region: source_region)
+            s3_target = Aws::S3::Client.new(region: region)
+
+            ancestry_resp = s3_source.get_object(bucket: source_bucket, key: "registry/images/#{image_id}/ancestry")
+            # Ancestry includes self
+            JSON.load(ancestry_resp.body.read).each do |image|
+              sync_prefix(s3_source, s3_target, bucket, "registry/images/#{image}/", source_bucket)
+            end
+          end
+
+          @private
+          def sync_prefix(source_client, target_client, target_bucket, prefix, source_bucket)
+            keys = []
+            img_resp = source_client.list_objects(bucket: source_bucket, prefix: prefix)
+
+            loop do
+              img_resp.contents.each do |item|
+                keys << item.key
+              end
+              if img_resp.last_page?
+                break
+              else
+                img_resp.next_page
+              end
+            end
+            sync_keys(target_client, target_bucket, keys, source_bucket)
+          end
+
+          def sync_keys(target_client, target_bucket, keys, source_bucket)
+            keys.each do |key|
+              @config.logger.info "Syncing key #{source_bucket}/#{key} to bucket #{target_bucket}"
+              target_client.copy_object(acl: 'bucket-owner-full-control',
+                                        bucket: target_bucket,
+                                        key: key,
+                                        copy_source: "#{source_bucket}/#{key}")
+            end
+          end
         end
       end
     end

data/lib/docker/registry/sync/version.rb

@@ -1,7 +1,7 @@
 module Docker
   module Registry
     module Sync
-      VERSION = '0.1.3'
+      VERSION = '0.1.4'
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: docker-registry-sync
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Brian Oldfield