docker-registry-sync 0.1.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MTUwYjg1N2FmMjBmMzYyZjQ2OTJiNDZiY2ZjMWI2ZTFiOGRiNGVjYg==
+  data.tar.gz: !binary |-
+    MzhhYWY5OTkxMmNiMDczYTU3NjBhMjI3ZTg3MWVhZmMwZDYxNmFkMw==
+SHA512:
+  metadata.gz: !binary |-
+    NjNlYjg4OWYwNWM4MDM3ZmJkNjY3M2M3ZDk3OGUzOTZjMWZmYzQzYjUzNDQ1
+    ZWNkYTk1MzMwMGYxM2IzOGQzOTEwODFhYWRmYjhlNGExMDNmZDliOTNhM2Zj
+    ZWU5Y2YzZGQ2MTlkMGVmZGM2MzYzNjgyOTdmY2E3OWE0OTQ3ZTA=
+  data.tar.gz: !binary |-
+    ZmYxY2NjZTUyZjNjMGI0M2FmNDExZTU4MmM1NGM5MGZlMzczNTlmMmY0NjE3
+    YzM1MzA1Y2M5Y2ExNTJlMjk3YTg3YWVmZmEzOTU2MmE1ZjAxNTgzYmQ1ZmZh
+    NjYyNDk2N2IxMzNhMWRiYjk5MjNkMGM2ZDkxYWZiODUzYTcwZDk=

data/.gitignore

@@ -0,0 +1,8 @@
+*.gem
+*.swp
+bin/*
+!bin/docker-registry-sync
+.bundle/
+coverage/
+install.sh
+test.rb

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/bin/docker-registry-sync

@@ -0,0 +1,104 @@
+#!/usr/bin/env ruby
+require 'optparse'
+require 'docker/registry/sync'
+
+include Docker::Registry::Sync
+
+def verify_opts(opts, command)
+  source_defined = !opts[:source_bucket].nil?
+  dest_defined = !opts[:target_buckets].nil?
+  queue_defined = !opts[:sqs_queue].nil?
+  if command == 'sync'
+    source_defined && dest_defined
+  elsif command == 'queue-sync'
+    source_defined && dest_defined && queue_defined
+  elsif command == 'run-sync'
+    queue_defined
+  else
+    false
+  end
+end
+
+options = {}
+opt_parser = OptionParser.new do |opts|
+  opts.banner = <<EOC
+docker-registry-sync [options] sync <image>:<tag>
+docker-registry-sync [options] queue-sync <image>:<tag>
+docker-registry-sync [options] run-sync
+
+Available commands:
+  sync -- Sync docker image from one source registry S3 bucket to one or more S3 buckets.
+
+Options:
+EOC
+  options[:source_bucket] = nil
+  opts.on('-s SOUCE', '--source-bucket SOURCE', 'Primary docker registry S3 bucket. Pass as <region>:<bucket> [required]') do |b|
+    options[:source_bucket] = b
+  end
+
+  options[:target_buckets] = nil
+  opts.on('-t TARGETS', '--target-buckets TARGETS', 'S3 buckets to sync to, comma separated. Pass as <region>:<bucket>[,<region>:<bucket>]  [required]') do |t|
+    options[:target_buckets] = t
+  end
+
+  options[:sqs_queue] = nil
+  opts.on('-q SQS_QUEUE', '--queue SQS_QUEUE', 'SQS queue url used to enqueue sync job. Pass as <region>:<uri> (do not include schema) [required if using queue-sync or run-sync]') do |b|
+    options[:sqs_queue] = b
+  end
+
+  options[:unset_proxy] = false
+  opts.on(nil, '--unset-proxy', "Use if 'http_proxy' is set in your environment, but you don't want to use it...") do |_u|
+    options[:unset_proxy] = true
+  end
+end
+
+opt_parser.version = VERSION
+opt_parser.parse!
+
+cmd = ARGV[0]
+if cmd.nil?
+  puts "Command must be supplied!\n\n"
+  puts opt_parser
+  exit 1
+end
+
+image = ARGV[1]
+if image.nil? && !['queue-sync', 'run-sync'].include?(cmd)
+  puts "Image and tag to sync must be supplied!\n\n"
+  puts opt_parser
+  exit 1
+end
+
+ENV['http_proxy'] = nil if options[:unset_proxy]
+
+unless verify_opts(options, cmd)
+  puts "Both '--source-bucket' and '--target-buckets' must be defined for 'sync' and 'queue-sync' commands\n"
+  puts "'--queue' must be defined for 'queue-sync' and 'run-sync' commands\n\n"
+  puts opt_parser
+  exit 1
+end
+
+CMD.configure(
+  options[:source_bucket],
+  options[:target_buckets],
+  options[:sqs_queue]
+)
+
+ec = 1
+
+case cmd
+when 'sync'
+  image, tag = image.split(':')
+  ec = CMD.sync(image, tag)
+when 'queue-sync'
+  image, tag = image.split(':')
+  ec = CMD.queue_sync(image, tag)
+when 'run-sync'
+  ec = CMD.run_sync
+else
+  puts "Unknown command: #{cmd}"
+  puts opt_parser
+  ec = 1
+end
+
+exit(ec)

data/docker-registry-sync.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'docker/registry/sync'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'docker-registry-sync'
+  spec.version       = Docker::Registry::Sync::VERSION
+  spec.authors       = ['Brian Oldfield']
+  spec.email         = ['brian.oldfield@socrata.com']
+  spec.summary       = 'Sync data between S3 two distinct docker registries using S3 data storage and SNS as a messaging system.'
+  spec.description   = 'Sync data between S3 two distinct docker registries using S3 data storage and SNS as a messaging system.'
+  spec.homepage      = 'http://github.com/socrata-platform/docker-registry-sync'
+  spec.license       = 'Apache'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/docker-registry-sync}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.7'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.3'
+  spec.add_development_dependency 'simplecov', '~> 0.10'
+  spec.add_development_dependency 'simplecov-console', '~> 0.2'
+  spec.add_development_dependency 'webmock', '~> 1.21'
+  spec.add_development_dependency 'rack', '~> 1.6'
+  spec.add_dependency 'aws-sdk', '~> 2.0'
+end

data/lib/docker/registry/sync.rb

@@ -0,0 +1,18 @@
+require 'docker/registry/sync/cmd'
+require 'docker/registry/sync/configuration'
+require 'docker/registry/sync/version'
+
+module Docker
+  module Registry
+    module Sync
+      class << self
+        attr_accessor :config
+
+        def configure
+          self.config ||= Docker::Registry::Sync::Configuration.new
+          yield self.config
+        end
+      end
+    end
+  end
+end

data/lib/docker/registry/sync/cmd.rb

@@ -0,0 +1,118 @@
+require 'json'
+require 'docker/registry/sync/s3'
+require 'docker/registry/sync/sqs'
+
+module Docker
+  module Registry
+    module Sync
+      module CMD
+        include Docker::Registry::Sync
+
+        class << self
+          def configure(source_bucket, target_buckets, sqs_queue)
+            source_region, source_bucket = source_bucket.split(':')
+            target_buckets = target_buckets.split(',').collect { |bucket| bucket.split(':') }
+            sqs_region, sqs_uri = sqs_queue.split(':')
+            Docker::Registry::Sync.configure do |config|
+              config.source_bucket = source_bucket
+              config.source_region = source_region
+              config.target_buckets = target_buckets
+              config.sqs_region = sqs_region
+              config.sqs_url = "https://#{sqs_uri}"
+            end
+            @config = Docker::Registry::Sync.config
+          end
+
+          def configure_signal_handlers
+            @terminated = false
+
+            Signal.trap('INT') do
+              @config.logger.error 'Received INT signal...'
+              @terminated = true
+            end
+            Signal.trap('TERM') do
+              @config.logger.error 'Received TERM signal...'
+              @terminated = true
+            end
+          end
+
+          def sync(image, tag)
+            success = false
+            @config.target_buckets.each do |region, bucket|
+              if image_exists?(image, bucket, region)
+                success = sync_tag(image, tag, bucket, region)
+              else
+                success = sync_image(image, bucket, region)
+              end
+            end
+            success ? 0 : 1
+          end
+
+          def queue_sync(image, tag)
+            msgs = @config.target_buckets.map do |region, bucket|
+              JSON.dump(retries: 0,
+                        image: image,
+                        tag: tag,
+                        source: {
+                          bucket: @config.source_bucket,
+                          region: @config.source_region
+                        },
+                        target: {
+                          bucket: bucket,
+                          region: region
+                        })
+            end
+            send_message_batch(msgs) ? 0 : 1
+          end
+
+          def run_sync
+            ec = 1
+            configure_signal_handlers
+            begin
+              @config.logger.info 'Polling queue for images to sync...'
+              sqs = Aws::SQS::Client.new(region: @config.sqs_region)
+              resp = sqs.receive_message(
+                queue_url: @config.sqs_url,
+                max_number_of_messages: 1,
+                visibility_timeout: 900, # Give ourselves 15min to sync the image
+                wait_time_seconds: 10, # Wait a maximum of 10s for a new message
+              )
+              @config.logger.info "SQS returned #{resp.messages.length} new images to sync..."
+              if resp.messages.length == 1
+                message = resp.messages[0]
+                data = JSON.load(message.body)
+                @config.logger.info "Image sync data:  #{data}"
+
+                if image_exists?(data['image'], data['target']['bucket'], data['target']['region'])
+                  @config.logger.info("Syncing tag: #{data['image']}:#{data['tag']} to #{data['target']['region']}:#{data['target']['bucket']}")
+                  if sync_tag(data['image'], data['tag'], data['target']['bucket'], data['target']['region'], data['source']['bucket'], data['source']['region'])
+                    @config.logger.info("Finished syncing tag: #{data['image']}:#{data['tag']} to #{data['target']['region']}:#{data['target']['bucket']}")
+                    finalize_message(message.receipt_handle)
+                  else
+                    @config.logger.info("Falied to sync tag, leaving on queue: #{data['image']}:#{data['tag']} to #{data['target']['region']}:#{data['target']['bucket']}")
+                  end
+                else
+                  @config.logger.info("Syncing image: #{data['image']} to #{data['target']['region']}:#{data['target']['bucket']}")
+                  if sync_image(data['image'], data['target']['bucket'], data['target']['region'], data['source']['bucket'], data['source']['region'])
+                    @config.logger.info("Finished syncing image: #{data['image']} to #{data['target']['region']}:#{data['target']['bucket']}")
+                    finalize_message(message.receipt_handle)
+                  else
+                    @config.logger.error("Failed to sync image, leaving on queue: #{data['image']} to #{data['target']['region']}:#{data['target']['bucket']}")
+                  end
+                end
+              end
+              ec = 0
+              sleep @config.empty_queue_sleep_time unless @terminated
+            rescue Exception => e
+              @config.logger.error "An unknown error occurred while monitoring queue: #{e}"
+              @config.logger.error 'Exiting...'
+              @terminated = true
+              ec = 1
+            end until @terminated
+            ec
+          end
+        end
+      end
+    end
+  end
+end

data/lib/docker/registry/sync/configuration.rb

@@ -0,0 +1,73 @@
+module Docker
+  module Registry
+    module Sync
+      module STDLogLvl
+        class << self
+          def debug
+            1
+          end
+
+          def info
+            2
+          end
+
+          def error
+            3
+          end
+
+          def off
+            4
+          end
+        end
+      end
+
+      module STDLogger
+        class << self
+          def do_log?(requested_lvl, curr_lvl)
+            curr_lvl = Docker::Registry::Sync::STDLogLvl.send(curr_lvl.to_sym)
+            requested_lvl = Docker::Registry::Sync::STDLogLvl.send(requested_lvl.to_sym)
+            requested_lvl >= curr_lvl
+          end
+
+          def debug(msg)
+            if do_log?(:debug, Docker::Registry::Sync.config.log_level)
+              STDOUT.puts "[DEBUG] #{msg}"
+            end
+          end
+
+          def info(msg)
+            if do_log?(:info, Docker::Registry::Sync.config.log_level)
+              STDOUT.puts "[INFO] #{msg}"
+            end
+          end
+
+          def error(msg)
+            if do_log?(:error, Docker::Registry::Sync.config.log_level)
+              STDERR.puts "[ERROR] #{msg}"
+            end
+          end
+        end
+      end
+
+      class Configuration
+        attr_accessor :source_bucket, :source_region, :target_buckets, :sqs_region, :sqs_url, :empty_queue_sleep_time
+        attr_accessor :log_level, :logger
+
+        def initialize
+          @source_bucket = nil
+          @source_region = nil
+
+          @target_buckets = nil
+
+          @sqs_region = nil
+          @sqs_url = nil
+
+          @empty_queue_sleep_time = 5
+
+          @log_level = :debug
+          @logger = Docker::Registry::Sync::STDLogger
+        end
+      end
+    end
+  end
+end

data/lib/docker/registry/sync/s3.rb

@@ -0,0 +1,100 @@
+require 'aws-sdk'
+
+module Docker
+  module Registry
+    module Sync
+      module CMD
+        include Docker::Registry::Sync
+
+        class << self
+          def image_exists?(image, bucket, region)
+            s3 = Aws::S3::Client.new(region: region)
+            begin
+              s3.head_object(bucket: bucket, key: "registry/repositories/#{image}/_index_images")
+            rescue Aws::S3::Errors::NotFound
+              false
+            else
+              true
+            end
+          end
+
+          def sync_prefix(source_client, target_client, target_bucket, prefix, source_bucket = nil)
+            source_bucket ||= @config.source_bucket
+            keys = []
+            img_resp = source_client.list_objects(bucket: source_bucket, prefix: prefix)
+
+            loop do
+              img_resp.contents.each do |item|
+                keys << item.key
+              end
+              if img_resp.last_page?
+                break
+              else
+                img_resp.next_page
+              end
+            end
+            sync_keys(target_client, target_bucket, keys)
+          end
+
+          def sync_keys(target_client, target_bucket, keys, source_bucket = nil)
+            source_bucket ||= @config.source_bucket
+            keys.each do |key|
+              @config.logger.info "Syncing key #{source_bucket}/#{key} to bucket #{target_bucket}"
+              target_client.copy_object(acl: 'bucket-owner-full-control',
+                                        bucket: target_bucket,
+                                        key: key,
+                                        copy_source: "#{source_bucket}/#{key}")
+            end
+          end
+
+          def sync_tag(image, tag, bucket, region, source_bucket = nil, source_region = nil)
+            source_region ||= @config.source_region
+            source_bucket ||= @config.source_bucket
+
+            s3_source = Aws::S3::Client.new(region: source_region)
+            s3_target = Aws::S3::Client.new(region: region)
+
+            begin
+              keys = ["tag#{tag}_json", "tag_#{tag}", '_index_images'].map do |key|
+                "registry/repositories/#{image}/#{key}"
+              end
+              sync_keys(s3_target, bucket, keys)
+
+              img_id_resp = s3_source.get_object(bucket: source_bucket, key: "registry/repositories/#{image}/tag_#{tag}")
+              img_prefix = "registry/images/#{img_id_resp.body.read}/"
+              sync_prefix(s3_source, s3_target, bucket, img_prefix)
+            rescue Exception => e
+              @config.logger.error "An unexpected error occoured while syncing tag #{image}:#{tag}: #{e}"
+              false
+            else
+              true
+            end
+          end
+
+          def sync_image(image, bucket, region, source_bucket = nil, source_region = nil)
+            source_region ||= @config.source_region
+            source_bucket ||= @config.source_bucket
+            s3_source = Aws::S3::Client.new(region: source_region)
+            s3_target = Aws::S3::Client.new(region: region)
+
+            begin
+              rep_prefix = "registry/repositories/#{image}/"
+              sync_prefix(s3_source, s3_target, bucket, rep_prefix)
+
+              img_index_resp = s3_source.get_object(bucket: source_bucket, key: "registry/repositories/#{image}/_index_images")
+              JSON.load(img_index_resp.body.read).each do |image|
+                image_prefix = "registry/images/#{image['id']}/"
+                sync_prefix(s3_source, s3_target, bucket, image_prefix)
+              end
+            rescue Exception => e
+              @config.logger.error "An unexpected error occoured while syncing image #{image}: #{e}"
+              false
+            else
+              true
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/docker/registry/sync/sqs.rb

@@ -0,0 +1,48 @@
+require 'digest'
+require 'aws-sdk'
+
+module Docker
+  module Registry
+    module Sync
+      module CMD
+        include Docker::Registry::Sync
+
+        class << self
+          def send_message_batch(messages, retries = 5)
+            if retries <= 0
+              success = false
+              messages.each do |msg|
+                @config.logger.Error "Failed to Enqueue message: #{msg}"
+              end
+            else
+              entries = messages.map do |msg|
+                @config.logger.info "Enqueuing message: #{msg}"
+                { id: Digest::MD5.hexdigest(msg), message_body: msg }
+              end
+              sqs = Aws::SQS::Client.new(region: @config.sqs_region)
+              resp = sqs.send_message_batch(queue_url: @config.sqs_url,
+                                            entries: entries)
+              if resp.failed.length > 0
+                rerun = resp.failed.map do |failed|
+                  @config.logger.warn "Failed to Enqueue message, re-enqueuing: #{msg}"
+                  messages.select { |m| Digest::MD5.hexdigest(m) == failed.id }[0]
+                end
+                sleep 1 # short sleep before trying again...
+                success = send_message_batch(rerun, retries - 1)
+              else
+                success = true
+              end
+            end
+            success
+          end
+
+          def finalize_message(receipt_handle)
+            sqs = Aws::SQS::Client.new(region: @config.sqs_region)
+            resp = sqs.delete_message(queue_url: @config.sqs_url,
+                                      receipt_handle: receipt_handle)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/docker/registry/sync/version.rb

@@ -0,0 +1,7 @@
+module Docker
+  module Registry
+    module Sync
+      VERSION = '0.1.0'
+    end
+  end
+end

metadata

@@ -0,0 +1,170 @@
+--- !ruby/object:Gem::Specification
+name: docker-registry-sync
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Brian Oldfield
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-09-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: simplecov-console
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.21'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.21'
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: Sync data between S3 two distinct docker registries using S3 data storage
+  and SNS as a messaging system.
+email:
+- brian.oldfield@socrata.com
+executables:
+- docker-registry-sync
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.md
+- bin/docker-registry-sync
+- docker-registry-sync.gemspec
+- lib/docker/registry/sync.rb
+- lib/docker/registry/sync/cmd.rb
+- lib/docker/registry/sync/configuration.rb
+- lib/docker/registry/sync/s3.rb
+- lib/docker/registry/sync/sqs.rb
+- lib/docker/registry/sync/version.rb
+homepage: http://github.com/socrata-platform/docker-registry-sync
+licenses:
+- Apache
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: Sync data between S3 two distinct docker registries using S3 data storage
+  and SNS as a messaging system.
+test_files: []