docker-registry-sync 0.2.11 → 0.2.12

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NzRjNGI0YjY1ZTYzNDY5ZTljZmE4NGY5MjI3MmI4M2VhMWZiYmU4YQ==
+    MzRkMjE5NGZlODE5MTMxM2ZmMWFmNzBiMGNiZDE3OWVlNmZmMDQ4Mw==
   data.tar.gz: !binary |-
-    Nzk4MjJmZmM0OWIzZjY4MTAxZmI3OTljNGJiYmVhMDc2ODcxMjM3Yg==
+    MDI5MDVmM2Q4MzM5Y2Q2M2ZlNDQyYzU1Y2M5OWMyMGFhNDEyNzcyNg==
 SHA512:
   metadata.gz: !binary |-
-    ZmJmMWQ1YTUxZTkyYTJlNzRjODVmMzk4MTYyZjc2YjJkNmI5ZTc1ODFiOTdl
-    MjI5ODNhMjdlNThjYzhkYTIzZDZmNThmYzM5NzVlZTk1MDg1ODg2NmEwOTIz
-    YmJmMmI5OTVhMDExZDBlYzg1ZGYyMGU4NWM0N2I2MmEyMjkyMzA=
+    OWRmZTdhNTNlMmY1ZTIzNmIyOGIzNGJhOTE5ZDAwMzI3YTlhYWU2ZmMyMzU5
+    NDhlMzk5Zjc1MGFlM2VmNTdhMzk0NjEzYWQ0MWQwNGY4MTY4MjkyYzEyMDdk
+    ZGI4OGU5NzBhZjVlNzI0NzVmN2EzMWIzNjZmZjViMWU3ZTliZGI=
   data.tar.gz: !binary |-
-    M2ZmOGI3ZmY4NmRjYjFjMjNlZjViZWI1NGI4ODZkZDUwODcyNjFmZjkwYWQx
-    YzQzNmJiYjBiMjM4ODI2NzEyMzczZjI5ZjE2OWFmY2RiOTljOTM5MjFhYzI0
-    OTJjZjhlZjgwZTIyMTM3YzcwNjkxYjQ4MmZhOTBiZDk5YTM5MGY=
+    MGNmMmExZjBiODk1MzI0ZWY1NjUxZGNkYzZiOGQyMGUwZTY1YjA2ZjNjYmNk
+    MTBmYzhmOTAzYjU5Y2E5ZWQ4NTM0NWU0ZmIyOTNhZGNkYmI4OWRjZTkyOGFj
+    MGVjYzM5MDk1OTIyYzdjNGM4ZDU2MDI5ZjFhZjYwZDQxMDE5ODU=

data/.gitignore

@@ -3,6 +3,7 @@
 bin/*
 !bin/docker-registry-sync
 .bundle/
+Gemfile.lock
 coverage/
 install.sh
 test.rb

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/lib/docker/registry/sync/cmd.rb

@@ -11,7 +11,9 @@ module Docker
         include Docker::Registry::Sync
 
         class << self
-          def configure(source_bucket, target_buckets, sqs_queue, use_sse, source_uses_sse, pool)
+          attr_accessor :config, :producer_finished, :work_queue, :status_queue, :threads
+
+          def configure(source_bucket, target_buckets, sqs_queue, use_sse, source_uses_sse, pool, log_level = :debug)
             unless source_bucket.nil?
               source_region, source_bucket = source_bucket.split(':')
             else
@@ -41,6 +43,7 @@ module Docker
               config.sqs_region = sqs_region
               config.pool_size = pool
               config.sqs_url = "https://#{sqs_uri}"
+              config.log_level = log_level
             end
             @config = Docker::Registry::Sync.config
           end
@@ -66,7 +69,7 @@ module Docker
             end
           end
 
-          def start_workers
+          def configure_workers
             @threads = Array.new(@config.pool_size)
             @work_queue = Queue.new
             @status_queue = Queue.new
@@ -75,6 +78,9 @@ module Docker
             @threads_available = @threads.new_cond
 
             @producer_finished = false
+          end
+
+          def start_workers
             @consumer_thread = Thread.new do
               sync_key_consumer
             end
@@ -85,6 +91,7 @@ module Docker
               @producer_finished = true
             end
             @consumer_thread.join
+            @consumer_thread = nil
             @threads.each { |t| t.join unless t.nil? }
             @config.logger.info "Processing job results..."
 
@@ -103,6 +110,7 @@ module Docker
 
           def sync(image, tag)
             configure_signal_handlers
+            configure_workers
             start_workers
             success = false
             @config.target_buckets.each do |region, bucket, sse|
@@ -153,6 +161,7 @@ module Docker
                 @config.logger.info "Image sync data:  #{data}"
 
                 if image_exists?(data['image'], data['target']['bucket'], data['target']['region'])
+                  configure_workers
                   start_workers
                   @config.logger.info("Syncing tag: #{data['image']}:#{data['tag']} to #{data['target']['region']}:#{data['target']['bucket']}")
                   success = sync_tag(data['image'], data['tag'], data['target']['bucket'], data['target']['region'], data['target']['sse'], data['source']['bucket'], data['source']['region'])
@@ -165,6 +174,7 @@ module Docker
                     @config.logger.info("Falied to sync tag, leaving on queue: #{data['image']}:#{data['tag']} to #{data['target']['region']}:#{data['target']['bucket']}")
                   end
                 else
+                  configure_workers
                   start_workers
                   success = sync_repo(data['image'], data['target']['bucket'], data['target']['region'], data['target']['sse'], data['source']['bucket'], data['source']['region'])
                   success &&= finalize_workers
@@ -179,12 +189,13 @@ module Docker
               end
               ec = 0
               sleep @config.empty_queue_sleep_time unless @terminated
-            rescue => e
+            rescue StandardError => e
               @config.logger.error "An unknown error occurred while monitoring queue: #{e}"
               @config.logger.error e.traceback
               @config.logger.error 'Exiting...'
               @terminated = true
               ec = 1
+              finalize_workers # make sure there are no hangers-on
             end until @terminated
             ec
           end

data/lib/docker/registry/sync/s3.rb

@@ -34,7 +34,7 @@ module Docker
 
               img_id = s3_source.get_object(bucket: source_bucket, key: "registry/repositories/#{image}/tag_#{tag}").body.read
               sync_image(img_id, bucket, region, sse, source_bucket, source_region)
-            rescue => e
+            rescue StandardError => e
               @config.logger.error "An unexpected error occoured while syncing tag #{image}:#{tag}: #{e}"
               @config.logger.error e.backtrace
               false
@@ -57,7 +57,7 @@ module Docker
               JSON.load(img_index_resp.body.read).each do |image|
                 sync_image(image['id'], bucket, region, sse, source_bucket, source_region)
               end
-            rescue => e
+            rescue StandardError => e
               @config.logger.error "An unexpected error occoured while syncing repo #{repo}: #{e}"
               @config.logger.error e.backtrace
               false
@@ -114,9 +114,7 @@ module Docker
               if @config.source_sse
                 opts[:copy_source_sse_customer_algorithm] = 'AES256'
               end
-              @threads.synchronize do
-                @work_queue << opts
-              end
+              @work_queue << opts
               sleep 0.1
             end
           end
@@ -134,9 +132,7 @@ module Docker
               t_index = @threads.rindex { |t| t.nil? || t.status == false || t['finished'].nil? == false }
 
               begin
-                opts = @threads.synchronize do
-                  @work_queue.pop(true)
-                end
+                opts = @work_queue.pop(true)
               rescue ThreadError
                 @config.logger.info "No work found on the queue, sleeping..."
                 sleep 1
@@ -152,14 +148,12 @@ module Docker
                       target_client.copy_object(opts)
                       success = true
                       @config.logger.info "Worker finished syncing key: #{opts[:key]}"
-                    rescue => e
+                    rescue StandardError => e
                       @config.logger.error "An unknown error occoured while copying object in s3: #{e}"
                       @config.logger.error e.backtrace
                     ensure
                       Thread.current['finished'] = true
-                      @threads.synchronize do
-                        @status_queue << success
-                      end
+                      @status_queue << success
                     end
                   end
                 else

data/lib/docker/registry/sync/version.rb

@@ -1,7 +1,7 @@
 module Docker
   module Registry
     module Sync
-      VERSION = '0.2.11'
+      VERSION = '0.2.12'
     end
   end
 end

data/spec/s3_spec.rb

@@ -0,0 +1,148 @@
+require 'json'
+require 'spec_helper'
+require 'docker/registry/sync'
+require 'aws-sdk'
+
+def stub_s3_response(action, resp)
+  Aws.config[:s3] = { stub_responses: {} }
+  Aws.config[:s3][:stub_responses][action] = resp
+end
+
+describe 'Docker::Register::Sync::CMD' '#sync_tag' do
+  before do
+    Docker::Registry::Sync::CMD.configure(
+      'us-west-2:test-source-bucket',
+      'us-west-2:test-target-bucket',
+      'us-west-2:test-sqs-queue',
+      false,
+      false,
+      2,
+      :off
+    )
+    Docker::Registry::Sync::CMD.configure_workers
+  end
+
+  context 'syncing tags' do
+    it 'successfully syncs' do
+      img_id = 'sync-successful'
+      stub_s3_response(:get_object, { body: img_id })
+      allow(Docker::Registry::Sync::CMD).to receive(:sync_image).with(
+        img_id,
+        'test-target-bucket',
+        'us-west-2',
+        false,
+        'test-source-bucket',
+        'us-west-2'
+      )
+      allow(Docker::Registry::Sync::CMD).to receive(:sync_keys)
+      res = Docker::Registry::Sync::CMD.sync_tag('sync-image', 'sync-tag', 'test-target-bucket', 'us-west-2', false)
+      expect(res).to be true
+    end
+
+    it 'S3 API failure' do
+      img_id = 'sync-unsuccessful'
+      stub_s3_response(:get_object, RuntimeError.new('api failure'))
+      allow(Docker::Registry::Sync::CMD).to receive(:sync_keys)
+      res = Docker::Registry::Sync::CMD.sync_tag('sync-image', 'sync-tag', 'test-target-bucket', 'us-west-2', false)
+      expect(res).to be false
+    end
+  end
+end
+
+describe 'Docker::Register::Sync::CMD' '#sync_repo' do
+  before do
+    Docker::Registry::Sync::CMD.configure(
+      'us-west-2:test-source-bucket',
+      'us-west-2:test-target-bucket',
+      'us-west-2:test-sqs-queue',
+      false,
+      false,
+      2,
+      :off
+    )
+    Docker::Registry::Sync::CMD.configure_workers
+  end
+
+  context 'syncing repositories' do
+    it 'successfully syncs' do
+      images = [{'id' => 'image-one'}, {'id' => 'image-two'}]
+      allow(Docker::Registry::Sync::CMD).to receive(:sync_prefix)
+      images.each do |i|
+        allow(Docker::Registry::Sync::CMD).to receive(:sync_image).with(i['id'],'test-target-bucket', 'us-west-2', false, 'test-source-bucket', 'us-west-2')
+      end
+
+      stub_s3_response(:get_object, { body: JSON.dump(images)})
+      res = Docker::Registry::Sync::CMD.sync_repo('sync-repo', 'test-target-bucket', 'us-west-2', false)
+      expect(res).to be true
+    end
+
+    it 'S3 API failure' do
+      images = [{'id' => 'image-one'}, {'id' => 'image-two'}]
+      allow(Docker::Registry::Sync::CMD).to receive(:sync_prefix)
+      images.each do |i|
+        allow(Docker::Registry::Sync::CMD).to receive(:sync_image).with(i['id'],'test-target-bucket', 'us-west-2', false, 'test-source-bucket', 'us-west-2')
+      end
+
+      stub_s3_response(:get_object, RuntimeError.new('api failure'))
+      res = Docker::Registry::Sync::CMD.sync_repo('sync-repo', 'test-target-bucket', 'us-west-2', false)
+      expect(res).to be false
+    end
+  end
+end
+
+describe 'Docker::Register::Sync::CMD' '#sync_key_consumer' do
+  before do
+    Docker::Registry::Sync::CMD.configure(
+      'us-west-2:test-source-bucket',
+      'us-west-2:test-target-bucket',
+      'us-west-2:test-sqs-queue',
+      false,
+      false,
+      1,
+      :off
+    )
+    Docker::Registry::Sync::CMD.configure_workers
+  end
+
+  context 'runs' do
+    it 'successfully exits with no work and producer is finished' do
+      t = Thread.new do
+        Docker::Registry::Sync::CMD.sync_key_consumer
+      end
+      Docker::Registry::Sync::CMD.producer_finished = true
+      t.join
+      expect(Docker::Registry::Sync::CMD.work_queue.length).to be 0
+      expect(t.status).to be false
+    end
+
+    it 'successfully exits when a child encounters an exception' do
+      Docker::Registry::Sync::CMD.work_queue << {
+        region: 'us-west-2',
+        key: 'exception-raiser',
+        bucket: 'exception-raiser',
+        copy_source: 'exception-raiser',
+      }
+      Docker::Registry::Sync::CMD.work_queue << {
+        region: 'us-west-2',
+        key: 'exception-raiser',
+        bucket: 'exception-raiser',
+        copy_source: 'exception-raiser',
+      }
+      Docker::Registry::Sync::CMD.producer_finished = true
+      stub_s3_response(:copy_object, RuntimeError.new('api failure'))
+      t = Thread.new do
+        Docker::Registry::Sync::CMD.sync_key_consumer
+      end
+
+      t.join
+      Docker::Registry::Sync::CMD.threads.each { |t| t.join }
+      expect(t.status).to be false
+      expect(Docker::Registry::Sync::CMD.threads[0].status).to be false
+
+      expect(Docker::Registry::Sync::CMD.work_queue.length).to be 0
+      expect(Docker::Registry::Sync::CMD.status_queue.length).to be 2
+      expect(Docker::Registry::Sync::CMD.status_queue.pop).to be false
+      expect(Docker::Registry::Sync::CMD.status_queue.pop).to be false
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,66 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+require 'webmock/rspec'
+require 'rack'
+require 'simplecov'
+require 'simplecov-console'
+
+WebMock.disable_net_connect!(allow_localhost: false)
+DIR = File.expand_path(File.dirname(__FILE__))
+
+class S3ApiRack
+  def call(env)
+    code, ret = 500, 'The tests have changed, fix the rack'
+    [code, { 'Content-Type' => 'application/json' }, [ret]]
+  end
+end
+
+RSpec.configure do |config|
+  config.expect_with :rspec do |expectations|
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+  config.mock_with :rspec do |mocks|
+    mocks.verify_partial_doubles = true
+  end
+  # 
+  config.before(:each) do
+    # Leaving in place to prevent accidental api calls
+    stub_request(:any, /127.0.0.1\//).
+      to_rack(S3ApiRack.new)
+  end
+
+end
+
+SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
+  SimpleCov::Formatter::HTMLFormatter,
+  SimpleCov::Formatter::Console
+]
+SimpleCov.minimum_coverage(80)
+SimpleCov.start
+
+def capture_stdout(&block)
+  original_stdout = $stdout
+  $stdout = fake = StringIO.new
+  begin
+    yield
+  ensure
+    $stdout = original_stdout
+  end
+  #fake.string
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: docker-registry-sync
 version: !ruby/object:Gem::Version
-  version: 0.2.11
+  version: 0.2.12
 platform: ruby
 authors:
 - Brian Oldfield
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-22 00:00:00.000000000 Z
+date: 2015-10-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -132,6 +132,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .rspec
 - Gemfile
 - README.md
 - bin/docker-registry-sync
@@ -142,6 +143,8 @@ files:
 - lib/docker/registry/sync/s3.rb
 - lib/docker/registry/sync/sqs.rb
 - lib/docker/registry/sync/version.rb
+- spec/s3_spec.rb
+- spec/spec_helper.rb
 homepage: http://github.com/socrata-platform/docker-registry-sync
 licenses:
 - Apache
@@ -167,4 +170,6 @@ signing_key:
 specification_version: 4
 summary: Sync data between S3 two distinct docker registries using S3 data storage
   and SNS as a messaging system.
-test_files: []
+test_files:
+- spec/s3_spec.rb
+- spec/spec_helper.rb