indocker 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 90d16fd262c9576fcdd7e9f442bd6706533b37c195a8c95f991dd6ac4d7daf0a
-  data.tar.gz: 9a4d6fb758745112611f62879a6c8409bc8f2f3bfc9355b83b8022e3951ec111
+  metadata.gz: f5c86b947ae3162d9b81812db303c76b6825a1adc397cee5e3495bbd68437e96
+  data.tar.gz: 6dd6c07468dad622731efeb333a55279dabf18e74a940e62fa2308d572636713
 SHA512:
-  metadata.gz: 12faa98ac4b95b8f9d7dbfad174a8fc797082c320eeff858bedfad1f136fd60024f9800915fb9e725c55717565833f6c36b83666ccace129a83ef1fb9e631791
-  data.tar.gz: ae35e8ca9212ebb76c63dd43348aa1fe20cbd462809e54660c20f693a0bcb4201d01b57bf68941c597c396ad68a2532e5fe0a2ac3077bd3d5578e9db1798a6e0
+  metadata.gz: 5546b2240618feacfe4efaa8c6b8ae82e58dde32eb383ff8cf7fd7295cf50caae8833ae0c8205031dea06bf7d7acb536935b0235c425bb5bf8506cb6cb22e7ef
+  data.tar.gz: e9d0d5ae9c6ee1b114d7551e954a282775a62c20840dc4c3a51675659f7ee99e5faa2691fb450e3e4c1ab1757b61d3459bc6aa1e0e67be0b75dd9d2de3036c74

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    indocker (0.4.1)
+    indocker (0.5.0)
       net-ssh
 
 GEM

data/lib/indocker/container_deployer.rb

@@ -12,30 +12,37 @@ class Indocker::ContainerDeployer
 
     @deployed_containers = Hash.new(false)
     @deployed_servers = {}
+
+    @server_locks = {}
+    @server_locks_mutex = Mutex.new
   end
 
   def deploy(container, force_restart, skip_force_restart, progress)
     return if @deployed_containers[container]
 
-    # Deploy the same container to all of its servers in parallel.
+    # Deploy the container to all of its servers in parallel, each server in its
+    # own thread with its own ssh connection.
     #
-    # Each thread opens its OWN ssh connection (create_connection!), so no
-    # net-ssh session is shared between threads. We join all threads before
-    # returning, so container dependency ordering (depends_on) is preserved:
-    # a dependent container never starts deploying until every server of its
-    # dependency has finished.
-    threads = container.servers.map do |server|
-      progress.start_deploying_container(container, server)
-
-      Thread.new do
-        deploy_to_server(container, server, force_restart, skip_force_restart, progress)
+    # Per-server exclusion: only one container may deploy on a given server at a
+    # time, so several container types don't spike CPU/memory by starting at once
+    # on the same host. We grab the lock of every target server before deploying.
+    # Locks are always taken in a consistent (sorted) order, so concurrent
+    # deployments of containers with overlapping server sets can't deadlock.
+    ordered_servers = container.servers.sort_by { |server| server_lock_key(server) }
+    locks = ordered_servers.map { |server| server_lock(server) }.uniq
+
+    with_locks(locks) do
+      threads = container.servers.map do |server|
+        Thread.new do
+          deploy_to_server(container, server, force_restart, skip_force_restart, progress)
+        end
       end
-    end
 
-    results = threads.map(&:value)
+      results = threads.map(&:value)
 
-    if results.any? { |exit_code| exit_code != 0 }
-      exit 1
+      if results.any? { |exit_code| exit_code != 0 }
+        exit 1
+      end
     end
 
     @deployed_containers[container] = true
@@ -54,6 +61,8 @@ class Indocker::ContainerDeployer
   # connection, so it must not touch shared mutable state except the
   # thread-safe `progress`.
   def deploy_to_server(container, server, force_restart, skip_force_restart, progress)
+    progress.start_deploying_container(container, server)
+
     deploy_server = @server_pool.create_connection!(server)
     @logger.info("Deploying container: #{container.name.to_s.green} to #{server.user}@#{server.host}")
 
@@ -73,4 +82,26 @@ class Indocker::ContainerDeployer
 
     result.exit_code
   end
-end
+
+  def server_lock_key(server)
+    [server.host, server.port, server.user]
+  end
+
+  # Returns a process-wide singleton Mutex for the given server, so the same lock
+  # is shared across every container targeting that server.
+  def server_lock(server)
+    key = server_lock_key(server)
+
+    @server_locks_mutex.synchronize do
+      @server_locks[key] ||= Mutex.new
+    end
+  end
+
+  def with_locks(locks, &block)
+    if locks.empty?
+      block.call
+    else
+      locks.first.synchronize { with_locks(locks.drop(1), &block) }
+    end
+  end
+end

data/lib/indocker/launchers/configuration_deployer.rb

@@ -88,27 +88,16 @@ class Indocker::Launchers::ConfigurationDeployer
 
     update_crontab_redeploy_rules(configuration, build_servers.first)
 
-    containers.uniq.each do |container|
-      recursively_deploy_container(
-        configuration,
-        deployer,
-        build_server_pool,
-        container,
-        containers,
-        deployment_policy.skip_build,
-        deployment_policy.skip_deploy,
-        deployment_policy.force_restart,
-        deployment_policy.skip_force_restart
-      )
-    end
-
-    Thread
-      .list
-      .each { |t|
-        if t != Thread.current
-          t.join if !t.stop?
-        end
-      }
+    pipeline_build_and_deploy(
+      configuration,
+      deployer,
+      build_server_pool,
+      containers.uniq,
+      deployment_policy.skip_build,
+      deployment_policy.skip_deploy,
+      deployment_policy.force_restart,
+      deployment_policy.skip_force_restart
+    )
   ensure
     build_server_pool.close_sessions if build_server_pool
     deployer.close_sessions if deployer
@@ -306,40 +295,86 @@ class Indocker::Launchers::ConfigurationDeployer
     @compiled_images[image] = true
   end
 
-  def recursively_deploy_container(configuration, deployer, build_server_pool, container,
-    containers, skip_build, skip_deploy, force_restart, skip_force_restart)
-
-    container.dependent_containers.each do |container|
-      recursively_deploy_container(
-        configuration,
-        deployer,
-        build_server_pool,
-        container,
-        containers,
-        skip_build,
-        skip_deploy,
-        force_restart,
-        skip_force_restart
-      )
+  # Producer/consumer pipeline that decouples image builds from deployments.
+  #
+  # The producer builds images one after another on the build server (serially,
+  # in dependency order) and, as soon as an image is built & pushed, hands its
+  # container to the deploy stage — so the next image is already building while
+  # the previous container deploys.
+  #
+  # The consumer starts a deploy per built container. Each deploy first waits for
+  # all of the container's dependencies to finish deploying (depends_on ordering),
+  # then deploys. Independent containers deploy concurrently; per-server exclusion
+  # (one container at a time per host) is enforced inside ContainerDeployer.
+  def pipeline_build_and_deploy(configuration, deployer, build_server_pool, containers,
+    skip_build, skip_deploy, force_restart, skip_force_restart)
+
+    ordered_containers = build_deploy_order(containers)
+
+    deploy_queue   = Queue.new
+    deploy_threads = {}
+
+    consumer = Thread.new do
+      while (container = deploy_queue.pop) != :done
+        # Dependencies are always built (and therefore queued) before their
+        # dependents, so their deploy threads already exist here.
+        dependency_threads = container
+          .dependent_containers
+          .map { |dependency| deploy_threads[dependency] }
+          .compact
+
+        deploy_threads[container] = Thread.new do
+          dependency_threads.each(&:join)
+          deploy_container(deployer, container, force_restart, skip_force_restart)
+        end
+      end
     end
 
-    return if !containers.include?(container)
+    ordered_containers.each do |container|
+      @progress.start_building_container(container)
 
-    @progress.start_building_container(container)
+      if !skip_build
+        build_server = build_server_pool.get
 
-    if !skip_build
-      build_server = build_server_pool.get
+        build_server.set_busy(true)
+        compile_image(configuration, container.image, build_server)
+        build_server.set_busy(false)
+      end
 
-      build_server.set_busy(true)
-      compile_image(configuration, container.image, build_server)
-      build_server.set_busy(false)
+      @progress.finish_building_container(container)
+
+      deploy_queue.push(container) if !skip_deploy
     end
 
-    @progress.finish_building_container(container)
+    deploy_queue.push(:done)
+    consumer.join
+    deploy_threads.values.each(&:join)
+  end
+
+  # Returns the containers to deploy in dependency order (each container's
+  # dependencies come before it), deduplicated. Containers that are only pulled
+  # in as dependencies but are not part of the deploy set are used for ordering
+  # but not included.
+  def build_deploy_order(containers)
+    ordered = []
+    visited = {}
 
-    if !skip_deploy
-      deploy_container(deployer, container, force_restart, skip_force_restart)
+    containers.each do |container|
+      collect_deploy_order(container, containers, ordered, visited)
     end
+
+    ordered
+  end
+
+  def collect_deploy_order(container, containers, ordered, visited)
+    return if visited[container]
+    visited[container] = true
+
+    container.dependent_containers.each do |dependency|
+      collect_deploy_order(dependency, containers, ordered, visited)
+    end
+
+    ordered << container if containers.include?(container)
   end
 
   def deploy_container(deployer, container, force_restart, skip_force_restart)

data/lib/indocker/version.rb

@@ -1,3 +1,3 @@
 module Indocker
-  VERSION = "0.4.1"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: indocker
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Ruslan Gatiyatov