diffbroker 0.11.0 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5edf0d3084adb3e7b1e8a9f55dda5062efa3a5207d8f4ca7d7860141acef3ec7
-  data.tar.gz: 85899028c0a4d827e3df60c530298f9713e4a33e4ccf5ec4397fae0d58a6c37a
+  metadata.gz: 5b3251c8cf1dc34ca4cfda98157ac33e2cdd3181d2b7464b6732a73b6731de43
+  data.tar.gz: cf1c80fb6862c480b58442d6432ef38d05e7817a44f5813642b402d2b1450a7e
 SHA512:
-  metadata.gz: 7b7ca4efd11472c69863a386ebf10336402eac96f998a8665457f20a7f6b395af3a3814925915e1881d0f018c906344a0eccd45cdd0d9557fc4f44fc0a3891be
-  data.tar.gz: d763d542d2a6c27adad2bf78cf3b6f0bb1488b0f384bb90fef6990b27c142af472078409f5886be43c92fe6df7b158e847547720018f28ce0efe57ab7060b4dc
+  metadata.gz: 4561d11449c3387cd2f97b44bb60b2f4bf0f3259ae5d5bc97fb7c13764c00c68a60f787fec09f45a876dccb1c861edd4d7f8acf24448fec0d2f16b3878e0e336
+  data.tar.gz: 79116f0dff8028b473d6c526606cca711c091e1b71bdf7fdc304482d41e2ca9d171e3d0aa01b1396c0c4e22870abf1300b8aa2360ed2a66ccbf566ea6e8cfab5

data/exe/diffbroker

@@ -3,6 +3,9 @@
 
 require_relative "../lib/diffbroker"
 
+# A daemon's log is read when something is wrong, so lines must reach the file as they happen, not when a buffer fills.
+$stdout.sync = $stderr.sync = true
+
 begin
   exit Diffbroker::CLI.new(ARGV).run
 rescue Interrupt

data/lib/diffbroker/cable_client.rb

@@ -49,17 +49,35 @@ module Diffbroker
     POLL_EVERY = 1
     SELECT_TIMEOUT = 1
     CONNECT_TIMEOUT = 10
+    SUBSCRIBE_TIMEOUT = 15
+    WRITE_TIMEOUT = 15
     READ_CHUNK = 16_384
 
+    # Writes never block for ever on a peer that vanished (a laptop that slept, a dropped Wi-Fi): after WRITE_TIMEOUT the
+    # connection is an error and the daemon reconnects.
     SocketAdapter = Struct.new(:url, :io) do
-      def write(data) = io.write(data)
+      def write(data)
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + WRITE_TIMEOUT
+        until data.empty?
+          written = io.write_nonblock(data, exception: false)
+          case written
+          when Integer then data = data.byteslice(written..)
+          when :wait_readable, :wait_writable
+            left = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            raise Errno::ETIMEDOUT, "write to #{url}" if left <= 0
+
+            written == :wait_readable ? io.wait_readable(left) : io.wait_writable(left)
+          end
+        end
+      end
     end
 
-    def initialize(url:, token:, origin:, protocol: Protocol.new)
+    def initialize(url:, token:, origin:, protocol: Protocol.new, subscribe_timeout: SUBSCRIBE_TIMEOUT)
       @url = url
       @token = token
       @origin = origin
       @protocol = protocol
+      @subscribe_timeout = subscribe_timeout
       @outbox = Queue.new
       @wake_reader, @wake_writer = IO.pipe
     end
@@ -91,8 +109,9 @@ module Diffbroker
       pump
       final_flush
     ensure
-      @io.close if @io && !@io.closed?
-      [ @wake_reader, @wake_writer ].each(&:close)
+      close_quietly(@io)
+      close_quietly(@wake_reader)
+      close_quietly(@wake_writer)
     end
 
     private
@@ -147,6 +166,7 @@ module Diffbroker
     def stale!
       if @welcomed
         @stopped = true if monotonic - @last_ping > STALE_AFTER
+        raise HandshakeError, "The server never confirmed the device channel subscription" if !@subscribed && monotonic - @welcomed_at > @subscribe_timeout
       elsif monotonic - @started > STALE_AFTER
         raise HandshakeError, "No welcome from #{@url} within #{STALE_AFTER} s"
       end
@@ -171,7 +191,7 @@ module Diffbroker
       case kind
       when :welcome
         @welcomed = true
-        @last_ping = monotonic
+        @welcomed_at = @last_ping = monotonic
         @driver.text(@protocol.subscribe)
       when :ping then @last_ping = monotonic
       when :confirmed
@@ -186,6 +206,19 @@ module Diffbroker
       end
     end
 
+    # Best effort: lets the OS notice a peer that vanished without a goodbye.
+    def keep_alive(tcp)
+      tcp.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true)
+    rescue SystemCallError
+      nil
+    end
+
+    def close_quietly(io)
+      io.close if io && !io.closed?
+    rescue IOError, SystemCallError, OpenSSL::SSL::SSLError
+      nil
+    end
+
     def wake
       @wake_writer.write_nonblock(".", exception: false)
     rescue IOError
@@ -196,7 +229,8 @@ module Diffbroker
       uri = URI(@url)
       raise Error, "Invalid cable URL #{@url.inspect}" unless %w[ws wss].include?(uri.scheme) && uri.host
 
-      tcp = Socket.tcp(uri.host, uri.port, connect_timeout: CONNECT_TIMEOUT)
+      tcp = Socket.tcp(uri.host, uri.port, connect_timeout: CONNECT_TIMEOUT, resolv_timeout: CONNECT_TIMEOUT)
+      keep_alive(tcp)
       return tcp unless uri.scheme == "wss"
 
       context = OpenSSL::SSL::SSLContext.new
@@ -204,10 +238,26 @@ module Diffbroker
       ssl = OpenSSL::SSL::SSLSocket.new(tcp, context)
       ssl.hostname = uri.host
       ssl.sync_close = true
-      ssl.connect
+      connect_tls(ssl)
       ssl
     end
 
+    # The TLS handshake has no timeout of its own: a captive portal or a stalled VPN would hold the daemon for ever.
+    def connect_tls(ssl)
+      deadline = monotonic + CONNECT_TIMEOUT
+      loop do
+        case ssl.connect_nonblock(exception: false)
+        when :wait_readable then ssl.wait_readable([ deadline - monotonic, 0 ].max)
+        when :wait_writable then ssl.wait_writable([ deadline - monotonic, 0 ].max)
+        else return
+        end
+        raise Errno::ETIMEDOUT, "TLS handshake with #{@url}" if monotonic > deadline
+      end
+    rescue StandardError
+      close_quietly(ssl)
+      raise
+    end
+
     def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
   end
 end

data/lib/diffbroker/cli.rb

@@ -36,10 +36,13 @@ module Diffbroker
       Agent activity is visible only to you; only status is sent, never prompts, file contents or tool input.
     TEXT
 
-    def initialize(argv, out: $stdout, err: $stderr, env: ENV, home: Dir.home, input: $stdin, maintenance: nil, lifecycle: nil)
+    def initialize(argv, out: $stdout, err: $stderr, env: ENV, home: Dir.home, input: $stdin, maintenance: nil, lifecycle: nil, reconciler: nil,
+                   sleeper: ->(seconds) { sleep(seconds) })
       @argv = argv.dup
       @maintenance = maintenance
       @lifecycle = lifecycle
+      @reconciler = reconciler
+      @sleeper = sleeper
       @out = out
       @err = err
       @env = env
@@ -134,6 +137,7 @@ module Diffbroker
       raise Error, "This machine is not paired. Run `diffbroker start` in a terminal to pair it." unless config.paired?
 
       @out.puts "Diff Broker companion #{VERSION} → #{config.server} (agent: #{config.agent}, terminal: #{config.terminal})"
+      heal_service_definition
       Daemon.build(config, out: @out).run
     rescue Daemon::Rejected
       # The pairing is dead. Forget it and remove the service, so nothing restarts a daemon that can never connect.
@@ -142,11 +146,20 @@ module Diffbroker
       raise
     end
 
+    # A daemon the service manager started under an old definition (another launcher or Ruby, or the old restart policy)
+    # corrects the file for the next load. Only the file: the running job keeps going.
+    def heal_service_definition
+      service = ServiceInstaller.new(home: @home, env: @env)
+      service.rewrite! if lifecycle.service_process? && service.installed? && service.stale?
+    rescue StandardError
+      nil
+    end
+
     def set_up(name)
       forget_dead_pairing if config.paired? && pairing_rejected?
       Login.new(config:, out: @out, name:).run unless config.paired?
+      remove_legacy_service
       install_background_service
-      @out.puts "Diff Broker is running in the background. Open #{config.server} to see it connected."
     rescue Error => e
       raise unless service_unavailable?(e)
 
@@ -155,8 +168,16 @@ module Diffbroker
       run_daemon
     end
 
-    # True only when the server answered "unauthorized". An unreachable server proves nothing, so the pairing is kept.
+    # True only when the server answered "unauthorized" twice, a few seconds apart (a deploy or a restore can say it once).
+    # An unreachable server proves nothing, so the pairing is kept.
     def pairing_rejected?
+      return false unless ping_unauthorized?
+
+      @sleeper.call(3)
+      ping_unauthorized?
+    end
+
+    def ping_unauthorized?
       Api.new(server: config.server, token: config.device_token).get("/device/ping")
       false
     rescue Api::HttpError => e
@@ -171,16 +192,37 @@ module Diffbroker
       ServiceInstaller.new(home: @home, env: @env).uninstall!
     end
 
-    # Installs what is missing and starts it. A service that is already installed is only started (installing again
-    # would restart a daemon that may be running a task).
+    # Installs what is missing, then makes sure the newest companion runs and is connected (Reconciler): an installed
+    # service is not just started, because starting a running job does nothing and the daemon in it may be an older version.
     def install_background_service
       service = ServiceInstaller.new(home: @home, env: @env)
-      return lifecycle.launch if service.installed?
+      install_service unless service.installed?
+      raise Error, "The companion is not connected yet." unless reconciler(service).call
+    end
 
-      install_service
+    def service_unavailable?(error)
+      error.is_a?(ServiceInstaller::Unavailable) || error.message.include?("not found on PATH") || error.message.include?("systemctl")
     end
 
-    def service_unavailable?(error) = error.message.include?("not found on PATH") || error.message.include?("systemctl")
+    # The service of the companion's old name (gitbroker) keeps crash-looping against a domain that no longer answers.
+    def remove_legacy_service
+      @out.puts "Removed the background service of the companion's old name (gitbroker)." if ServiceInstaller.new(home: @home, env: @env).remove_legacy!
+    rescue Error
+      nil
+    end
+
+    def reconciler(service = ServiceInstaller.new(home: @home, env: @env), version: VERSION)
+      @reconciler ||= Reconciler.new(service:, lifecycle:, state: DaemonState.new(dir: config.dir), out: @out, server: config.server, version:,
+                                     on_refresh: method(:refresh_integrations))
+    end
+
+    # A rewritten service runs a new launcher: the Claude Code hooks and the start link name it too.
+    def refresh_integrations
+      refresh_hooks
+      UrlHandlerInstaller.new(home: @home).install!
+    rescue Error
+      nil
+    end
 
     def install_service
       raise Error, "This machine is not paired. Run `diffbroker start` first." unless config.paired?
@@ -200,7 +242,8 @@ module Diffbroker
     # service, or the daemon itself when none is installed. Safe to run when it is already running.
     def launch
       raise Error, "This machine is not paired. Run `diffbroker start` first." unless config.paired?
-      lifecycle.launch
+
+      reconciler.call(verify: false)
     end
 
     def lifecycle = @lifecycle ||= Lifecycle.new(dir: config.dir, home: @home, service: ServiceInstaller.new(home: @home, env: @env))
@@ -214,8 +257,8 @@ module Diffbroker
     def restart
       raise Error, "This machine is not paired. Run `diffbroker start` first." unless config.paired?
 
-      lifecycle.restart
       @out.puts "Restarting the companion."
+      raise Error, "The companion is not connected yet." unless reconciler.call(force: true)
     end
 
     def maintenance = @maintenance ||= Maintenance.new(hooks: hooks_installer)
@@ -226,8 +269,14 @@ module Diffbroker
       maintenance.upgrade!
       return @out.puts("Upgraded. Run `diffbroker start` to pair this machine.") unless config.paired?
 
-      lifecycle.restart
       @out.puts "Upgraded. Restarting the companion."
+      raise Error, "The companion is not connected yet." unless reconciler(version: newest_installed_version).call(force: true)
+    end
+
+    # This process runs the old code, so the version to wait for is the newest one on disk.
+    def newest_installed_version
+      Gem::Specification.reset
+      Gem::Specification.find_all_by_name("diffbroker").map(&:version).max&.to_s || VERSION
     end
 
     def uninstall
@@ -249,6 +298,7 @@ module Diffbroker
       @out.puts "Paired:     #{config.paired? ? "yes (device #{config.device_id})" : "no (run `diffbroker start`)"}"
       @out.puts "Agent:      #{config.agent}"
       @out.puts "Terminal:   #{config.terminal}"
+      daemon_status
       @out.puts "Hooks:      #{hooks_installer.installed? ? "installed" : "not installed (diffbroker install-hooks)"}"
       @out.puts "Roots:      #{config.roots.join(", ")} (plus the projects Claude Code and Codex know)"
       @out.puts "Worktrees:  #{config.worktree_root}"
@@ -258,6 +308,29 @@ module Diffbroker
       repositories.each { @out.puts "  #{_1["full_name"]}  #{_1["path"]}" }
     end
 
+    # What the daemon wrote about itself (DaemonState), next to what this installed version and service would run.
+    def daemon_status
+      data = DaemonState.new(dir: config.dir).read
+      pid = data&.dig("pid")
+      running = pid && Process.kill(0, pid.to_i) && true
+      if running
+        line = "running (pid #{pid}, #{data["state"]}"
+        line += ", version #{data["version"]} but #{VERSION} is installed: run `diffbroker start`" if data["version"] != VERSION
+        @out.puts "Daemon:     #{line})"
+      elsif (legacy = lifecycle.running_pid)
+        @out.puts "Daemon:     running an older version (pid #{legacy}) that reports no state: run `diffbroker start` to replace it"
+      else
+        @out.puts "Daemon:     not running (run `diffbroker start`)#{" — last state: #{data["state"]}" if data}"
+      end
+      error = data&.dig("last_error")
+      @out.puts "Last error: #{error["message"]} (#{error["kind"]})" if error
+      service = ServiceInstaller.new(home: @home, env: @env)
+      @out.puts "Service:    #{service.installed? ? (service.stale? ? "installed but out of date: run `diffbroker start`" : "installed") : "not installed"}"
+      @out.puts "Log:        #{lifecycle.log_path}"
+    rescue SystemCallError
+      @out.puts "Daemon:     not running (run `diffbroker start`)"
+    end
+
     # `git worktree remove` without --force refuses a worktree with modified or untracked files, so work is never lost.
     def clean
       worktree = Worktree.new(root: config.worktree_root)

data/lib/diffbroker/daemon.rb

@@ -3,6 +3,8 @@
 require "openssl"
 require "socket"
 require "set"
+require "timeout"
+require "rbconfig"
 
 module Diffbroker
   # `diffbroker start`: keep the device connected, report checkouts, claim and run tasks, and relay the status of
@@ -10,6 +12,17 @@ module Diffbroker
   class Daemon
     class Rejected < Error; end
 
+    # The daemon's own log lines carry the time, so a log that stops can be read against the clock.
+    class Stamped
+      def initialize(out, clock: -> { Time.now })
+        @out = out
+        @clock = clock
+      end
+
+      def puts(*lines) = lines.flatten.each { @out.puts("#{@clock.call.strftime("%F %T")} #{_1}") }
+      def flush = @out.flush
+    end
+
     RESCAN_EVERY = 300
     USAGE_EVERY = 300
     MAX_BACKOFF = 60
@@ -26,9 +39,21 @@ module Diffbroker
 
       new(scanner: config.repo_scanner, roots_changed: roots_changed(config), device_roots: config.device_roots, ignored_roots: config.ignored_roots,
           client_factory: -> { CableClient.new(url: config.cable_url, token: config.device_token, origin: config.server) },
-          task_runner_factory: ->(report) { TaskRunner.new(config:, report:, out:) }, out:, relay:, process_scanner:, agent: config.agent,
+          task_runner_factory: ->(report) { TaskRunner.new(config:, report:, out:) }, out: Stamped.new(out), relay:, process_scanner:, agent: config.agent,
           plan_usage: PlanUsage.new(agent: config.agent),
-          lifecycle: Lifecycle.new(dir: config.dir, server: config.server), reaper: Reaper.new(root: config.worktree_root))
+          lifecycle: Lifecycle.new(dir: config.dir, server: config.server), reaper: Reaper.new(root: config.worktree_root),
+          state: DaemonState.new(dir: config.dir), lock: DaemonLock.new(dir: config.dir), server: config.server,
+          rejection_confirmed: -> { server_rejects?(config) }, jitter: ->(seconds) { seconds * rand(0.8..1.25) })
+    end
+
+    # True only when the server itself answers 401 to the device token. An unreachable or erroring server proves nothing.
+    def self.server_rejects?(config)
+      Api.new(server: config.server, token: config.device_token).get("/device/ping")
+      false
+    rescue Api::HttpError => e
+      e.status == 401
+    rescue StandardError
+      false
     end
 
     # The server's "roots" message: remember the folders in config.yml (so they survive a restart without the server) and
@@ -56,7 +81,8 @@ module Diffbroker
 
     def initialize(scanner:, client_factory:, task_runner_factory:, out: $stdout, sleeper: ->(seconds) { sleep(seconds) },
                    clock: -> { Time.now }, spawner: ->(&block) { Thread.new(&block) }, relay: nil, process_scanner: nil, lifecycle: nil,
-                   maintenance: Maintenance.new, roots_changed: nil, device_roots: [], ignored_roots: [], agent: "claude", reaper: nil, plan_usage: nil)
+                   maintenance: Maintenance.new, roots_changed: nil, device_roots: [], ignored_roots: [], agent: "claude", reaper: nil, plan_usage: nil,
+                   state: nil, lock: nil, rejection_confirmed: nil, server: nil, jitter: ->(seconds) { seconds })
       @scanner = scanner
       @device_roots = device_roots
       @ignored_roots = ignored_roots
@@ -74,6 +100,11 @@ module Diffbroker
       @agent = agent
       @reaper = reaper
       @plan_usage = plan_usage
+      @state = state
+      @lock = lock
+      @rejection_confirmed = rejection_confirmed
+      @server = server
+      @jitter = jitter
       @repositories = []
       @running = Set.new
       @running_lock = Mutex.new
@@ -82,32 +113,65 @@ module Diffbroker
     def run
       @backoff = 1
       @rejections = 0
+      # One daemon per settings directory. A second one (a hand-started daemon next to the service, a double click on
+      # "Start companion") leaves quietly and untouched: the first keeps the relay socket and the connection.
+      if @lock && !@lock.acquire
+        @out.puts "Another diffbroker daemon already runs here; leaving it alone."
+        return
+      end
+      @owner = true
       @relay&.start
       @lifecycle&.write_pid
+      @state&.update(pid: Process.pid, version: VERSION, ruby: RbConfig.ruby, server: @server, started_at: @clock.call.to_i,
+                     service_managed: @lifecycle ? @lifecycle.service_process? : false, state: "starting", busy_tasks: 0, error: nil)
       sweep_strays
       until @stopping
         begin
           @client = @client_factory.call
+          @state&.update(state: "connecting")
           wire(@client)
           @client.run
-        rescue CableClient::Unauthorized
-          @rejections += 1
-          raise Rejected, "Diff Broker no longer knows this device (revoked?). Run `diffbroker start` to pair again." if @rejections >= REJECTIONS_BEFORE_GIVING_UP
-
-          @out.puts "Diff Broker rejected this device (#{@rejections} of #{REJECTIONS_BEFORE_GIVING_UP}); trying again."
-        rescue Error, SystemCallError, IOError, SocketError, OpenSSL::SSL::SSLError => e
-          @out.puts "Connection problem: #{e.message}"
+          @out.puts "Connection closed; reconnecting." unless @stopping
+        rescue CableClient::Unauthorized => e
+          give_up_or_retry(e)
+        rescue Error, SystemCallError, IOError, SocketError, OpenSSL::SSL::SSLError, Timeout::Error => e
+          failed(e)
+        rescue StandardError, ScriptError => e
+          # A bug or a half-loaded gem after an upgrade must cost one reconnect, never the daemon: with no service
+          # manager nothing would bring it back.
+          failed(e, backtrace: true)
         end
         break if @stopping
 
-        @sleeper.call(@backoff)
+        @state&.update(state: "backoff")
+        @sleeper.call(@jitter.call(@backoff))
         @backoff = [ @backoff * 2, MAX_BACKOFF ].min
       end
     ensure
-      @relay&.stop
-      @lifecycle&.clear_pid
-      # Only now, with the connection closed, so two daemons never share the device.
-      @lifecycle.spawn_daemon if @respawn && @lifecycle
+      if @owner
+        @relay&.stop
+        @lifecycle&.clear_pid
+        @state&.clear(pid: Process.pid) unless @rejected # a rejected daemon leaves its reason for `status`
+        @lock&.release
+        # Only now, with the connection closed, so two daemons never share the device.
+        @lifecycle.spawn_daemon if @respawn && @lifecycle
+      end
+    end
+
+    # What went wrong, as a kind a person can act on and the message to show next to it.
+    def self.classify(error)
+      message = error.message.to_s
+      status = message[/response code: (\d+)/, 1]&.to_i
+      kind = case error
+      when SocketError then "dns"
+      when Errno::ECONNREFUSED then "refused"
+      when OpenSSL::SSL::SSLError then "tls"
+      when Errno::ETIMEDOUT, Timeout::Error then "timeout"
+      when CableClient::HandshakeError then "handshake"
+      when Errno::ECONNRESET, Errno::EPIPE, EOFError, IOError, SystemCallError then "network"
+      else "internal"
+      end
+      { kind:, message: message.lines.first.to_s.strip[0, 300], status: }.compact
     end
 
     def stop
@@ -121,10 +185,38 @@ module Diffbroker
 
     private
 
+    def failed(error, backtrace: false)
+      info = self.class.classify(error)
+      @state&.update(error: info.merge(at: @clock.call.to_i))
+      @out.puts "Connection problem (#{info[:kind]}): #{info[:message]}"
+      @out.puts error.backtrace.first(5).map { "  #{_1}" } if backtrace && error.backtrace
+    end
+
+    # The server said "unauthorized". Three in a row are not proof: a deploy or a restore can answer that for a moment.
+    # The pairing is only dead when the server's own HTTP check (GET /device/ping) says 401 too.
+    def give_up_or_retry(error)
+      @rejections += 1
+      if @rejections < REJECTIONS_BEFORE_GIVING_UP
+        @out.puts "Diff Broker rejected this device (#{@rejections} of #{REJECTIONS_BEFORE_GIVING_UP}); trying again."
+        return
+      end
+
+      if @rejection_confirmed.nil? || @rejection_confirmed.call
+        @rejected = true
+        @state&.update(state: "rejected", error: { kind: "unauthorized", message: error.message, at: @clock.call.to_i })
+        raise Rejected, "Diff Broker no longer knows this device (revoked?). Run `diffbroker start` to pair again."
+      end
+
+      @rejections = 0
+      @state&.update(error: { kind: "unauthorized", message: "rejected on the cable, but the server still knows this device", at: @clock.call.to_i })
+      @out.puts "Diff Broker rejected the connection, but the server still knows this device; trying again."
+    end
+
     def wire(client)
       client.on_subscribed do
         @backoff = 1
         @rejections = 0
+        @state&.update(state: "connected", last_connect_at: @clock.call.to_i, error: nil)
         rescan(client)
         @usage_at = nil
         client.perform("hello", "version" => VERSION, "platform" => Diffbroker.platform,
@@ -271,6 +363,7 @@ module Diffbroker
     def start_task(id, message)
       return unless @running_lock.synchronize { !@upgrading && @running.add?(id) } # the upgrade's restart would kill it
 
+      @state&.update(busy_tasks: @running_lock.synchronize { @running.size })
       repositories = @repositories
       @spawner.call do
         @task_runner.run(message, repositories:)
@@ -278,6 +371,7 @@ module Diffbroker
         @out.puts "Task #{id} crashed: #{e.class}: #{e.message}"
       ensure
         upgrade_now = @running_lock.synchronize { @running.delete(id) && @pending_upgrade && @running.empty? }
+        @state&.update(busy_tasks: @running_lock.synchronize { @running.size })
         upgrade if upgrade_now
         sweep_strays
       end

data/lib/diffbroker/daemon_lock.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Diffbroker
+  # One daemon per settings directory. An exclusive flock held for the daemon's life: the kernel releases it when the
+  # process dies, however it dies, so there is never a stale lock to clean up (unlike a pid file).
+  class DaemonLock
+    FILE = "daemon.lock"
+
+    def initialize(dir:)
+      @dir = dir
+    end
+
+    def path = File.join(@dir, FILE)
+
+    # True when this process now holds the lock.
+    def acquire
+      FileUtils.mkdir_p(@dir, mode: 0o700)
+      @file = File.open(path, File::RDWR | File::CREAT, 0o600)
+      return true if @file.flock(File::LOCK_EX | File::LOCK_NB)
+
+      @file.close
+      @file = nil
+      false
+    end
+
+    def release
+      @file&.flock(File::LOCK_UN)
+      @file&.close
+      @file = nil
+    end
+  end
+end

data/lib/diffbroker/daemon_state.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+
+module Diffbroker
+  # What the running daemon is doing, as a small JSON file next to config.yml. The daemon writes it on every transition;
+  # `diffbroker start` / `status` read it to tell a connected daemon of this version from a stale, dead or failing one.
+  # Writes are atomic (temp file + rename), so a reader never sees half a file.
+  class DaemonState
+    FILE = "daemon.state"
+
+    def initialize(dir:)
+      @dir = dir
+    end
+
+    def path = File.join(@dir, FILE)
+
+    def read
+      data = JSON.parse(File.read(path))
+      data.is_a?(Hash) ? data : nil
+    rescue SystemCallError, JSON::ParserError
+      nil
+    end
+
+    # `error: nil` clears the last error; a Hash sets it. Other keys are merged in as given.
+    def update(**changes)
+      current = read || {}
+      error = changes.key?(:error) ? changes.delete(:error) : current["last_error"]
+      data = current.merge(changes.transform_keys(&:to_s))
+      error ? data["last_error"] = error.transform_keys(&:to_s) : data.delete("last_error")
+      write(data)
+    end
+
+    # Only the owner removes it, so a late-exiting old daemon never erases a newer daemon's state.
+    def clear(pid:)
+      File.delete(path) if read&.dig("pid") == pid
+    rescue SystemCallError
+      nil
+    end
+
+    private
+
+    def write(data)
+      FileUtils.mkdir_p(@dir, mode: 0o700)
+      temp = "#{path}.#{Process.pid}.tmp"
+      File.write(temp, JSON.generate(data), perm: 0o600)
+      File.rename(temp, path)
+    rescue SystemCallError
+      nil # the state file is a convenience; a read-only home never stops the daemon
+    end
+  end
+end

data/lib/diffbroker/lifecycle.rb

@@ -9,6 +9,8 @@ module Diffbroker
   # restarted through its manager, otherwise it would resurrect a stopped daemon (KeepAlive / Restart=always).
   class Lifecycle
     PID_FILE = "daemon.pid"
+    # `diffbroker start` / `launch` as a process: a recycled pid that is `tail -f ~/.config/diffbroker/companion.log` is not it.
+    DAEMON_COMMAND = %r{(?:\A|[/\s])diffbroker\s+(?:start|launch)\b}
     WAIT = 10
 
     def initialize(dir:, home: Dir.home, service: ServiceInstaller.new(home:), runner: Runner.new, exe: ServiceInstaller.launcher,
@@ -87,11 +89,14 @@ module Diffbroker
       return nil if pid <= 1 || pid == @pid
 
       Process.kill(0, pid)
-      @runner.capture("ps", "-p", pid.to_s, "-o", "command=").include?("diffbroker") ? pid : nil
+      @runner.capture("ps", "-p", pid.to_s, "-o", "command=").match?(DAEMON_COMMAND) ? pid : nil
     rescue SystemCallError
       nil
     end
 
+    # Ends a daemon started outside the service manager (and waits for it to go).
+    def terminate_daemon = terminate_running
+
     private
 
     def terminate_running

data/lib/diffbroker/reconciler.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module Diffbroker
+  # Brings this machine to "the newest companion is running and connected", whatever it looks like now: an older daemon
+  # still running after `gem install`, a service definition that names a deleted launcher or another Ruby, a daemon
+  # started by hand next to the service, nothing running at all. Safe to run any number of times: a machine that is
+  # already right is left untouched. `diffbroker start`, `launch`, `restart` and `upgrade` all end here.
+  class Reconciler
+    INTERVAL = 1
+
+    # `attempts` x INTERVAL seconds are given to the daemon to report "connected" before it counts as a failure.
+    def initialize(service:, lifecycle:, state:, out:, server:, sleeper: ->(seconds) { sleep(seconds) }, alive: method(:process_alive?),
+                   attempts: 20, version: VERSION, on_refresh: nil)
+      @service = service
+      @lifecycle = lifecycle
+      @state = state
+      @out = out
+      @server = server
+      @sleeper = sleeper
+      @alive = alive
+      @attempts = attempts
+      @version = version
+      @on_refresh = on_refresh
+    end
+
+    # True when the daemon is (or, with verify: false, was asked to be) running the current version. `force` restarts a
+    # current daemon too (`diffbroker restart`, the end of an upgrade).
+    def call(force: false, verify: true)
+      current = running_daemon
+      if current && busy?(current) && (force || stale?(current))
+        @out.puts "The companion is running a task; leaving it until the task ends. Run this again then."
+        return true
+      end
+
+      act(current, force:)
+      verify ? wait_until_connected : true
+    end
+
+    private
+
+    def act(current, force:)
+      if @service.installed? && @service.stale?
+        # launchd keeps the definition it loaded, so the new one only counts after the job is loaded again; that ends the
+        # daemon it was running and starts the new one.
+        @lifecycle.terminate_daemon if current && !current["service_managed"]
+        @service.refresh!
+        @on_refresh&.call
+        @out.puts "Updated the background service so it runs this version."
+      elsif current && (force || stale?(current))
+        restart(current)
+      elsif !current
+        @lifecycle.launch
+      end
+    end
+
+    # Under a service the manager restarts the daemon (it loads the newest gem); a daemon started by hand is ended first,
+    # because the manager does not know it and would leave it running next to the new one.
+    def restart(current)
+      return @lifecycle.restart unless @service.installed?
+
+      @lifecycle.terminate_daemon unless current["service_managed"]
+      done = current["service_managed"] ? @service.restart! : @service.start!
+      @lifecycle.spawn_daemon unless done
+    end
+
+    # Not the current version, aimed at another server, left by a rejection, or too old to have a state file.
+    def stale?(current)
+      current["legacy"] || current["version"] != @version || current["server"] != @server || current["state"] == "rejected"
+    end
+
+    def busy?(current) = current["busy_tasks"].to_i.positive?
+
+    # The live daemon as its state file describes it; a pid file without a state file is a daemon from before this
+    # release, which has to be restarted once to start writing one.
+    def running_daemon
+      data = @state.read
+      return data if data && data["pid"] && @alive.call(data["pid"])
+
+      pid = @lifecycle.running_pid
+      pid ? { "pid" => pid, "legacy" => true } : nil
+    end
+
+    def verified?
+      data = @state.read
+      connected = data && data["state"] == "connected" && data["version"] == @version && data["pid"] && @alive.call(data["pid"])
+      @out.puts "Connected (companion #{@version}, pid #{data["pid"]})." if connected
+      connected
+    end
+
+    def wait_until_connected
+      @attempts.times do
+        return true if verified?
+
+        @sleeper.call(INTERVAL)
+      end
+      return true if verified?
+
+      report_failure
+      false
+    end
+
+    def report_failure
+      data = @state.read || {}
+      error = data["last_error"]
+      @out.puts "The companion did not connect within #{@attempts * INTERVAL} seconds."
+      @out.puts "Last problem: #{error["message"]} (#{error["kind"]})" if error
+      @out.puts "State: #{data["state"] || "no daemon is running"}#{" (version #{data["version"]}, wanted #{@version})" if data["version"] && data["version"] != @version}"
+      @out.puts "Log: #{@lifecycle.log_path}"
+      @out.puts "It keeps trying in the background; `diffbroker status` shows what it sees."
+    end
+
+    def process_alive?(pid)
+      Process.kill(0, pid.to_i)
+      true
+    rescue Errno::EPERM
+      true
+    rescue SystemCallError
+      false
+    end
+  end
+end

data/lib/diffbroker/service_installer.rb

@@ -7,6 +7,11 @@ module Diffbroker
   # Runs `diffbroker start` in the background as the logged-in user (launchd on macOS, systemd --user on Linux).
   class ServiceInstaller
     LABEL = "broker.diff.companion"
+    LEGACY_LABEL = "broker.git.companion"
+    # The service manager refused (no GUI session over ssh, no systemd user bus): the caller falls back to a daemon of its own.
+    class Unavailable < Error; end
+
+    SETTLE_ATTEMPTS = 20
     EXE = File.join(Diffbroker::ROOT, "exe", "diffbroker")
 
     # What the service and the "Start companion" app run. RubyGems' own wrapper activates the newest installed diffbroker
@@ -27,7 +32,8 @@ module Diffbroker
     def self.pinned?(launcher = self.launcher) = launcher == EXE
 
     def initialize(platform: Diffbroker.platform, home: Dir.home, ruby: RbConfig.ruby, exe: ServiceInstaller.launcher, env: ENV, runner: Runner.new,
-                   uid: Process.uid)
+                   uid: Process.uid, sleeper: ->(seconds) { sleep(seconds) })
+      @sleeper = sleeper
       @platform = platform
       @home = home
       @ruby = ruby
@@ -39,7 +45,63 @@ module Diffbroker
 
     def install! = @platform == "macos" ? install_launchd! : install_systemd!
 
-    def installed? = File.exist?(@platform == "macos" ? launchd_path : systemd_path)
+    def installed? = File.exist?(definition_path)
+
+    # True when the installed definition would not start what this process is: another Ruby or launcher, a launcher or Ruby
+    # that no longer exists (`gem cleanup` removed the version a pinned definition names), or a definition from before
+    # the duplicate-safe restart policy. PATH is left out: it differs between a terminal and the service, so comparing it
+    # would call every definition stale.
+    def stale?
+      return false unless installed?
+
+      text = File.read(definition_path)
+      !text.include?(program_arguments) || !text.include?(restart_policy_marker) || !File.exist?(@ruby) || !File.exist?(@exe)
+    rescue SystemCallError
+      true
+    end
+
+    # The companion was called gitbroker before the rename. Its service points at a domain that no longer answers and
+    # restarts for ever, so it goes. Returns true when one was there.
+    def remove_legacy!
+      if @platform == "macos"
+        path = File.join(@home, "Library", "LaunchAgents", "#{LEGACY_LABEL}.plist")
+        return false unless File.exist?(path)
+
+        File.delete(path)
+        @runner.success?("launchctl", "bootout", "gui/#{@uid}/#{LEGACY_LABEL}")
+      else
+        path = File.join(@home, ".config", "systemd", "user", "gitbroker.service")
+        return false unless File.exist?(path)
+
+        @runner.success?("systemctl", "--user", "disable", "--now", "gitbroker.service")
+        File.delete(path)
+        @runner.success?("systemctl", "--user", "daemon-reload")
+      end
+      true
+    end
+
+    # Writes the current definition without touching the manager (the running job keeps what it loaded).
+    def rewrite!
+      path = definition_path
+      FileUtils.mkdir_p(File.dirname(path))
+      FileUtils.mkdir_p(File.join(@home, "Library", "Logs")) if @platform == "macos"
+      File.write(path, @platform == "macos" ? plist : unit)
+      path
+    end
+
+    # Writes the current definition and makes the manager use it. launchd keeps the plist it loaded, so the job is booted
+    # out and loaded again (this ends a running daemon); systemd re-reads its unit and the next restart uses it.
+    def refresh!
+      path = rewrite!
+      if @platform == "macos"
+        reload_launchd!(path)
+      else
+        @runner.run!("systemctl", "--user", "daemon-reload")
+      end
+      path
+    rescue Error => e
+      raise Unavailable, e.message
+    end
 
     # Starts the installed service if it is not running (a running one is left alone). False when there is no
     # service or the manager refused, so the caller can start the daemon itself.
@@ -117,7 +179,12 @@ module Diffbroker
           <key>RunAtLoad</key>
           <true/>
           <key>KeepAlive</key>
-          <true/>
+          <dict>
+            <key>SuccessfulExit</key>
+            <false/>
+          </dict>
+          <key>ThrottleInterval</key>
+          <integer>30</integer>
           <key>StandardOutPath</key>
           <string>#{log}</string>
           <key>StandardErrorPath</key>
@@ -132,10 +199,11 @@ module Diffbroker
         [Unit]
         Description=Diff Broker companion (runs your own agents on your pull requests)
         After=network-online.target
+        StartLimitIntervalSec=0
 
         [Service]
         ExecStart="#{@ruby}" "#{@exe}" start
-        Restart=always
+        Restart=on-failure
         RestartSec=5
         Environment="PATH=#{@env["PATH"]}"
         Environment="SHELL=#{@env["SHELL"] || "/bin/bash"}"
@@ -158,22 +226,40 @@ module Diffbroker
     def systemd_path = File.join(@home, ".config", "systemd", "user", "diffbroker.service")
 
     def install_launchd!
-      path = launchd_path
-      FileUtils.mkdir_p(File.dirname(path))
-      FileUtils.mkdir_p(File.join(@home, "Library", "Logs"))
-      File.write(path, plist)
-      @runner.success?("launchctl", "bootout", "gui/#{@uid}/#{LABEL}") # not loaded yet is fine
-      @runner.run!("launchctl", "bootstrap", "gui/#{@uid}", path)
+      path = rewrite!
+      reload_launchd!(path)
       path
+    rescue Error => e
+      raise Unavailable, e.message
+    end
+
+    # bootout answers before the job is gone, and bootstrap onto a job that is still there fails ("Bootstrap failed: 5"),
+    # so wait until launchd no longer knows the label.
+    def reload_launchd!(path)
+      @runner.success?("launchctl", "bootout", target) # not loaded yet is fine
+      SETTLE_ATTEMPTS.times do
+        break unless @runner.success?("launchctl", "print", target)
+
+        @sleeper.call(0.25)
+      end
+      @runner.run!("launchctl", "bootstrap", "gui/#{@uid}", path)
     end
 
     def install_systemd!
-      path = systemd_path
-      FileUtils.mkdir_p(File.dirname(path))
-      File.write(path, unit)
+      path = rewrite!
       @runner.run!("systemctl", "--user", "daemon-reload")
       @runner.run!("systemctl", "--user", "enable", "--now", "diffbroker.service")
       path
+    rescue Error => e
+      raise Unavailable, e.message
     end
+
+    def definition_path = @platform == "macos" ? launchd_path : systemd_path
+
+    def program_arguments
+      @platform == "macos" ? [ @ruby, @exe, "start" ].map { "    <string>#{xml(_1)}</string>" }.join("\n") : %(ExecStart="#{@ruby}" "#{@exe}" start)
+    end
+
+    def restart_policy_marker = @platform == "macos" ? "<key>SuccessfulExit</key>" : "Restart=on-failure"
   end
 end

data/lib/diffbroker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Diffbroker
-  VERSION = "0.11.0"
+  VERSION = "0.12.0"
 end

data/lib/diffbroker.rb

@@ -21,6 +21,8 @@ module Diffbroker
   autoload :CLI, File.expand_path("diffbroker/cli", __dir__)
   autoload :Config, File.expand_path("diffbroker/config", __dir__)
   autoload :Daemon, File.expand_path("diffbroker/daemon", __dir__)
+  autoload :DaemonLock, File.expand_path("diffbroker/daemon_lock", __dir__)
+  autoload :DaemonState, File.expand_path("diffbroker/daemon_state", __dir__)
   autoload :ExplainVerifier, File.expand_path("diffbroker/explain_verifier", __dir__)
   autoload :GhCheck, File.expand_path("diffbroker/gh_check", __dir__)
   autoload :GitContext, File.expand_path("diffbroker/git_context", __dir__)
@@ -35,6 +37,7 @@ module Diffbroker
   autoload :PlanUsage, File.expand_path("diffbroker/plan_usage", __dir__)
   autoload :ProcessScanner, File.expand_path("diffbroker/process_scanner", __dir__)
   autoload :Reaper, File.expand_path("diffbroker/reaper", __dir__)
+  autoload :Reconciler, File.expand_path("diffbroker/reconciler", __dir__)
   autoload :RepoScanner, File.expand_path("diffbroker/repo_scanner", __dir__)
   autoload :RunScript, File.expand_path("diffbroker/run_script", __dir__)
   autoload :Runner, File.expand_path("diffbroker/runner", __dir__)

data/plugin/.claude-plugin/plugin.json

@@ -1,5 +1,5 @@
 {
   "name": "diffbroker",
-  "version": "0.11.0",
+  "version": "0.12.0",
   "description": "Skills for working on Diff Broker agent tasks: the diffbroker MCP tools, write rules and report_task."
 }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: diffbroker
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.12.0
 platform: ruby
 authors:
 - Yuri Sidorov (@newstler)
@@ -45,6 +45,8 @@ files:
 - lib/diffbroker/cli.rb
 - lib/diffbroker/config.rb
 - lib/diffbroker/daemon.rb
+- lib/diffbroker/daemon_lock.rb
+- lib/diffbroker/daemon_state.rb
 - lib/diffbroker/explain_verifier.rb
 - lib/diffbroker/gh_check.rb
 - lib/diffbroker/git_context.rb
@@ -59,6 +61,7 @@ files:
 - lib/diffbroker/plan_usage.rb
 - lib/diffbroker/process_scanner.rb
 - lib/diffbroker/reaper.rb
+- lib/diffbroker/reconciler.rb
 - lib/diffbroker/repo_scanner.rb
 - lib/diffbroker/run_script.rb
 - lib/diffbroker/runner.rb