@apralabs/apra-fleet 0.4.4 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (268) hide show
  1. package/README.md +49 -4
  2. package/dist/agents/backlog-groomer.md +5 -0
  3. package/dist/agents/ci-watcher.md +5 -0
  4. package/dist/agents/deployer.md +20 -2
  5. package/dist/agents/doer.md +38 -2
  6. package/dist/agents/harvester.md +12 -9
  7. package/dist/agents/integ-test-runner.md +5 -0
  8. package/dist/agents/kb-reconciler.md +6 -0
  9. package/dist/agents/plan-reviewer.md +5 -0
  10. package/dist/agents/planner.md +5 -0
  11. package/dist/agents/regression-test-runner.md +5 -0
  12. package/dist/agents/reviewer.md +67 -15
  13. package/dist/agents/schemas/deployer-output.json +5 -0
  14. package/dist/agents/schemas/doer-output.json +78 -2
  15. package/dist/agents/schemas/reviewer-output.json +104 -2
  16. package/dist/cli/auth.js +5 -1
  17. package/dist/cli/beads-pin.js +6 -7
  18. package/dist/cli/claude-mod.d.ts +75 -0
  19. package/dist/cli/claude-mod.js +288 -0
  20. package/dist/cli/dolt-install.d.ts +1 -1
  21. package/dist/cli/dolt-install.js +5 -1
  22. package/dist/cli/fleet-se-guard.d.ts +8 -0
  23. package/dist/cli/fleet-se-guard.js +19 -3
  24. package/dist/cli/install-guard.d.ts +9 -0
  25. package/dist/cli/install-guard.js +24 -0
  26. package/dist/cli/install.d.ts +12 -0
  27. package/dist/cli/install.js +180 -20
  28. package/dist/cli/kb-profile.d.ts +16 -0
  29. package/dist/cli/kb-profile.js +81 -0
  30. package/dist/cli/start.js +3 -0
  31. package/dist/cli/uninstall.js +13 -3
  32. package/dist/cli/update.d.ts +4 -0
  33. package/dist/cli/update.js +19 -3
  34. package/dist/commands/kb-server.d.ts +10 -1
  35. package/dist/commands/kb-server.js +285 -11
  36. package/dist/index.js +22 -1
  37. package/dist/os/git-executable.d.ts +57 -0
  38. package/dist/os/git-executable.js +85 -0
  39. package/dist/os/go-concurrency.d.ts +44 -0
  40. package/dist/os/go-concurrency.js +53 -0
  41. package/dist/os/index.d.ts +1 -0
  42. package/dist/os/index.js +1 -0
  43. package/dist/os/linux.d.ts +2 -0
  44. package/dist/os/linux.js +10 -1
  45. package/dist/os/os-commands.d.ts +25 -0
  46. package/dist/os/os-commands.js +2 -0
  47. package/dist/os/windows-executable.d.ts +53 -0
  48. package/dist/os/windows-executable.js +185 -0
  49. package/dist/os/windows-gitbash.d.ts +23 -14
  50. package/dist/os/windows-gitbash.js +40 -16
  51. package/dist/os/windows.d.ts +2 -0
  52. package/dist/os/windows.js +37 -6
  53. package/dist/paths.d.ts +10 -1
  54. package/dist/paths.js +19 -2
  55. package/dist/services/agy-project.d.ts +7 -1
  56. package/dist/services/agy-project.js +7 -2
  57. package/dist/services/auth-socket.d.ts +18 -0
  58. package/dist/services/auth-socket.js +30 -4
  59. package/dist/services/credential-store.d.ts +33 -0
  60. package/dist/services/credential-store.js +79 -2
  61. package/dist/services/http-transport.d.ts +29 -0
  62. package/dist/services/http-transport.js +166 -37
  63. package/dist/services/kb-tool-profile.d.ts +36 -0
  64. package/dist/services/kb-tool-profile.js +88 -0
  65. package/dist/services/knowledge/bible-blob-id.js +10 -1
  66. package/dist/services/knowledge/bible-import.d.ts +108 -0
  67. package/dist/services/knowledge/bible-import.js +137 -4
  68. package/dist/services/knowledge/file-hash.js +6 -5
  69. package/dist/services/knowledge/http-provider.d.ts +303 -20
  70. package/dist/services/knowledge/http-provider.js +795 -125
  71. package/dist/services/knowledge/kb-config.d.ts +65 -1
  72. package/dist/services/knowledge/kb-config.js +143 -3
  73. package/dist/services/knowledge/kb-http-auth-state.d.ts +26 -0
  74. package/dist/services/knowledge/kb-http-auth-state.js +40 -0
  75. package/dist/services/knowledge/kb-providers.d.ts +9 -2
  76. package/dist/services/knowledge/kb-providers.js +67 -37
  77. package/dist/services/knowledge/kb-removed-scope-keys.js +1 -1
  78. package/dist/services/knowledge/kb-self.d.ts +122 -9
  79. package/dist/services/knowledge/kb-self.js +196 -36
  80. package/dist/services/knowledge/member-bible-view.js +16 -3
  81. package/dist/services/knowledge/path-validation.d.ts +7 -0
  82. package/dist/services/knowledge/path-validation.js +11 -0
  83. package/dist/services/knowledge/project-slug.js +3 -2
  84. package/dist/services/knowledge/repository-header.d.ts +77 -0
  85. package/dist/services/knowledge/repository-header.js +233 -0
  86. package/dist/services/knowledge/require-sqlite-project.d.ts +1 -1
  87. package/dist/services/knowledge/require-sqlite-project.js +1 -1
  88. package/dist/services/knowledge/sqlite-provider.d.ts +169 -0
  89. package/dist/services/knowledge/sqlite-provider.js +416 -4
  90. package/dist/services/knowledge/types.d.ts +15 -2
  91. package/dist/services/member-busy.d.ts +12 -0
  92. package/dist/services/member-busy.js +20 -0
  93. package/dist/services/member-config-io.d.ts +3 -2
  94. package/dist/services/member-config-io.js +7 -4
  95. package/dist/services/member-fleet-install.js +1 -0
  96. package/dist/services/member-tool-allowlist.d.ts +1 -1
  97. package/dist/services/member-tool-allowlist.js +8 -4
  98. package/dist/services/port-holder.js +2 -1
  99. package/dist/services/service-manager/types.d.ts +17 -1
  100. package/dist/services/service-manager/windows.d.ts +7 -2
  101. package/dist/services/service-manager/windows.js +28 -11
  102. package/dist/services/service-notice.d.ts +7 -0
  103. package/dist/services/service-notice.js +19 -0
  104. package/dist/services/session-registry.d.ts +13 -0
  105. package/dist/services/session-registry.js +13 -0
  106. package/dist/services/session-roots.d.ts +67 -0
  107. package/dist/services/session-roots.js +126 -0
  108. package/dist/services/statusline.d.ts +21 -0
  109. package/dist/services/statusline.js +97 -1
  110. package/dist/services/supervisor-service-cleanup.js +2 -1
  111. package/dist/services/tool-registry.js +108 -20
  112. package/dist/services/tool-scope.d.ts +34 -1
  113. package/dist/services/tool-scope.js +27 -2
  114. package/dist/services/vcs/azure-devops.js +10 -0
  115. package/dist/services/vcs/bitbucket.d.ts +22 -1
  116. package/dist/services/vcs/bitbucket.js +163 -17
  117. package/dist/services/vcs/types.d.ts +25 -2
  118. package/dist/services/watch/project-resolver.js +2 -1
  119. package/dist/skills/pm/cost.md +5 -1
  120. package/dist/skills/pm/index.md +9 -9
  121. package/dist/skills/pm/sprint.md +2 -1
  122. package/dist/tools/check-status.d.ts +51 -0
  123. package/dist/tools/check-status.js +124 -12
  124. package/dist/tools/code-intelligence-freshness.js +1 -1
  125. package/dist/tools/code-intelligence-gitnexus.d.ts +19 -0
  126. package/dist/tools/code-intelligence-gitnexus.js +166 -9
  127. package/dist/tools/code-intelligence-readiness.d.ts +58 -1
  128. package/dist/tools/code-intelligence-readiness.js +93 -8
  129. package/dist/tools/code-intelligence-reindex.d.ts +25 -1
  130. package/dist/tools/code-intelligence-reindex.js +76 -4
  131. package/dist/tools/code-intelligence.d.ts +1 -1
  132. package/dist/tools/code-intelligence.js +1 -1
  133. package/dist/tools/execute-command.js +14 -8
  134. package/dist/tools/execute-prompt.js +6 -2
  135. package/dist/tools/fleet-snapshot.d.ts +21 -0
  136. package/dist/tools/fleet-snapshot.js +78 -0
  137. package/dist/tools/kb-bible-commit.d.ts +25 -11
  138. package/dist/tools/kb-bible-commit.js +142 -33
  139. package/dist/tools/kb-capture.d.ts +2 -2
  140. package/dist/tools/kb-context.js +47 -27
  141. package/dist/tools/kb-demote.d.ts +27 -0
  142. package/dist/tools/kb-demote.js +37 -0
  143. package/dist/tools/kb-export.d.ts +35 -3
  144. package/dist/tools/kb-export.js +56 -13
  145. package/dist/tools/kb-feedback.d.ts +2 -0
  146. package/dist/tools/kb-feedback.js +17 -7
  147. package/dist/tools/kb-harvest.js +28 -9
  148. package/dist/tools/kb-import.d.ts +18 -2
  149. package/dist/tools/kb-import.js +24 -9
  150. package/dist/tools/kb-invalidate.d.ts +11 -3
  151. package/dist/tools/kb-invalidate.js +80 -7
  152. package/dist/tools/kb-list.js +38 -10
  153. package/dist/tools/kb-promote.js +6 -4
  154. package/dist/tools/kb-query.d.ts +3 -0
  155. package/dist/tools/kb-query.js +181 -37
  156. package/dist/tools/kb-session-prime.d.ts +4 -4
  157. package/dist/tools/kb-session-prime.js +89 -9
  158. package/dist/tools/kb-setup.d.ts +20 -0
  159. package/dist/tools/kb-setup.js +149 -11
  160. package/dist/tools/list-members.js +1 -1
  161. package/dist/tools/member-detail.js +7 -3
  162. package/dist/tools/provision-vcs-auth.d.ts +28 -3
  163. package/dist/tools/provision-vcs-auth.js +53 -7
  164. package/dist/tools/receive-files.js +7 -4
  165. package/dist/tools/send-files.js +5 -4
  166. package/dist/tools/send-message.d.ts +3 -0
  167. package/dist/tools/send-message.js +15 -2
  168. package/dist/tools/stop-prompt.js +3 -1
  169. package/dist/tools/vcs-credential-exec.d.ts +45 -3
  170. package/dist/tools/vcs-credential-exec.js +90 -11
  171. package/dist/utils/process-utils.js +5 -1
  172. package/dist/utils/redact-known-secrets.d.ts +17 -0
  173. package/dist/utils/redact-known-secrets.js +70 -0
  174. package/dist/utils/skill-matrix.js +1 -1
  175. package/dist/utils/vcs-provider-detect.js +2 -2
  176. package/dist/workflows/auto-sprint.js +245 -9
  177. package/package.json +10 -7
  178. package/packages/apra-fleet-client/package.json +1 -1
  179. package/packages/apra-fleet-client/src/client/api.mjs +172 -19
  180. package/packages/apra-fleet-client/src/client/reconnecting-transport.mjs +35 -2
  181. package/packages/apra-fleet-client/src/client/transport.mjs +6 -0
  182. package/packages/apra-fleet-se/apra-pm/agents/backlog-groomer.md +5 -0
  183. package/packages/apra-fleet-se/apra-pm/agents/ci-watcher.md +5 -0
  184. package/packages/apra-fleet-se/apra-pm/agents/deployer.md +20 -2
  185. package/packages/apra-fleet-se/apra-pm/agents/doer.md +38 -2
  186. package/packages/apra-fleet-se/apra-pm/agents/harvester.md +12 -9
  187. package/packages/apra-fleet-se/apra-pm/agents/integ-test-runner.md +5 -0
  188. package/packages/apra-fleet-se/apra-pm/agents/kb-reconciler.md +6 -0
  189. package/packages/apra-fleet-se/apra-pm/agents/plan-reviewer.md +5 -0
  190. package/packages/apra-fleet-se/apra-pm/agents/planner.md +5 -0
  191. package/packages/apra-fleet-se/apra-pm/agents/regression-test-runner.md +5 -0
  192. package/packages/apra-fleet-se/apra-pm/agents/reviewer.md +67 -15
  193. package/packages/apra-fleet-se/apra-pm/agents/schemas/deployer-output.json +5 -0
  194. package/packages/apra-fleet-se/apra-pm/agents/schemas/doer-output.json +78 -2
  195. package/packages/apra-fleet-se/apra-pm/agents/schemas/reviewer-output.json +104 -2
  196. package/packages/apra-fleet-se/apra-pm/skills/pm/cost.md +5 -1
  197. package/packages/apra-fleet-se/apra-pm/skills/pm/index.md +9 -9
  198. package/packages/apra-fleet-se/apra-pm/skills/pm/sprint.md +2 -1
  199. package/packages/apra-fleet-se/bin/cli.mjs +104 -31
  200. package/packages/apra-fleet-se/bin/serve.mjs +46 -8
  201. package/packages/apra-fleet-se/fleet-sprint/abort.mjs +184 -32
  202. package/packages/apra-fleet-se/fleet-sprint/bd-sync-command.mjs +69 -0
  203. package/packages/apra-fleet-se/fleet-sprint/beads-children.mjs +21 -0
  204. package/packages/apra-fleet-se/fleet-sprint/beads-lock-probe.mjs +176 -0
  205. package/packages/apra-fleet-se/fleet-sprint/beads-reachability.mjs +126 -0
  206. package/packages/apra-fleet-se/fleet-sprint/beads-scope.mjs +158 -1
  207. package/packages/apra-fleet-se/fleet-sprint/contracts.mjs +82 -2
  208. package/packages/apra-fleet-se/fleet-sprint/coordination.mjs +23 -6
  209. package/packages/apra-fleet-se/fleet-sprint/decision-notify.mjs +338 -0
  210. package/packages/apra-fleet-se/fleet-sprint/decision-wait.mjs +731 -0
  211. package/packages/apra-fleet-se/fleet-sprint/decisions.mjs +372 -0
  212. package/packages/apra-fleet-se/fleet-sprint/dispatch-role.mjs +44 -11
  213. package/packages/apra-fleet-se/fleet-sprint/docs/README.md +4 -1
  214. package/packages/apra-fleet-se/fleet-sprint/dolt-settle.mjs +9 -3
  215. package/packages/apra-fleet-se/fleet-sprint/dolt-sync.mjs +314 -19
  216. package/packages/apra-fleet-se/fleet-sprint/errors.mjs +72 -4
  217. package/packages/apra-fleet-se/fleet-sprint/git-sync.mjs +7 -1
  218. package/packages/apra-fleet-se/fleet-sprint/git-topology.mjs +112 -1
  219. package/packages/apra-fleet-se/fleet-sprint/guarded-modules.mjs +36 -2
  220. package/packages/apra-fleet-se/fleet-sprint/kb.mjs +736 -101
  221. package/packages/apra-fleet-se/fleet-sprint/member-init-probe.mjs +1 -1
  222. package/packages/apra-fleet-se/fleet-sprint/member-provisioning.mjs +7 -1
  223. package/packages/apra-fleet-se/fleet-sprint/phases/deploy.mjs +49 -2
  224. package/packages/apra-fleet-se/fleet-sprint/phases/develop.mjs +206 -3
  225. package/packages/apra-fleet-se/fleet-sprint/phases/final-review.mjs +61 -3
  226. package/packages/apra-fleet-se/fleet-sprint/phases/harvest.mjs +81 -31
  227. package/packages/apra-fleet-se/fleet-sprint/phases/plan.mjs +7 -3
  228. package/packages/apra-fleet-se/fleet-sprint/phases/publish-pr.mjs +39 -14
  229. package/packages/apra-fleet-se/fleet-sprint/phases/re-review.mjs +17 -1
  230. package/packages/apra-fleet-se/fleet-sprint/phases/review.mjs +18 -3
  231. package/packages/apra-fleet-se/fleet-sprint/pr-body.mjs +14 -1
  232. package/packages/apra-fleet-se/fleet-sprint/prompts.mjs +32 -5
  233. package/packages/apra-fleet-se/fleet-sprint/runner.js +824 -98
  234. package/packages/apra-fleet-se/fleet-sprint/se-os-commands.mjs +4 -2
  235. package/packages/apra-fleet-se/fleet-sprint/se-posix.mjs +54 -1
  236. package/packages/apra-fleet-se/fleet-sprint/se-windows.mjs +29 -2
  237. package/packages/apra-fleet-se/fleet-sprint/set-aside.mjs +288 -0
  238. package/packages/apra-fleet-se/fleet-sprint/skills/fleet-supervisor/SKILL.md +3 -2
  239. package/packages/apra-fleet-se/fleet-sprint/sprint-args.mjs +20 -0
  240. package/packages/apra-fleet-se/fleet-sprint/sprint-report.mjs +21 -0
  241. package/packages/apra-fleet-se/fleet-sprint/sprint-state.mjs +9 -0
  242. package/packages/apra-fleet-se/fleet-sprint/vcs-auth.mjs +605 -32
  243. package/packages/apra-fleet-se/fleet-sprint/vcs-providers/bitbucket.mjs +811 -17
  244. package/packages/apra-fleet-se/fleet-sprint/vcs-providers/index.mjs +89 -1
  245. package/packages/apra-fleet-se/fleet-sprint/viewer-extensions.mjs +225 -2
  246. package/packages/apra-fleet-se/fleet-sprint/worklists.mjs +59 -3
  247. package/packages/apra-fleet-se/src/supervisor/api.mjs +148 -8
  248. package/packages/apra-fleet-se/src/supervisor/dashboard.mjs +6 -3
  249. package/packages/apra-fleet-se/src/supervisor/dolt-mutex.mjs +30 -10
  250. package/packages/apra-fleet-se/src/supervisor/fleet-members.mjs +17 -4
  251. package/packages/apra-fleet-se/src/supervisor/history.mjs +8 -1
  252. package/packages/apra-fleet-se/src/supervisor/id-allocator.mjs +92 -2
  253. package/packages/apra-fleet-se/src/supervisor/proxy.mjs +26 -1
  254. package/packages/apra-fleet-se/src/supervisor/server.mjs +151 -8
  255. package/packages/apra-fleet-se/src/supervisor/spawner.mjs +11 -9
  256. package/packages/apra-fleet-se/src/supervisor/watchdog.mjs +7 -1
  257. package/packages/apra-fleet-workflow/package.json +2 -1
  258. package/packages/apra-fleet-workflow/src/viewer/index.mjs +86 -5
  259. package/packages/apra-fleet-workflow/src/viewer/request-guard.mjs +222 -0
  260. package/packages/apra-fleet-workflow/src/workflow/index.mjs +34 -2
  261. package/plugins/claude-code/fleet-status/.claude-plugin/plugin.json +7 -0
  262. package/plugins/claude-code/fleet-status/hooks/hooks.json +1 -0
  263. package/plugins/claude-code/fleet-status/hooks/register.tsx +906 -0
  264. package/plugins/claude-code/fleet-status/types/index.d.ts +88 -0
  265. package/skills/fleet/auth-bitbucket.md +47 -20
  266. package/skills/fleet/onboarding.md +1 -1
  267. package/skills/fleet/skill-matrix.md +3 -3
  268. package/version.json +1 -1
@@ -85,7 +85,9 @@
85
85
  * ASCII only.
86
86
  */
87
87
 
88
- import { DoltDivergedError, DoltSyncError, UnpublishedSchemaMigrationsError } from './errors.mjs';
88
+ import { DoltDivergedError, DoltSyncError, DoltWindowsSpawnLimitError, DoltLockHeldError, UnpublishedSchemaMigrationsError } from './errors.mjs';
89
+ import { bdSyncCommandFor, normalizeMemberTarget, POSIX_MEMBER_TARGET } from './bd-sync-command.mjs';
90
+ import { buildLockHolderProbe, parseLockHolderProbe, describeLockProcess } from './beads-lock-probe.mjs';
89
91
  import { classifyFailure, toDoltVerdict, commandBinary } from './vcs-module.mjs';
90
92
  import { buildSettleCallback } from './dolt-settle.mjs';
91
93
 
@@ -683,6 +685,241 @@ export function doltBackoffDelayMs(attempt, baseMs = DOLT_BACKOFF_BASE_MS, maxMs
683
685
 
684
686
  const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
685
687
 
688
+ // ---------------------------------------------------------------------------
689
+ // Member OS/shell for the bd sync command string
690
+ // ---------------------------------------------------------------------------
691
+ //
692
+ // A Windows member runs every `bd dolt pull`/`bd dolt push` with a bounded
693
+ // GOMAXPROCS (see bd-sync-command.mjs for why and for the exact strings), so
694
+ // runDoltStep needs the member's { os, shell }. Two sources, in order:
695
+ // 1. `opts.memberTarget` on the call -- an { os, shell } object or a
696
+ // `(member) => { os, shell }` function (tests, and any caller that has
697
+ // already resolved the member);
698
+ // 2. the sprint-wide resolver installed once at sprint start with
699
+ // setMemberTargetResolver() -- the runner wires the cached
700
+ // member_detail-backed resolveMemberTarget there, so no extra
701
+ // member_detail round trip happens per sync once a member is resolved.
702
+ // With neither, or when resolution fails, the member is treated as POSIX and
703
+ // the command strings stay byte-identical to the pre-existing ones.
704
+
705
+ /** @type {((member: string) => Promise<{ os: string, shell: string }>|{ os: string, shell: string })|null} */
706
+ let memberTargetResolver = null;
707
+
708
+ /**
709
+ * Install (or clear, with null) the sprint-wide member { os, shell } resolver
710
+ * runDoltStep falls back to when a call passes no `memberTarget`.
711
+ *
712
+ * @param {Function|null} resolver
713
+ */
714
+ export function setMemberTargetResolver(resolver) {
715
+ memberTargetResolver = typeof resolver === 'function' ? resolver : null;
716
+ }
717
+
718
+ /**
719
+ * Resolve `member`'s { os, shell } for command construction. Never throws: a
720
+ * failed lookup is logged and degrades to POSIX (the unchanged strings).
721
+ *
722
+ * @param {string} member
723
+ * @param {object|Function|undefined} memberTarget
724
+ * @param {Function} log
725
+ * @returns {Promise<{ os: string, shell: string }>}
726
+ */
727
+ async function resolveDoltMemberTarget(member, memberTarget, log) {
728
+ const source = (memberTarget !== undefined && memberTarget !== null) ? memberTarget : memberTargetResolver;
729
+ if (!source) return POSIX_MEMBER_TARGET;
730
+ try {
731
+ const target = typeof source === 'function' ? await source(member) : source;
732
+ const normalized = normalizeMemberTarget(target);
733
+ return normalized.os ? normalized : POSIX_MEMBER_TARGET;
734
+ } catch (err) {
735
+ log(`[Dolt] could not resolve the OS/shell of member '${member}' (${(err && err.message) || err}); building its bd sync commands for a POSIX shell.`);
736
+ return POSIX_MEMBER_TARGET;
737
+ }
738
+ }
739
+
740
+ /** Remedy text of the typed Windows spawn-limit sync failure. */
741
+ export const WINDOWS_SPAWN_LIMIT_REMEDY =
742
+ 'Remedy: on that Windows member, stop other process-heavy work (or wait for it to finish) and re-run the sync; '
743
+ + 'if it recurs, shrink the environment block the member\'s processes inherit (deduplicate PATH, drop oversized variables) '
744
+ + 'and run the fleet service in an interactive session so it does not exhaust the non-interactive desktop heap.';
745
+
746
+ /**
747
+ * The typed terminal error for a step that ended on the Windows
748
+ * process-creation limit (runDoltStep's `spawnLimit` flag).
749
+ *
750
+ * @param {string} member
751
+ * @param {string} operation - e.g. 'pull', 'push', 'push-reconcile'
752
+ * @param {{ error: string }} step
753
+ * @returns {DoltWindowsSpawnLimitError}
754
+ */
755
+ function windowsSpawnLimitSyncError(member, operation, step) {
756
+ return new DoltWindowsSpawnLimitError(
757
+ `[Dolt] D-${operation} for member '${member}' failed: Windows refused to start bd's git child processes (the Windows process-creation limit, `
758
+ + `CreateProcess ERROR_NOT_ENOUGH_MEMORY) on every attempt until the transient retry budget ran out. Not a credential, remote or data problem. `
759
+ + `${WINDOWS_SPAWN_LIMIT_REMEDY} Raw: ${step.error}`,
760
+ { member, doltOutput: step.error, details: { operation } },
761
+ );
762
+ }
763
+
764
+ /**
765
+ * The typed terminal error for a step whose beads lock was held by a foreign
766
+ * process (runDoltStep's 'lock-held' kind).
767
+ *
768
+ * @param {string} member
769
+ * @param {string} operation
770
+ * @param {{ error: string, lockHolders?: object[] }} step
771
+ * @returns {DoltLockHeldError}
772
+ */
773
+ function lockHeldSyncError(member, operation, step) {
774
+ const holders = step.lockHolders || [];
775
+ return new DoltLockHeldError(
776
+ `[Dolt] D-${operation} for member '${member}' failed: a beads lock on that member is held by another process that started before this sync -- `
777
+ + `${holders.map(describeLockProcess).join('; ') || 'holder unknown'}. The sync was waiting on that lock silently. `
778
+ + 'Remedy: stop that process on the member (it is not part of this sync) or wait for it to exit, then re-run the sync.',
779
+ { member, doltOutput: step.error, details: { operation, holders } },
780
+ );
781
+ }
782
+
783
+ /**
784
+ * The typed terminal error a failed step maps to, or null for a generic
785
+ * failure (the caller then throws its own DoltSyncError).
786
+ *
787
+ * @param {string} member
788
+ * @param {string} operation
789
+ * @param {{ kind?: string, spawnLimit?: boolean, error: string }} step
790
+ * @returns {DoltSyncError|null}
791
+ */
792
+ function typedStepError(member, operation, step) {
793
+ if (step.kind === 'lock-held') return lockHeldSyncError(member, operation, step);
794
+ if (step.spawnLimit) return windowsSpawnLimitSyncError(member, operation, step);
795
+ return null;
796
+ }
797
+
798
+ // ---------------------------------------------------------------------------
799
+ // Lock-holder watchdog for a silent `bd dolt pull`/`bd dolt push`
800
+ // ---------------------------------------------------------------------------
801
+ //
802
+ // A bd sync that waits on a beads lock prints nothing until it gets the lock,
803
+ // so a lock held by some other long-lived process on the member used to look
804
+ // like a silent hang ending in the DOLT_STEP_TIMEOUT_S inactivity kill. Each
805
+ // attempt that is still in flight after DOLT_LOCK_PROBE_AFTER_MS gets exactly
806
+ // ONE lock-holder probe (beads-lock-probe.mjs): a separate short command on
807
+ // the same member, run while the sync is still waiting. Its result is logged.
808
+ //
809
+ // FOREIGN vs OWN holder. The bd process this attempt started legitimately
810
+ // holds the lock while it works, and a slow incremental pull can run for
811
+ // minutes. So a holder only counts as contention when it is provably older
812
+ // than this attempt: its process age exceeds the attempt's elapsed time by
813
+ // more than DOLT_LOCK_HOLDER_AGE_SLACK_S. Both are durations (member clock vs
814
+ // orchestrator clock), so no clock sync is assumed. An own or unknown-age
815
+ // holder is logged and the attempt keeps waiting.
816
+ //
817
+ // THE BOUND. A foreign holder fails the attempt at once with a typed
818
+ // DoltLockHeldError naming it: at most DOLT_LOCK_PROBE_AFTER_MS plus the
819
+ // probe's own timeout (60s + 30s = 90s) after the attempt started, instead of
820
+ // the 600s inactivity kill. 60s is far past a healthy sync (seconds; the
821
+ // incident's retry landed in 4s) yet short enough to name the holder while it
822
+ // is still alive. A lock-held failure is not retried: the holder is long-lived
823
+ // by construction, so a retry would just wait on it again.
824
+ //
825
+ // The abandoned sync command keeps running on the member (there is no
826
+ // cancel channel); its promise is handed to `onAbandonedAttempt` so the push
827
+ // bracket can hold the push mutex until it ends (see doltPushAfter).
828
+ //
829
+ // Side effects of running the probe concurrently: the member's statusline
830
+ // busy marker may flip to idle when the probe ends while the sync is still in
831
+ // flight, and a resumed run's journal replay sees the probe as an extra
832
+ // command whose position depends on timing, so replay diverges at that point
833
+ // and continues live.
834
+
835
+ /** How long a bd sync attempt may stay silent before its lock-holder probe. */
836
+ export const DOLT_LOCK_PROBE_AFTER_MS = 60_000;
837
+
838
+ /** Timeout of the lock-holder probe command itself. */
839
+ export const DOLT_LOCK_PROBE_TIMEOUT_S = 30;
840
+
841
+ /** A holder must be at least this much older than the attempt to count as foreign. */
842
+ export const DOLT_LOCK_HOLDER_AGE_SLACK_S = 5;
843
+
844
+ const defaultLockProbeTimer = {
845
+ set(fn, ms) {
846
+ const handle = setTimeout(fn, ms);
847
+ if (handle && typeof handle.unref === 'function') handle.unref();
848
+ return handle;
849
+ },
850
+ clear(handle) {
851
+ clearTimeout(handle);
852
+ },
853
+ };
854
+
855
+ /**
856
+ * Run the lock-holder probe once for a sync attempt that is still in flight,
857
+ * log what it found, and return the foreign holders. Never throws for an
858
+ * ordinary failure (a failed probe is logged and reports no holder); a
859
+ * cancellation still propagates.
860
+ *
861
+ * @returns {Promise<{ foreignHolders: object[] }>}
862
+ */
863
+ async function probeLockHolders({ command, member, target, label, log, attemptStartedAt, now }) {
864
+ const elapsedS = Math.max(0, Math.round((now() - attemptStartedAt) / 1000));
865
+ let res;
866
+ try {
867
+ res = await command(buildLockHolderProbe(target), {
868
+ member_name: member, silent: true, failSoft: true,
869
+ label: `beads lock-holder probe for '${member}'`, timeout_s: DOLT_LOCK_PROBE_TIMEOUT_S,
870
+ });
871
+ } catch (err) {
872
+ if (err && err.name === 'CancelledError') throw err;
873
+ res = { ok: false, output: '', error: (err && err.message) || String(err) };
874
+ }
875
+ if (!res || !res.ok) {
876
+ log(`[Dolt] ${label} has produced no result after ${elapsedS}s on member '${member}'; the beads lock-holder probe failed (${(res && res.error) || 'no result'}), so the holders are unknown. Still waiting.`);
877
+ return { foreignHolders: [] };
878
+ }
879
+ const parsed = parseLockHolderProbe(res.output);
880
+ const foreignHolders = parsed.holders.filter((h) => h.ageS !== null && h.ageS > elapsedS + DOLT_LOCK_HOLDER_AGE_SLACK_S);
881
+ const parts = [`[Dolt] ${label} has produced no result after ${elapsedS}s on member '${member}'.`];
882
+ if (parsed.noLookup) parts.push(`Beads lock holders cannot be listed on this member: ${parsed.noLookup}.`);
883
+ if (parsed.holders.length > 0) parts.push(`Beads lock holders: ${parsed.holders.map(describeLockProcess).join('; ')}.`);
884
+ else if (!parsed.noLookup) parts.push('Beads lock holders: none found.');
885
+ if (parsed.candidates.length > 0) parts.push(`Running bd/dolt processes (candidates): ${parsed.candidates.map(describeLockProcess).join('; ')}.`);
886
+ parts.push(foreignHolders.length > 0
887
+ ? `Held by a process that started before this sync (more than ${DOLT_LOCK_HOLDER_AGE_SLACK_S}s older than the ${elapsedS}s attempt) -- failing fast.`
888
+ : 'No lock held by a process older than this sync -- still waiting.');
889
+ log(parts.join(' '));
890
+ return { foreignHolders };
891
+ }
892
+
893
+ /**
894
+ * Await one sync attempt under the lock-holder watchdog. Resolves to
895
+ * `{ res }` when the command finishes (probe or not), or to `{ lockHeld }`
896
+ * when the probe names a foreign holder first.
897
+ */
898
+ async function awaitSyncAttempt({ pending, command, member, target, label, log, attemptStartedAt, now, lockProbeAfterMs, lockProbeTimer }) {
899
+ if (!(Number(lockProbeAfterMs) > 0)) return { res: await pending };
900
+ const timer = lockProbeTimer || defaultLockProbeTimer;
901
+ let fired = false;
902
+ let handle = null;
903
+ const probed = new Promise((resolve, reject) => {
904
+ handle = timer.set(() => {
905
+ fired = true;
906
+ probeLockHolders({ command, member, target, label, log, attemptStartedAt, now }).then(resolve, reject);
907
+ }, Number(lockProbeAfterMs));
908
+ });
909
+ const first = await Promise.race([
910
+ pending.then((res) => ({ done: true, res })),
911
+ probed.then((probe) => ({ done: false, probe })),
912
+ ]);
913
+ if (first.done) {
914
+ if (!fired) timer.clear(handle);
915
+ // A probe that was already running finishes and logs on its own.
916
+ probed.catch(() => {});
917
+ return { res: first.res };
918
+ }
919
+ if (first.probe.foreignHolders.length > 0) return { lockHeld: first.probe.foreignHolders };
920
+ return { res: await pending };
921
+ }
922
+
686
923
  /**
687
924
  * Run a single `bd dolt` command via the injected command() with failSoft,
688
925
  * retrying ONLY transient failures up to `maxTransientRetries` times, with a
@@ -709,7 +946,13 @@ const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
709
946
  *
710
947
  * @returns {Promise<{ ok: boolean, output: string, error: string|null, kind?: 'no-remote'|'empty-remote'|'remote-unreachable'|'diverged'|'auth'|'transient'|'missing-tool'|'unknown', missingTool?: string }>}
711
948
  */
712
- async function runDoltStep({ command, member, cmd, label, log, maxTransientRetries, onAuthFailure, sleep = defaultSleep, backoffBaseMs = DOLT_BACKOFF_BASE_MS, timeoutS = DOLT_STEP_TIMEOUT_S, spawnOutageBudgetMs = DOLT_SPAWN_OUTAGE_BUDGET_MS, now = Date.now }) {
949
+ async function runDoltStep({ command, member, cmd, label, log, maxTransientRetries, onAuthFailure, sleep = defaultSleep, backoffBaseMs = DOLT_BACKOFF_BASE_MS, timeoutS = DOLT_STEP_TIMEOUT_S, spawnOutageBudgetMs = DOLT_SPAWN_OUTAGE_BUDGET_MS, now = Date.now, memberTarget, lockProbeAfterMs = DOLT_LOCK_PROBE_AFTER_MS, lockProbeTimer, onAbandonedAttempt }) {
950
+ // `cmd` stays the LOGICAL command (classification, commandBinary, labels);
951
+ // `issued` is the member-bound string, which on a Windows member bounds
952
+ // GOMAXPROCS for bd (resolved once per step, not per attempt). The same
953
+ // target builds the lock-holder probe.
954
+ const target = await resolveDoltMemberTarget(member, memberTarget, log);
955
+ const issued = bdSyncCommandFor(cmd, target);
713
956
  let attempt = 0;
714
957
  let authHealAttempted = false;
715
958
  /** The info the self-heal was invoked with, reported back on a failed retry. */
@@ -726,7 +969,20 @@ async function runDoltStep({ command, member, cmd, label, log, maxTransientRetri
726
969
  : DOLT_SPAWN_OUTAGE_MAX_RETRIES;
727
970
  // eslint-disable-next-line no-constant-condition
728
971
  while (true) {
729
- const res = await command(cmd, { member_name: member, silent: true, failSoft: true, label, timeout_s: timeoutS });
972
+ const attemptStartedAt = now();
973
+ const pending = Promise.resolve(command(issued, { member_name: member, silent: true, failSoft: true, label, timeout_s: timeoutS }));
974
+ const waited = await awaitSyncAttempt({ pending, command, member, target, label, log, attemptStartedAt, now, lockProbeAfterMs, lockProbeTimer });
975
+ if (waited.lockHeld) {
976
+ // The sync command is still waiting on the member; stop waiting
977
+ // for it here and hand it to the caller (the push bracket keeps
978
+ // its mutex until it ends). Not retried -- see the watchdog notes.
979
+ pending.catch(() => {});
980
+ if (typeof onAbandonedAttempt === 'function') onAbandonedAttempt(pending);
981
+ const error = `beads lock held by another process: ${waited.lockHeld.map(describeLockProcess).join('; ')}`;
982
+ log(`[Dolt] ${label} FAILED (lock-held) for member '${member}' -- not retrying while that process holds the lock: ${error}`);
983
+ return { ok: false, output: '', error, kind: 'lock-held', lockHolders: waited.lockHeld, selfHealed: authHealAttempted };
984
+ }
985
+ const res = waited.res;
730
986
  if (res && res.ok) {
731
987
  if (authHealAttempted) log(`[Dolt] self-heal recovered: ${label} succeeded for member '${member}' on the retry after re-provisioning credentials.`);
732
988
  return res;
@@ -823,7 +1079,11 @@ async function runDoltStep({ command, member, cmd, label, log, maxTransientRetri
823
1079
  }
824
1080
  }
825
1081
  log(`[Dolt] ${label} FAILED (${kind}) -- reads/writes for member '${member}' may be stale until this is resolved. Raw: ${error}`);
826
- return { ok: false, output: res ? res.output : '', error, kind, selfHealed: authHealAttempted };
1082
+ // A transient failure that is still the Windows process-creation
1083
+ // limit after the retry budget is flagged so the caller throws the
1084
+ // typed DoltWindowsSpawnLimitError instead of a generic sync failure.
1085
+ const spawnLimit = kind === 'transient' && WINDOWS_SPAWN_LIMIT_RE.test(String(rawError || ''));
1086
+ return { ok: false, output: res ? res.output : '', error, kind, selfHealed: authHealAttempted, ...(spawnLimit ? { spawnLimit: true } : {}) };
827
1087
  }
828
1088
  }
829
1089
 
@@ -1433,7 +1693,7 @@ export async function doltPullBefore(member, opts = {}) {
1433
1693
  // `maxTransientRetries` is deliberately NOT defaulted here: runDoltStep()
1434
1694
  // must see whether the caller passed it (an explicit value caps both
1435
1695
  // retry ladders; undefined selects each ladder's own default).
1436
- const { command, log = () => {}, maxTransientRetries, checkSyncRemoteConfigured, skipPull = false, onAuthFailure, sleep, backoffBaseMs, timeoutS, settle, remoteTipFingerprint = true, spawnOutageBudgetMs, now } = opts;
1696
+ const { command, log = () => {}, maxTransientRetries, checkSyncRemoteConfigured, skipPull = false, onAuthFailure, sleep, backoffBaseMs, timeoutS, settle, remoteTipFingerprint = true, spawnOutageBudgetMs, now, memberTarget, lockProbeAfterMs, lockProbeTimer, onAbandonedAttempt } = opts;
1437
1697
  if (typeof command !== 'function') {
1438
1698
  throw new Error("doltPullBefore requires an injected command() in opts");
1439
1699
  }
@@ -1479,7 +1739,7 @@ export async function doltPullBefore(member, opts = {}) {
1479
1739
 
1480
1740
  const pull = await runDoltStep({
1481
1741
  command, member, cmd: 'bd dolt pull',
1482
- label: `D-pull for '${member}'`, log, maxTransientRetries, onAuthFailure, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now,
1742
+ label: `D-pull for '${member}'`, log, maxTransientRetries, onAuthFailure, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now, memberTarget, lockProbeAfterMs, lockProbeTimer, onAbandonedAttempt,
1483
1743
  });
1484
1744
  if (!pull.ok) {
1485
1745
  if (pull.kind === 'no-remote') {
@@ -1537,6 +1797,8 @@ export async function doltPullBefore(member, opts = {}) {
1537
1797
  { member, doltOutput: pull.error, remoteUrl: url },
1538
1798
  );
1539
1799
  }
1800
+ const typedPullError = typedStepError(member, 'pull', pull);
1801
+ if (typedPullError) throw typedPullError;
1540
1802
  throw new DoltSyncError(
1541
1803
  `[Dolt] D-pull failed for member '${member}': ${pull.error}`,
1542
1804
  { member, doltOutput: pull.error },
@@ -1735,7 +1997,15 @@ export async function doltPushAfter(member, opts = {}) {
1735
1997
  // doltPullBefore. `remoteTipFingerprint` is accepted for call-site parity
1736
1998
  // with doltPullBefore but has nothing to switch off on the push side any
1737
1999
  // more: a push never probes and never records (it only forgets).
1738
- const { command, pushBeads = true, log = () => {}, maxTransientRetries, mutex, sprintId, checkSyncRemoteConfigured, onAuthFailure, sleep, backoffBaseMs, timeoutS, settle, renewIntervalMs = DOLT_MUTEX_RENEW_INTERVAL_MS, spawnOutageBudgetMs, now } = opts;
2000
+ const { command, pushBeads = true, log = () => {}, maxTransientRetries, mutex, sprintId, checkSyncRemoteConfigured, onAuthFailure, sleep, backoffBaseMs, timeoutS, settle, renewIntervalMs = DOLT_MUTEX_RENEW_INTERVAL_MS, spawnOutageBudgetMs, now, memberTarget, lockProbeAfterMs, lockProbeTimer } = opts;
2001
+ // Sync commands this push gave up on while they were still running on the
2002
+ // member (a lock-held fail-fast, see runDoltStep). The push mutex is kept
2003
+ // until they end -- see the release in the finally below.
2004
+ const abandonedAttempts = [];
2005
+ const onAbandonedAttempt = (pending) => {
2006
+ abandonedAttempts.push(pending);
2007
+ if (typeof opts.onAbandonedAttempt === 'function') opts.onAbandonedAttempt(pending);
2008
+ };
1739
2009
  if (typeof command !== 'function') {
1740
2010
  throw new Error("doltPushAfter requires an injected command() in opts");
1741
2011
  }
@@ -1804,13 +2074,27 @@ export async function doltPushAfter(member, opts = {}) {
1804
2074
  landedCheck = await beginDoltPushLandedCheck(member, { command, log });
1805
2075
  return await doltPushGuarded();
1806
2076
  } finally {
1807
- if (renewTimer) clearInterval(renewTimer);
1808
- if (grant && mutex && typeof mutex.release === 'function') {
1809
- try {
1810
- await mutex.release(grant.token);
1811
- } catch (relErr) {
1812
- log(`[Dolt] mutex release after D-push for member '${member}' failed (non-fatal; lease will expire): ${relErr.message}`);
2077
+ const releaseMutex = async () => {
2078
+ if (renewTimer) clearInterval(renewTimer);
2079
+ if (grant && mutex && typeof mutex.release === 'function') {
2080
+ try {
2081
+ await mutex.release(grant.token);
2082
+ } catch (relErr) {
2083
+ log(`[Dolt] mutex release after D-push for member '${member}' failed (non-fatal; lease will expire): ${relErr.message}`);
2084
+ }
1813
2085
  }
2086
+ };
2087
+ if (abandonedAttempts.length > 0 && grant) {
2088
+ // A `bd dolt push` this bracket stopped waiting for is still
2089
+ // running on the member and could still publish once the lock
2090
+ // frees. Releasing the mutex now would let another writer push
2091
+ // concurrently with it, so the mutex (and its lease renewal) is
2092
+ // held until that command ends -- bounded by its own command
2093
+ // timeout -- while this call returns immediately.
2094
+ log(`[Dolt] D-push for member '${member}' stopped waiting for a sync command that is still running on the member; keeping the push mutex until that command ends so it cannot publish concurrently with another writer.`);
2095
+ Promise.allSettled(abandonedAttempts).then(releaseMutex);
2096
+ } else {
2097
+ await releaseMutex();
1814
2098
  }
1815
2099
  }
1816
2100
 
@@ -1865,8 +2149,13 @@ export async function doltPushAfter(member, opts = {}) {
1865
2149
  log(`[Dolt] ${label} reported success but the push did NOT land: ${verdict.why}. Retrying the push once.`);
1866
2150
  const retry = await runDoltStep({
1867
2151
  command, member, cmd: 'bd dolt push',
1868
- label: `D-push retry after an unlanded push for '${member}'`, log, maxTransientRetries, onAuthFailure, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now,
2152
+ label: `D-push retry after an unlanded push for '${member}'`, log, maxTransientRetries, onAuthFailure, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now, memberTarget, lockProbeAfterMs, lockProbeTimer, onAbandonedAttempt,
1869
2153
  });
2154
+ const typedRetryError = retry.ok ? null : typedStepError(member, 'push retry after an unlanded push', retry);
2155
+ if (typedRetryError) {
2156
+ clearLastSyncedTip(member);
2157
+ throw typedRetryError;
2158
+ }
1870
2159
  const firstWhy = verdict.why;
1871
2160
  verdict = retry.ok
1872
2161
  ? await checkDoltPushLanded(member, { command, check: landedCheck })
@@ -1894,7 +2183,7 @@ export async function doltPushAfter(member, opts = {}) {
1894
2183
  async function doltPushGuarded() {
1895
2184
  let push = await runDoltStep({
1896
2185
  command, member, cmd: 'bd dolt push',
1897
- label: `D-push for '${member}'`, log, maxTransientRetries, onAuthFailure, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now,
2186
+ label: `D-push for '${member}'`, log, maxTransientRetries, onAuthFailure, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now, memberTarget, lockProbeAfterMs, lockProbeTimer, onAbandonedAttempt,
1898
2187
  });
1899
2188
  if (push.ok) {
1900
2189
  return await confirmPushLanded({ ok: true, member, pushed: true, reconciled: false }, `D-push for member '${member}'`);
@@ -1943,6 +2232,8 @@ export async function doltPushAfter(member, opts = {}) {
1943
2232
  { member, doltOutput: push.error, remoteUrl: url },
1944
2233
  );
1945
2234
  }
2235
+ const typedPushError = typedStepError(member, 'push', push);
2236
+ if (typedPushError) throw typedPushError;
1946
2237
  throw new DoltSyncError(
1947
2238
  `[Dolt] D-push for member '${member}' failed: ${push.error}`,
1948
2239
  { member, doltOutput: push.error },
@@ -1955,7 +2246,7 @@ export async function doltPushAfter(member, opts = {}) {
1955
2246
  log(`[Dolt] D-push for member '${member}' was rejected (another writer pushed first); reconciling with a single D-pull then one re-push (first-successful-pusher-wins).`);
1956
2247
  const reconcile = await runDoltStep({
1957
2248
  command, member, cmd: 'bd dolt pull',
1958
- label: `D-push reconcile pull for '${member}'`, log, maxTransientRetries, onAuthFailure, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now,
2249
+ label: `D-push reconcile pull for '${member}'`, log, maxTransientRetries, onAuthFailure, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now, memberTarget, lockProbeAfterMs, lockProbeTimer, onAbandonedAttempt,
1959
2250
  });
1960
2251
  if (!reconcile.ok) {
1961
2252
  if (reconcile.kind === 'diverged') {
@@ -1967,6 +2258,8 @@ export async function doltPushAfter(member, opts = {}) {
1967
2258
  'push-reconcile',
1968
2259
  );
1969
2260
  }
2261
+ const typedReconcileError = typedStepError(member, 'push reconcile pull', reconcile);
2262
+ if (typedReconcileError) throw typedReconcileError;
1970
2263
  throw new DoltSyncError(
1971
2264
  `[Dolt] D-push reconcile pull for member '${member}' failed: ${reconcile.error}`,
1972
2265
  { member, doltOutput: reconcile.error },
@@ -1979,7 +2272,7 @@ export async function doltPushAfter(member, opts = {}) {
1979
2272
  landedCheck = await beginDoltPushLandedCheck(member, { command, log });
1980
2273
  push = await runDoltStep({
1981
2274
  command, member, cmd: 'bd dolt push',
1982
- label: `D-push re-push after reconcile for '${member}'`, log, maxTransientRetries, onAuthFailure, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now,
2275
+ label: `D-push re-push after reconcile for '${member}'`, log, maxTransientRetries, onAuthFailure, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now, memberTarget, lockProbeAfterMs, lockProbeTimer, onAbandonedAttempt,
1983
2276
  });
1984
2277
  if (push.ok) {
1985
2278
  return await confirmPushLanded({ ok: true, member, pushed: true, reconciled: true }, `D-push re-push after reconcile for member '${member}'`);
@@ -2041,7 +2334,7 @@ export async function doltPushAfter(member, opts = {}) {
2041
2334
 
2042
2335
  const secondReconcile = await runDoltStep({
2043
2336
  command, member, cmd: 'bd dolt pull',
2044
- label: `D-push second reconcile pull for '${member}'`, log, maxTransientRetries, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now,
2337
+ label: `D-push second reconcile pull for '${member}'`, log, maxTransientRetries, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now, memberTarget, lockProbeAfterMs, lockProbeTimer, onAbandonedAttempt,
2045
2338
  });
2046
2339
  if (!secondReconcile.ok) {
2047
2340
  if (secondReconcile.kind === 'diverged') {
@@ -2053,6 +2346,8 @@ export async function doltPushAfter(member, opts = {}) {
2053
2346
  'push-reconcile',
2054
2347
  );
2055
2348
  }
2349
+ const typedSecondReconcileError = typedStepError(member, 'push second reconcile pull', secondReconcile);
2350
+ if (typedSecondReconcileError) throw typedSecondReconcileError;
2056
2351
  throw new DoltSyncError(
2057
2352
  `[Dolt] D-push second reconcile pull for member '${member}' failed: ${secondReconcile.error}`,
2058
2353
  { member, doltOutput: secondReconcile.error },
@@ -2062,7 +2357,7 @@ export async function doltPushAfter(member, opts = {}) {
2062
2357
  landedCheck = await beginDoltPushLandedCheck(member, { command, log });
2063
2358
  push = await runDoltStep({
2064
2359
  command, member, cmd: 'bd dolt push',
2065
- label: `D-push second re-push after reconcile for '${member}'`, log, maxTransientRetries, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now,
2360
+ label: `D-push second re-push after reconcile for '${member}'`, log, maxTransientRetries, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now, memberTarget, lockProbeAfterMs, lockProbeTimer, onAbandonedAttempt,
2066
2361
  });
2067
2362
  if (push.ok) {
2068
2363
  return await confirmPushLanded({ ok: true, member, pushed: true, reconciled: true }, `D-push second re-push after reconcile for member '${member}'`);
@@ -148,17 +148,23 @@ export class SprintPlanRejectedError extends WorkflowError {
148
148
  * @property {string[]} [blockerIds] - apra-fleet-mjo: the bead ids still open
149
149
  * at/above goal priority when the stall was declared, i.e. exactly what an
150
150
  * operator must clear. Counts alone cannot answer "blocked on what?".
151
+ * Also names every bead set aside for the run (doer-BLOCKED and its
152
+ * dependents), plus the pseudo-id 'deploy' when the deploy is set aside.
153
+ * @property {Record<string, string>} [blockerReasons] - blocker id -> reason
154
+ * (with first-seen cycle/round) for the set-aside blockers.
155
+ * @property {object[]} [setAside] - the run's set-aside records, rendered as
156
+ * the "Set aside this run" report section (see ./set-aside.mjs).
151
157
  */
152
158
  export class StalledSprintError extends WorkflowError {
153
159
  /**
154
160
  * @param {string} message
155
- * @param {{ staleCycles?: number, closedCountHistory?: number[], highWaterClosedCount?: number, thrashIds?: string[], blockerIds?: string[], cycle?: number, details?: object, cause?: unknown }} [opts]
161
+ * @param {{ staleCycles?: number, closedCountHistory?: number[], highWaterClosedCount?: number, thrashIds?: string[], blockerIds?: string[], blockerReasons?: Record<string, string>, setAside?: object[], waited?: object[], cycle?: number, details?: object, cause?: unknown }} [opts]
156
162
  */
157
163
  constructor(message, opts = {}) {
158
- const { staleCycles = null, closedCountHistory = [], highWaterClosedCount = null, thrashIds = [], blockerIds = [], cycle, details, cause } = opts;
164
+ const { staleCycles = null, closedCountHistory = [], highWaterClosedCount = null, thrashIds = [], blockerIds = [], blockerReasons = {}, setAside = [], waited = [], cycle, details, cause } = opts;
159
165
  super(message, {
160
166
  code: 'SPRINT_STALLED',
161
- details: { staleCycles, closedCountHistory, highWaterClosedCount, thrashIds, blockerIds, cycle, ...details },
167
+ details: { staleCycles, closedCountHistory, highWaterClosedCount, thrashIds, blockerIds, blockerReasons, setAside, cycle, ...details },
162
168
  cause,
163
169
  });
164
170
  this.staleCycles = staleCycles;
@@ -166,6 +172,11 @@ export class StalledSprintError extends WorkflowError {
166
172
  this.highWaterClosedCount = highWaterClosedCount;
167
173
  this.thrashIds = thrashIds;
168
174
  this.blockerIds = blockerIds;
175
+ this.blockerReasons = blockerReasons;
176
+ this.setAside = setAside;
177
+ // The run's wait records (./set-aside.mjs collectWaitEntries): the
178
+ // "Waited this run" line of an [ABORTED] PR body.
179
+ this.waited = waited;
169
180
  }
170
181
  }
171
182
 
@@ -356,6 +367,52 @@ export class DoltSyncError extends WorkflowError {
356
367
  }
357
368
  }
358
369
 
370
+ /** Stable error code of DoltWindowsSpawnLimitError. */
371
+ export const DOLT_WINDOWS_SPAWN_LIMIT_CODE = 'DOLT_WINDOWS_SPAWN_LIMIT';
372
+
373
+ /**
374
+ * A D-pull/D-push that kept failing because Windows refused to start bd/dolt's
375
+ * git child processes (CreateProcess ERROR_NOT_ENOUGH_MEMORY, the Windows
376
+ * process-creation limit) until its transient retry budget ran out. A
377
+ * DoltSyncError subtype, so every existing DoltSyncError handler (degrade/fatal
378
+ * policy, classifySyncError -> 'transient') still applies; the stable `code`
379
+ * lets a caller tell it apart from a generic sync failure.
380
+ */
381
+ export class DoltWindowsSpawnLimitError extends DoltSyncError {
382
+ /**
383
+ * @param {string} message
384
+ * @param {{ member?: string|null, doltOutput?: string|null, details?: object, cause?: unknown }} [opts]
385
+ */
386
+ constructor(message, opts = {}) {
387
+ super(message, { ...opts, details: { kind: 'transient', cause: 'windows-spawn-limit', ...opts.details } });
388
+ this.name = 'DoltWindowsSpawnLimitError';
389
+ this.code = DOLT_WINDOWS_SPAWN_LIMIT_CODE;
390
+ }
391
+ }
392
+
393
+ /** Stable error code of DoltLockHeldError. */
394
+ export const DOLT_BEADS_LOCK_HELD_CODE = 'DOLT_BEADS_LOCK_HELD';
395
+
396
+ /**
397
+ * A D-pull/D-push that was still waiting after the lock-holder watchdog fired
398
+ * while a process OTHER than this sync (one that started before the sync
399
+ * attempt did) held a beads lock on the member. Raised in bounded time instead
400
+ * of waiting out the inactivity timeout; `details.holders` lists each holder
401
+ * ({ pid, ageS, lock, commandLine }). A DoltSyncError subtype, so every
402
+ * existing DoltSyncError handler still applies.
403
+ */
404
+ export class DoltLockHeldError extends DoltSyncError {
405
+ /**
406
+ * @param {string} message
407
+ * @param {{ member?: string|null, doltOutput?: string|null, details?: object, cause?: unknown }} [opts]
408
+ */
409
+ constructor(message, opts = {}) {
410
+ super(message, { ...opts, details: { kind: 'lock-held', ...opts.details } });
411
+ this.name = 'DoltLockHeldError';
412
+ this.code = DOLT_BEADS_LOCK_HELD_CODE;
413
+ }
414
+ }
415
+
359
416
  /**
360
417
  * Thrown by preflightBeadsHealthGate() when the backlog member's beads clone
361
418
  * holds local schema migrations bd auto-applied but never published (the
@@ -431,6 +488,16 @@ export class DoltBinaryUnavailableError extends WorkflowError {
431
488
  // AgentDispatchError -- checked FIRST below since it can't be fooled by
432
489
  // auth-like noise in an unrelated failure's message text; the regex remains
433
490
  // as a fallback for older/mocked errors that only ever set `.message`.
491
+ // The server refuses a dispatch (no LLM call made) when the member's stored
492
+ // credential env vars cannot be delivered without a command line: a relay
493
+ // member or one with SFTP disabled. Deterministic -- the member's channel does
494
+ // not change between attempts -- so it is non-retryable. It is deliberately
495
+ // NOT an LLM-auth failure (isAuthDispatchError): the credential is present
496
+ // but undeliverable, so provision_llm_auth self-heal cannot fix it; the
497
+ // operator must enable SFTP or clear/provide the credential on the member.
498
+ export const SECRET_DELIVERY_UNAVAILABLE_REASON = 'secret_delivery_unavailable';
499
+ export const SECRET_DELIVERY_UNAVAILABLE_REMEDY = 'enable SFTP on the member, or clear its stored credentials (provision_llm_auth with clear_stored_credentials) and provide the credential on the member itself';
500
+
434
501
  const NON_RETRYABLE_DISPATCH_RE = /authentication failed|failed to authenticate|oauth session expired|could not be refreshed|not logged in|workspace not trusted|has not been trusted/i;
435
502
 
436
503
  /**
@@ -443,7 +510,8 @@ const NON_RETRYABLE_DISPATCH_RE = /authentication failed|failed to authenticate|
443
510
  export function isNonRetryableDispatchError(err) {
444
511
  const reason = err?.details?.reason;
445
512
  if (reason === 'auth' || reason === 'workspace_not_trusted'
446
- || reason === 'preflight_auth_missing' || reason === 'preflight_auth_expired') return true;
513
+ || reason === 'preflight_auth_missing' || reason === 'preflight_auth_expired'
514
+ || reason === SECRET_DELIVERY_UNAVAILABLE_REASON) return true;
447
515
  return NON_RETRYABLE_DISPATCH_RE.test(String(err?.message ?? ''));
448
516
  }
449
517
 
@@ -320,7 +320,13 @@ export function createSyncBrackets({ setPauseGuard } = {}) {
320
320
  // failure (it may have committed and/or pushed), so run the full
321
321
  // pre-dispatch sync in syncMemberBefore's resetToRemoteTip mode -- fetch
322
322
  // and reset onto the remote tip BEFORE the doer can commit, resuming on
323
- // published work instead of re-implementing it and diverging.
323
+ // published work instead of re-implementing it and diverging. The
324
+ // prior attempt's unpublished commits and uncommitted TRACKED changes
325
+ // are first pinned on a LOCAL, never-pushed
326
+ // fleet-salvage/<branch-slug>/<short sha> side ref (git stash create +
327
+ // git branch -f), without moving the sprint branch or touching
328
+ // untracked files; a salvage failure aborts the bracket with
329
+ // GitSyncError instead of resetting (see salvageBeforeReset).
324
330
  // skipPreDispatchSync and resumeOntoRemoteTip encode opposite assumptions
325
331
  // about whether the prior attempt published anything and MUST never be
326
332
  // passed together. Nothing in the code enforces this today.