@apralabs/apra-fleet 0.4.4 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -4
- package/dist/agents/backlog-groomer.md +5 -0
- package/dist/agents/ci-watcher.md +5 -0
- package/dist/agents/deployer.md +20 -2
- package/dist/agents/doer.md +38 -2
- package/dist/agents/harvester.md +12 -9
- package/dist/agents/integ-test-runner.md +5 -0
- package/dist/agents/kb-reconciler.md +6 -0
- package/dist/agents/plan-reviewer.md +5 -0
- package/dist/agents/planner.md +5 -0
- package/dist/agents/regression-test-runner.md +5 -0
- package/dist/agents/reviewer.md +67 -15
- package/dist/agents/schemas/deployer-output.json +5 -0
- package/dist/agents/schemas/doer-output.json +78 -2
- package/dist/agents/schemas/reviewer-output.json +104 -2
- package/dist/cli/auth.js +5 -1
- package/dist/cli/beads-pin.js +6 -7
- package/dist/cli/claude-mod.d.ts +75 -0
- package/dist/cli/claude-mod.js +288 -0
- package/dist/cli/dolt-install.d.ts +1 -1
- package/dist/cli/dolt-install.js +5 -1
- package/dist/cli/fleet-se-guard.d.ts +8 -0
- package/dist/cli/fleet-se-guard.js +19 -3
- package/dist/cli/install-guard.d.ts +9 -0
- package/dist/cli/install-guard.js +24 -0
- package/dist/cli/install.d.ts +12 -0
- package/dist/cli/install.js +180 -20
- package/dist/cli/kb-profile.d.ts +16 -0
- package/dist/cli/kb-profile.js +81 -0
- package/dist/cli/start.js +3 -0
- package/dist/cli/uninstall.js +13 -3
- package/dist/cli/update.d.ts +4 -0
- package/dist/cli/update.js +19 -3
- package/dist/commands/kb-server.d.ts +10 -1
- package/dist/commands/kb-server.js +285 -11
- package/dist/index.js +22 -1
- package/dist/os/git-executable.d.ts +57 -0
- package/dist/os/git-executable.js +85 -0
- package/dist/os/go-concurrency.d.ts +44 -0
- package/dist/os/go-concurrency.js +53 -0
- package/dist/os/index.d.ts +1 -0
- package/dist/os/index.js +1 -0
- package/dist/os/linux.d.ts +2 -0
- package/dist/os/linux.js +10 -1
- package/dist/os/os-commands.d.ts +25 -0
- package/dist/os/os-commands.js +2 -0
- package/dist/os/windows-executable.d.ts +53 -0
- package/dist/os/windows-executable.js +185 -0
- package/dist/os/windows-gitbash.d.ts +23 -14
- package/dist/os/windows-gitbash.js +40 -16
- package/dist/os/windows.d.ts +2 -0
- package/dist/os/windows.js +37 -6
- package/dist/paths.d.ts +10 -1
- package/dist/paths.js +19 -2
- package/dist/services/agy-project.d.ts +7 -1
- package/dist/services/agy-project.js +7 -2
- package/dist/services/auth-socket.d.ts +18 -0
- package/dist/services/auth-socket.js +30 -4
- package/dist/services/credential-store.d.ts +33 -0
- package/dist/services/credential-store.js +79 -2
- package/dist/services/http-transport.d.ts +29 -0
- package/dist/services/http-transport.js +166 -37
- package/dist/services/kb-tool-profile.d.ts +36 -0
- package/dist/services/kb-tool-profile.js +88 -0
- package/dist/services/knowledge/bible-blob-id.js +10 -1
- package/dist/services/knowledge/bible-import.d.ts +108 -0
- package/dist/services/knowledge/bible-import.js +137 -4
- package/dist/services/knowledge/file-hash.js +6 -5
- package/dist/services/knowledge/http-provider.d.ts +303 -20
- package/dist/services/knowledge/http-provider.js +795 -125
- package/dist/services/knowledge/kb-config.d.ts +65 -1
- package/dist/services/knowledge/kb-config.js +143 -3
- package/dist/services/knowledge/kb-http-auth-state.d.ts +26 -0
- package/dist/services/knowledge/kb-http-auth-state.js +40 -0
- package/dist/services/knowledge/kb-providers.d.ts +9 -2
- package/dist/services/knowledge/kb-providers.js +67 -37
- package/dist/services/knowledge/kb-removed-scope-keys.js +1 -1
- package/dist/services/knowledge/kb-self.d.ts +122 -9
- package/dist/services/knowledge/kb-self.js +196 -36
- package/dist/services/knowledge/member-bible-view.js +16 -3
- package/dist/services/knowledge/path-validation.d.ts +7 -0
- package/dist/services/knowledge/path-validation.js +11 -0
- package/dist/services/knowledge/project-slug.js +3 -2
- package/dist/services/knowledge/repository-header.d.ts +77 -0
- package/dist/services/knowledge/repository-header.js +233 -0
- package/dist/services/knowledge/require-sqlite-project.d.ts +1 -1
- package/dist/services/knowledge/require-sqlite-project.js +1 -1
- package/dist/services/knowledge/sqlite-provider.d.ts +169 -0
- package/dist/services/knowledge/sqlite-provider.js +416 -4
- package/dist/services/knowledge/types.d.ts +15 -2
- package/dist/services/member-busy.d.ts +12 -0
- package/dist/services/member-busy.js +20 -0
- package/dist/services/member-config-io.d.ts +3 -2
- package/dist/services/member-config-io.js +7 -4
- package/dist/services/member-fleet-install.js +1 -0
- package/dist/services/member-tool-allowlist.d.ts +1 -1
- package/dist/services/member-tool-allowlist.js +8 -4
- package/dist/services/port-holder.js +2 -1
- package/dist/services/service-manager/types.d.ts +17 -1
- package/dist/services/service-manager/windows.d.ts +7 -2
- package/dist/services/service-manager/windows.js +28 -11
- package/dist/services/service-notice.d.ts +7 -0
- package/dist/services/service-notice.js +19 -0
- package/dist/services/session-registry.d.ts +13 -0
- package/dist/services/session-registry.js +13 -0
- package/dist/services/session-roots.d.ts +67 -0
- package/dist/services/session-roots.js +126 -0
- package/dist/services/statusline.d.ts +21 -0
- package/dist/services/statusline.js +97 -1
- package/dist/services/supervisor-service-cleanup.js +2 -1
- package/dist/services/tool-registry.js +108 -20
- package/dist/services/tool-scope.d.ts +34 -1
- package/dist/services/tool-scope.js +27 -2
- package/dist/services/vcs/azure-devops.js +10 -0
- package/dist/services/vcs/bitbucket.d.ts +22 -1
- package/dist/services/vcs/bitbucket.js +163 -17
- package/dist/services/vcs/types.d.ts +25 -2
- package/dist/services/watch/project-resolver.js +2 -1
- package/dist/skills/pm/cost.md +5 -1
- package/dist/skills/pm/index.md +9 -9
- package/dist/skills/pm/sprint.md +2 -1
- package/dist/tools/check-status.d.ts +51 -0
- package/dist/tools/check-status.js +124 -12
- package/dist/tools/code-intelligence-freshness.js +1 -1
- package/dist/tools/code-intelligence-gitnexus.d.ts +19 -0
- package/dist/tools/code-intelligence-gitnexus.js +166 -9
- package/dist/tools/code-intelligence-readiness.d.ts +58 -1
- package/dist/tools/code-intelligence-readiness.js +93 -8
- package/dist/tools/code-intelligence-reindex.d.ts +25 -1
- package/dist/tools/code-intelligence-reindex.js +76 -4
- package/dist/tools/code-intelligence.d.ts +1 -1
- package/dist/tools/code-intelligence.js +1 -1
- package/dist/tools/execute-command.js +14 -8
- package/dist/tools/execute-prompt.js +6 -2
- package/dist/tools/fleet-snapshot.d.ts +21 -0
- package/dist/tools/fleet-snapshot.js +78 -0
- package/dist/tools/kb-bible-commit.d.ts +25 -11
- package/dist/tools/kb-bible-commit.js +142 -33
- package/dist/tools/kb-capture.d.ts +2 -2
- package/dist/tools/kb-context.js +47 -27
- package/dist/tools/kb-demote.d.ts +27 -0
- package/dist/tools/kb-demote.js +37 -0
- package/dist/tools/kb-export.d.ts +35 -3
- package/dist/tools/kb-export.js +56 -13
- package/dist/tools/kb-feedback.d.ts +2 -0
- package/dist/tools/kb-feedback.js +17 -7
- package/dist/tools/kb-harvest.js +28 -9
- package/dist/tools/kb-import.d.ts +18 -2
- package/dist/tools/kb-import.js +24 -9
- package/dist/tools/kb-invalidate.d.ts +11 -3
- package/dist/tools/kb-invalidate.js +80 -7
- package/dist/tools/kb-list.js +38 -10
- package/dist/tools/kb-promote.js +6 -4
- package/dist/tools/kb-query.d.ts +3 -0
- package/dist/tools/kb-query.js +181 -37
- package/dist/tools/kb-session-prime.d.ts +4 -4
- package/dist/tools/kb-session-prime.js +89 -9
- package/dist/tools/kb-setup.d.ts +20 -0
- package/dist/tools/kb-setup.js +149 -11
- package/dist/tools/list-members.js +1 -1
- package/dist/tools/member-detail.js +7 -3
- package/dist/tools/provision-vcs-auth.d.ts +28 -3
- package/dist/tools/provision-vcs-auth.js +53 -7
- package/dist/tools/receive-files.js +7 -4
- package/dist/tools/send-files.js +5 -4
- package/dist/tools/send-message.d.ts +3 -0
- package/dist/tools/send-message.js +15 -2
- package/dist/tools/stop-prompt.js +3 -1
- package/dist/tools/vcs-credential-exec.d.ts +45 -3
- package/dist/tools/vcs-credential-exec.js +90 -11
- package/dist/utils/process-utils.js +5 -1
- package/dist/utils/redact-known-secrets.d.ts +17 -0
- package/dist/utils/redact-known-secrets.js +70 -0
- package/dist/utils/skill-matrix.js +1 -1
- package/dist/utils/vcs-provider-detect.js +2 -2
- package/dist/workflows/auto-sprint.js +245 -9
- package/package.json +10 -7
- package/packages/apra-fleet-client/package.json +1 -1
- package/packages/apra-fleet-client/src/client/api.mjs +172 -19
- package/packages/apra-fleet-client/src/client/reconnecting-transport.mjs +35 -2
- package/packages/apra-fleet-client/src/client/transport.mjs +6 -0
- package/packages/apra-fleet-se/apra-pm/agents/backlog-groomer.md +5 -0
- package/packages/apra-fleet-se/apra-pm/agents/ci-watcher.md +5 -0
- package/packages/apra-fleet-se/apra-pm/agents/deployer.md +20 -2
- package/packages/apra-fleet-se/apra-pm/agents/doer.md +38 -2
- package/packages/apra-fleet-se/apra-pm/agents/harvester.md +12 -9
- package/packages/apra-fleet-se/apra-pm/agents/integ-test-runner.md +5 -0
- package/packages/apra-fleet-se/apra-pm/agents/kb-reconciler.md +6 -0
- package/packages/apra-fleet-se/apra-pm/agents/plan-reviewer.md +5 -0
- package/packages/apra-fleet-se/apra-pm/agents/planner.md +5 -0
- package/packages/apra-fleet-se/apra-pm/agents/regression-test-runner.md +5 -0
- package/packages/apra-fleet-se/apra-pm/agents/reviewer.md +67 -15
- package/packages/apra-fleet-se/apra-pm/agents/schemas/deployer-output.json +5 -0
- package/packages/apra-fleet-se/apra-pm/agents/schemas/doer-output.json +78 -2
- package/packages/apra-fleet-se/apra-pm/agents/schemas/reviewer-output.json +104 -2
- package/packages/apra-fleet-se/apra-pm/skills/pm/cost.md +5 -1
- package/packages/apra-fleet-se/apra-pm/skills/pm/index.md +9 -9
- package/packages/apra-fleet-se/apra-pm/skills/pm/sprint.md +2 -1
- package/packages/apra-fleet-se/bin/cli.mjs +104 -31
- package/packages/apra-fleet-se/bin/serve.mjs +46 -8
- package/packages/apra-fleet-se/fleet-sprint/abort.mjs +184 -32
- package/packages/apra-fleet-se/fleet-sprint/bd-sync-command.mjs +69 -0
- package/packages/apra-fleet-se/fleet-sprint/beads-children.mjs +21 -0
- package/packages/apra-fleet-se/fleet-sprint/beads-lock-probe.mjs +176 -0
- package/packages/apra-fleet-se/fleet-sprint/beads-reachability.mjs +126 -0
- package/packages/apra-fleet-se/fleet-sprint/beads-scope.mjs +158 -1
- package/packages/apra-fleet-se/fleet-sprint/contracts.mjs +82 -2
- package/packages/apra-fleet-se/fleet-sprint/coordination.mjs +23 -6
- package/packages/apra-fleet-se/fleet-sprint/decision-notify.mjs +338 -0
- package/packages/apra-fleet-se/fleet-sprint/decision-wait.mjs +731 -0
- package/packages/apra-fleet-se/fleet-sprint/decisions.mjs +372 -0
- package/packages/apra-fleet-se/fleet-sprint/dispatch-role.mjs +44 -11
- package/packages/apra-fleet-se/fleet-sprint/docs/README.md +4 -1
- package/packages/apra-fleet-se/fleet-sprint/dolt-settle.mjs +9 -3
- package/packages/apra-fleet-se/fleet-sprint/dolt-sync.mjs +314 -19
- package/packages/apra-fleet-se/fleet-sprint/errors.mjs +72 -4
- package/packages/apra-fleet-se/fleet-sprint/git-sync.mjs +7 -1
- package/packages/apra-fleet-se/fleet-sprint/git-topology.mjs +112 -1
- package/packages/apra-fleet-se/fleet-sprint/guarded-modules.mjs +36 -2
- package/packages/apra-fleet-se/fleet-sprint/kb.mjs +736 -101
- package/packages/apra-fleet-se/fleet-sprint/member-init-probe.mjs +1 -1
- package/packages/apra-fleet-se/fleet-sprint/member-provisioning.mjs +7 -1
- package/packages/apra-fleet-se/fleet-sprint/phases/deploy.mjs +49 -2
- package/packages/apra-fleet-se/fleet-sprint/phases/develop.mjs +206 -3
- package/packages/apra-fleet-se/fleet-sprint/phases/final-review.mjs +61 -3
- package/packages/apra-fleet-se/fleet-sprint/phases/harvest.mjs +81 -31
- package/packages/apra-fleet-se/fleet-sprint/phases/plan.mjs +7 -3
- package/packages/apra-fleet-se/fleet-sprint/phases/publish-pr.mjs +39 -14
- package/packages/apra-fleet-se/fleet-sprint/phases/re-review.mjs +17 -1
- package/packages/apra-fleet-se/fleet-sprint/phases/review.mjs +18 -3
- package/packages/apra-fleet-se/fleet-sprint/pr-body.mjs +14 -1
- package/packages/apra-fleet-se/fleet-sprint/prompts.mjs +32 -5
- package/packages/apra-fleet-se/fleet-sprint/runner.js +824 -98
- package/packages/apra-fleet-se/fleet-sprint/se-os-commands.mjs +4 -2
- package/packages/apra-fleet-se/fleet-sprint/se-posix.mjs +54 -1
- package/packages/apra-fleet-se/fleet-sprint/se-windows.mjs +29 -2
- package/packages/apra-fleet-se/fleet-sprint/set-aside.mjs +288 -0
- package/packages/apra-fleet-se/fleet-sprint/skills/fleet-supervisor/SKILL.md +3 -2
- package/packages/apra-fleet-se/fleet-sprint/sprint-args.mjs +20 -0
- package/packages/apra-fleet-se/fleet-sprint/sprint-report.mjs +21 -0
- package/packages/apra-fleet-se/fleet-sprint/sprint-state.mjs +9 -0
- package/packages/apra-fleet-se/fleet-sprint/vcs-auth.mjs +605 -32
- package/packages/apra-fleet-se/fleet-sprint/vcs-providers/bitbucket.mjs +811 -17
- package/packages/apra-fleet-se/fleet-sprint/vcs-providers/index.mjs +89 -1
- package/packages/apra-fleet-se/fleet-sprint/viewer-extensions.mjs +225 -2
- package/packages/apra-fleet-se/fleet-sprint/worklists.mjs +59 -3
- package/packages/apra-fleet-se/src/supervisor/api.mjs +148 -8
- package/packages/apra-fleet-se/src/supervisor/dashboard.mjs +6 -3
- package/packages/apra-fleet-se/src/supervisor/dolt-mutex.mjs +30 -10
- package/packages/apra-fleet-se/src/supervisor/fleet-members.mjs +17 -4
- package/packages/apra-fleet-se/src/supervisor/history.mjs +8 -1
- package/packages/apra-fleet-se/src/supervisor/id-allocator.mjs +92 -2
- package/packages/apra-fleet-se/src/supervisor/proxy.mjs +26 -1
- package/packages/apra-fleet-se/src/supervisor/server.mjs +151 -8
- package/packages/apra-fleet-se/src/supervisor/spawner.mjs +11 -9
- package/packages/apra-fleet-se/src/supervisor/watchdog.mjs +7 -1
- package/packages/apra-fleet-workflow/package.json +2 -1
- package/packages/apra-fleet-workflow/src/viewer/index.mjs +86 -5
- package/packages/apra-fleet-workflow/src/viewer/request-guard.mjs +222 -0
- package/packages/apra-fleet-workflow/src/workflow/index.mjs +34 -2
- package/plugins/claude-code/fleet-status/.claude-plugin/plugin.json +7 -0
- package/plugins/claude-code/fleet-status/hooks/hooks.json +1 -0
- package/plugins/claude-code/fleet-status/hooks/register.tsx +906 -0
- package/plugins/claude-code/fleet-status/types/index.d.ts +88 -0
- package/skills/fleet/auth-bitbucket.md +47 -20
- package/skills/fleet/onboarding.md +1 -1
- package/skills/fleet/skill-matrix.md +3 -3
- package/version.json +1 -1
|
@@ -85,7 +85,9 @@
|
|
|
85
85
|
* ASCII only.
|
|
86
86
|
*/
|
|
87
87
|
|
|
88
|
-
import { DoltDivergedError, DoltSyncError, UnpublishedSchemaMigrationsError } from './errors.mjs';
|
|
88
|
+
import { DoltDivergedError, DoltSyncError, DoltWindowsSpawnLimitError, DoltLockHeldError, UnpublishedSchemaMigrationsError } from './errors.mjs';
|
|
89
|
+
import { bdSyncCommandFor, normalizeMemberTarget, POSIX_MEMBER_TARGET } from './bd-sync-command.mjs';
|
|
90
|
+
import { buildLockHolderProbe, parseLockHolderProbe, describeLockProcess } from './beads-lock-probe.mjs';
|
|
89
91
|
import { classifyFailure, toDoltVerdict, commandBinary } from './vcs-module.mjs';
|
|
90
92
|
import { buildSettleCallback } from './dolt-settle.mjs';
|
|
91
93
|
|
|
@@ -683,6 +685,241 @@ export function doltBackoffDelayMs(attempt, baseMs = DOLT_BACKOFF_BASE_MS, maxMs
|
|
|
683
685
|
|
|
684
686
|
const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
685
687
|
|
|
688
|
+
// ---------------------------------------------------------------------------
|
|
689
|
+
// Member OS/shell for the bd sync command string
|
|
690
|
+
// ---------------------------------------------------------------------------
|
|
691
|
+
//
|
|
692
|
+
// A Windows member runs every `bd dolt pull`/`bd dolt push` with a bounded
|
|
693
|
+
// GOMAXPROCS (see bd-sync-command.mjs for why and for the exact strings), so
|
|
694
|
+
// runDoltStep needs the member's { os, shell }. Two sources, in order:
|
|
695
|
+
// 1. `opts.memberTarget` on the call -- an { os, shell } object or a
|
|
696
|
+
// `(member) => { os, shell }` function (tests, and any caller that has
|
|
697
|
+
// already resolved the member);
|
|
698
|
+
// 2. the sprint-wide resolver installed once at sprint start with
|
|
699
|
+
// setMemberTargetResolver() -- the runner wires the cached
|
|
700
|
+
// member_detail-backed resolveMemberTarget there, so no extra
|
|
701
|
+
// member_detail round trip happens per sync once a member is resolved.
|
|
702
|
+
// With neither, or when resolution fails, the member is treated as POSIX and
|
|
703
|
+
// the command strings stay byte-identical to the pre-existing ones.
|
|
704
|
+
|
|
705
|
+
/** @type {((member: string) => Promise<{ os: string, shell: string }>|{ os: string, shell: string })|null} */
|
|
706
|
+
let memberTargetResolver = null;
|
|
707
|
+
|
|
708
|
+
/**
|
|
709
|
+
* Install (or clear, with null) the sprint-wide member { os, shell } resolver
|
|
710
|
+
* runDoltStep falls back to when a call passes no `memberTarget`.
|
|
711
|
+
*
|
|
712
|
+
* @param {Function|null} resolver
|
|
713
|
+
*/
|
|
714
|
+
export function setMemberTargetResolver(resolver) {
|
|
715
|
+
memberTargetResolver = typeof resolver === 'function' ? resolver : null;
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
/**
|
|
719
|
+
* Resolve `member`'s { os, shell } for command construction. Never throws: a
|
|
720
|
+
* failed lookup is logged and degrades to POSIX (the unchanged strings).
|
|
721
|
+
*
|
|
722
|
+
* @param {string} member
|
|
723
|
+
* @param {object|Function|undefined} memberTarget
|
|
724
|
+
* @param {Function} log
|
|
725
|
+
* @returns {Promise<{ os: string, shell: string }>}
|
|
726
|
+
*/
|
|
727
|
+
async function resolveDoltMemberTarget(member, memberTarget, log) {
|
|
728
|
+
const source = (memberTarget !== undefined && memberTarget !== null) ? memberTarget : memberTargetResolver;
|
|
729
|
+
if (!source) return POSIX_MEMBER_TARGET;
|
|
730
|
+
try {
|
|
731
|
+
const target = typeof source === 'function' ? await source(member) : source;
|
|
732
|
+
const normalized = normalizeMemberTarget(target);
|
|
733
|
+
return normalized.os ? normalized : POSIX_MEMBER_TARGET;
|
|
734
|
+
} catch (err) {
|
|
735
|
+
log(`[Dolt] could not resolve the OS/shell of member '${member}' (${(err && err.message) || err}); building its bd sync commands for a POSIX shell.`);
|
|
736
|
+
return POSIX_MEMBER_TARGET;
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
/** Remedy text of the typed Windows spawn-limit sync failure. */
|
|
741
|
+
export const WINDOWS_SPAWN_LIMIT_REMEDY =
|
|
742
|
+
'Remedy: on that Windows member, stop other process-heavy work (or wait for it to finish) and re-run the sync; '
|
|
743
|
+
+ 'if it recurs, shrink the environment block the member\'s processes inherit (deduplicate PATH, drop oversized variables) '
|
|
744
|
+
+ 'and run the fleet service in an interactive session so it does not exhaust the non-interactive desktop heap.';
|
|
745
|
+
|
|
746
|
+
/**
|
|
747
|
+
* The typed terminal error for a step that ended on the Windows
|
|
748
|
+
* process-creation limit (runDoltStep's `spawnLimit` flag).
|
|
749
|
+
*
|
|
750
|
+
* @param {string} member
|
|
751
|
+
* @param {string} operation - e.g. 'pull', 'push', 'push-reconcile'
|
|
752
|
+
* @param {{ error: string }} step
|
|
753
|
+
* @returns {DoltWindowsSpawnLimitError}
|
|
754
|
+
*/
|
|
755
|
+
function windowsSpawnLimitSyncError(member, operation, step) {
|
|
756
|
+
return new DoltWindowsSpawnLimitError(
|
|
757
|
+
`[Dolt] D-${operation} for member '${member}' failed: Windows refused to start bd's git child processes (the Windows process-creation limit, `
|
|
758
|
+
+ `CreateProcess ERROR_NOT_ENOUGH_MEMORY) on every attempt until the transient retry budget ran out. Not a credential, remote or data problem. `
|
|
759
|
+
+ `${WINDOWS_SPAWN_LIMIT_REMEDY} Raw: ${step.error}`,
|
|
760
|
+
{ member, doltOutput: step.error, details: { operation } },
|
|
761
|
+
);
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
/**
|
|
765
|
+
* The typed terminal error for a step whose beads lock was held by a foreign
|
|
766
|
+
* process (runDoltStep's 'lock-held' kind).
|
|
767
|
+
*
|
|
768
|
+
* @param {string} member
|
|
769
|
+
* @param {string} operation
|
|
770
|
+
* @param {{ error: string, lockHolders?: object[] }} step
|
|
771
|
+
* @returns {DoltLockHeldError}
|
|
772
|
+
*/
|
|
773
|
+
function lockHeldSyncError(member, operation, step) {
|
|
774
|
+
const holders = step.lockHolders || [];
|
|
775
|
+
return new DoltLockHeldError(
|
|
776
|
+
`[Dolt] D-${operation} for member '${member}' failed: a beads lock on that member is held by another process that started before this sync -- `
|
|
777
|
+
+ `${holders.map(describeLockProcess).join('; ') || 'holder unknown'}. The sync was waiting on that lock silently. `
|
|
778
|
+
+ 'Remedy: stop that process on the member (it is not part of this sync) or wait for it to exit, then re-run the sync.',
|
|
779
|
+
{ member, doltOutput: step.error, details: { operation, holders } },
|
|
780
|
+
);
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
/**
|
|
784
|
+
* The typed terminal error a failed step maps to, or null for a generic
|
|
785
|
+
* failure (the caller then throws its own DoltSyncError).
|
|
786
|
+
*
|
|
787
|
+
* @param {string} member
|
|
788
|
+
* @param {string} operation
|
|
789
|
+
* @param {{ kind?: string, spawnLimit?: boolean, error: string }} step
|
|
790
|
+
* @returns {DoltSyncError|null}
|
|
791
|
+
*/
|
|
792
|
+
function typedStepError(member, operation, step) {
|
|
793
|
+
if (step.kind === 'lock-held') return lockHeldSyncError(member, operation, step);
|
|
794
|
+
if (step.spawnLimit) return windowsSpawnLimitSyncError(member, operation, step);
|
|
795
|
+
return null;
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
// ---------------------------------------------------------------------------
|
|
799
|
+
// Lock-holder watchdog for a silent `bd dolt pull`/`bd dolt push`
|
|
800
|
+
// ---------------------------------------------------------------------------
|
|
801
|
+
//
|
|
802
|
+
// A bd sync that waits on a beads lock prints nothing until it gets the lock,
|
|
803
|
+
// so a lock held by some other long-lived process on the member used to look
|
|
804
|
+
// like a silent hang ending in the DOLT_STEP_TIMEOUT_S inactivity kill. Each
|
|
805
|
+
// attempt that is still in flight after DOLT_LOCK_PROBE_AFTER_MS gets exactly
|
|
806
|
+
// ONE lock-holder probe (beads-lock-probe.mjs): a separate short command on
|
|
807
|
+
// the same member, run while the sync is still waiting. Its result is logged.
|
|
808
|
+
//
|
|
809
|
+
// FOREIGN vs OWN holder. The bd process this attempt started legitimately
|
|
810
|
+
// holds the lock while it works, and a slow incremental pull can run for
|
|
811
|
+
// minutes. So a holder only counts as contention when it is provably older
|
|
812
|
+
// than this attempt: its process age exceeds the attempt's elapsed time by
|
|
813
|
+
// more than DOLT_LOCK_HOLDER_AGE_SLACK_S. Both are durations (member clock vs
|
|
814
|
+
// orchestrator clock), so no clock sync is assumed. An own or unknown-age
|
|
815
|
+
// holder is logged and the attempt keeps waiting.
|
|
816
|
+
//
|
|
817
|
+
// THE BOUND. A foreign holder fails the attempt at once with a typed
|
|
818
|
+
// DoltLockHeldError naming it: at most DOLT_LOCK_PROBE_AFTER_MS plus the
|
|
819
|
+
// probe's own timeout (60s + 30s = 90s) after the attempt started, instead of
|
|
820
|
+
// the 600s inactivity kill. 60s is far past a healthy sync (seconds; the
|
|
821
|
+
// incident's retry landed in 4s) yet short enough to name the holder while it
|
|
822
|
+
// is still alive. A lock-held failure is not retried: the holder is long-lived
|
|
823
|
+
// by construction, so a retry would just wait on it again.
|
|
824
|
+
//
|
|
825
|
+
// The abandoned sync command keeps running on the member (there is no
|
|
826
|
+
// cancel channel); its promise is handed to `onAbandonedAttempt` so the push
|
|
827
|
+
// bracket can hold the push mutex until it ends (see doltPushAfter).
|
|
828
|
+
//
|
|
829
|
+
// Side effects of running the probe concurrently: the member's statusline
|
|
830
|
+
// busy marker may flip to idle when the probe ends while the sync is still in
|
|
831
|
+
// flight, and a resumed run's journal replay sees the probe as an extra
|
|
832
|
+
// command whose position depends on timing, so replay diverges at that point
|
|
833
|
+
// and continues live.
|
|
834
|
+
|
|
835
|
+
/** How long a bd sync attempt may stay silent before its lock-holder probe. */
|
|
836
|
+
export const DOLT_LOCK_PROBE_AFTER_MS = 60_000;
|
|
837
|
+
|
|
838
|
+
/** Timeout of the lock-holder probe command itself. */
|
|
839
|
+
export const DOLT_LOCK_PROBE_TIMEOUT_S = 30;
|
|
840
|
+
|
|
841
|
+
/** A holder must be at least this much older than the attempt to count as foreign. */
|
|
842
|
+
export const DOLT_LOCK_HOLDER_AGE_SLACK_S = 5;
|
|
843
|
+
|
|
844
|
+
const defaultLockProbeTimer = {
|
|
845
|
+
set(fn, ms) {
|
|
846
|
+
const handle = setTimeout(fn, ms);
|
|
847
|
+
if (handle && typeof handle.unref === 'function') handle.unref();
|
|
848
|
+
return handle;
|
|
849
|
+
},
|
|
850
|
+
clear(handle) {
|
|
851
|
+
clearTimeout(handle);
|
|
852
|
+
},
|
|
853
|
+
};
|
|
854
|
+
|
|
855
|
+
/**
|
|
856
|
+
* Run the lock-holder probe once for a sync attempt that is still in flight,
|
|
857
|
+
* log what it found, and return the foreign holders. Never throws for an
|
|
858
|
+
* ordinary failure (a failed probe is logged and reports no holder); a
|
|
859
|
+
* cancellation still propagates.
|
|
860
|
+
*
|
|
861
|
+
* @returns {Promise<{ foreignHolders: object[] }>}
|
|
862
|
+
*/
|
|
863
|
+
async function probeLockHolders({ command, member, target, label, log, attemptStartedAt, now }) {
|
|
864
|
+
const elapsedS = Math.max(0, Math.round((now() - attemptStartedAt) / 1000));
|
|
865
|
+
let res;
|
|
866
|
+
try {
|
|
867
|
+
res = await command(buildLockHolderProbe(target), {
|
|
868
|
+
member_name: member, silent: true, failSoft: true,
|
|
869
|
+
label: `beads lock-holder probe for '${member}'`, timeout_s: DOLT_LOCK_PROBE_TIMEOUT_S,
|
|
870
|
+
});
|
|
871
|
+
} catch (err) {
|
|
872
|
+
if (err && err.name === 'CancelledError') throw err;
|
|
873
|
+
res = { ok: false, output: '', error: (err && err.message) || String(err) };
|
|
874
|
+
}
|
|
875
|
+
if (!res || !res.ok) {
|
|
876
|
+
log(`[Dolt] ${label} has produced no result after ${elapsedS}s on member '${member}'; the beads lock-holder probe failed (${(res && res.error) || 'no result'}), so the holders are unknown. Still waiting.`);
|
|
877
|
+
return { foreignHolders: [] };
|
|
878
|
+
}
|
|
879
|
+
const parsed = parseLockHolderProbe(res.output);
|
|
880
|
+
const foreignHolders = parsed.holders.filter((h) => h.ageS !== null && h.ageS > elapsedS + DOLT_LOCK_HOLDER_AGE_SLACK_S);
|
|
881
|
+
const parts = [`[Dolt] ${label} has produced no result after ${elapsedS}s on member '${member}'.`];
|
|
882
|
+
if (parsed.noLookup) parts.push(`Beads lock holders cannot be listed on this member: ${parsed.noLookup}.`);
|
|
883
|
+
if (parsed.holders.length > 0) parts.push(`Beads lock holders: ${parsed.holders.map(describeLockProcess).join('; ')}.`);
|
|
884
|
+
else if (!parsed.noLookup) parts.push('Beads lock holders: none found.');
|
|
885
|
+
if (parsed.candidates.length > 0) parts.push(`Running bd/dolt processes (candidates): ${parsed.candidates.map(describeLockProcess).join('; ')}.`);
|
|
886
|
+
parts.push(foreignHolders.length > 0
|
|
887
|
+
? `Held by a process that started before this sync (more than ${DOLT_LOCK_HOLDER_AGE_SLACK_S}s older than the ${elapsedS}s attempt) -- failing fast.`
|
|
888
|
+
: 'No lock held by a process older than this sync -- still waiting.');
|
|
889
|
+
log(parts.join(' '));
|
|
890
|
+
return { foreignHolders };
|
|
891
|
+
}
|
|
892
|
+
|
|
893
|
+
/**
|
|
894
|
+
* Await one sync attempt under the lock-holder watchdog. Resolves to
|
|
895
|
+
* `{ res }` when the command finishes (probe or not), or to `{ lockHeld }`
|
|
896
|
+
* when the probe names a foreign holder first.
|
|
897
|
+
*/
|
|
898
|
+
async function awaitSyncAttempt({ pending, command, member, target, label, log, attemptStartedAt, now, lockProbeAfterMs, lockProbeTimer }) {
|
|
899
|
+
if (!(Number(lockProbeAfterMs) > 0)) return { res: await pending };
|
|
900
|
+
const timer = lockProbeTimer || defaultLockProbeTimer;
|
|
901
|
+
let fired = false;
|
|
902
|
+
let handle = null;
|
|
903
|
+
const probed = new Promise((resolve, reject) => {
|
|
904
|
+
handle = timer.set(() => {
|
|
905
|
+
fired = true;
|
|
906
|
+
probeLockHolders({ command, member, target, label, log, attemptStartedAt, now }).then(resolve, reject);
|
|
907
|
+
}, Number(lockProbeAfterMs));
|
|
908
|
+
});
|
|
909
|
+
const first = await Promise.race([
|
|
910
|
+
pending.then((res) => ({ done: true, res })),
|
|
911
|
+
probed.then((probe) => ({ done: false, probe })),
|
|
912
|
+
]);
|
|
913
|
+
if (first.done) {
|
|
914
|
+
if (!fired) timer.clear(handle);
|
|
915
|
+
// A probe that was already running finishes and logs on its own.
|
|
916
|
+
probed.catch(() => {});
|
|
917
|
+
return { res: first.res };
|
|
918
|
+
}
|
|
919
|
+
if (first.probe.foreignHolders.length > 0) return { lockHeld: first.probe.foreignHolders };
|
|
920
|
+
return { res: await pending };
|
|
921
|
+
}
|
|
922
|
+
|
|
686
923
|
/**
|
|
687
924
|
* Run a single `bd dolt` command via the injected command() with failSoft,
|
|
688
925
|
* retrying ONLY transient failures up to `maxTransientRetries` times, with a
|
|
@@ -709,7 +946,13 @@ const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
|
709
946
|
*
|
|
710
947
|
* @returns {Promise<{ ok: boolean, output: string, error: string|null, kind?: 'no-remote'|'empty-remote'|'remote-unreachable'|'diverged'|'auth'|'transient'|'missing-tool'|'unknown', missingTool?: string }>}
|
|
711
948
|
*/
|
|
712
|
-
async function runDoltStep({ command, member, cmd, label, log, maxTransientRetries, onAuthFailure, sleep = defaultSleep, backoffBaseMs = DOLT_BACKOFF_BASE_MS, timeoutS = DOLT_STEP_TIMEOUT_S, spawnOutageBudgetMs = DOLT_SPAWN_OUTAGE_BUDGET_MS, now = Date.now }) {
|
|
949
|
+
async function runDoltStep({ command, member, cmd, label, log, maxTransientRetries, onAuthFailure, sleep = defaultSleep, backoffBaseMs = DOLT_BACKOFF_BASE_MS, timeoutS = DOLT_STEP_TIMEOUT_S, spawnOutageBudgetMs = DOLT_SPAWN_OUTAGE_BUDGET_MS, now = Date.now, memberTarget, lockProbeAfterMs = DOLT_LOCK_PROBE_AFTER_MS, lockProbeTimer, onAbandonedAttempt }) {
|
|
950
|
+
// `cmd` stays the LOGICAL command (classification, commandBinary, labels);
|
|
951
|
+
// `issued` is the member-bound string, which on a Windows member bounds
|
|
952
|
+
// GOMAXPROCS for bd (resolved once per step, not per attempt). The same
|
|
953
|
+
// target builds the lock-holder probe.
|
|
954
|
+
const target = await resolveDoltMemberTarget(member, memberTarget, log);
|
|
955
|
+
const issued = bdSyncCommandFor(cmd, target);
|
|
713
956
|
let attempt = 0;
|
|
714
957
|
let authHealAttempted = false;
|
|
715
958
|
/** The info the self-heal was invoked with, reported back on a failed retry. */
|
|
@@ -726,7 +969,20 @@ async function runDoltStep({ command, member, cmd, label, log, maxTransientRetri
|
|
|
726
969
|
: DOLT_SPAWN_OUTAGE_MAX_RETRIES;
|
|
727
970
|
// eslint-disable-next-line no-constant-condition
|
|
728
971
|
while (true) {
|
|
729
|
-
const
|
|
972
|
+
const attemptStartedAt = now();
|
|
973
|
+
const pending = Promise.resolve(command(issued, { member_name: member, silent: true, failSoft: true, label, timeout_s: timeoutS }));
|
|
974
|
+
const waited = await awaitSyncAttempt({ pending, command, member, target, label, log, attemptStartedAt, now, lockProbeAfterMs, lockProbeTimer });
|
|
975
|
+
if (waited.lockHeld) {
|
|
976
|
+
// The sync command is still waiting on the member; stop waiting
|
|
977
|
+
// for it here and hand it to the caller (the push bracket keeps
|
|
978
|
+
// its mutex until it ends). Not retried -- see the watchdog notes.
|
|
979
|
+
pending.catch(() => {});
|
|
980
|
+
if (typeof onAbandonedAttempt === 'function') onAbandonedAttempt(pending);
|
|
981
|
+
const error = `beads lock held by another process: ${waited.lockHeld.map(describeLockProcess).join('; ')}`;
|
|
982
|
+
log(`[Dolt] ${label} FAILED (lock-held) for member '${member}' -- not retrying while that process holds the lock: ${error}`);
|
|
983
|
+
return { ok: false, output: '', error, kind: 'lock-held', lockHolders: waited.lockHeld, selfHealed: authHealAttempted };
|
|
984
|
+
}
|
|
985
|
+
const res = waited.res;
|
|
730
986
|
if (res && res.ok) {
|
|
731
987
|
if (authHealAttempted) log(`[Dolt] self-heal recovered: ${label} succeeded for member '${member}' on the retry after re-provisioning credentials.`);
|
|
732
988
|
return res;
|
|
@@ -823,7 +1079,11 @@ async function runDoltStep({ command, member, cmd, label, log, maxTransientRetri
|
|
|
823
1079
|
}
|
|
824
1080
|
}
|
|
825
1081
|
log(`[Dolt] ${label} FAILED (${kind}) -- reads/writes for member '${member}' may be stale until this is resolved. Raw: ${error}`);
|
|
826
|
-
|
|
1082
|
+
// A transient failure that is still the Windows process-creation
|
|
1083
|
+
// limit after the retry budget is flagged so the caller throws the
|
|
1084
|
+
// typed DoltWindowsSpawnLimitError instead of a generic sync failure.
|
|
1085
|
+
const spawnLimit = kind === 'transient' && WINDOWS_SPAWN_LIMIT_RE.test(String(rawError || ''));
|
|
1086
|
+
return { ok: false, output: res ? res.output : '', error, kind, selfHealed: authHealAttempted, ...(spawnLimit ? { spawnLimit: true } : {}) };
|
|
827
1087
|
}
|
|
828
1088
|
}
|
|
829
1089
|
|
|
@@ -1433,7 +1693,7 @@ export async function doltPullBefore(member, opts = {}) {
|
|
|
1433
1693
|
// `maxTransientRetries` is deliberately NOT defaulted here: runDoltStep()
|
|
1434
1694
|
// must see whether the caller passed it (an explicit value caps both
|
|
1435
1695
|
// retry ladders; undefined selects each ladder's own default).
|
|
1436
|
-
const { command, log = () => {}, maxTransientRetries, checkSyncRemoteConfigured, skipPull = false, onAuthFailure, sleep, backoffBaseMs, timeoutS, settle, remoteTipFingerprint = true, spawnOutageBudgetMs, now } = opts;
|
|
1696
|
+
const { command, log = () => {}, maxTransientRetries, checkSyncRemoteConfigured, skipPull = false, onAuthFailure, sleep, backoffBaseMs, timeoutS, settle, remoteTipFingerprint = true, spawnOutageBudgetMs, now, memberTarget, lockProbeAfterMs, lockProbeTimer, onAbandonedAttempt } = opts;
|
|
1437
1697
|
if (typeof command !== 'function') {
|
|
1438
1698
|
throw new Error("doltPullBefore requires an injected command() in opts");
|
|
1439
1699
|
}
|
|
@@ -1479,7 +1739,7 @@ export async function doltPullBefore(member, opts = {}) {
|
|
|
1479
1739
|
|
|
1480
1740
|
const pull = await runDoltStep({
|
|
1481
1741
|
command, member, cmd: 'bd dolt pull',
|
|
1482
|
-
label: `D-pull for '${member}'`, log, maxTransientRetries, onAuthFailure, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now,
|
|
1742
|
+
label: `D-pull for '${member}'`, log, maxTransientRetries, onAuthFailure, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now, memberTarget, lockProbeAfterMs, lockProbeTimer, onAbandonedAttempt,
|
|
1483
1743
|
});
|
|
1484
1744
|
if (!pull.ok) {
|
|
1485
1745
|
if (pull.kind === 'no-remote') {
|
|
@@ -1537,6 +1797,8 @@ export async function doltPullBefore(member, opts = {}) {
|
|
|
1537
1797
|
{ member, doltOutput: pull.error, remoteUrl: url },
|
|
1538
1798
|
);
|
|
1539
1799
|
}
|
|
1800
|
+
const typedPullError = typedStepError(member, 'pull', pull);
|
|
1801
|
+
if (typedPullError) throw typedPullError;
|
|
1540
1802
|
throw new DoltSyncError(
|
|
1541
1803
|
`[Dolt] D-pull failed for member '${member}': ${pull.error}`,
|
|
1542
1804
|
{ member, doltOutput: pull.error },
|
|
@@ -1735,7 +1997,15 @@ export async function doltPushAfter(member, opts = {}) {
|
|
|
1735
1997
|
// doltPullBefore. `remoteTipFingerprint` is accepted for call-site parity
|
|
1736
1998
|
// with doltPullBefore but has nothing to switch off on the push side any
|
|
1737
1999
|
// more: a push never probes and never records (it only forgets).
|
|
1738
|
-
const { command, pushBeads = true, log = () => {}, maxTransientRetries, mutex, sprintId, checkSyncRemoteConfigured, onAuthFailure, sleep, backoffBaseMs, timeoutS, settle, renewIntervalMs = DOLT_MUTEX_RENEW_INTERVAL_MS, spawnOutageBudgetMs, now } = opts;
|
|
2000
|
+
const { command, pushBeads = true, log = () => {}, maxTransientRetries, mutex, sprintId, checkSyncRemoteConfigured, onAuthFailure, sleep, backoffBaseMs, timeoutS, settle, renewIntervalMs = DOLT_MUTEX_RENEW_INTERVAL_MS, spawnOutageBudgetMs, now, memberTarget, lockProbeAfterMs, lockProbeTimer } = opts;
|
|
2001
|
+
// Sync commands this push gave up on while they were still running on the
|
|
2002
|
+
// member (a lock-held fail-fast, see runDoltStep). The push mutex is kept
|
|
2003
|
+
// until they end -- see the release in the finally below.
|
|
2004
|
+
const abandonedAttempts = [];
|
|
2005
|
+
const onAbandonedAttempt = (pending) => {
|
|
2006
|
+
abandonedAttempts.push(pending);
|
|
2007
|
+
if (typeof opts.onAbandonedAttempt === 'function') opts.onAbandonedAttempt(pending);
|
|
2008
|
+
};
|
|
1739
2009
|
if (typeof command !== 'function') {
|
|
1740
2010
|
throw new Error("doltPushAfter requires an injected command() in opts");
|
|
1741
2011
|
}
|
|
@@ -1804,13 +2074,27 @@ export async function doltPushAfter(member, opts = {}) {
|
|
|
1804
2074
|
landedCheck = await beginDoltPushLandedCheck(member, { command, log });
|
|
1805
2075
|
return await doltPushGuarded();
|
|
1806
2076
|
} finally {
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
2077
|
+
const releaseMutex = async () => {
|
|
2078
|
+
if (renewTimer) clearInterval(renewTimer);
|
|
2079
|
+
if (grant && mutex && typeof mutex.release === 'function') {
|
|
2080
|
+
try {
|
|
2081
|
+
await mutex.release(grant.token);
|
|
2082
|
+
} catch (relErr) {
|
|
2083
|
+
log(`[Dolt] mutex release after D-push for member '${member}' failed (non-fatal; lease will expire): ${relErr.message}`);
|
|
2084
|
+
}
|
|
1813
2085
|
}
|
|
2086
|
+
};
|
|
2087
|
+
if (abandonedAttempts.length > 0 && grant) {
|
|
2088
|
+
// A `bd dolt push` this bracket stopped waiting for is still
|
|
2089
|
+
// running on the member and could still publish once the lock
|
|
2090
|
+
// frees. Releasing the mutex now would let another writer push
|
|
2091
|
+
// concurrently with it, so the mutex (and its lease renewal) is
|
|
2092
|
+
// held until that command ends -- bounded by its own command
|
|
2093
|
+
// timeout -- while this call returns immediately.
|
|
2094
|
+
log(`[Dolt] D-push for member '${member}' stopped waiting for a sync command that is still running on the member; keeping the push mutex until that command ends so it cannot publish concurrently with another writer.`);
|
|
2095
|
+
Promise.allSettled(abandonedAttempts).then(releaseMutex);
|
|
2096
|
+
} else {
|
|
2097
|
+
await releaseMutex();
|
|
1814
2098
|
}
|
|
1815
2099
|
}
|
|
1816
2100
|
|
|
@@ -1865,8 +2149,13 @@ export async function doltPushAfter(member, opts = {}) {
|
|
|
1865
2149
|
log(`[Dolt] ${label} reported success but the push did NOT land: ${verdict.why}. Retrying the push once.`);
|
|
1866
2150
|
const retry = await runDoltStep({
|
|
1867
2151
|
command, member, cmd: 'bd dolt push',
|
|
1868
|
-
label: `D-push retry after an unlanded push for '${member}'`, log, maxTransientRetries, onAuthFailure, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now,
|
|
2152
|
+
label: `D-push retry after an unlanded push for '${member}'`, log, maxTransientRetries, onAuthFailure, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now, memberTarget, lockProbeAfterMs, lockProbeTimer, onAbandonedAttempt,
|
|
1869
2153
|
});
|
|
2154
|
+
const typedRetryError = retry.ok ? null : typedStepError(member, 'push retry after an unlanded push', retry);
|
|
2155
|
+
if (typedRetryError) {
|
|
2156
|
+
clearLastSyncedTip(member);
|
|
2157
|
+
throw typedRetryError;
|
|
2158
|
+
}
|
|
1870
2159
|
const firstWhy = verdict.why;
|
|
1871
2160
|
verdict = retry.ok
|
|
1872
2161
|
? await checkDoltPushLanded(member, { command, check: landedCheck })
|
|
@@ -1894,7 +2183,7 @@ export async function doltPushAfter(member, opts = {}) {
|
|
|
1894
2183
|
async function doltPushGuarded() {
|
|
1895
2184
|
let push = await runDoltStep({
|
|
1896
2185
|
command, member, cmd: 'bd dolt push',
|
|
1897
|
-
label: `D-push for '${member}'`, log, maxTransientRetries, onAuthFailure, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now,
|
|
2186
|
+
label: `D-push for '${member}'`, log, maxTransientRetries, onAuthFailure, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now, memberTarget, lockProbeAfterMs, lockProbeTimer, onAbandonedAttempt,
|
|
1898
2187
|
});
|
|
1899
2188
|
if (push.ok) {
|
|
1900
2189
|
return await confirmPushLanded({ ok: true, member, pushed: true, reconciled: false }, `D-push for member '${member}'`);
|
|
@@ -1943,6 +2232,8 @@ export async function doltPushAfter(member, opts = {}) {
|
|
|
1943
2232
|
{ member, doltOutput: push.error, remoteUrl: url },
|
|
1944
2233
|
);
|
|
1945
2234
|
}
|
|
2235
|
+
const typedPushError = typedStepError(member, 'push', push);
|
|
2236
|
+
if (typedPushError) throw typedPushError;
|
|
1946
2237
|
throw new DoltSyncError(
|
|
1947
2238
|
`[Dolt] D-push for member '${member}' failed: ${push.error}`,
|
|
1948
2239
|
{ member, doltOutput: push.error },
|
|
@@ -1955,7 +2246,7 @@ export async function doltPushAfter(member, opts = {}) {
|
|
|
1955
2246
|
log(`[Dolt] D-push for member '${member}' was rejected (another writer pushed first); reconciling with a single D-pull then one re-push (first-successful-pusher-wins).`);
|
|
1956
2247
|
const reconcile = await runDoltStep({
|
|
1957
2248
|
command, member, cmd: 'bd dolt pull',
|
|
1958
|
-
label: `D-push reconcile pull for '${member}'`, log, maxTransientRetries, onAuthFailure, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now,
|
|
2249
|
+
label: `D-push reconcile pull for '${member}'`, log, maxTransientRetries, onAuthFailure, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now, memberTarget, lockProbeAfterMs, lockProbeTimer, onAbandonedAttempt,
|
|
1959
2250
|
});
|
|
1960
2251
|
if (!reconcile.ok) {
|
|
1961
2252
|
if (reconcile.kind === 'diverged') {
|
|
@@ -1967,6 +2258,8 @@ export async function doltPushAfter(member, opts = {}) {
|
|
|
1967
2258
|
'push-reconcile',
|
|
1968
2259
|
);
|
|
1969
2260
|
}
|
|
2261
|
+
const typedReconcileError = typedStepError(member, 'push reconcile pull', reconcile);
|
|
2262
|
+
if (typedReconcileError) throw typedReconcileError;
|
|
1970
2263
|
throw new DoltSyncError(
|
|
1971
2264
|
`[Dolt] D-push reconcile pull for member '${member}' failed: ${reconcile.error}`,
|
|
1972
2265
|
{ member, doltOutput: reconcile.error },
|
|
@@ -1979,7 +2272,7 @@ export async function doltPushAfter(member, opts = {}) {
|
|
|
1979
2272
|
landedCheck = await beginDoltPushLandedCheck(member, { command, log });
|
|
1980
2273
|
push = await runDoltStep({
|
|
1981
2274
|
command, member, cmd: 'bd dolt push',
|
|
1982
|
-
label: `D-push re-push after reconcile for '${member}'`, log, maxTransientRetries, onAuthFailure, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now,
|
|
2275
|
+
label: `D-push re-push after reconcile for '${member}'`, log, maxTransientRetries, onAuthFailure, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now, memberTarget, lockProbeAfterMs, lockProbeTimer, onAbandonedAttempt,
|
|
1983
2276
|
});
|
|
1984
2277
|
if (push.ok) {
|
|
1985
2278
|
return await confirmPushLanded({ ok: true, member, pushed: true, reconciled: true }, `D-push re-push after reconcile for member '${member}'`);
|
|
@@ -2041,7 +2334,7 @@ export async function doltPushAfter(member, opts = {}) {
|
|
|
2041
2334
|
|
|
2042
2335
|
const secondReconcile = await runDoltStep({
|
|
2043
2336
|
command, member, cmd: 'bd dolt pull',
|
|
2044
|
-
label: `D-push second reconcile pull for '${member}'`, log, maxTransientRetries, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now,
|
|
2337
|
+
label: `D-push second reconcile pull for '${member}'`, log, maxTransientRetries, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now, memberTarget, lockProbeAfterMs, lockProbeTimer, onAbandonedAttempt,
|
|
2045
2338
|
});
|
|
2046
2339
|
if (!secondReconcile.ok) {
|
|
2047
2340
|
if (secondReconcile.kind === 'diverged') {
|
|
@@ -2053,6 +2346,8 @@ export async function doltPushAfter(member, opts = {}) {
|
|
|
2053
2346
|
'push-reconcile',
|
|
2054
2347
|
);
|
|
2055
2348
|
}
|
|
2349
|
+
const typedSecondReconcileError = typedStepError(member, 'push second reconcile pull', secondReconcile);
|
|
2350
|
+
if (typedSecondReconcileError) throw typedSecondReconcileError;
|
|
2056
2351
|
throw new DoltSyncError(
|
|
2057
2352
|
`[Dolt] D-push second reconcile pull for member '${member}' failed: ${secondReconcile.error}`,
|
|
2058
2353
|
{ member, doltOutput: secondReconcile.error },
|
|
@@ -2062,7 +2357,7 @@ export async function doltPushAfter(member, opts = {}) {
|
|
|
2062
2357
|
landedCheck = await beginDoltPushLandedCheck(member, { command, log });
|
|
2063
2358
|
push = await runDoltStep({
|
|
2064
2359
|
command, member, cmd: 'bd dolt push',
|
|
2065
|
-
label: `D-push second re-push after reconcile for '${member}'`, log, maxTransientRetries, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now,
|
|
2360
|
+
label: `D-push second re-push after reconcile for '${member}'`, log, maxTransientRetries, sleep, backoffBaseMs, timeoutS, spawnOutageBudgetMs, now, memberTarget, lockProbeAfterMs, lockProbeTimer, onAbandonedAttempt,
|
|
2066
2361
|
});
|
|
2067
2362
|
if (push.ok) {
|
|
2068
2363
|
return await confirmPushLanded({ ok: true, member, pushed: true, reconciled: true }, `D-push second re-push after reconcile for member '${member}'`);
|
|
@@ -148,17 +148,23 @@ export class SprintPlanRejectedError extends WorkflowError {
|
|
|
148
148
|
* @property {string[]} [blockerIds] - apra-fleet-mjo: the bead ids still open
|
|
149
149
|
* at/above goal priority when the stall was declared, i.e. exactly what an
|
|
150
150
|
* operator must clear. Counts alone cannot answer "blocked on what?".
|
|
151
|
+
* Also names every bead set aside for the run (doer-BLOCKED and its
|
|
152
|
+
* dependents), plus the pseudo-id 'deploy' when the deploy is set aside.
|
|
153
|
+
* @property {Record<string, string>} [blockerReasons] - blocker id -> reason
|
|
154
|
+
* (with first-seen cycle/round) for the set-aside blockers.
|
|
155
|
+
* @property {object[]} [setAside] - the run's set-aside records, rendered as
|
|
156
|
+
* the "Set aside this run" report section (see ./set-aside.mjs).
|
|
151
157
|
*/
|
|
152
158
|
export class StalledSprintError extends WorkflowError {
|
|
153
159
|
/**
|
|
154
160
|
* @param {string} message
|
|
155
|
-
* @param {{ staleCycles?: number, closedCountHistory?: number[], highWaterClosedCount?: number, thrashIds?: string[], blockerIds?: string[], cycle?: number, details?: object, cause?: unknown }} [opts]
|
|
161
|
+
* @param {{ staleCycles?: number, closedCountHistory?: number[], highWaterClosedCount?: number, thrashIds?: string[], blockerIds?: string[], blockerReasons?: Record<string, string>, setAside?: object[], waited?: object[], cycle?: number, details?: object, cause?: unknown }} [opts]
|
|
156
162
|
*/
|
|
157
163
|
constructor(message, opts = {}) {
|
|
158
|
-
const { staleCycles = null, closedCountHistory = [], highWaterClosedCount = null, thrashIds = [], blockerIds = [], cycle, details, cause } = opts;
|
|
164
|
+
const { staleCycles = null, closedCountHistory = [], highWaterClosedCount = null, thrashIds = [], blockerIds = [], blockerReasons = {}, setAside = [], waited = [], cycle, details, cause } = opts;
|
|
159
165
|
super(message, {
|
|
160
166
|
code: 'SPRINT_STALLED',
|
|
161
|
-
details: { staleCycles, closedCountHistory, highWaterClosedCount, thrashIds, blockerIds, cycle, ...details },
|
|
167
|
+
details: { staleCycles, closedCountHistory, highWaterClosedCount, thrashIds, blockerIds, blockerReasons, setAside, cycle, ...details },
|
|
162
168
|
cause,
|
|
163
169
|
});
|
|
164
170
|
this.staleCycles = staleCycles;
|
|
@@ -166,6 +172,11 @@ export class StalledSprintError extends WorkflowError {
|
|
|
166
172
|
this.highWaterClosedCount = highWaterClosedCount;
|
|
167
173
|
this.thrashIds = thrashIds;
|
|
168
174
|
this.blockerIds = blockerIds;
|
|
175
|
+
this.blockerReasons = blockerReasons;
|
|
176
|
+
this.setAside = setAside;
|
|
177
|
+
// The run's wait records (./set-aside.mjs collectWaitEntries): the
|
|
178
|
+
// "Waited this run" line of an [ABORTED] PR body.
|
|
179
|
+
this.waited = waited;
|
|
169
180
|
}
|
|
170
181
|
}
|
|
171
182
|
|
|
@@ -356,6 +367,52 @@ export class DoltSyncError extends WorkflowError {
|
|
|
356
367
|
}
|
|
357
368
|
}
|
|
358
369
|
|
|
370
|
+
/** Stable error code of DoltWindowsSpawnLimitError. */
|
|
371
|
+
export const DOLT_WINDOWS_SPAWN_LIMIT_CODE = 'DOLT_WINDOWS_SPAWN_LIMIT';
|
|
372
|
+
|
|
373
|
+
/**
|
|
374
|
+
* A D-pull/D-push that kept failing because Windows refused to start bd/dolt's
|
|
375
|
+
* git child processes (CreateProcess ERROR_NOT_ENOUGH_MEMORY, the Windows
|
|
376
|
+
* process-creation limit) until its transient retry budget ran out. A
|
|
377
|
+
* DoltSyncError subtype, so every existing DoltSyncError handler (degrade/fatal
|
|
378
|
+
* policy, classifySyncError -> 'transient') still applies; the stable `code`
|
|
379
|
+
* lets a caller tell it apart from a generic sync failure.
|
|
380
|
+
*/
|
|
381
|
+
export class DoltWindowsSpawnLimitError extends DoltSyncError {
|
|
382
|
+
/**
|
|
383
|
+
* @param {string} message
|
|
384
|
+
* @param {{ member?: string|null, doltOutput?: string|null, details?: object, cause?: unknown }} [opts]
|
|
385
|
+
*/
|
|
386
|
+
constructor(message, opts = {}) {
|
|
387
|
+
super(message, { ...opts, details: { kind: 'transient', cause: 'windows-spawn-limit', ...opts.details } });
|
|
388
|
+
this.name = 'DoltWindowsSpawnLimitError';
|
|
389
|
+
this.code = DOLT_WINDOWS_SPAWN_LIMIT_CODE;
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
/** Stable error code of DoltLockHeldError. */
|
|
394
|
+
export const DOLT_BEADS_LOCK_HELD_CODE = 'DOLT_BEADS_LOCK_HELD';
|
|
395
|
+
|
|
396
|
+
/**
|
|
397
|
+
* A D-pull/D-push that was still waiting after the lock-holder watchdog fired
|
|
398
|
+
* while a process OTHER than this sync (one that started before the sync
|
|
399
|
+
* attempt did) held a beads lock on the member. Raised in bounded time instead
|
|
400
|
+
* of waiting out the inactivity timeout; `details.holders` lists each holder
|
|
401
|
+
* ({ pid, ageS, lock, commandLine }). A DoltSyncError subtype, so every
|
|
402
|
+
* existing DoltSyncError handler still applies.
|
|
403
|
+
*/
|
|
404
|
+
export class DoltLockHeldError extends DoltSyncError {
|
|
405
|
+
/**
|
|
406
|
+
* @param {string} message
|
|
407
|
+
* @param {{ member?: string|null, doltOutput?: string|null, details?: object, cause?: unknown }} [opts]
|
|
408
|
+
*/
|
|
409
|
+
constructor(message, opts = {}) {
|
|
410
|
+
super(message, { ...opts, details: { kind: 'lock-held', ...opts.details } });
|
|
411
|
+
this.name = 'DoltLockHeldError';
|
|
412
|
+
this.code = DOLT_BEADS_LOCK_HELD_CODE;
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
|
|
359
416
|
/**
|
|
360
417
|
* Thrown by preflightBeadsHealthGate() when the backlog member's beads clone
|
|
361
418
|
* holds local schema migrations bd auto-applied but never published (the
|
|
@@ -431,6 +488,16 @@ export class DoltBinaryUnavailableError extends WorkflowError {
|
|
|
431
488
|
// AgentDispatchError -- checked FIRST below since it can't be fooled by
|
|
432
489
|
// auth-like noise in an unrelated failure's message text; the regex remains
|
|
433
490
|
// as a fallback for older/mocked errors that only ever set `.message`.
|
|
491
|
+
// The server refuses a dispatch (no LLM call made) when the member's stored
|
|
492
|
+
// credential env vars cannot be delivered without a command line: a relay
|
|
493
|
+
// member or one with SFTP disabled. Deterministic -- the member's channel does
|
|
494
|
+
// not change between attempts -- so it is non-retryable. It is deliberately
|
|
495
|
+
// NOT an LLM-auth failure (isAuthDispatchError): the credential is present
|
|
496
|
+
// but undeliverable, so provision_llm_auth self-heal cannot fix it; the
|
|
497
|
+
// operator must enable SFTP or clear/provide the credential on the member.
|
|
498
|
+
export const SECRET_DELIVERY_UNAVAILABLE_REASON = 'secret_delivery_unavailable';
|
|
499
|
+
export const SECRET_DELIVERY_UNAVAILABLE_REMEDY = 'enable SFTP on the member, or clear its stored credentials (provision_llm_auth with clear_stored_credentials) and provide the credential on the member itself';
|
|
500
|
+
|
|
434
501
|
const NON_RETRYABLE_DISPATCH_RE = /authentication failed|failed to authenticate|oauth session expired|could not be refreshed|not logged in|workspace not trusted|has not been trusted/i;
|
|
435
502
|
|
|
436
503
|
/**
|
|
@@ -443,7 +510,8 @@ const NON_RETRYABLE_DISPATCH_RE = /authentication failed|failed to authenticate|
|
|
|
443
510
|
export function isNonRetryableDispatchError(err) {
|
|
444
511
|
const reason = err?.details?.reason;
|
|
445
512
|
if (reason === 'auth' || reason === 'workspace_not_trusted'
|
|
446
|
-
|| reason === 'preflight_auth_missing' || reason === 'preflight_auth_expired'
|
|
513
|
+
|| reason === 'preflight_auth_missing' || reason === 'preflight_auth_expired'
|
|
514
|
+
|| reason === SECRET_DELIVERY_UNAVAILABLE_REASON) return true;
|
|
447
515
|
return NON_RETRYABLE_DISPATCH_RE.test(String(err?.message ?? ''));
|
|
448
516
|
}
|
|
449
517
|
|
|
@@ -320,7 +320,13 @@ export function createSyncBrackets({ setPauseGuard } = {}) {
|
|
|
320
320
|
// failure (it may have committed and/or pushed), so run the full
|
|
321
321
|
// pre-dispatch sync in syncMemberBefore's resetToRemoteTip mode -- fetch
|
|
322
322
|
// and reset onto the remote tip BEFORE the doer can commit, resuming on
|
|
323
|
-
// published work instead of re-implementing it and diverging.
|
|
323
|
+
// published work instead of re-implementing it and diverging. The
|
|
324
|
+
// prior attempt's unpublished commits and uncommitted TRACKED changes
|
|
325
|
+
// are first pinned on a LOCAL, never-pushed
|
|
326
|
+
// fleet-salvage/<branch-slug>/<short sha> side ref (git stash create +
|
|
327
|
+
// git branch -f), without moving the sprint branch or touching
|
|
328
|
+
// untracked files; a salvage failure aborts the bracket with
|
|
329
|
+
// GitSyncError instead of resetting (see salvageBeforeReset).
|
|
324
330
|
// skipPreDispatchSync and resumeOntoRemoteTip encode opposite assumptions
|
|
325
331
|
// about whether the prior attempt published anything and MUST never be
|
|
326
332
|
// passed together. Nothing in the code enforces this today.
|