akm-cli 0.9.0-rc.9 → 0.9.1-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1770 -44
- package/README.md +38 -47
- package/SECURITY.md +14 -1
- package/STABILITY.md +499 -0
- package/dist/akm +148 -35
- package/dist/{akm-migrate-storage → akm-migrate} +6 -9
- package/dist/assets/hints/cli-hints-full.md +223 -95
- package/dist/assets/hints/cli-hints-short.md +85 -22
- package/dist/assets/improve-strategies/default.json +1 -1
- package/dist/assets/improve-strategies/reflect-distill.json +1 -1
- package/dist/assets/prompts/memory-infer-user.md +2 -3
- package/dist/assets/stash-skeleton/README.md +6 -5
- package/dist/assets/stash-skeleton/facts/conventions/assets/agent.md +2 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/command.md +2 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/fact.md +2 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/knowledge.md +2 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/lesson.md +2 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/memory.md +2 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/script.md +2 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/skill.md +2 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/workflow.md +2 -0
- package/dist/assets/stash-skeleton/facts/conventions/backlinks.md +2 -0
- package/dist/assets/stash-skeleton/facts/conventions/domains.md +2 -0
- package/dist/assets/stash-skeleton/facts/conventions/organization.md +20 -9
- package/dist/assets/tasks/core/extract.yml +1 -1
- package/dist/assets/tasks/core/version-check.yml +1 -1
- package/dist/assets/tasks/improve/akm-graph-refresh-weekly.yml +5 -0
- package/dist/assets/tasks/improve/akm-improve-catchup.yml +8 -0
- package/dist/assets/tasks/improve/akm-improve-consolidate.yml +5 -0
- package/dist/assets/tasks/improve/akm-improve-frequent.yml +5 -0
- package/dist/assets/tasks/improve/akm-improve-nightly.yml +5 -0
- package/dist/assets/templates/html/health.html +1 -3
- package/dist/assets/workflows/workflow-template.md +32 -15
- package/dist/cli/invocation.js +40 -15
- package/dist/cli/parse-args.js +0 -22
- package/dist/cli/retired-commands.js +121 -0
- package/dist/cli/shared.js +154 -22
- package/dist/cli/unknown-flags.js +236 -0
- package/dist/cli-node.mjs +2 -1
- package/dist/cli.js +696 -258
- package/dist/commands/agent/agent-dispatch.js +14 -3
- package/dist/commands/agent/contribute-cli.js +74 -89
- package/dist/commands/completions.js +79 -22
- package/dist/commands/config-cli.js +17 -150
- package/dist/commands/env/env-cli.js +59 -143
- package/dist/commands/env/env.js +12 -163
- package/dist/commands/env/marker-path.js +6 -0
- package/dist/commands/env/secret-cli.js +36 -66
- package/dist/commands/env/secret.js +24 -57
- package/dist/commands/feedback-cli.js +148 -88
- package/dist/commands/health/accept-rate.js +58 -0
- package/dist/commands/health/advisories.js +3 -4
- package/dist/commands/health/checks.js +85 -23
- package/dist/commands/health/html-report.js +7 -10
- package/dist/commands/health/improve-metrics.js +25 -83
- package/dist/commands/health/llm-usage.js +2 -1
- package/dist/commands/health/md-report.js +5 -9
- package/dist/commands/health/metrics.js +62 -20
- package/dist/commands/health/renderers.js +47 -0
- package/dist/commands/health/report-view-model.js +4 -5
- package/dist/commands/health/stash-exposure.js +1 -1
- package/dist/commands/health/surfaces.js +4 -122
- package/dist/commands/health/task-runs.js +3 -67
- package/dist/commands/health/types-improve.js +7 -0
- package/dist/commands/health.js +160 -35
- package/dist/commands/improve/anti-collapse.js +2 -2
- package/dist/commands/improve/autonomy-gate.js +68 -0
- package/dist/commands/improve/collapse-detector.js +41 -40
- package/dist/commands/improve/consolidate/eligibility.js +1 -23
- package/dist/commands/improve/consolidate/merge.js +4 -0
- package/dist/commands/improve/consolidate.js +140 -1000
- package/dist/commands/improve/distill/promote-memory.js +12 -12
- package/dist/commands/improve/distill/quality-gate.js +11 -6
- package/dist/commands/improve/distill.js +58 -69
- package/dist/commands/improve/eligibility.js +101 -55
- package/dist/commands/improve/extract-cli.js +14 -133
- package/dist/commands/improve/improve-cli.js +98 -114
- package/dist/commands/improve/improve-result-file.js +1 -28
- package/dist/commands/improve/improve-strategies.js +8 -5
- package/dist/commands/improve/improve.js +254 -101
- package/dist/commands/improve/locks.js +7 -0
- package/dist/commands/improve/loop-stages.js +182 -20
- package/dist/commands/improve/memory/derived-ref.js +45 -43
- package/dist/commands/improve/memory/memory-belief.js +1 -1
- package/dist/commands/improve/memory/memory-contradiction-detect.js +4 -12
- package/dist/commands/improve/memory/memory-improve.js +14 -5
- package/dist/commands/improve/outcome-loop.js +22 -65
- package/dist/commands/improve/preparation.js +114 -123
- package/dist/commands/improve/proactive-maintenance.js +2 -5
- package/dist/commands/improve/reflect.js +56 -160
- package/dist/commands/improve/run-context.js +5 -0
- package/dist/commands/improve/salience.js +11 -122
- package/dist/commands/improve/session-asset.js +4 -0
- package/dist/commands/improve/source-identity.js +10 -38
- package/dist/commands/lint/base-linter.js +51 -131
- package/dist/commands/lint/env-key-rules.js +31 -47
- package/dist/commands/lint/index.js +444 -88
- package/dist/commands/lint/types.js +22 -1
- package/dist/commands/{events.js → log.js} +33 -38
- package/dist/commands/migrate-cli.js +92 -12
- package/dist/commands/migration-tool.js +46 -0
- package/dist/commands/observability-cli.js +70 -209
- package/dist/commands/proposal/drain.js +101 -29
- package/dist/commands/proposal/proposal-cli.js +76 -48
- package/dist/commands/proposal/proposal.js +54 -18
- package/dist/commands/proposal/propose-cli.js +88 -0
- package/dist/commands/proposal/propose.js +23 -15
- package/dist/commands/proposal/repository.js +718 -279
- package/dist/commands/proposal/validators/proposal-quality-validators.js +2 -8
- package/dist/commands/proposal/validators/proposal-validators.js +55 -7
- package/dist/commands/proposal/validators/proposals.js +4 -7
- package/dist/commands/read/curate.js +34 -53
- package/dist/commands/read/knowledge.js +150 -95
- package/dist/commands/read/registry-search.js +2 -2
- package/dist/commands/read/remember-cli.js +42 -15
- package/dist/commands/read/search-cli.js +180 -78
- package/dist/commands/read/search.js +58 -43
- package/dist/commands/read/show.js +197 -141
- package/dist/commands/registry-cli.js +12 -51
- package/dist/commands/remember.js +14 -57
- package/dist/commands/sources/add-cli.js +100 -31
- package/dist/commands/sources/bundle-cli.js +166 -0
- package/dist/commands/sources/bundle-config-ops.js +7 -2
- package/dist/commands/sources/info.js +30 -7
- package/dist/commands/sources/init.js +12 -12
- package/dist/commands/sources/installed-stashes.js +387 -98
- package/dist/commands/sources/schema-repair.js +3 -2
- package/dist/commands/sources/self-update.js +131 -38
- package/dist/commands/sources/source-add.js +72 -17
- package/dist/commands/sources/source-clone.js +129 -45
- package/dist/commands/sources/source-manage.js +43 -23
- package/dist/commands/sources/sources-cli.js +57 -208
- package/dist/commands/sources/stash-cli.js +46 -53
- package/dist/commands/tasks/tasks-cli.js +91 -97
- package/dist/commands/tasks/tasks.js +284 -423
- package/dist/commands/workflow-cli.js +181 -450
- package/dist/core/abort-deadline.js +28 -0
- package/dist/core/adapter/adapters/agent-skills-adapter.js +67 -5
- package/dist/core/adapter/adapters/akm-adapter.js +60 -38
- package/dist/core/adapter/adapters/akm-lint.js +109 -42
- package/dist/core/adapter/adapters/akm-metadata.js +15 -44
- package/dist/core/adapter/adapters/akm-task-adapter.js +54 -31
- package/dist/core/adapter/adapters/akm-workflow-adapter.js +55 -71
- package/dist/core/adapter/adapters/dotenv-adapter.js +1 -1
- package/dist/core/adapter/adapters/generic-files-adapter.js +2 -0
- package/dist/core/adapter/adapters/index.js +6 -6
- package/dist/core/adapter/adapters/llm-wiki-adapter.js +14 -8
- package/dist/core/adapter/adapters/okf-adapter.js +187 -19
- package/dist/core/adapter/adapters/shared.js +3 -19
- package/dist/core/adapter/adapters/tool-dir-shared.js +13 -6
- package/dist/core/adapter/adapters/website-snapshot-adapter.js +1 -0
- package/dist/core/adapter/detect-adapter.js +17 -0
- package/dist/core/adapter/recognize-match.js +6 -4
- package/dist/core/adapter/validate-context.js +214 -0
- package/dist/core/asset/akm-markdown.js +63 -0
- package/dist/core/asset/asset-placement.js +20 -6
- package/dist/core/asset/asset-ref.js +11 -9
- package/dist/core/asset/frontmatter-lint.js +30 -0
- package/dist/core/asset/frontmatter.js +29 -9
- package/dist/core/asset/markdown.js +40 -51
- package/dist/core/asset/resolve-ref.js +89 -18
- package/dist/core/asset/stash-meta.js +1 -1
- package/dist/core/bundle-id.js +51 -0
- package/dist/core/common.js +218 -44
- package/dist/core/concurrent.js +32 -0
- package/dist/core/config/config-io.js +12 -1
- package/dist/core/config/config-schema.js +35 -8
- package/dist/core/config/config-sources.js +55 -11
- package/dist/core/config/config-walker.js +25 -9
- package/dist/core/config/config.js +9 -48
- package/dist/core/config/experimental.js +21 -0
- package/dist/core/config/schema/embedding.js +5 -1
- package/dist/core/config/schema/experimental.js +30 -0
- package/dist/core/config/schema/improve-processes.js +0 -6
- package/dist/core/config/schema/improve.js +21 -3
- package/dist/core/config/schema/index-config.js +8 -15
- package/dist/core/config/schema/output.js +4 -1
- package/dist/core/config/schema/setup.js +9 -18
- package/dist/core/config/schema/sources-bundles.js +49 -33
- package/dist/core/config/schema/workflow.js +14 -3
- package/dist/core/env-secret-ref.js +76 -46
- package/dist/core/errors.js +43 -12
- package/dist/core/events.js +76 -152
- package/dist/core/file-change.js +6 -5
- package/dist/core/file-lock.js +7 -1
- package/dist/core/fs-txn.js +83 -7
- package/dist/core/git-message.js +2 -2
- package/dist/core/improve-result.js +6 -100
- package/dist/core/json-schema.js +327 -9
- package/dist/core/lesson-lint.js +1 -17
- package/dist/core/logs-db.js +2 -1
- package/dist/core/loopback.js +89 -0
- package/dist/core/migration-operation.js +33 -2
- package/dist/core/mutation-target.js +78 -0
- package/dist/core/parse.js +4 -1
- package/dist/core/path-access.js +107 -0
- package/dist/core/paths.js +17 -20
- package/dist/core/recognition-util.js +12 -14
- package/dist/core/redaction.js +98 -17
- package/dist/core/spawn-env.js +234 -0
- package/dist/core/standards/resolve-standards-context.js +2 -14
- package/dist/core/standards/resolve-stash-standards.js +2 -2
- package/dist/core/standards/resolve-type-conventions.js +2 -2
- package/dist/core/state/migrations.js +41 -18
- package/dist/core/state-db-scope.js +134 -0
- package/dist/core/state-db.js +5 -14
- package/dist/core/structured.js +1 -1
- package/dist/core/subprocess.js +182 -36
- package/dist/core/text-truncation.js +9 -5
- package/dist/core/type-presentation.js +3 -3
- package/dist/core/warn.js +0 -3
- package/dist/core/write-provenance.js +85 -0
- package/dist/core/write-source.js +778 -95
- package/dist/indexer/bundle-identity-guard.js +3 -2
- package/dist/indexer/db/graph-db.js +17 -30
- package/dist/indexer/ensure-index.js +11 -3
- package/dist/indexer/graph/graph-boost.js +9 -34
- package/dist/indexer/graph/graph-extraction.js +8 -5
- package/dist/indexer/index-writer-lock.js +53 -17
- package/dist/indexer/index-written-assets.js +33 -24
- package/dist/indexer/indexer.js +551 -253
- package/dist/indexer/installations.js +14 -96
- package/dist/indexer/passes/dir-staleness.js +16 -9
- package/dist/indexer/passes/memory-inference.js +15 -9
- package/dist/indexer/passes/metadata.js +113 -47
- package/dist/indexer/scan/doc-to-entry.js +38 -1
- package/dist/indexer/scan/drain-dir.js +13 -23
- package/dist/indexer/search/db-search.js +107 -69
- package/dist/indexer/search/fts-query.js +47 -24
- package/dist/indexer/search/ranking-contributors.js +42 -20
- package/dist/indexer/search/ranking.js +18 -99
- package/dist/indexer/search/search-fields.js +7 -2
- package/dist/indexer/search/search-source.js +82 -93
- package/dist/indexer/usage/usage-events.js +0 -89
- package/dist/indexer/walk/file-context.js +2 -1
- package/dist/indexer/walk/matchers.js +30 -43
- package/dist/indexer/walk/path-resolver.js +7 -2
- package/dist/indexer/walk/walker.js +38 -12
- package/dist/integrations/agent/builders.js +0 -6
- package/dist/integrations/agent/config.js +2 -2
- package/dist/integrations/agent/detect.js +49 -19
- package/dist/integrations/agent/engine-fallback.js +76 -0
- package/dist/integrations/agent/engine-resolution.js +24 -11
- package/dist/integrations/agent/model-aliases.js +1 -1
- package/dist/integrations/agent/profiles.js +23 -1
- package/dist/integrations/agent/prompts.js +12 -8
- package/dist/integrations/agent/runner-dispatch.js +4 -2
- package/dist/integrations/agent/runner.js +0 -1
- package/dist/integrations/agent/spawn.js +20 -93
- package/dist/integrations/github.js +1 -1
- package/dist/integrations/harnesses/aider/agent-builder.js +6 -4
- package/dist/integrations/harnesses/amazonq/agent-builder.js +7 -4
- package/dist/integrations/harnesses/claude/session-log.js +0 -10
- package/dist/integrations/harnesses/codex/agent-builder.js +5 -2
- package/dist/integrations/harnesses/copilot/agent-builder.js +5 -3
- package/dist/integrations/harnesses/gemini/agent-builder.js +5 -3
- package/dist/integrations/harnesses/index.js +3 -7
- package/dist/integrations/harnesses/opencode/agent-builder.js +21 -2
- package/dist/integrations/harnesses/opencode/session-log.js +0 -15
- package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +13 -4
- package/dist/integrations/harnesses/openhands/agent-builder.js +9 -6
- package/dist/integrations/harnesses/pi/agent-builder.js +6 -4
- package/dist/integrations/lockfile.js +132 -8
- package/dist/integrations/session-logs/index.js +3 -28
- package/dist/llm/client.js +136 -100
- package/dist/llm/embedders/remote.js +13 -5
- package/dist/llm/feature-gate.js +4 -12
- package/dist/llm/graph-extract.js +5 -11
- package/dist/llm/memory-infer.js +144 -1
- package/dist/llm/metadata-enhance.js +5 -7
- package/dist/llm/structured-call.js +1 -1
- package/dist/llm/usage-persist.js +26 -5
- package/dist/llm/usage-telemetry.js +25 -2
- package/dist/output/cli-hints.js +1 -2
- package/dist/output/context.js +22 -7
- package/dist/output/format-exempt.js +80 -0
- package/dist/output/generic-render.js +259 -0
- package/dist/output/render-registry.js +57 -0
- package/dist/output/renderers.js +14 -36
- package/dist/output/shapes/curate.js +10 -1
- package/dist/output/shapes/events.js +12 -7
- package/dist/output/shapes/helpers.js +56 -83
- package/dist/output/shapes/migrate.js +8 -0
- package/dist/output/shapes/passthrough.js +7 -41
- package/dist/output/shapes/proposal/producer.js +15 -7
- package/dist/output/shapes.js +2 -9
- package/dist/output/text/{init.js → bundle-create.js} +3 -1
- package/dist/output/text/bundle-show.js +7 -0
- package/dist/output/text/command-format.js +164 -96
- package/dist/output/text/env.js +1 -3
- package/dist/output/text/events.js +8 -7
- package/dist/output/text/health-format.js +103 -0
- package/dist/output/text/health.js +7 -0
- package/dist/output/text/helpers.js +10 -8
- package/dist/output/text/lint-format.js +56 -0
- package/dist/{migrate-storage-node.mjs → output/text/lint.js} +2 -5
- package/dist/output/text/migrate.js +88 -0
- package/dist/output/text/proposal/producer.js +4 -2
- package/dist/output/text/proposal-format.js +44 -72
- package/dist/output/text/registry-commands.js +1 -2
- package/dist/output/text/show-directives.js +15 -7
- package/dist/output/text/status-list.js +32 -0
- package/dist/output/text/{save.js → sync.js} +2 -2
- package/dist/output/text/workflow-format.js +24 -203
- package/dist/output/text/workflow.js +1 -7
- package/dist/output/text.js +16 -17
- package/dist/registry/factory.js +4 -6
- package/dist/registry/origin-resolve.js +16 -27
- package/dist/registry/providers/skills-sh.js +3 -3
- package/dist/registry/providers/static-index.js +13 -23
- package/dist/registry/resolve.js +42 -7
- package/dist/registry/semver.js +34 -84
- package/dist/runtime.js +2 -23
- package/dist/scripts/akm-migrate-node.js +60972 -0
- package/dist/scripts/akm-migrate.js +60310 -0
- package/dist/setup/detect.js +42 -15
- package/dist/setup/registry-stash-loader.js +2 -2
- package/dist/setup/setup.js +236 -136
- package/dist/setup/steps/connection.js +7 -9
- package/dist/setup/steps/platforms.js +9 -9
- package/dist/setup/steps/semantic.js +15 -3
- package/dist/setup/steps/sources.js +12 -13
- package/dist/setup/steps/stashdir.js +2 -3
- package/dist/setup/steps/tasks.js +237 -120
- package/dist/sources/freshness.js +1 -1
- package/dist/sources/provider-factory.js +11 -17
- package/dist/sources/providers/filesystem.js +2 -3
- package/dist/sources/providers/git-install.js +278 -34
- package/dist/sources/providers/git-provider.js +25 -23
- package/dist/sources/providers/git-stash.js +414 -106
- package/dist/sources/providers/git.js +2 -2
- package/dist/sources/providers/npm.js +16 -19
- package/dist/sources/providers/provider-utils.js +7 -4
- package/dist/sources/providers/sync-from-ref.js +3 -9
- package/dist/sources/providers/website.js +6 -1
- package/dist/sources/resolve.js +6 -5
- package/dist/sources/snapshot-fetchers/bluesky.js +146 -0
- package/dist/sources/snapshot-fetchers/content-extract.js +566 -0
- package/dist/sources/snapshot-fetchers/fetcher-util.js +41 -0
- package/dist/sources/snapshot-fetchers/github.js +100 -0
- package/dist/sources/snapshot-fetchers/host-guard.js +291 -0
- package/dist/sources/snapshot-fetchers/registry.js +17 -1
- package/dist/sources/snapshot-fetchers/robots.js +348 -0
- package/dist/sources/snapshot-fetchers/rss.js +282 -0
- package/dist/sources/snapshot-fetchers/secret-seam.js +42 -0
- package/dist/sources/snapshot-fetchers/website-ingest.js +686 -282
- package/dist/sources/snapshot-fetchers/x.js +910 -0
- package/dist/storage/database.js +13 -1
- package/dist/storage/engines/sqlite-migrations.js +23 -111
- package/dist/storage/managed-db.js +21 -2
- package/dist/storage/repositories/canaries-repository.js +1 -1
- package/dist/storage/repositories/events-repository.js +27 -11
- package/dist/storage/repositories/improve-runs-repository.js +6 -12
- package/dist/storage/repositories/index-connection.js +44 -8
- package/dist/storage/repositories/index-entries-repository.js +157 -241
- package/dist/storage/repositories/index-entry-mapper.js +15 -11
- package/dist/storage/repositories/index-fts-repository.js +5 -2
- package/dist/storage/repositories/index-llm-cache-repository.js +0 -1
- package/dist/storage/repositories/index-meta-repository.js +2 -3
- package/dist/storage/repositories/index-schema.js +10 -25
- package/dist/storage/repositories/index-utility-repository.js +15 -28
- package/dist/storage/repositories/index-vec-repository.js +6 -1
- package/dist/storage/repositories/outcome-repository.js +119 -0
- package/dist/storage/repositories/proposals-repository.js +296 -59
- package/dist/storage/repositories/registry-cache.js +19 -0
- package/dist/storage/repositories/salience-repository.js +172 -0
- package/dist/storage/repositories/task-history-repository.js +15 -13
- package/dist/storage/repositories/workflow-runs-repository.js +118 -53
- package/dist/tasks/backends/cron.js +105 -15
- package/dist/tasks/backends/index.js +1 -1
- package/dist/tasks/backends/launchd.js +85 -38
- package/dist/tasks/backends/schtasks.js +135 -15
- package/dist/tasks/embedded.js +56 -40
- package/dist/tasks/log-redaction.js +156 -0
- package/dist/tasks/parser.js +87 -160
- package/dist/tasks/resolve-akm-bin.js +137 -59
- package/dist/tasks/runner.js +209 -44
- package/dist/tasks/scheduler-invocation.js +220 -10
- package/dist/tasks/schema.js +89 -1
- package/dist/tasks/task-id.js +1 -3
- package/dist/tasks/validator.js +20 -6
- package/dist/workflows/authoring/authoring.js +94 -143
- package/dist/workflows/authoring/scope-key.js +1 -1
- package/dist/workflows/concurrency-policy.js +95 -1
- package/dist/workflows/exec/dispatch-redaction.js +114 -0
- package/dist/workflows/exec/exec-unit.js +542 -0
- package/dist/workflows/exec/frozen-judge.js +130 -32
- package/dist/workflows/exec/native-executor.js +513 -274
- package/dist/workflows/exec/param-secrets.js +12 -11
- package/dist/workflows/exec/run-workflow.js +496 -237
- package/dist/workflows/exec/step-work.js +616 -412
- package/dist/workflows/exec/unit-dispatch.js +34 -0
- package/dist/workflows/exec/unit-writer.js +53 -13
- package/dist/workflows/exec/worktree.js +454 -41
- package/dist/workflows/ir/compile.js +166 -271
- package/dist/workflows/ir/freeze.js +119 -42
- package/dist/workflows/ir/params.js +135 -11
- package/dist/workflows/ir/plan-hash.js +1 -1
- package/dist/workflows/ir/schema.js +129 -45
- package/dist/workflows/parser.js +1070 -307
- package/dist/workflows/program/expressions.js +20 -208
- package/dist/workflows/program/schema.js +31 -10
- package/dist/workflows/renderer.js +123 -68
- package/dist/workflows/resource-limits.js +184 -0
- package/dist/workflows/runtime/checkin.js +3 -3
- package/dist/workflows/runtime/plan-classifier.js +16 -75
- package/dist/workflows/runtime/runs.js +331 -132
- package/dist/workflows/runtime/unit-checkin.js +1 -1
- package/dist/workflows/runtime/unit-phases.js +2 -2
- package/dist/workflows/runtime/workflow-asset-loader.js +232 -83
- package/dist/workflows/schema.js +1 -11
- package/dist/workflows/validate-summary.js +47 -38
- package/dist/workflows/validator.js +21 -62
- package/docs/README.md +110 -0
- package/docs/migration/README.md +8 -0
- package/docs/migration/release-notes/0.7.0.md +11 -11
- package/docs/migration/release-notes/0.9.0.md +209 -27
- package/docs/migration/v0.7-to-v0.8.md +48 -48
- package/docs/migration/v0.8-to-v0.9.md +567 -209
- package/docs/migration/v0.9.0-troubleshooting.md +561 -0
- package/docs/reference/README.md +19 -0
- package/docs/reference/bundle-types.md +19 -0
- package/docs/reference/cli.md +2342 -0
- package/docs/reference/configuration.md +371 -0
- package/docs/reference/data-and-telemetry.md +129 -46
- package/docs/reference/supported-formats.md +50 -0
- package/docs/reference/workflow-schema.md +1014 -0
- package/docs/reference/workflows.md +51 -0
- package/package.json +31 -13
- package/schemas/akm-asset-envelope.json +93 -0
- package/schemas/akm-config.json +89 -128
- package/schemas/akm-task.json +27 -5
- package/schemas/akm-workflow.json +157 -81
- package/dist/assets/tasks/core/backup.yml +0 -5
- package/dist/assets/tasks/graph-refresh-weekly.yml +0 -10
- package/dist/cli/config-migrate.js +0 -1878
- package/dist/cli/config-validate.js +0 -41
- package/dist/commands/backup-cli.js +0 -56
- package/dist/commands/bundle/bundle-cli.js +0 -68
- package/dist/commands/bundle/bundle.js +0 -219
- package/dist/commands/graph/graph-cli.js +0 -124
- package/dist/commands/graph/graph.js +0 -489
- package/dist/commands/improve/extract-watch.js +0 -140
- package/dist/commands/mv-cli.js +0 -1221
- package/dist/commands/sources/history.js +0 -201
- package/dist/commands/tasks/default-tasks.js +0 -186
- package/dist/core/migration-backup.js +0 -1231
- package/dist/indexer/usage/unmigrated-vaults-guard.js +0 -95
- package/dist/llm/memory-infer-impl.js +0 -138
- package/dist/migrate/legacy/config-source-migration.js +0 -257
- package/dist/migrate/legacy/content-migration.js +0 -350
- package/dist/migrate/legacy/legacy-layout.js +0 -779
- package/dist/migrate/legacy/legacy-paths.js +0 -25
- package/dist/migrate/legacy/legacy-stash-json.js +0 -80
- package/dist/migrate/legacy/proposal-fs-import.js +0 -168
- package/dist/migrate/legacy/task-target-ref-migration.js +0 -278
- package/dist/migrate/legacy/three-db-cutover.js +0 -845
- package/dist/migrate/legacy/workflow-migrations-bodies.js +0 -52
- package/dist/migrate/legacy/workflow-migrations-frozen.js +0 -21
- package/dist/migrate/legacy-ref-grammar.js +0 -214
- package/dist/output/shapes/distill.js +0 -14
- package/dist/output/shapes/history.js +0 -11
- package/dist/output/text/distill.js +0 -6
- package/dist/output/text/enable-disable.js +0 -8
- package/dist/output/text/history.js +0 -6
- package/dist/registry/build-index.js +0 -382
- package/dist/schemas/akm-config.json +0 -4704
- package/dist/schemas/akm-task.json +0 -87
- package/dist/schemas/akm-workflow.json +0 -372
- package/dist/scripts/migrate-storage.js +0 -3816
- package/dist/workflows/authoring/workflow-program-template.yaml +0 -31
- package/dist/workflows/cli.js +0 -53
- package/dist/workflows/exec/brief.js +0 -481
- package/dist/workflows/exec/report.js +0 -1460
- package/dist/workflows/exec/watch.js +0 -116
- package/dist/workflows/program/parser.js +0 -813
- package/dist/workflows/program/project.js +0 -104
|
@@ -2,50 +2,52 @@
|
|
|
2
2
|
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
4
|
/**
|
|
5
|
-
* Native executor — executes
|
|
5
|
+
* Native executor — executes one frozen step subgraph (`IrStepPlan.root`) on
|
|
6
6
|
* the local machine: fan-out through the scheduler, schema-validated
|
|
7
7
|
* structured output through `runStructured` (core/structured.ts), per-unit
|
|
8
8
|
* persistence through the serialized writer queue, and `workflow_unit_*`
|
|
9
9
|
* events for observability.
|
|
10
10
|
*
|
|
11
|
-
* Data flow (
|
|
12
|
-
*
|
|
13
|
-
* —
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
11
|
+
* Data flow (workflow-format-unification, spec §2.3): there is NO
|
|
12
|
+
* interpolation language. A unit's instructions are the step's body prose
|
|
13
|
+
* BYTE-EXACT — prose is never scanned for reference syntax, so a literal `${{`
|
|
14
|
+
* in a body is content, not grammar. Data reaches the unit as ATTACHED
|
|
15
|
+
* STRUCTURED CONTEXT rather than string splices: `buildUnitPrompt`
|
|
16
|
+
* (`exec/step-work.ts`) wraps the verbatim instructions with JSON blocks for
|
|
17
|
+
* the run params, a map unit's item + index, and the artifacts named by the
|
|
18
|
+
* step's `inputs:`. Because nothing is ever substituted INTO the prose, the P1
|
|
19
|
+
* `{{item}}` re-scan injection class is structurally impossible.
|
|
20
|
+
*
|
|
21
|
+
* References survive only in the three whole-value FRONTMATTER positions the
|
|
22
|
+
* closed two-root grammar occupies (`program/expressions.ts`): `map.over`,
|
|
23
|
+
* `route.input`, and each `inputs[]` entry. They resolve ONCE per step against
|
|
24
|
+
* `{ params, stepOutputs }`. There is NO ambient key search: a
|
|
25
|
+
* `steps.<id>.output.<path>` reference addresses INTO that step's recorded
|
|
22
26
|
* output explicitly.
|
|
23
27
|
*
|
|
24
|
-
* Step outputs (
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
* object as-is.
|
|
28
|
+
* Step outputs (`steps.<id>.output…`): every engine-executed step journals a
|
|
29
|
+
* promoted ARTIFACT under `evidence.output` — the solo unit's result/text, the
|
|
30
|
+
* collect reducer's per-item array, or the vote reducer's winner — and that
|
|
31
|
+
* artifact is what the reference scope exposes ({@link projectStepOutput}).
|
|
32
|
+
* The documented addressing (`steps.discover.output.files`) therefore resolves
|
|
33
|
+
* against real step results, never the raw evidence envelope (peer review R1).
|
|
34
|
+
* Steps completed manually (no `output` key in their evidence) expose their
|
|
35
|
+
* recorded evidence object as-is.
|
|
33
36
|
*
|
|
34
37
|
* Empty free-text outputs (peer review): a SUCCESSFUL schemaless unit that
|
|
35
38
|
* returns the empty string is normalized to "no output" — {@link dispatchUnit}
|
|
36
|
-
* drops the falsy `text`, `finishUnit` journals `result_json = NULL`, and
|
|
37
|
-
* durable-row reuse
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
* `EMPTY_OUTPUT` driver-parity golden pins it). Consequences that follow from
|
|
39
|
+
* drops the falsy `text`, `finishUnit` journals `result_json = NULL`, and
|
|
40
|
+
* durable-row reuse rehydrates the same absence (`unitOutcomeFromRow`). This is
|
|
41
|
+
* the ONLY empty-output resolution: `''` never survives into the journal, so the
|
|
42
|
+
* live artifact cannot diverge from the artifact a resume rebuilds from the same
|
|
43
|
+
* rows (the byte-identical-graph cardinal rule). Consequences that follow from
|
|
42
44
|
* "empty == absent", not special-cased anywhere:
|
|
43
45
|
* - a SOLO empty step promotes `output = null` (the unit's absent text ??
|
|
44
46
|
* null); a `collect` fan-out promotes `null` in that item's slot.
|
|
45
|
-
* - A downstream
|
|
46
|
-
* resolves against `null` and fails LOUDLY at
|
|
47
|
-
* (`… resolved to null`) — a deterministic
|
|
48
|
-
*
|
|
47
|
+
* - A downstream `steps.x.output` reference to an empty solo step therefore
|
|
48
|
+
* resolves against `null` and fails LOUDLY at reference resolution
|
|
49
|
+
* (`… resolved to null`) — a deterministic whole-step failure, never a
|
|
50
|
+
* silent empty string.
|
|
49
51
|
* - A SCHEMA unit is unaffected by this normalization: an empty response is
|
|
50
52
|
* not parseable JSON, so `runStructured` fails it (`parse_error`) — an
|
|
51
53
|
* empty output can never satisfy a declared schema as a silent `null`.
|
|
@@ -74,9 +76,7 @@
|
|
|
74
76
|
* never a silent re-dispatch — under a frozen plan the same identity must
|
|
75
77
|
* reproduce the same inputs, so a mismatch means the journal (or params
|
|
76
78
|
* row) was tampered with. Failed/running/missing rows dispatch live.
|
|
77
|
-
* -
|
|
78
|
-
* no back-compat shim: those rows simply never match a content-derived id
|
|
79
|
-
* and are ignored (the step re-runs cleanly on top of them).
|
|
79
|
+
* - Rows with unrelated ids never match a content-derived id and are ignored.
|
|
80
80
|
*
|
|
81
81
|
* Gate loops (addendum, R2 `gate.max_loops`): when the engine re-executes a
|
|
82
82
|
* step subgraph after a gate rejection, it threads the judge's feedback in as
|
|
@@ -129,24 +129,28 @@
|
|
|
129
129
|
* - This module NEVER writes step rows: advancing the gated spine is the
|
|
130
130
|
* engine loop's job (`run-workflow.ts`) via `completeWorkflowStep`.
|
|
131
131
|
*/
|
|
132
|
-
import { deepMergeConfig } from "../../core/config/deep-merge.js";
|
|
133
|
-
import { ConfigError } from "../../core/errors.js";
|
|
134
132
|
import { appendEvent } from "../../core/events.js";
|
|
135
133
|
import { validateJsonSchemaSubset } from "../../core/json-schema.js";
|
|
136
|
-
import { collectSensitiveValues, isEnvPassthroughValueSafeToExpose, redactSensitiveValue } from "../../core/redaction.js";
|
|
137
134
|
import { runStructured } from "../../core/structured.js";
|
|
138
135
|
import { warn } from "../../core/warn.js";
|
|
139
|
-
import {
|
|
136
|
+
import { insertEventStrict } from "../../storage/repositories/events-repository.js";
|
|
137
|
+
import { withWorkflowRunsConnection, withWorkflowRunsRepo, } from "../../storage/repositories/workflow-runs-repository.js";
|
|
138
|
+
import { WORKFLOW_UNIT_DIAGNOSTIC_CLIP } from "../resource-limits.js";
|
|
139
|
+
// The ONE dispatch redaction contract, shared with the gate-judge path
|
|
140
|
+
// (exec/frozen-judge.ts). Consumers import the leaf directly — this module is
|
|
141
|
+
// not a second front door onto the seam.
|
|
142
|
+
import { collectWorkflowDispatchSensitiveValues, redactUnitOutcome } from "./dispatch-redaction.js";
|
|
143
|
+
// The exec (shell) unit runner — a leaf that owns argv spawning, containment,
|
|
144
|
+
// and the process-outcome → failure-reason mapping.
|
|
145
|
+
import { runExecUnit } from "./exec-unit.js";
|
|
140
146
|
import { LIFETIME_UNIT_CAP, scheduleUnits, UnitCapExceededError } from "./scheduler.js";
|
|
141
|
-
// Shared step semantics — the ONE implementation consumed by
|
|
142
|
-
// (this module + run-workflow.ts)
|
|
143
|
-
//
|
|
144
|
-
import {
|
|
147
|
+
// Shared step semantics — the ONE implementation consumed by the engine
|
|
148
|
+
// (this module + run-workflow.ts) on both the fresh-execution and the resume
|
|
149
|
+
// path. This module dispatches; step-work.ts owns the pure decisions.
|
|
150
|
+
import { clip, computeStepWorkList, reduceEmptyStep, reduceStepOutcomes, stepOutputsFromEvidence, unitOutcomeFromRow, } from "./step-work.js";
|
|
151
|
+
import { materializeFrozenLlm, } from "./unit-dispatch.js";
|
|
145
152
|
import { enqueueUnitWrite } from "./unit-writer.js";
|
|
146
153
|
import { assertGitWorkTree, cleanupUnitWorktree, createUnitWorktree } from "./worktree.js";
|
|
147
|
-
// Re-exported for existing consumers (run-workflow.ts, tests) that import these
|
|
148
|
-
// from native-executor; they now live in the shared step-work module.
|
|
149
|
-
export { buildArtifactSummary, DEFAULT_UNIT_TIMEOUT_MS, projectStepOutput };
|
|
150
154
|
/**
|
|
151
155
|
* Mutable per-step dispatch budget: the lifetime unit cap PLUS the declared
|
|
152
156
|
* run-level budget ceilings (`budget.max_units` / `budget.max_tokens`,
|
|
@@ -211,47 +215,115 @@ class DispatchBudget {
|
|
|
211
215
|
this.onExceeded?.();
|
|
212
216
|
}
|
|
213
217
|
}
|
|
214
|
-
function classifyUnitReuse(workUnit,
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
218
|
+
function classifyUnitReuse(workUnit, completedRows, gateLoop) {
|
|
219
|
+
const inputHash = workUnit.inputHash;
|
|
220
|
+
// Scan EVERY journaled attempt row of this unit (`<base>` / `<base>~r<N>`
|
|
221
|
+
// for ANY N), not just the attempts the CURRENT retry policy allows:
|
|
222
|
+
// retry/onError are deliberately excluded from the input hash (step-work.ts)
|
|
223
|
+
// precisely so completed rows stay valid across policy changes — a run
|
|
224
|
+
// re-invoked with a lowered retry.max must still find the `~rN` row a prior
|
|
225
|
+
// invocation completed beyond the new max, never re-dispatch finished work.
|
|
226
|
+
// A completed hash-matching row anywhere wins (reuse); a completed loop-1
|
|
227
|
+
// row with a DIFFERENT hash — and no matching sibling — is replay divergence.
|
|
228
|
+
let divergedAttemptId;
|
|
229
|
+
for (const prior of completedRows?.get(workUnit.journalBaseId) ?? []) {
|
|
225
230
|
if (prior.input_hash === inputHash)
|
|
226
231
|
return { kind: "reuse", row: prior };
|
|
227
232
|
// Gate-loop rows are NOT replay-deterministic (the prompt embeds a fresh
|
|
228
233
|
// judge output): a stale loop-N row with a different hash re-dispatches
|
|
229
234
|
// live. Divergence only guards loop-1 rows, whose inputs ARE a pure
|
|
230
235
|
// function of (frozen plan, params, journaled results).
|
|
231
|
-
if (gateLoop
|
|
232
|
-
|
|
233
|
-
return { kind: "diverge", attemptId };
|
|
236
|
+
if (gateLoop <= 1)
|
|
237
|
+
divergedAttemptId ??= prior.unit_id;
|
|
234
238
|
}
|
|
239
|
+
if (divergedAttemptId !== undefined)
|
|
240
|
+
return { kind: "diverge", attemptId: divergedAttemptId };
|
|
235
241
|
return { kind: "dispatch" };
|
|
236
242
|
}
|
|
243
|
+
function indexCompletedRows(rows) {
|
|
244
|
+
const index = new Map();
|
|
245
|
+
for (const row of rows) {
|
|
246
|
+
if (row.status !== "completed")
|
|
247
|
+
continue;
|
|
248
|
+
const base = row.unit_id.replace(/~r\d+$/, "");
|
|
249
|
+
const forBase = index.get(base);
|
|
250
|
+
if (forBase)
|
|
251
|
+
forBase.push(row);
|
|
252
|
+
else
|
|
253
|
+
index.set(base, [row]);
|
|
254
|
+
}
|
|
255
|
+
return index;
|
|
256
|
+
}
|
|
237
257
|
/**
|
|
238
|
-
*
|
|
239
|
-
*
|
|
240
|
-
*
|
|
241
|
-
*
|
|
258
|
+
* Execute one step plan natively. Never throws for unit-level failures.
|
|
259
|
+
*
|
|
260
|
+
* The whole step runs inside ONE state.db connection scope
|
|
261
|
+
* ({@link withWorkflowRunsConnection}): the journal read, every unit's
|
|
262
|
+
* insert/finish transaction, and every `workflow_unit_*` event share a single
|
|
263
|
+
* handle for the step's lifetime instead of opening and closing state.db twice
|
|
264
|
+
* per unit plus twice per unit's events. The scope closes the handle when the
|
|
265
|
+
* step settles (success, failure, or throw), so there is no handle to leak and
|
|
266
|
+
* no lifetime that outlives the step. Everything inside keeps its existing
|
|
267
|
+
* transaction boundaries — see `core/state-db-scope.ts` for why sharing a
|
|
268
|
+
* handle across concurrently-scheduled units is safe here.
|
|
242
269
|
*/
|
|
243
|
-
function
|
|
244
|
-
return
|
|
270
|
+
export function executeStepPlan(plan, ctx) {
|
|
271
|
+
return withWorkflowRunsConnection(() => executeStepPlanInConnection(plan, ctx));
|
|
245
272
|
}
|
|
246
|
-
/**
|
|
247
|
-
|
|
273
|
+
/**
|
|
274
|
+
* Open the step's dispatch budget and the abort signal it trips.
|
|
275
|
+
*
|
|
276
|
+
* Budget ceilings (addendum R2): when the frozen plan declares a budget,
|
|
277
|
+
* dispatch runs under an AbortController CHAINED onto `ctx.signal` — hitting a
|
|
278
|
+
* ceiling aborts pending and in-flight dispatches, and the step fails hard.
|
|
279
|
+
* Without a budget the context signal passes through untouched (the no-budget
|
|
280
|
+
* path is byte-identical to pre-R2 behavior).
|
|
281
|
+
*
|
|
282
|
+
* The returned {@link DispatchBudget} is seeded with the run's journaled
|
|
283
|
+
* dispatch count and token total and consumed per ACTUAL dispatch inside
|
|
284
|
+
* `runUnit` — never for durable-row reuses, so resuming a large
|
|
285
|
+
* partially-completed fan-out works.
|
|
286
|
+
*
|
|
287
|
+
* `unchainSignal` MUST be called when dispatch finishes (the caller's
|
|
288
|
+
* `finally`) so the upstream abort listener is removed.
|
|
289
|
+
*/
|
|
290
|
+
function openDispatchBudget(ctx, dispatched) {
|
|
291
|
+
const declaredBudget = ctx.budget && (ctx.budget.maxUnits !== undefined || ctx.budget.maxTokens !== undefined) ? ctx.budget : undefined;
|
|
292
|
+
let signal = ctx.signal;
|
|
293
|
+
let onExceeded;
|
|
294
|
+
let unchainSignal;
|
|
295
|
+
if (declaredBudget) {
|
|
296
|
+
const controller = new AbortController();
|
|
297
|
+
const upstream = ctx.signal;
|
|
298
|
+
if (upstream) {
|
|
299
|
+
if (upstream.aborted) {
|
|
300
|
+
controller.abort();
|
|
301
|
+
}
|
|
302
|
+
else {
|
|
303
|
+
const onUpstreamAbort = () => controller.abort();
|
|
304
|
+
upstream.addEventListener("abort", onUpstreamAbort, { once: true });
|
|
305
|
+
unchainSignal = () => upstream.removeEventListener("abort", onUpstreamAbort);
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
signal = controller.signal;
|
|
309
|
+
onExceeded = () => controller.abort();
|
|
310
|
+
}
|
|
311
|
+
const budget = new DispatchBudget(dispatched, {
|
|
312
|
+
tokensUsed: ctx.tokensUsed ?? 0,
|
|
313
|
+
...(declaredBudget ? { budget: declaredBudget } : {}),
|
|
314
|
+
...(onExceeded ? { onExceeded } : {}),
|
|
315
|
+
});
|
|
316
|
+
return { signal, budget, unchainSignal };
|
|
317
|
+
}
|
|
318
|
+
async function executeStepPlanInConnection(plan, ctx) {
|
|
248
319
|
const dispatched = ctx.unitsDispatched ?? 0;
|
|
249
320
|
// Work-list computation is the SHARED, PURE decision (step-work.ts): resolve
|
|
250
321
|
// the fan-out list, derive content-derived unit ids, assemble each unit's
|
|
251
|
-
// prompt, and hash its resolved input.
|
|
252
|
-
//
|
|
253
|
-
//
|
|
254
|
-
//
|
|
322
|
+
// prompt, and hash its resolved input. A resume recomputes the identical list
|
|
323
|
+
// from the same frozen plan — that shared pure implementation is what lets
|
|
324
|
+
// journaled rows be matched instead of re-executed. This module owns only the
|
|
325
|
+
// impure remainder: env/worktree preflight, durable-row reuse, dispatch,
|
|
326
|
+
// journaling, budget.
|
|
255
327
|
const workList = computeStepWorkList(plan, {
|
|
256
328
|
runId: ctx.runId,
|
|
257
329
|
params: ctx.params,
|
|
@@ -266,10 +338,9 @@ export async function executeStepPlan(plan, ctx) {
|
|
|
266
338
|
const { template, reducer, isFanOut, items, units: workUnits } = workList.list;
|
|
267
339
|
if (items.length === 0) {
|
|
268
340
|
// Empty fan-out: the promoted artifact is the degenerate empty value, honored
|
|
269
|
-
// against the step's declared output schema. `reduceEmptyStep`
|
|
270
|
-
// decision
|
|
271
|
-
//
|
|
272
|
-
// artifact + schema verdict.
|
|
341
|
+
// against the step's declared output schema. `reduceEmptyStep` (step-work.ts)
|
|
342
|
+
// owns that decision so a zero-unit step promotes the identical artifact +
|
|
343
|
+
// schema verdict every time the spine reaches it.
|
|
273
344
|
return { ...reduceEmptyStep(plan, reducer), unitsDispatched: dispatched };
|
|
274
345
|
}
|
|
275
346
|
const dispatcher = ctx.dispatcher ?? defaultUnitDispatcher;
|
|
@@ -279,10 +350,7 @@ export async function executeStepPlan(plan, ctx) {
|
|
|
279
350
|
// is reused, not re-dispatched — a crash-resume must never double-issue
|
|
280
351
|
// side-effecting work. Loading the rows up front is what lets us skip the
|
|
281
352
|
// dispatch prerequisites below when nothing will actually dispatch.
|
|
282
|
-
const
|
|
283
|
-
for (const row of await withWorkflowRunsRepo((repo) => repo.getUnitsForStep(ctx.runId, plan.stepId))) {
|
|
284
|
-
existingUnits.set(row.unit_id, row);
|
|
285
|
-
}
|
|
353
|
+
const completedRows = indexCompletedRows(await withWorkflowRunsRepo((repo) => repo.getUnitsForStep(ctx.runId, plan.stepId)));
|
|
286
354
|
// Reviewer finding #2: env resolution and worktree preflight are DISPATCH
|
|
287
355
|
// prerequisites, so they must run only when a unit will actually dispatch. A
|
|
288
356
|
// fully-journaled step whose units all reuse completed rows must resume to
|
|
@@ -291,7 +359,12 @@ export async function executeStepPlan(plan, ctx) {
|
|
|
291
359
|
// to hand back a cached result. The predicate mirrors runUnit's reuse
|
|
292
360
|
// decision exactly (shared classifyUnitReuse).
|
|
293
361
|
const gateLoop = ctx.gateLoop ?? 1;
|
|
294
|
-
|
|
362
|
+
// Classify every unit ONCE. The gate below and each unit's own dispatch then
|
|
363
|
+
// read the SAME decision rather than recomputing it from inputs that must be
|
|
364
|
+
// identical — the agreement the gate depends on is structural, not a property
|
|
365
|
+
// two call sites have to keep re-establishing.
|
|
366
|
+
const reuseDecisions = workUnits.map((unit) => classifyUnitReuse(unit, completedRows, gateLoop));
|
|
367
|
+
const willDispatch = reuseDecisions.some((decision) => decision.kind === "dispatch");
|
|
295
368
|
// Env bindings resolve once per step, before any dispatch; a binding error
|
|
296
369
|
// fails the whole step cleanly rather than N units racing into it. Skipped
|
|
297
370
|
// entirely when nothing will dispatch.
|
|
@@ -326,41 +399,23 @@ export async function executeStepPlan(plan, ctx) {
|
|
|
326
399
|
}
|
|
327
400
|
worktreeBase = base;
|
|
328
401
|
}
|
|
329
|
-
//
|
|
330
|
-
//
|
|
331
|
-
//
|
|
332
|
-
//
|
|
333
|
-
//
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
const upstream = ctx.signal;
|
|
341
|
-
if (upstream) {
|
|
342
|
-
if (upstream.aborted) {
|
|
343
|
-
controller.abort();
|
|
344
|
-
}
|
|
345
|
-
else {
|
|
346
|
-
const onUpstreamAbort = () => controller.abort();
|
|
347
|
-
upstream.addEventListener("abort", onUpstreamAbort, { once: true });
|
|
348
|
-
unchainSignal = () => upstream.removeEventListener("abort", onUpstreamAbort);
|
|
349
|
-
}
|
|
350
|
-
}
|
|
351
|
-
signal = controller.signal;
|
|
352
|
-
onExceeded = () => controller.abort();
|
|
353
|
-
}
|
|
354
|
-
// Lifetime-cap + declared-budget accounting: seeded with the run's
|
|
355
|
-
// journaled dispatch count and token total, consumed per ACTUAL dispatch
|
|
356
|
-
// inside runUnit — never for durable-row reuses, so resuming a large
|
|
357
|
-
// partially-completed fan-out works.
|
|
358
|
-
const budget = new DispatchBudget(dispatched, {
|
|
359
|
-
tokensUsed: ctx.tokensUsed ?? 0,
|
|
360
|
-
...(declaredBudget ? { budget: declaredBudget } : {}),
|
|
361
|
-
...(onExceeded ? { onExceeded } : {}),
|
|
362
|
-
});
|
|
402
|
+
// The values that must never survive into the journal. Collected once per
|
|
403
|
+
// step, not once per unit: the frozen engine and its SDK fallback are
|
|
404
|
+
// resolved from the step template and handed unchanged to every unit, and
|
|
405
|
+
// `env` is step-wide, so a fan-out was re-scanning process.env — and
|
|
406
|
+
// re-inspecting every passthrough value — once per unit, including for units
|
|
407
|
+
// that go on to reuse a journaled row and dispatch nothing at all.
|
|
408
|
+
const sensitiveValues = willDispatch && workUnits[0] ? collectWorkflowDispatchSensitiveValues(workUnits[0], env) : [];
|
|
409
|
+
// Budget ceilings + lifetime-cap accounting, and the budget-chained abort
|
|
410
|
+
// signal they trip. Extracted verbatim (behavior-identical) — see
|
|
411
|
+
// {@link openDispatchBudget}.
|
|
412
|
+
const { signal, budget, unchainSignal } = openDispatchBudget(ctx, dispatched);
|
|
363
413
|
let outcomes;
|
|
414
|
+
// Worktree removals started by finished units and awaited below, before this
|
|
415
|
+
// step reports anything. Cleanup is best-effort, but "the step resolved" must
|
|
416
|
+
// still mean "its clean worktrees are gone" — only the WAIT moves off the
|
|
417
|
+
// unit's scheduler slot, not the guarantee.
|
|
418
|
+
const pendingWorktreeCleanups = [];
|
|
364
419
|
const selectedEngine = template.invocation ? ctx.engines?.[template.invocation.engine] : undefined;
|
|
365
420
|
const selectedLlmEngine = selectedEngine?.kind === "llm"
|
|
366
421
|
? selectedEngine
|
|
@@ -368,7 +423,7 @@ export async function executeStepPlan(plan, ctx) {
|
|
|
368
423
|
? ctx.engines?.[selectedEngine.fallbackLlmEngine]
|
|
369
424
|
: undefined;
|
|
370
425
|
try {
|
|
371
|
-
outcomes = await scheduleUnits(workUnits, (workUnit) => runUnit({
|
|
426
|
+
outcomes = await scheduleUnits(workUnits, (workUnit, index) => runUnit({
|
|
372
427
|
plan,
|
|
373
428
|
workUnit,
|
|
374
429
|
env,
|
|
@@ -376,7 +431,9 @@ export async function executeStepPlan(plan, ctx) {
|
|
|
376
431
|
ctx,
|
|
377
432
|
signal,
|
|
378
433
|
dispatcher,
|
|
379
|
-
|
|
434
|
+
reuse: reuseDecisions[index],
|
|
435
|
+
sensitiveValues,
|
|
436
|
+
pendingWorktreeCleanups,
|
|
380
437
|
budget,
|
|
381
438
|
}), {
|
|
382
439
|
concurrency: workList.list.concurrency,
|
|
@@ -387,6 +444,9 @@ export async function executeStepPlan(plan, ctx) {
|
|
|
387
444
|
}
|
|
388
445
|
finally {
|
|
389
446
|
unchainSignal?.();
|
|
447
|
+
// The barrier. `allSettled` because each task already swallowed its own
|
|
448
|
+
// failure into a warn — nothing here can fail the step.
|
|
449
|
+
await Promise.allSettled(pendingWorktreeCleanups);
|
|
390
450
|
}
|
|
391
451
|
// Declared budget ceilings and the lifetime cap are hard backstops: a step
|
|
392
452
|
// that hit one FAILS regardless of on_error policy (a capped run must never
|
|
@@ -412,6 +472,19 @@ export async function executeStepPlan(plan, ctx) {
|
|
|
412
472
|
.map((u) => u.error ?? `replay divergence: unit "${u.unitId}" was journaled with different inputs`)
|
|
413
473
|
.join(" "));
|
|
414
474
|
}
|
|
475
|
+
// A journal-write failure is likewise HARD regardless of on_error: the
|
|
476
|
+
// unit dispatched (spent tokens, ran side effects) but its result could not
|
|
477
|
+
// be persisted, so completing the step would promote an artifact the
|
|
478
|
+
// journal cannot rebuild on resume — and the stuck-`running` row would
|
|
479
|
+
// wedge or double-dispatch a later invocation. The summary carries the
|
|
480
|
+
// per-unit cause verbatim.
|
|
481
|
+
const unjournaled = units.filter((u) => u.failureReason === "journal_write_failed");
|
|
482
|
+
if (unjournaled.length > 0) {
|
|
483
|
+
return {
|
|
484
|
+
...failedStep(budget.used, unjournaled.map((u) => u.error ?? `unit "${u.unitId}" result could not be journaled`).join(" ")),
|
|
485
|
+
tokensUsed: budget.tokens,
|
|
486
|
+
};
|
|
487
|
+
}
|
|
415
488
|
// Failure policy + reducer + typed-artifact validation are the SHARED
|
|
416
489
|
// post-dispatch decision (`reduceStepOutcomes`): `onError: "fail"` (default)
|
|
417
490
|
// fails the step on any unit failure, `"continue"` records failures and lets
|
|
@@ -431,54 +504,53 @@ export async function executeStepPlan(plan, ctx) {
|
|
|
431
504
|
async function runUnit(input) {
|
|
432
505
|
const { plan, workUnit, env, ctx, dispatcher } = input;
|
|
433
506
|
const unitId = workUnit.unitId;
|
|
434
|
-
//
|
|
435
|
-
//
|
|
436
|
-
//
|
|
437
|
-
//
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
if (!workUnit.engine || !workUnit.invocation) {
|
|
442
|
-
return {
|
|
443
|
-
unitId,
|
|
444
|
-
ok: false,
|
|
445
|
-
failureReason: "dispatch_error",
|
|
446
|
-
error: `unit "${unitId}" has no frozen engine snapshot and cannot be dispatched`,
|
|
447
|
-
};
|
|
448
|
-
}
|
|
507
|
+
// Engine/exec presence is a WHOLE-LIST invariant, never a per-unit condition:
|
|
508
|
+
// computeStepWorkList fails the entire step before it builds any unit when a
|
|
509
|
+
// template carries neither an `exec` spec nor an `invocation`, and when an
|
|
510
|
+
// `invocation` names an engine absent from the frozen catalog — and
|
|
511
|
+
// executeStepPlanInConnection returns on `!workList.ok` without reaching here.
|
|
512
|
+
// So every unit below carries either `exec` (a child process, naming no
|
|
513
|
+
// engine) or `engine` + `invocation`.
|
|
449
514
|
// The prompt (and therefore the input hash) was built once with the BASE
|
|
450
515
|
// unit id by computeStepWorkList: a retry re-dispatches the SAME input, the
|
|
451
516
|
// `~r<n>` suffix is journal bookkeeping only.
|
|
452
|
-
const { prompt, inputHash } = workUnit
|
|
453
|
-
const sensitiveValues =
|
|
517
|
+
const { prompt, inputHash } = workUnit;
|
|
518
|
+
const sensitiveValues = input.sensitiveValues;
|
|
454
519
|
const request = {
|
|
455
520
|
runId: ctx.runId,
|
|
456
521
|
stepId: plan.stepId,
|
|
457
522
|
unitId,
|
|
458
523
|
nodeId: workUnit.nodeId,
|
|
459
524
|
prompt,
|
|
460
|
-
engine: workUnit.engine,
|
|
525
|
+
...(workUnit.engine ? { engine: workUnit.engine } : {}),
|
|
461
526
|
...(workUnit.fallbackEngine ? { fallbackEngine: workUnit.fallbackEngine } : {}),
|
|
462
|
-
invocation: workUnit.invocation,
|
|
527
|
+
...(workUnit.invocation ? { invocation: workUnit.invocation } : {}),
|
|
528
|
+
...(workUnit.exec ? { exec: workUnit.exec } : {}),
|
|
529
|
+
...(workUnit.execContext ? { execContext: workUnit.execContext } : {}),
|
|
530
|
+
// A NON-isolated exec unit spawns in the engine invocation's working
|
|
531
|
+
// directory. `dispatchJournaledAttempt` overwrites this with the unit's
|
|
532
|
+
// fresh worktree when `isolation: worktree` is in play. Only exec units get
|
|
533
|
+
// it: handing an agent unit a cwd it never had would change harness
|
|
534
|
+
// behavior, and the agent path already takes its cwd from its profile.
|
|
535
|
+
...(workUnit.exec && ctx.workDir !== undefined ? { cwd: ctx.workDir } : {}),
|
|
463
536
|
timeoutMs: workUnit.timeoutMs,
|
|
464
537
|
...(workUnit.schema ? { schema: workUnit.schema } : {}),
|
|
465
538
|
...(env ? { env } : {}),
|
|
466
539
|
...(sensitiveValues.length > 0 ? { sensitiveValues } : {}),
|
|
467
540
|
...(input.signal ? { signal: input.signal } : {}),
|
|
468
541
|
};
|
|
469
|
-
// Bounded retry
|
|
542
|
+
// Bounded retry: attempt 0 journals under the base
|
|
470
543
|
// journal id (`<unitId>`, or `<unitId>~l<loop>` in a gate loop — computed by
|
|
471
544
|
// the shared work-list), retry attempt N under `<baseId>~r<N>`. Every attempt
|
|
472
545
|
// keeps its own row. Retries only fire when the failure reason is in
|
|
473
546
|
// `retry.on`.
|
|
474
547
|
const retry = workUnit.retry;
|
|
475
548
|
const maxAttempts = 1 + Math.max(0, retry?.max ?? 0);
|
|
476
|
-
const gateLoop = ctx.gateLoop ?? 1;
|
|
477
549
|
const journalBaseId = workUnit.journalBaseId;
|
|
478
550
|
const attemptIdFor = (attempt) => (attempt === 0 ? journalBaseId : `${journalBaseId}~r${attempt}`);
|
|
479
|
-
// Durable-row reuse
|
|
480
|
-
//
|
|
481
|
-
// what happens here
|
|
551
|
+
// Durable-row reuse — literally the decision executeStepPlan's preflight gate
|
|
552
|
+
// counted, handed down rather than recomputed, so the gate cannot disagree
|
|
553
|
+
// with what happens here. A completed row with the matching input hash IS the
|
|
482
554
|
// result: return it without touching rows, dispatching, or re-emitting events
|
|
483
555
|
// (a crash-resume must never double-issue work). A completed loop-1 row with
|
|
484
556
|
// a DIFFERENT hash is replay divergence (under a frozen plan the same
|
|
@@ -486,11 +558,11 @@ async function runUnit(input) {
|
|
|
486
558
|
// tampered with; executeStepPlan promotes this to a hard step failure
|
|
487
559
|
// regardless of on_error). Stale gate-loop rows, failed/running/missing rows,
|
|
488
560
|
// and pre-release R1 positional ids all fall through and dispatch live.
|
|
489
|
-
const reuse =
|
|
561
|
+
const reuse = input.reuse;
|
|
490
562
|
if (reuse.kind === "reuse") {
|
|
491
563
|
// Identity in the durable step evidence is the CONTENT-derived base id, not
|
|
492
|
-
// the `~r<n>` attempt row it was reused from — the
|
|
493
|
-
//
|
|
564
|
+
// the `~r<n>` attempt row it was reused from — the work list only ever knows
|
|
565
|
+
// base ids, so evidence.units[].unitId stays stable across retries+resumes.
|
|
494
566
|
return reuseCompletedUnit(unitId, reuse.row, workUnit.schema !== undefined);
|
|
495
567
|
}
|
|
496
568
|
if (reuse.kind === "diverge") {
|
|
@@ -504,6 +576,14 @@ async function runUnit(input) {
|
|
|
504
576
|
}
|
|
505
577
|
let outcome;
|
|
506
578
|
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
|
579
|
+
if (input.signal?.aborted) {
|
|
580
|
+
return (outcome ?? {
|
|
581
|
+
unitId,
|
|
582
|
+
ok: false,
|
|
583
|
+
failureReason: "aborted",
|
|
584
|
+
error: "unit was not dispatched because the workflow invocation was interrupted",
|
|
585
|
+
});
|
|
586
|
+
}
|
|
507
587
|
const attemptId = attemptIdFor(attempt);
|
|
508
588
|
// Lifetime cap + declared budget ceilings, consumed per ACTUAL dispatch
|
|
509
589
|
// (reuses above returned before reaching here). Refusal fails this unit
|
|
@@ -527,6 +607,7 @@ async function runUnit(input) {
|
|
|
527
607
|
attemptId,
|
|
528
608
|
inputHash,
|
|
529
609
|
...(input.worktreeBase !== undefined ? { worktreeBase: input.worktreeBase } : {}),
|
|
610
|
+
pendingWorktreeCleanups: input.pendingWorktreeCleanups,
|
|
530
611
|
});
|
|
531
612
|
// The journal ROW keeps the `~r<n>`/`~l<loop>` attempt id (dispatchJournaledAttempt
|
|
532
613
|
// wrote it), but the returned outcome's identity in the DURABLE step evidence is
|
|
@@ -542,6 +623,8 @@ async function runUnit(input) {
|
|
|
542
623
|
input.budget.addTokens(outcome.tokens);
|
|
543
624
|
if (outcome.ok)
|
|
544
625
|
return outcome;
|
|
626
|
+
if (input.signal?.aborted)
|
|
627
|
+
return outcome;
|
|
545
628
|
const reason = outcome.failureReason;
|
|
546
629
|
if (!retry || reason === undefined || !retry.on.includes(reason))
|
|
547
630
|
return outcome;
|
|
@@ -549,6 +632,52 @@ async function runUnit(input) {
|
|
|
549
632
|
// maxAttempts >= 1, so outcome is always set by the loop above.
|
|
550
633
|
return outcome;
|
|
551
634
|
}
|
|
635
|
+
/**
|
|
636
|
+
* What a finished attempt writes to `workflow_run_steps`' unit row
|
|
637
|
+
* `result_json` — the ONE durable, human-facing surface for a dispatch outcome
|
|
638
|
+
* (`akm workflow status --units` reads exactly this, and the step summary is
|
|
639
|
+
* built from the same text).
|
|
640
|
+
*
|
|
641
|
+
* A SUCCESS journals its promoted value, unchanged.
|
|
642
|
+
*
|
|
643
|
+
* A FAILURE journals its DIAGNOSTIC. Before this, only `result`/`text` were
|
|
644
|
+
* written: `outcome.error` — the one field that says WHY — reached nothing
|
|
645
|
+
* durable, because `buildEvidence` deliberately drops it from the deterministic
|
|
646
|
+
* evidence graph and nothing else persisted it. For an engine unit that mostly
|
|
647
|
+
* cost detail; for an `exec` unit it lost the diagnostic entirely, since a
|
|
648
|
+
* command that fails and explains itself on stderr with empty stdout left
|
|
649
|
+
* `status --units` showing a bare `non_zero_exit`.
|
|
650
|
+
*
|
|
651
|
+
* Three constraints hold:
|
|
652
|
+
*
|
|
653
|
+
* - REDACTION — the caller journals only `redactUnitOutcome(...)` output, so
|
|
654
|
+
* `error` has already been through the shared dispatch redaction contract
|
|
655
|
+
* (`exec/dispatch-redaction.ts`) with this dispatch's resolved `env:`
|
|
656
|
+
* values. It is scrubbed by construction, exactly like `text`.
|
|
657
|
+
* - BOUNDS — clipped to {@link WORKFLOW_UNIT_DIAGNOSTIC_CLIP}, the same bound
|
|
658
|
+
* `status --units` renders with, so a runaway command cannot use the journal
|
|
659
|
+
* as its log file.
|
|
660
|
+
* - HASHES — `result_json` is an OUTPUT. The unit input hash
|
|
661
|
+
* (`computeUnitInputHash`) is computed from plan-frozen INPUTS only
|
|
662
|
+
* (template bytes, item, declared inputs, params, dispatch/invocation/exec
|
|
663
|
+
* snapshots, env ref names, isolation), and reuse compares the stored
|
|
664
|
+
* `input_hash` against that. Nothing here is a hash preimage input, so no
|
|
665
|
+
* completed unit re-dispatches because of it.
|
|
666
|
+
*
|
|
667
|
+
* Partial output on a failed unit is kept ALONGSIDE the diagnostic rather than
|
|
668
|
+
* replacing it: a tool that fails after printing its real complaint on stdout
|
|
669
|
+
* is common, and the reason lives on whichever stream that tool chose.
|
|
670
|
+
*/
|
|
671
|
+
function journaledUnitResultJson(outcome) {
|
|
672
|
+
if (outcome.result !== undefined)
|
|
673
|
+
return JSON.stringify(outcome.result);
|
|
674
|
+
if (outcome.ok)
|
|
675
|
+
return outcome.text ? JSON.stringify(outcome.text) : null;
|
|
676
|
+
const parts = [outcome.error, outcome.text].filter((part) => Boolean(part && part.trim()));
|
|
677
|
+
if (parts.length === 0)
|
|
678
|
+
return null;
|
|
679
|
+
return JSON.stringify(clip(parts.join("\n--- unit output ---\n"), WORKFLOW_UNIT_DIAGNOSTIC_CLIP));
|
|
680
|
+
}
|
|
552
681
|
/** Journal one dispatch attempt: insert row, events, dispatch, finish row. */
|
|
553
682
|
async function dispatchJournaledAttempt(input) {
|
|
554
683
|
const { plan, workUnit, ctx, dispatcher, attemptId, inputHash } = input;
|
|
@@ -559,35 +688,57 @@ async function dispatchJournaledAttempt(input) {
|
|
|
559
688
|
// row — nothing was dispatched (same contract as an expression failure).
|
|
560
689
|
let worktreePath;
|
|
561
690
|
if (input.worktreeBase !== undefined) {
|
|
562
|
-
const created = createUnitWorktree(input.worktreeBase, ctx.runId, attemptId);
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
691
|
+
const created = await createUnitWorktree(input.worktreeBase, ctx.runId, attemptId);
|
|
692
|
+
// Reported BEFORE the failure check: when the worktree could not be minted,
|
|
693
|
+
// the leftover moved aside is the only copy of the prior attempt's
|
|
694
|
+
// uncollected work, so that is exactly when its path must not be swallowed.
|
|
566
695
|
if (created.preservedLeftover !== undefined) {
|
|
567
696
|
// Never destroy a dirty (or unverifiable) leftover from a prior
|
|
568
697
|
// invocation of the same attempt — it was moved aside instead.
|
|
569
698
|
warn(`Workflow unit ${attemptId}: a previous attempt left uncollected work in its isolation worktree; ` +
|
|
570
699
|
`preserved at ${created.preservedLeftover}`);
|
|
571
700
|
}
|
|
701
|
+
if (!created.ok) {
|
|
702
|
+
return { unitId: request.unitId, ok: false, failureReason: "worktree_failed", error: created.error };
|
|
703
|
+
}
|
|
572
704
|
worktreePath = created.path;
|
|
573
705
|
request = { ...request, cwd: worktreePath };
|
|
574
706
|
}
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
707
|
+
const startedAt = new Date().toISOString();
|
|
708
|
+
try {
|
|
709
|
+
await enqueueUnitWrite(async () => {
|
|
710
|
+
await withWorkflowRunsRepo((repo) => repo.insertUnit({
|
|
711
|
+
runId: ctx.runId,
|
|
712
|
+
unitId: attemptId,
|
|
713
|
+
stepId: plan.stepId,
|
|
714
|
+
nodeId: workUnit.nodeId,
|
|
715
|
+
parentUnitId: workUnit.isFanOut ? `${plan.stepId}.map` : null,
|
|
716
|
+
phase: null,
|
|
717
|
+
runner: workUnit.runner,
|
|
718
|
+
engine: request.engine?.name ?? null,
|
|
719
|
+
model: request.invocation?.model ?? null,
|
|
720
|
+
inputHash,
|
|
721
|
+
worktreePath: worktreePath ?? null,
|
|
722
|
+
startedAt,
|
|
723
|
+
}));
|
|
724
|
+
});
|
|
725
|
+
}
|
|
726
|
+
catch (err) {
|
|
727
|
+
// A failed dispatch-row insert means NOTHING dispatched (the row is the
|
|
728
|
+
// dispatch's precondition) — fail the unit with the real cause instead of
|
|
729
|
+
// letting the throw escape into the scheduler, where a swallowed worker
|
|
730
|
+
// error is indistinguishable from "never claimed" and used to be
|
|
731
|
+
// misreported as an aborted, never-dispatched unit.
|
|
732
|
+
if (worktreePath !== undefined && input.worktreeBase !== undefined) {
|
|
733
|
+
await cleanupUnitWorktree(input.worktreeBase, worktreePath);
|
|
734
|
+
}
|
|
735
|
+
return {
|
|
736
|
+
unitId: request.unitId,
|
|
737
|
+
ok: false,
|
|
738
|
+
failureReason: "dispatch_error",
|
|
739
|
+
error: `unit "${attemptId}" could not journal its dispatch row (nothing was dispatched): ${message(err)}`,
|
|
740
|
+
};
|
|
741
|
+
}
|
|
591
742
|
// Ids/status only — instructions and results are workflow-authored content
|
|
592
743
|
// and stay out of the events stream (07 P1-B).
|
|
593
744
|
appendEvent({
|
|
@@ -596,51 +747,143 @@ async function dispatchJournaledAttempt(input) {
|
|
|
596
747
|
metadata: { runId: ctx.runId, stepId: plan.stepId, unitId: attemptId },
|
|
597
748
|
});
|
|
598
749
|
const outcome = redactUnitOutcome(await dispatchUnit(request, dispatcher), request.sensitiveValues ?? []);
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
750
|
+
const finishedAt = new Date().toISOString();
|
|
751
|
+
// A dispatched unit's outcome is NEVER silently discarded. The single-driver
|
|
752
|
+
// guard lives at the ROW level (`finishUnitFromDispatch`: still `running`,
|
|
753
|
+
// still this dispatch's `started_at`): a stolen run's new driver re-dispatches
|
|
754
|
+
// through insertUnit, which REPLACES the row with a fresh started_at, so the
|
|
755
|
+
// stale driver's finish matches nothing and can never clobber the new
|
|
756
|
+
// driver's live dispatch. A row that IS still ours is finished with the real
|
|
757
|
+
// result even when the run went non-active or the lease moved mid-flight —
|
|
758
|
+
// dropping it would leave the row `running` and make a later resume
|
|
759
|
+
// re-dispatch side-effecting work that already ran and already spent tokens.
|
|
760
|
+
// Persisting a unit result never advances the run; spine advancement stays
|
|
761
|
+
// lease-guarded in completeWorkflowStep.
|
|
762
|
+
let journalError;
|
|
763
|
+
try {
|
|
764
|
+
await enqueueUnitWrite(() => withWorkflowRunsRepo((repo) => repo.immediateTransaction((db) => {
|
|
765
|
+
const finished = repo.finishUnitFromDispatch({
|
|
766
|
+
runId: ctx.runId,
|
|
767
|
+
unitId: attemptId,
|
|
768
|
+
status: outcome.ok ? "completed" : "failed",
|
|
769
|
+
resultJson: journaledUnitResultJson(outcome),
|
|
770
|
+
tokens: outcome.tokens ?? null,
|
|
771
|
+
failureReason: outcome.failureReason ?? null,
|
|
772
|
+
// Harness-native session id (P2): journaled so resume can replay the
|
|
773
|
+
// harness's own context cache (e.g. `codex exec resume <id>`).
|
|
774
|
+
sessionId: outcome.sessionId ?? null,
|
|
775
|
+
finishedAt,
|
|
776
|
+
dispatchStartedAt: startedAt,
|
|
777
|
+
});
|
|
778
|
+
if (!finished) {
|
|
779
|
+
if (!repo.getUnit(ctx.runId, attemptId)) {
|
|
780
|
+
// The dispatch row vanished (run deleted mid-flight, journal
|
|
781
|
+
// tampered): the same journaling-bug contract finishUnit throws
|
|
782
|
+
// for — surfaced below as a journal-write failure, never a no-op.
|
|
783
|
+
throw new Error(`finishUnit updated no row: no unit "${attemptId}" exists for run "${ctx.runId}". ` +
|
|
784
|
+
`The dispatch row this invocation inserted is gone, so the unit's terminal state cannot be journaled.`);
|
|
785
|
+
}
|
|
786
|
+
// The row exists but is no longer this dispatch's `running` row:
|
|
787
|
+
// another engine invocation re-dispatched (or finished) the unit
|
|
788
|
+
// after taking the run. Its journal owns the unit now — writing
|
|
789
|
+
// would clobber a live dispatch — so the outcome is surfaced
|
|
790
|
+
// loudly instead of silently dropped.
|
|
791
|
+
warn(`Workflow unit ${attemptId} (run ${ctx.runId}) ${outcome.ok ? "completed" : `failed (${outcome.failureReason ?? "error"})`}, ` +
|
|
792
|
+
`but its journal row was re-dispatched by another engine invocation mid-flight — refusing to overwrite ` +
|
|
793
|
+
`the new driver's row. This dispatch's result is not journaled.`);
|
|
794
|
+
return;
|
|
795
|
+
}
|
|
796
|
+
const run = repo.getRunById(ctx.runId);
|
|
797
|
+
if (run?.status !== "active" ||
|
|
798
|
+
(ctx.leaseHolder !== undefined && run.engine_lease_holder !== ctx.leaseHolder)) {
|
|
799
|
+
warn(`Workflow unit ${attemptId} (run ${ctx.runId}): the run ` +
|
|
800
|
+
`${run?.status !== "active" ? `is now ${run?.status ?? "gone"}` : `lease moved to ${run?.engine_lease_holder ?? "(nobody)"}`} ` +
|
|
801
|
+
`while the unit was in flight; its result was journaled so a resume reuses it instead of re-dispatching.`);
|
|
802
|
+
}
|
|
803
|
+
insertEventStrict(db, {
|
|
804
|
+
eventType: "workflow_unit_finished",
|
|
805
|
+
ts: finishedAt,
|
|
806
|
+
ref: ctx.workflowRef,
|
|
807
|
+
metadata: {
|
|
808
|
+
runId: ctx.runId,
|
|
809
|
+
stepId: plan.stepId,
|
|
810
|
+
unitId: attemptId,
|
|
811
|
+
status: outcome.ok ? "completed" : "failed",
|
|
812
|
+
...(outcome.failureReason ? { failureReason: outcome.failureReason } : {}),
|
|
813
|
+
...(outcome.tokens !== undefined ? { tokens: outcome.tokens } : {}),
|
|
814
|
+
},
|
|
815
|
+
});
|
|
816
|
+
})));
|
|
817
|
+
}
|
|
818
|
+
catch (err) {
|
|
819
|
+
journalError = err;
|
|
820
|
+
}
|
|
821
|
+
// Worktree lifecycle epilogue: a CLEAN worktree is removed; a DIRTY one is
|
|
822
|
+
// retained and logged — the unit left uncollected work, and its journaled
|
|
823
|
+
// worktree_path says where. Cleanup is best-effort observability, never a
|
|
824
|
+
// unit failure, so it is STARTED here and awaited at the step barrier: the
|
|
825
|
+
// removal serializes on the same per-repo chain as every sibling's
|
|
826
|
+
// `git worktree add`, and awaiting it in this unit's scheduler slot made a
|
|
827
|
+
// finished unit wait out other units' full checkouts before its worker could
|
|
828
|
+
// claim the next item.
|
|
633
829
|
if (worktreePath !== undefined && input.worktreeBase !== undefined) {
|
|
634
|
-
const
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
830
|
+
const worktreeBase = input.worktreeBase;
|
|
831
|
+
input.pendingWorktreeCleanups.push((async () => {
|
|
832
|
+
try {
|
|
833
|
+
const cleanup = await cleanupUnitWorktree(worktreeBase, worktreePath);
|
|
834
|
+
if (cleanup.dirty) {
|
|
835
|
+
warn(`Workflow unit ${attemptId} left uncommitted changes in its isolation worktree; retained at ${worktreePath}`);
|
|
836
|
+
}
|
|
837
|
+
else if (!cleanup.removed) {
|
|
838
|
+
warn(`Workflow unit ${attemptId}: could not clean up isolation worktree ${worktreePath}: ${cleanup.error}`);
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
catch (err) {
|
|
842
|
+
warn(`Workflow unit ${attemptId}: could not clean up isolation worktree ${worktreePath}: ${message(err)}`);
|
|
843
|
+
}
|
|
844
|
+
})());
|
|
845
|
+
}
|
|
846
|
+
// A journal-write failure AFTER a successful dispatch is its own loud
|
|
847
|
+
// failure class: the unit's work ran (and may have succeeded), but its
|
|
848
|
+
// terminal state could not be recorded, so the row may be stuck `running`.
|
|
849
|
+
// It must never masquerade as "not dispatched" — the outcome names the unit
|
|
850
|
+
// and the real cause, and executeStepPlan fails the step hard on it
|
|
851
|
+
// (out-of-taxonomy reason, so retry.on can never re-dispatch the work).
|
|
852
|
+
if (journalError !== undefined) {
|
|
853
|
+
return {
|
|
854
|
+
unitId: request.unitId,
|
|
855
|
+
ok: false,
|
|
856
|
+
failureReason: "journal_write_failed",
|
|
857
|
+
error: `unit "${attemptId}" dispatched and ${outcome.ok ? "completed" : `failed (${outcome.failureReason ?? "error"})`}, ` +
|
|
858
|
+
`but its result could not be journaled: ${message(journalError)}`,
|
|
859
|
+
...(outcome.tokens !== undefined ? { tokens: outcome.tokens } : {}),
|
|
860
|
+
...(outcome.sessionId !== undefined ? { sessionId: outcome.sessionId } : {}),
|
|
861
|
+
};
|
|
641
862
|
}
|
|
642
863
|
return outcome;
|
|
643
864
|
}
|
|
865
|
+
/**
|
|
866
|
+
* Strict JSON parse for an `exec` unit's declared-schema output: stdout must be
|
|
867
|
+
* EXACTLY one JSON value (leading/trailing whitespace tolerated, nothing else).
|
|
868
|
+
*
|
|
869
|
+
* Deliberately NOT `parseEmbeddedJsonResponse` — that scan strips code fences
|
|
870
|
+
* and hunts for a JSON island inside prose, which is the right forgiving
|
|
871
|
+
* behavior for an LLM and the wrong one for a command, where it would silently
|
|
872
|
+
* promote a JSON fragment found in unrelated log noise as the typed artifact.
|
|
873
|
+
* Returning `undefined` makes `runStructured` report `parse_error` (a real
|
|
874
|
+
* member of the retry taxonomy), so `retry.on: [parse_error]` still works.
|
|
875
|
+
*/
|
|
876
|
+
function parseExecJson(raw) {
|
|
877
|
+
const trimmed = raw.trim();
|
|
878
|
+
if (!trimmed)
|
|
879
|
+
return undefined;
|
|
880
|
+
try {
|
|
881
|
+
return JSON.parse(trimmed);
|
|
882
|
+
}
|
|
883
|
+
catch {
|
|
884
|
+
return undefined;
|
|
885
|
+
}
|
|
886
|
+
}
|
|
644
887
|
/** Transport failures surface as this sentinel so runStructured doesn't retry them. */
|
|
645
888
|
class UnitTransportError extends Error {
|
|
646
889
|
result;
|
|
@@ -682,6 +925,18 @@ async function dispatchUnit(request, dispatcher) {
|
|
|
682
925
|
const schema = request.schema;
|
|
683
926
|
const structured = await runStructured({
|
|
684
927
|
dispatch: dispatchOnce,
|
|
928
|
+
// A command is not re-promptable. Its stdout must be EXACTLY one JSON
|
|
929
|
+
// value: the default embedded-JSON scan (fences, think-blocks, "find the
|
|
930
|
+
// JSON inside the prose") is right for an LLM and wrong for a command,
|
|
931
|
+
// where it would pluck a JSON fragment out of unrelated log noise and
|
|
932
|
+
// promote it as the typed artifact. And it gets exactly ONE attempt —
|
|
933
|
+
// `runStructured`'s corrective retry re-dispatches with feedback, which
|
|
934
|
+
// for a command means running a SIDE-EFFECTING process a second time
|
|
935
|
+
// with byte-identical argv: it cannot produce different output, and it
|
|
936
|
+
// can produce a second deployment. Declared `retry:` still applies (the
|
|
937
|
+
// executor's own loop), because that is a policy the author opted into
|
|
938
|
+
// per failure reason.
|
|
939
|
+
...(request.exec ? { parse: parseExecJson, maxAttempts: 1 } : {}),
|
|
685
940
|
validate: (candidate) => {
|
|
686
941
|
const errors = validateJsonSchemaSubset(candidate, schema);
|
|
687
942
|
return errors.length === 0 ? { ok: true, value: candidate } : { ok: false, errors };
|
|
@@ -701,12 +956,12 @@ async function dispatchUnit(request, dispatcher) {
|
|
|
701
956
|
}
|
|
702
957
|
const text = await dispatchOnce();
|
|
703
958
|
// Normalize an EMPTY successful output to "no text". `finishUnit` journals
|
|
704
|
-
// result_json = NULL for a falsy text, so durable-reuse
|
|
705
|
-
//
|
|
706
|
-
//
|
|
707
|
-
//
|
|
708
|
-
//
|
|
709
|
-
//
|
|
959
|
+
// result_json = NULL for a falsy text, so durable-reuse rehydrates NO text
|
|
960
|
+
// from the row (unitOutcomeFromRow). Preserving `text: ""` only in this live
|
|
961
|
+
// outcome would make the LIVE step artifact ("") diverge from the artifact a
|
|
962
|
+
// resume rebuilds from the row (null) — the exact byte-identical-graph
|
|
963
|
+
// violation the cardinal rule forbids. Treating empty as absent keeps live
|
|
964
|
+
// dispatch and engine resume identical.
|
|
710
965
|
return { unitId: request.unitId, ok: true, ...(text ? { text } : {}), ...captured() };
|
|
711
966
|
}
|
|
712
967
|
catch (err) {
|
|
@@ -778,14 +1033,46 @@ async function resolveEnvBindings(refs) {
|
|
|
778
1033
|
export function buildAgentDispatchRequest(request, prompt) {
|
|
779
1034
|
return {
|
|
780
1035
|
prompt,
|
|
781
|
-
...(request.
|
|
782
|
-
...(request.invocation
|
|
1036
|
+
...(request.systemPrompt ? { systemPrompt: request.systemPrompt } : {}),
|
|
1037
|
+
...(request.invocation?.model ? { model: request.invocation.model } : {}),
|
|
1038
|
+
...(request.invocation?.model ? { modelIsExact: true } : {}),
|
|
783
1039
|
...(request.schema ? { schema: request.schema } : {}),
|
|
784
1040
|
};
|
|
785
1041
|
}
|
|
786
1042
|
export const defaultUnitDispatcher = async (request, feedback) => {
|
|
1043
|
+
// `exec` units are dispatched FIRST and separately: they name no engine, so
|
|
1044
|
+
// none of the engine-resolution below applies. `feedback` is deliberately
|
|
1045
|
+
// ignored — a corrective re-prompt is meaningless to a fixed argv, and
|
|
1046
|
+
// `dispatchUnit` already pins exec structured output to a single attempt.
|
|
1047
|
+
if (request.exec) {
|
|
1048
|
+
return runExecUnit({
|
|
1049
|
+
unitId: request.unitId,
|
|
1050
|
+
exec: request.exec,
|
|
1051
|
+
// Worktree isolation supplies `cwd`; otherwise the unit runs in the
|
|
1052
|
+
// engine invocation's own working directory.
|
|
1053
|
+
baseDir: request.cwd ?? process.cwd(),
|
|
1054
|
+
...(request.env ? { env: request.env } : {}),
|
|
1055
|
+
...(request.execContext ? { context: request.execContext } : {}),
|
|
1056
|
+
// A declared `output:` schema is what makes an output-cap overflow fatal:
|
|
1057
|
+
// stdout must then parse as exactly one JSON value, which a truncated
|
|
1058
|
+
// prefix cannot. Without one, overflow is marked in the artifact and the
|
|
1059
|
+
// command's own exit code decides the unit. See `exec-unit.ts`.
|
|
1060
|
+
...(request.schema ? { hasOutputSchema: true } : {}),
|
|
1061
|
+
timeoutMs: request.timeoutMs,
|
|
1062
|
+
...(request.signal ? { signal: request.signal } : {}),
|
|
1063
|
+
});
|
|
1064
|
+
}
|
|
1065
|
+
if (!request.engine || !request.invocation) {
|
|
1066
|
+
return {
|
|
1067
|
+
ok: false,
|
|
1068
|
+
text: "",
|
|
1069
|
+
failureReason: "dispatch_error",
|
|
1070
|
+
error: `unit "${request.unitId}" has neither a frozen engine snapshot nor an exec command to dispatch.`,
|
|
1071
|
+
};
|
|
1072
|
+
}
|
|
1073
|
+
const engineRequest = request;
|
|
787
1074
|
const prompt = feedback ? `${request.prompt}\n\n${feedback}` : request.prompt;
|
|
788
|
-
const resolved = frozenUnitRunner(
|
|
1075
|
+
const resolved = frozenUnitRunner(engineRequest);
|
|
789
1076
|
// `env` bindings can only reach a child process. The agent (CLI) runner
|
|
790
1077
|
// spawns one per call, and the sdk runner now injects them for real via the
|
|
791
1078
|
// env-keyed opencode server registry (sdk-runner.ts module doc, open seam
|
|
@@ -817,7 +1104,13 @@ export const defaultUnitDispatcher = async (request, feedback) => {
|
|
|
817
1104
|
const { chatCompletion, LlmCallError } = await import("../../llm/client.js");
|
|
818
1105
|
const connection = resolved.connection;
|
|
819
1106
|
try {
|
|
820
|
-
const
|
|
1107
|
+
const messages = request.systemPrompt
|
|
1108
|
+
? [
|
|
1109
|
+
{ role: "system", content: request.systemPrompt },
|
|
1110
|
+
{ role: "user", content: prompt },
|
|
1111
|
+
]
|
|
1112
|
+
: [{ role: "user", content: prompt }];
|
|
1113
|
+
const text = await chatCompletion(connection, messages, {
|
|
821
1114
|
timeoutMs: request.timeoutMs,
|
|
822
1115
|
...(request.signal ? { signal: request.signal } : {}),
|
|
823
1116
|
// Native structured output where the connection supports it; the
|
|
@@ -887,45 +1180,14 @@ export const defaultUnitDispatcher = async (request, feedback) => {
|
|
|
887
1180
|
...(result.usage ? { usage: result.usage } : {}),
|
|
888
1181
|
};
|
|
889
1182
|
};
|
|
890
|
-
function collectWorkflowDispatchSensitiveValues(workUnit, env) {
|
|
891
|
-
const values = new Set(Object.values(env ?? {}));
|
|
892
|
-
const addCredential = (engine) => {
|
|
893
|
-
if (!engine)
|
|
894
|
-
return;
|
|
895
|
-
if (engine.kind === "llm") {
|
|
896
|
-
for (const name of engine.credential?.names ?? []) {
|
|
897
|
-
const value = process.env[name]?.trim();
|
|
898
|
-
if (value)
|
|
899
|
-
values.add(value);
|
|
900
|
-
}
|
|
901
|
-
return;
|
|
902
|
-
}
|
|
903
|
-
for (const name of engine.envPassthrough) {
|
|
904
|
-
const value = process.env[name];
|
|
905
|
-
if (!isEnvPassthroughValueSafeToExpose(name, value) && value)
|
|
906
|
-
values.add(value);
|
|
907
|
-
}
|
|
908
|
-
};
|
|
909
|
-
addCredential(workUnit.engine);
|
|
910
|
-
addCredential(workUnit.fallbackEngine);
|
|
911
|
-
return collectSensitiveValues(values);
|
|
912
|
-
}
|
|
913
|
-
function redactUnitOutcome(outcome, sensitiveValues) {
|
|
914
|
-
const redacted = redactSensitiveValue(outcome, sensitiveValues);
|
|
915
|
-
if (outcome.failureReason !== undefined && redacted.failureReason !== outcome.failureReason) {
|
|
916
|
-
redacted.failureReason = "reported_failure";
|
|
917
|
-
}
|
|
918
|
-
return redacted;
|
|
919
|
-
}
|
|
920
1183
|
/**
|
|
921
1184
|
* Map a typed {@link import("../../llm/client.js").LlmCallErrorCode} into the
|
|
922
1185
|
* persisted `AgentFailureReason` taxonomy (agent/spawn.ts) — the ONLY
|
|
923
1186
|
* vocabulary `retry.on` accepts and the journal's `failure_reason` column
|
|
924
1187
|
* carries. Exhaustive over the code union (typecheck fails on drift):
|
|
925
1188
|
*
|
|
926
|
-
* - `timeout` → `timeout` (wall-clock expiry
|
|
927
|
-
*
|
|
928
|
-
* folds into its timeout code)
|
|
1189
|
+
* - `timeout` → `timeout` (wall-clock expiry)
|
|
1190
|
+
* - `aborted` → `aborted` (caller/budget cancellation)
|
|
929
1191
|
* - `rate_limited` → `llm_rate_limit` (HTTP 429 — the canonical transient)
|
|
930
1192
|
* - `parse_error` / `provider_html_error`
|
|
931
1193
|
* → `parse_error` (a response arrived but was not the
|
|
@@ -938,6 +1200,8 @@ function redactUnitOutcome(outcome, sensitiveValues) {
|
|
|
938
1200
|
*/
|
|
939
1201
|
export function llmFailureReasonFor(code) {
|
|
940
1202
|
switch (code) {
|
|
1203
|
+
case "aborted":
|
|
1204
|
+
return "aborted";
|
|
941
1205
|
case "timeout":
|
|
942
1206
|
return "timeout";
|
|
943
1207
|
case "rate_limited":
|
|
@@ -984,38 +1248,13 @@ function frozenUnitRunner(request) {
|
|
|
984
1248
|
const fallback = request.fallbackEngine ? materializeFrozenLlm(request.fallbackEngine, undefined) : undefined;
|
|
985
1249
|
return { kind: "sdk", profile, ...(fallback ? { fallbackConnection: fallback } : {}) };
|
|
986
1250
|
}
|
|
987
|
-
function materializeFrozenLlm(snapshot, invocation) {
|
|
988
|
-
let apiKey;
|
|
989
|
-
for (const name of snapshot.credential?.names ?? []) {
|
|
990
|
-
const candidate = process.env[name]?.trim();
|
|
991
|
-
if (candidate) {
|
|
992
|
-
apiKey = candidate;
|
|
993
|
-
break;
|
|
994
|
-
}
|
|
995
|
-
}
|
|
996
|
-
if (snapshot.credential?.required && !apiKey)
|
|
997
|
-
throw new ConfigError(`Required engine credential ${snapshot.credential.names[0]} is not set.`, "INVALID_CONFIG_FILE");
|
|
998
|
-
const base = {
|
|
999
|
-
provider: snapshot.provider,
|
|
1000
|
-
endpoint: snapshot.endpoint,
|
|
1001
|
-
model: invocation?.model ?? snapshot.model,
|
|
1002
|
-
...(snapshot.temperature !== undefined ? { temperature: snapshot.temperature } : {}),
|
|
1003
|
-
...(snapshot.maxTokens !== undefined ? { maxTokens: snapshot.maxTokens } : {}),
|
|
1004
|
-
...(snapshot.supportsJsonSchema !== undefined ? { supportsJsonSchema: snapshot.supportsJsonSchema } : {}),
|
|
1005
|
-
...(snapshot.extraParams ? { extraParams: snapshot.extraParams } : {}),
|
|
1006
|
-
...(snapshot.contextLength !== undefined ? { contextLength: snapshot.contextLength } : {}),
|
|
1007
|
-
...(snapshot.enableThinking !== undefined ? { enableThinking: snapshot.enableThinking } : {}),
|
|
1008
|
-
...(apiKey ? { apiKey } : {}),
|
|
1009
|
-
};
|
|
1010
|
-
return invocation?.llm ? deepMergeConfig(base, invocation.llm) : base;
|
|
1011
|
-
}
|
|
1012
1251
|
// ── Small helpers ────────────────────────────────────────────────────────────
|
|
1013
1252
|
/**
|
|
1014
1253
|
* Rehydrate a journaled completed unit row into a UnitOutcome (durable-row
|
|
1015
1254
|
* reuse). Delegates to the shared {@link unitOutcomeFromRow} — the reuse path
|
|
1016
1255
|
* only reaches here for completed rows (the caller guards `status ===
|
|
1017
|
-
* "completed"`), so
|
|
1018
|
-
*
|
|
1256
|
+
* "completed"`), so a reused unit contributes exactly what its original
|
|
1257
|
+
* dispatch did.
|
|
1019
1258
|
*/
|
|
1020
1259
|
function reuseCompletedUnit(unitId, row, hasSchema) {
|
|
1021
1260
|
return unitOutcomeFromRow(unitId, row, hasSchema);
|