akm-cli 0.9.1 → 0.9.2-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +103 -28
- package/README.md +3 -1
- package/SECURITY.md +1 -1
- package/STABILITY.md +1 -1
- package/dist/akm +2 -2
- package/dist/akm-migrate +2 -2
- package/dist/assets/hints/cli-hints-full.md +14 -9
- package/dist/assets/improve-strategies/proactive-maintenance.json +1 -1
- package/dist/assets/improve-strategies/reflect-distill.json +1 -1
- package/dist/assets/models.json +35 -0
- package/dist/assets/stash-skeleton/facts/conventions/backlinks.md +3 -4
- package/dist/assets/stash-skeleton/facts/conventions/organization.md +1 -3
- package/dist/assets/tasks/core/extract.yml +6 -5
- package/dist/assets/tasks/core/improve.yml +6 -5
- package/dist/assets/tasks/core/index-refresh.yml +6 -5
- package/dist/assets/tasks/core/sync.yml +6 -5
- package/dist/assets/tasks/core/version-check.yml +6 -5
- package/dist/assets/tasks/improve/akm-graph-refresh-weekly.yml +6 -5
- package/dist/assets/tasks/improve/akm-improve-catchup.yml +6 -5
- package/dist/assets/tasks/improve/akm-improve-consolidate.yml +6 -5
- package/dist/assets/tasks/improve/akm-improve-frequent.yml +6 -5
- package/dist/assets/tasks/improve/akm-improve-nightly.yml +6 -5
- package/dist/cli/confirm.js +2 -2
- package/dist/cli/parse-args.js +3 -24
- package/dist/cli/retired-commands.js +1 -1
- package/dist/cli/shared.js +2 -2
- package/dist/cli.js +11 -9
- package/dist/commands/agent/agent-dispatch.js +55 -89
- package/dist/commands/agent/contribute-cli.js +12 -45
- package/dist/commands/command/builtin-action.js +32 -0
- package/dist/commands/command/command-cli.js +99 -0
- package/dist/commands/command/command-execution.js +308 -0
- package/dist/commands/command/execution-source-loader.js +176 -0
- package/dist/commands/command/portable-template.js +60 -0
- package/dist/commands/config-cli.js +10 -4
- package/dist/commands/env/env.js +4 -2
- package/dist/commands/feedback-cli.js +1 -1
- package/dist/commands/health/checks.js +241 -29
- package/dist/commands/health/html-report.js +0 -14
- package/dist/commands/health/report-view-model.js +0 -1
- package/dist/commands/health/surfaces.js +6 -7
- package/dist/commands/health/types.js +0 -2
- package/dist/commands/health.js +63 -18
- package/dist/commands/improve/collapse-detector.js +5 -6
- package/dist/commands/improve/consolidate.js +251 -214
- package/dist/commands/improve/distill/promote-memory.js +71 -34
- package/dist/commands/improve/distill/quality-gate.js +17 -5
- package/dist/commands/improve/distill.js +232 -155
- package/dist/commands/improve/eligibility.js +112 -79
- package/dist/commands/improve/execution.js +57 -0
- package/dist/commands/improve/extract-cli.js +5 -5
- package/dist/commands/improve/extract-prompt.js +64 -22
- package/dist/commands/improve/extract.js +608 -360
- package/dist/commands/improve/improve-strategies.js +43 -14
- package/dist/commands/improve/improve.js +249 -29
- package/dist/commands/improve/loop-stages.js +11 -17
- package/dist/commands/improve/memory/memory-contradiction-detect.js +90 -66
- package/dist/commands/improve/outcome-loop.js +22 -38
- package/dist/commands/improve/planner.js +134 -0
- package/dist/commands/improve/preparation.js +730 -409
- package/dist/commands/improve/reflect.js +386 -223
- package/dist/commands/improve/run-context.js +3 -4
- package/dist/commands/improve/salience.js +6 -58
- package/dist/commands/improve/session-asset.js +12 -12
- package/dist/commands/lint/index.js +101 -29
- package/dist/commands/migrate-cli.js +11 -69
- package/dist/commands/migration-tool.js +6 -9
- package/dist/commands/models-cli.js +27 -0
- package/dist/commands/proposal/drain.js +258 -186
- package/dist/commands/proposal/proposal-cli.js +32 -10
- package/dist/commands/proposal/proposal.js +2 -5
- package/dist/commands/proposal/propose.js +192 -172
- package/dist/commands/proposal/repository.js +54 -91
- package/dist/commands/proposal/validators/proposal-validators.js +9 -7
- package/dist/commands/read/curate.js +53 -22
- package/dist/commands/read/registry-search.js +25 -9
- package/dist/commands/read/remember-cli.js +14 -2
- package/dist/commands/read/search.js +10 -4
- package/dist/commands/read/show.js +139 -153
- package/dist/commands/registry-cli.js +16 -7
- package/dist/commands/remember.js +33 -18
- package/dist/commands/sources/add-cli.js +19 -178
- package/dist/commands/sources/bundle-cli.js +15 -3
- package/dist/commands/sources/dangerous-env-audit.js +135 -0
- package/dist/commands/sources/info.js +2 -1
- package/dist/commands/sources/installed-stashes.js +901 -177
- package/dist/commands/sources/schema-repair.js +174 -95
- package/dist/commands/sources/self-update.js +30 -74
- package/dist/commands/sources/source-add.js +3 -5
- package/dist/commands/sources/sources-cli.js +2 -15
- package/dist/commands/sources/update-transaction.js +220 -0
- package/dist/commands/tasks/tasks-cli.js +3 -3
- package/dist/commands/tasks/tasks.js +736 -317
- package/dist/commands/workflow-cli.js +2 -2
- package/dist/core/adapter/adapters/agent-skills-adapter.js +3 -0
- package/dist/core/adapter/adapters/akm-adapter.js +85 -35
- package/dist/core/adapter/adapters/akm-lint.js +54 -39
- package/dist/core/adapter/adapters/akm-metadata.js +45 -45
- package/dist/core/adapter/adapters/akm-task-adapter.js +32 -49
- package/dist/core/adapter/adapters/akm-workflow-adapter.js +38 -23
- package/dist/core/adapter/adapters/dotenv-adapter.js +30 -1
- package/dist/core/adapter/adapters/generic-files-adapter.js +11 -0
- package/dist/core/adapter/adapters/index.js +0 -9
- package/dist/core/adapter/adapters/llm-wiki-adapter.js +4 -0
- package/dist/core/adapter/adapters/okf-adapter.js +4 -0
- package/dist/core/adapter/adapters/opencode-adapter.js +5 -8
- package/dist/core/adapter/adapters/tool-dir-shared.js +63 -6
- package/dist/core/adapter/adapters/website-snapshot-adapter.js +4 -0
- package/dist/core/adapter/execution-source.js +308 -0
- package/dist/core/adapter/recognize-match.js +36 -13
- package/dist/core/adapter/registry.js +0 -9
- package/dist/core/asset/stash-meta.js +94 -4
- package/dist/core/common.js +6 -11
- package/dist/core/config/config-io.js +3 -3
- package/dist/core/config/config-schema.js +18 -40
- package/dist/core/config/config-sources.js +11 -21
- package/dist/core/config/config-walker.js +31 -13
- package/dist/core/config/config.js +23 -26
- package/dist/core/config/schema/engines.js +8 -7
- package/dist/core/config/schema/improve-processes.js +29 -5
- package/dist/core/config/schema/index-config.js +0 -27
- package/dist/core/config/schema/primitives.js +1 -23
- package/dist/core/config/schema/sources-bundles.js +13 -16
- package/dist/core/errors.js +2 -0
- package/dist/core/events.js +68 -32
- package/dist/core/extra-params.js +1 -0
- package/dist/core/improve-result.js +315 -0
- package/dist/core/lesson-lint.js +0 -6
- package/dist/core/maintenance-barrier.js +4 -4
- package/dist/core/network-policy.js +152 -0
- package/dist/core/paths.js +1 -1
- package/dist/core/recognition-util.js +4 -4
- package/dist/core/registry-url.js +456 -0
- package/dist/core/state/migrations.js +161 -47
- package/dist/core/state-db.js +453 -80
- package/dist/core/system-error.js +32 -0
- package/dist/core/time.js +2 -12
- package/dist/core/write-source.js +0 -18
- package/dist/execution/directory-identity.js +52 -0
- package/dist/execution/executable-identity.js +107 -0
- package/dist/execution/guarded-source.js +398 -0
- package/dist/execution/json.js +95 -0
- package/dist/{commands/health/types-session-log.js → execution/limits.js} +2 -1
- package/dist/execution/record.js +55 -0
- package/dist/execution/resolved-request.js +730 -0
- package/dist/execution/source.js +320 -0
- package/dist/indexer/bundle-identity-guard.js +5 -4
- package/dist/indexer/db/graph-db.js +33 -0
- package/dist/indexer/graph/graph-boost.js +3 -4
- package/dist/indexer/graph/graph-extraction.js +562 -373
- package/dist/indexer/index-written-assets.js +78 -39
- package/dist/indexer/indexer.js +471 -432
- package/dist/indexer/installations.js +6 -0
- package/dist/indexer/lookup/adapter-concept-owner.js +283 -0
- package/dist/indexer/materialize-embeddings.js +155 -0
- package/dist/indexer/passes/memory-inference.js +227 -174
- package/dist/indexer/passes/metadata.js +263 -118
- package/dist/indexer/scan/doc-to-entry.js +7 -10
- package/dist/indexer/scan/drain-dir.js +51 -23
- package/dist/indexer/search/db-search.js +156 -50
- package/dist/indexer/search/fts-query.js +40 -40
- package/dist/indexer/search/ranking.js +36 -1
- package/dist/indexer/search/search-attribution.js +3 -1
- package/dist/indexer/search/search-fields.js +23 -14
- package/dist/indexer/search/search-hit-enrichers.js +1 -1
- package/dist/indexer/search/search-source.js +7 -16
- package/dist/indexer/search/semantic-status.js +10 -1
- package/dist/indexer/usage/show-usage.js +105 -0
- package/dist/indexer/usage/usage-events.js +7 -2
- package/dist/indexer/walk/matchers.js +40 -10
- package/dist/indexer/walk/path-resolver.js +5 -2
- package/dist/indexer/walk/walker.js +20 -2
- package/dist/integrations/agent/builder-shared.js +3 -6
- package/dist/integrations/agent/conversation-fallback.js +16 -0
- package/dist/integrations/agent/engine-resolution.js +87 -87
- package/dist/integrations/agent/execution-cascade.js +566 -0
- package/dist/integrations/agent/execution-definitions.js +211 -0
- package/dist/integrations/agent/execution-lowering.js +811 -0
- package/dist/integrations/agent/execution-preparation.js +67 -0
- package/dist/integrations/agent/index.js +0 -2
- package/dist/integrations/agent/inline-execution.js +74 -0
- package/dist/integrations/agent/model-map.js +515 -0
- package/dist/integrations/agent/persona-fallback.js +30 -0
- package/dist/integrations/agent/request-lowering.js +186 -0
- package/dist/integrations/agent/runner-dispatch.js +230 -37
- package/dist/integrations/agent/runner.js +12 -83
- package/dist/integrations/harnesses/aider/agent-builder.js +8 -0
- package/dist/integrations/harnesses/aider/index.js +0 -1
- package/dist/integrations/harnesses/amazonq/agent-builder.js +8 -0
- package/dist/integrations/harnesses/amazonq/index.js +0 -1
- package/dist/integrations/harnesses/claude/agent-builder.js +14 -1
- package/dist/integrations/harnesses/claude/index.js +1 -5
- package/dist/integrations/harnesses/claude/session-log.js +3 -33
- package/dist/integrations/harnesses/codex/agent-builder.js +8 -0
- package/dist/integrations/harnesses/codex/index.js +0 -1
- package/dist/integrations/harnesses/copilot/agent-builder.js +8 -0
- package/dist/integrations/harnesses/copilot/index.js +0 -1
- package/dist/integrations/harnesses/gemini/agent-builder.js +8 -0
- package/dist/integrations/harnesses/gemini/index.js +0 -1
- package/dist/integrations/harnesses/index.js +4 -44
- package/dist/integrations/harnesses/opencode/agent-builder.js +16 -9
- package/dist/integrations/harnesses/opencode/index.js +0 -2
- package/dist/integrations/harnesses/opencode/session-log.js +14 -204
- package/dist/integrations/harnesses/opencode-sdk/harness.js +12 -1
- package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +40 -42
- package/dist/integrations/harnesses/openhands/agent-builder.js +8 -0
- package/dist/integrations/harnesses/openhands/index.js +0 -1
- package/dist/integrations/harnesses/pi/agent-builder.js +8 -0
- package/dist/integrations/harnesses/pi/index.js +0 -1
- package/dist/integrations/harnesses/shared.js +0 -1
- package/dist/integrations/harnesses/types.js +1 -3
- package/dist/integrations/lockfile.js +82 -79
- package/dist/integrations/session-logs/index.js +6 -17
- package/dist/integrations/session-logs/provider-base.js +1 -29
- package/dist/llm/client.js +10 -5
- package/dist/llm/embedder.js +6 -7
- package/dist/llm/embedders/local.js +37 -88
- package/dist/llm/embedders/types.js +1 -1
- package/dist/llm/graph-extract.js +75 -50
- package/dist/llm/index-passes.js +43 -5
- package/dist/llm/memory-infer.js +8 -6
- package/dist/llm/metadata-enhance.js +5 -3
- package/dist/llm/structured-call.js +122 -25
- package/dist/output/format-exempt.js +1 -1
- package/dist/output/render-registry.js +0 -16
- package/dist/output/renderers.js +12 -7
- package/dist/output/shapes/curate.js +1 -0
- package/dist/output/shapes/helpers.js +10 -2
- package/dist/output/shapes/passthrough.js +2 -0
- package/dist/output/text/command-format.js +31 -33
- package/dist/output/text/health-format.js +1 -29
- package/dist/output/text/migrate.js +6 -56
- package/dist/output/text/proposal-format.js +16 -1
- package/dist/output/text/workflow-format.js +16 -0
- package/dist/registry/network.js +279 -0
- package/dist/registry/pinned-request-helper.js +247 -0
- package/dist/registry/pinned-transport.js +717 -0
- package/dist/registry/providers/skills-sh.js +18 -6
- package/dist/registry/providers/static-index.js +20 -7
- package/dist/registry/resolve.js +53 -28
- package/dist/scripts/akm-migrate-node.js +19334 -52269
- package/dist/scripts/akm-migrate.js +19270 -51612
- package/dist/setup/registry-stash-loader.js +64 -20
- package/dist/setup/semantic-assets.js +9 -34
- package/dist/setup/setup.js +12 -30
- package/dist/setup/source-identity.js +17 -0
- package/dist/setup/steps/sources.js +36 -15
- package/dist/setup/steps/tasks.js +39 -11
- package/dist/sources/providers/git-provider.js +3 -3
- package/dist/sources/providers/npm.js +2 -2
- package/dist/sources/providers/provider-utils.js +4 -3
- package/dist/sources/providers/website.js +11 -7
- package/dist/sources/snapshot-fetchers/host-guard.js +9 -136
- package/dist/sources/snapshot-fetchers/website-ingest.js +25 -109
- package/dist/sources/website-url.js +73 -0
- package/dist/storage/engines/sqlite-migrations.js +81 -26
- package/dist/storage/managed-db.js +27 -24
- package/dist/storage/repositories/events-repository.js +3 -0
- package/dist/storage/repositories/index-connection.js +42 -10
- package/dist/storage/repositories/index-entries-repository.js +203 -229
- package/dist/storage/repositories/index-entry-mapper.js +8 -12
- package/dist/storage/repositories/index-entry-schema.js +255 -0
- package/dist/storage/repositories/index-fts-repository.js +64 -71
- package/dist/storage/repositories/index-llm-cache-repository.js +8 -13
- package/dist/storage/repositories/index-meta-repository.js +0 -11
- package/dist/storage/repositories/index-schema.js +74 -350
- package/dist/storage/repositories/index-utility-repository.js +12 -17
- package/dist/storage/repositories/index-vec-repository.js +56 -7
- package/dist/storage/repositories/proposals-repository.js +4 -127
- package/dist/storage/repositories/registry-cache.js +2 -1
- package/dist/storage/repositories/task-history-repository.js +20 -40
- package/dist/storage/repositories/workflow-runs-repository.js +228 -129
- package/dist/storage/sqlite-read-snapshot.js +148 -0
- package/dist/tasks/backends/cron.js +170 -42
- package/dist/tasks/backends/index.js +1 -1
- package/dist/tasks/backends/launchd.js +787 -202
- package/dist/tasks/backends/schtasks.js +282 -83
- package/dist/tasks/embedded.js +7 -7
- package/dist/tasks/frozen-script.js +50 -0
- package/dist/tasks/resolve-akm-bin.js +5 -1
- package/dist/tasks/runner.js +239 -251
- package/dist/tasks/runtime-v3.js +281 -0
- package/dist/tasks/scheduler-binding.js +272 -0
- package/dist/tasks/scheduler-invocation.js +57 -43
- package/dist/tasks/scheduler-sync.js +654 -0
- package/dist/tasks/source-v3.js +752 -0
- package/dist/tasks/standalone-script-entry.js +5 -0
- package/dist/tasks/task-id.js +29 -0
- package/dist/workflows/authoring/authoring.js +15 -32
- package/dist/workflows/exec/dispatch-redaction.js +14 -8
- package/dist/workflows/exec/exec-unit.js +7 -28
- package/dist/workflows/exec/frozen-judge.js +57 -89
- package/dist/workflows/exec/lowering-notices.js +23 -0
- package/dist/workflows/exec/native-executor.js +301 -458
- package/dist/workflows/exec/param-secrets.js +4 -3
- package/dist/workflows/exec/run-workflow.js +26 -32
- package/dist/workflows/exec/step-work.js +105 -109
- package/dist/workflows/exec/unit-dispatch.js +103 -27
- package/dist/workflows/exec/unit-writer.js +3 -3
- package/dist/workflows/exec/worktree.js +2 -2
- package/dist/workflows/ir/compile.js +86 -72
- package/dist/workflows/ir/environment-v4.js +328 -0
- package/dist/workflows/ir/freeze-v4.js +122 -0
- package/dist/workflows/ir/plan-hash.js +13 -7
- package/dist/workflows/ir/schema-v4.js +525 -0
- package/dist/workflows/ir/schema.js +25 -284
- package/dist/workflows/ir/source-freeze-v4.js +506 -0
- package/dist/workflows/parser.js +27 -24
- package/dist/workflows/program/schema.js +1 -2
- package/dist/workflows/renderer.js +42 -29
- package/dist/workflows/resource-limits.js +4 -5
- package/dist/workflows/runtime/agent-identity.js +11 -13
- package/dist/workflows/runtime/plan-classifier.js +8 -8
- package/dist/workflows/runtime/runs.js +27 -43
- package/dist/workflows/runtime/workflow-asset-loader.js +45 -205
- package/dist/workflows/source-files.js +373 -0
- package/dist/workflows/source-ir/compile.js +196 -0
- package/dist/workflows/source-ir/github-yaml.js +577 -0
- package/dist/workflows/source-ir/ordering.js +38 -0
- package/dist/workflows/source-ir/program.js +50 -0
- package/dist/workflows/source-ir/result.js +26 -0
- package/dist/workflows/source-ir/schema.js +772 -0
- package/dist/workflows/source-ir/semantics.js +242 -0
- package/dist/workflows/source-ir/uses.js +14 -0
- package/docs/README.md +2 -0
- package/docs/migration/README.md +3 -1
- package/docs/migration/release-notes/0.9.2.md +55 -0
- package/docs/migration/release-notes/README.md +5 -0
- package/docs/migration/v0.8-to-v0.9.md +76 -1077
- package/docs/migration/v0.9.0-troubleshooting.md +104 -516
- package/docs/migration/v0.9.1-to-v0.9.2.md +150 -0
- package/docs/reference/README.md +1 -0
- package/docs/reference/cli.md +230 -98
- package/docs/reference/configuration.md +159 -36
- package/docs/reference/data-and-telemetry.md +19 -1
- package/docs/reference/supported-formats.md +23 -3
- package/docs/reference/tasks.md +182 -0
- package/docs/reference/workflow-schema.md +91 -40
- package/docs/reference/workflows.md +33 -6
- package/package.json +10 -6
- package/schemas/akm-config.json +372 -224
- package/schemas/akm-task.json +324 -80
- package/schemas/akm-workflow.json +6 -9
- package/dist/core/migration-operation.js +0 -75
- package/dist/integrations/agent/model-aliases.js +0 -74
- package/dist/tasks/parser.js +0 -380
- package/dist/tasks/schema.js +0 -123
- package/dist/tasks/validator.js +0 -80
- package/dist/workflows/ir/freeze.js +0 -320
- package/dist/workflows/runtime/document-cache.js +0 -13
|
@@ -78,34 +78,24 @@ function assertLockfilePathReadable(target) {
|
|
|
78
78
|
"records, and writing over them would destroy every bundle they track. Fix the ownership or mode of that " +
|
|
79
79
|
"path (or point AKM_DATA_DIR / XDG_DATA_HOME somewhere this user owns) and retry.", "DATA_DIR_UNREADABLE");
|
|
80
80
|
}
|
|
81
|
-
|
|
82
|
-
* Like {@link readLockfile}, but THROWS instead of silently degrading to `[]`
|
|
83
|
-
* when the on-disk lockfile exists yet is not parseable JSON or not a JSON
|
|
84
|
-
* array (R-012).
|
|
85
|
-
*
|
|
86
|
-
* `readLockfile`'s fail-open contract is intentional for READ paths — a
|
|
87
|
-
* corrupt lock degrades a managed bundle to "unmanaged" rather than erroring
|
|
88
|
-
* every read-only command (`list`, `installed-stashes`, …). But
|
|
89
|
-
* {@link upsertLockEntry} and {@link removeLockEntry} read the current
|
|
90
|
-
* entries and then WRITE `[...entries, change]` back out; if that read
|
|
91
|
-
* silently returned `[]` for a corrupt file, the write would silently
|
|
92
|
-
* replace the corrupt file with one containing only the single new/changed
|
|
93
|
-
* entry — permanently destroying every other surviving lock record. Write
|
|
94
|
-
* paths use this strict variant so a corrupt lockfile fails the operation
|
|
95
|
-
* loudly (matching the guard `mergeLockEntriesSync` already applies) instead
|
|
96
|
-
* of quietly deleting user state. A missing file is NOT corruption — there
|
|
97
|
-
* is nothing to preserve, so that case still returns `[]`. Entries that fail
|
|
98
|
-
* per-entry validation are still tolerated (filtered out), matching
|
|
99
|
-
* `readLockfile`'s existing shape-tolerant behavior.
|
|
100
|
-
*/
|
|
101
|
-
function readLockfileOrThrow() {
|
|
81
|
+
function readLockfileSnapshotOrThrow() {
|
|
102
82
|
const lockfilePath = getLockfilePath();
|
|
83
|
+
let fd;
|
|
103
84
|
let raw;
|
|
85
|
+
let mode;
|
|
104
86
|
try {
|
|
105
|
-
|
|
87
|
+
// One descriptor owns both byte and metadata observation so a rename or
|
|
88
|
+
// chmod between separate path-based calls cannot synthesize a generation
|
|
89
|
+
// that never existed on disk.
|
|
90
|
+
fd = fs.openSync(lockfilePath, "r");
|
|
91
|
+
raw = fs.readFileSync(fd, "utf8");
|
|
92
|
+
mode = fs.fstatSync(fd).mode & 0o777;
|
|
106
93
|
}
|
|
107
94
|
catch (err) {
|
|
108
95
|
rethrowIfTestIsolationError(err);
|
|
96
|
+
if (err.code === "ENOENT") {
|
|
97
|
+
return { entries: [], raw: null, mode: 0o600 };
|
|
98
|
+
}
|
|
109
99
|
// "Missing file" is the only failure with nothing to preserve. An
|
|
110
100
|
// UNREADABLE lockfile has everything to preserve and we cannot see it —
|
|
111
101
|
// degrading it to `[]` here is precisely the destructive overwrite this
|
|
@@ -114,7 +104,11 @@ function readLockfileOrThrow() {
|
|
|
114
104
|
// happy path costs no extra syscall and the answer describes the failure
|
|
115
105
|
// we actually got.
|
|
116
106
|
assertLockfilePathReadable(lockfilePath);
|
|
117
|
-
return [];
|
|
107
|
+
return { entries: [], raw: null, mode: 0o600 };
|
|
108
|
+
}
|
|
109
|
+
finally {
|
|
110
|
+
if (fd !== undefined)
|
|
111
|
+
fs.closeSync(fd);
|
|
118
112
|
}
|
|
119
113
|
let parsed;
|
|
120
114
|
try {
|
|
@@ -136,7 +130,23 @@ function readLockfileOrThrow() {
|
|
|
136
130
|
throw new ConfigError(`Refusing to modify lockfile ${lockfilePath}: ${invalid.length} existing entr${invalid.length === 1 ? "y is" : "ies are"} malformed. ` +
|
|
137
131
|
"Fix or remove the file by hand before retrying — those entries would otherwise be lost.", "INVALID_CONFIG_FILE");
|
|
138
132
|
}
|
|
139
|
-
return
|
|
133
|
+
return {
|
|
134
|
+
entries: parsed.filter(isValidLockfileEntry),
|
|
135
|
+
raw,
|
|
136
|
+
mode,
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
function readLockfileOrThrow() {
|
|
140
|
+
return readLockfileSnapshotOrThrow().entries;
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Read the exact lock generation that a source-lifecycle transaction may
|
|
144
|
+
* replace. Unlike {@link readLockfile}, this refuses corrupt, malformed, or
|
|
145
|
+
* unreadable state so an update can never treat state it could not snapshot as
|
|
146
|
+
* an empty generation.
|
|
147
|
+
*/
|
|
148
|
+
export function readLockfileForUpdate() {
|
|
149
|
+
return readLockfileSnapshotOrThrow();
|
|
140
150
|
}
|
|
141
151
|
/**
|
|
142
152
|
* The materialized content root recorded in the lock for a managed (git/npm)
|
|
@@ -147,9 +157,8 @@ function readLockfileOrThrow() {
|
|
|
147
157
|
* path (`resolveEntryContentDir` in indexer/search) and the command-layer WRITE
|
|
148
158
|
* path (`adaptConfiguredSource` in core/write-source): consulting it first makes
|
|
149
159
|
* a write land in exactly the directory a read walks. Returns `undefined` for a
|
|
150
|
-
* bundle with no lock `localRoot`
|
|
151
|
-
*
|
|
152
|
-
* callers fall back to the identical provider-path derivation.
|
|
160
|
+
* bundle with no lock `localRoot` or a non-managed type, so both callers fall
|
|
161
|
+
* back to the identical provider-path derivation.
|
|
153
162
|
*/
|
|
154
163
|
export function lockContentRootFor(bundleId, type) {
|
|
155
164
|
if (!bundleId || (type !== "git" && type !== "npm"))
|
|
@@ -191,6 +200,52 @@ export async function compareAndSwapLockfile(expected, desired) {
|
|
|
191
200
|
release();
|
|
192
201
|
}
|
|
193
202
|
}
|
|
203
|
+
/**
|
|
204
|
+
* Publish an update from one exact raw + parsed lockfile generation and return
|
|
205
|
+
* the exact bytes that were written. Formatting-only concurrent edits are a
|
|
206
|
+
* generation change here: rollback must never overwrite bytes it did not
|
|
207
|
+
* publish merely because they parse to an equivalent array.
|
|
208
|
+
*/
|
|
209
|
+
export async function publishLockfileUpdate(expected, desired) {
|
|
210
|
+
const release = await acquireLockSentinel();
|
|
211
|
+
try {
|
|
212
|
+
const current = readLockfileSnapshotOrThrow();
|
|
213
|
+
if (JSON.stringify(current.entries) !== JSON.stringify(expected.entries) ||
|
|
214
|
+
current.raw !== expected.raw ||
|
|
215
|
+
current.mode !== expected.mode) {
|
|
216
|
+
return null;
|
|
217
|
+
}
|
|
218
|
+
writeLockfileUnlocked(desired);
|
|
219
|
+
return readLockfileSnapshotOrThrow();
|
|
220
|
+
}
|
|
221
|
+
finally {
|
|
222
|
+
release();
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
/** Restore an exact raw snapshot after verifying the exact generation we published. */
|
|
226
|
+
export async function compareAndSwapLockfileSnapshot(expected, desired) {
|
|
227
|
+
const release = await acquireLockSentinel();
|
|
228
|
+
try {
|
|
229
|
+
const current = readLockfileSnapshotOrThrow();
|
|
230
|
+
if (JSON.stringify(current.entries) !== JSON.stringify(expected.entries) ||
|
|
231
|
+
current.raw !== expected.raw ||
|
|
232
|
+
current.mode !== expected.mode) {
|
|
233
|
+
return false;
|
|
234
|
+
}
|
|
235
|
+
const lockfilePath = getLockfilePath();
|
|
236
|
+
if (desired.raw === null) {
|
|
237
|
+
fs.rmSync(lockfilePath, { force: true });
|
|
238
|
+
}
|
|
239
|
+
else {
|
|
240
|
+
fs.mkdirSync(path.dirname(lockfilePath), { recursive: true });
|
|
241
|
+
writeFileAtomic(lockfilePath, desired.raw, desired.mode);
|
|
242
|
+
}
|
|
243
|
+
return true;
|
|
244
|
+
}
|
|
245
|
+
finally {
|
|
246
|
+
release();
|
|
247
|
+
}
|
|
248
|
+
}
|
|
194
249
|
export async function upsertLockEntry(entry) {
|
|
195
250
|
const release = await acquireLockSentinel();
|
|
196
251
|
try {
|
|
@@ -205,58 +260,6 @@ export async function upsertLockEntry(entry) {
|
|
|
205
260
|
release();
|
|
206
261
|
}
|
|
207
262
|
}
|
|
208
|
-
/**
|
|
209
|
-
* Synchronously upsert lock entries (merge by id) WITHOUT acquiring the async
|
|
210
|
-
* sentinel — for a caller already holding an exclusive lifecycle lock (e.g.
|
|
211
|
-
* migrate-apply's config lock + maintenance barrier) whose synchronous body
|
|
212
|
-
* cannot await the sentinel's retry loop. No-op for an empty list.
|
|
213
|
-
*/
|
|
214
|
-
function readLockEntriesForMigration() {
|
|
215
|
-
let existing = [];
|
|
216
|
-
const lockfilePath = getLockfilePath();
|
|
217
|
-
// `mergeLockEntriesSync` writes `existing` straight back out, so an
|
|
218
|
-
// unreadable lockfile read as absent would be overwritten with just the
|
|
219
|
-
// migrator's sparse entries (#791). This is also what
|
|
220
|
-
// `assertMigrationLockfileReadable` promises to have checked.
|
|
221
|
-
assertLockfilePathReadable(lockfilePath);
|
|
222
|
-
if (fs.existsSync(lockfilePath)) {
|
|
223
|
-
let raw;
|
|
224
|
-
try {
|
|
225
|
-
raw = JSON.parse(fs.readFileSync(lockfilePath, "utf8"));
|
|
226
|
-
}
|
|
227
|
-
catch (error) {
|
|
228
|
-
throw new ConfigError(`Cannot merge migration lock entries into unreadable lockfile ${lockfilePath}: ${error instanceof Error ? error.message : String(error)}.`, "INVALID_CONFIG_FILE");
|
|
229
|
-
}
|
|
230
|
-
if (!Array.isArray(raw) || !raw.every(isValidLockfileEntry)) {
|
|
231
|
-
throw new ConfigError(`Cannot merge migration lock entries into malformed lockfile ${lockfilePath}.`, "INVALID_CONFIG_FILE");
|
|
232
|
-
}
|
|
233
|
-
existing = raw;
|
|
234
|
-
}
|
|
235
|
-
return existing;
|
|
236
|
-
}
|
|
237
|
-
/** Validate the current lockfile before migrate-apply creates its backup or sentinel. */
|
|
238
|
-
export function assertMigrationLockfileReadable() {
|
|
239
|
-
readLockEntriesForMigration();
|
|
240
|
-
}
|
|
241
|
-
export function mergeLockEntriesSync(entries) {
|
|
242
|
-
const existing = readLockEntriesForMigration();
|
|
243
|
-
if (entries.length === 0)
|
|
244
|
-
return;
|
|
245
|
-
// MERGE per id, never replace: the migrator's entries are sparse (id/source/
|
|
246
|
-
// ref/localRoot), while an existing row may carry `resolvedVersion`,
|
|
247
|
-
// `resolvedRevision`, `integrity`, `installedAt`, … from a real install.
|
|
248
|
-
// Replacing the whole row discarded the user's recorded resolution/pin and
|
|
249
|
-
// left later update reporting comparing against an unknown prior version.
|
|
250
|
-
// Incoming DEFINED fields win; existing fields absent from the incoming
|
|
251
|
-
// entry are preserved.
|
|
252
|
-
const byId = new Map(existing.map((e) => [e.id, e]));
|
|
253
|
-
const merged = entries.map((incoming) => {
|
|
254
|
-
const prior = byId.get(incoming.id);
|
|
255
|
-
return prior ? { ...prior, ...incoming } : incoming;
|
|
256
|
-
});
|
|
257
|
-
const incomingIds = new Set(entries.map((e) => e.id));
|
|
258
|
-
writeLockfileUnlocked([...existing.filter((e) => !incomingIds.has(e.id)), ...merged]);
|
|
259
|
-
}
|
|
260
263
|
export async function removeLockEntry(id) {
|
|
261
264
|
// Returning early says "there is no lock record to remove", and the uninstall
|
|
262
265
|
// that called us reports success on that basis. Only an absent data dir earns
|
|
@@ -10,7 +10,7 @@ export { extractInlineRefMentions } from "./inline-refs.js";
|
|
|
10
10
|
// therefore one registry entry, never an edit here.
|
|
11
11
|
//
|
|
12
12
|
// Ordered by canonical id so the pre-derivation provider order
|
|
13
|
-
// ([claude
|
|
13
|
+
// ([claude, opencode] — visible in e.g. `extract --auto` result order)
|
|
14
14
|
// is preserved deterministically, independent of HARNESS_REGISTRY declaration
|
|
15
15
|
// order (which is pinned for JSON-schema enum stability).
|
|
16
16
|
//
|
|
@@ -25,10 +25,7 @@ export { extractInlineRefMentions } from "./inline-refs.js";
|
|
|
25
25
|
const HARNESSES = [...SESSION_LOG_HARNESSES]
|
|
26
26
|
.sort((a, b) => a.id.localeCompare(b.id))
|
|
27
27
|
.map((h) => h.sessionLogProvider());
|
|
28
|
-
//
|
|
29
|
-
// — a plain string the provider implementation sets independently of the
|
|
30
|
-
// harness registry (e.g. `ClaudeCodeProvider`'s `"claude-code"`) — must
|
|
31
|
-
// resolve back, via the id-normalization bridge, to a registry harness whose
|
|
28
|
+
// Every derived provider's `name` must exactly identify a registry harness whose
|
|
32
29
|
// `sessionLogs` capability is set. This is NOT expressible as a compile-time
|
|
33
30
|
// type constraint: `SessionLogHarness.name` is a runtime string with no
|
|
34
31
|
// static link to the `AkmHarness` that produced it, so a typo'd or
|
|
@@ -102,9 +99,8 @@ export function aggregateSessionEvents(events) {
|
|
|
102
99
|
* Extracted as a pure function (harnesses injected) so it is unit-testable
|
|
103
100
|
* without touching the real on-disk session-log locations.
|
|
104
101
|
*
|
|
105
|
-
* `maxSessionsPerHarness` bounds the
|
|
106
|
-
* session file
|
|
107
|
-
* with mtime ≥ sinceMs and skipped non-string content). On a machine with a
|
|
102
|
+
* `maxSessionsPerHarness` bounds the path: `readSession()` reads each
|
|
103
|
+
* session file in full. On a machine with a
|
|
108
104
|
* deep `~/.claude/projects` history a 30-day window can hold hundreds of
|
|
109
105
|
* multi-MB session files, and reading+parsing every one in full made the
|
|
110
106
|
* health command (`akm health`, which calls this synchronously) blow past its
|
|
@@ -117,16 +113,9 @@ export function collectSessionEvents(harnesses, sinceMs, maxSessionsPerHarness =
|
|
|
117
113
|
const events = [];
|
|
118
114
|
for (const harness of harnesses) {
|
|
119
115
|
try {
|
|
120
|
-
//
|
|
121
|
-
//
|
|
122
|
-
// surfaces nothing (e.g. a harness that wired readSession but whose
|
|
123
|
-
// listSessions returns empty on this machine) so we never regress
|
|
124
|
-
// coverage relative to the legacy scan.
|
|
116
|
+
// Enumerate sessions cheaply, then read each one's full structured event
|
|
117
|
+
// stream. There is no parallel flat-log parser.
|
|
125
118
|
const summaries = harness.listSessions({ sinceMs });
|
|
126
|
-
if (summaries.length === 0) {
|
|
127
|
-
events.push(...harness.readEvents({ sinceMs }));
|
|
128
|
-
continue;
|
|
129
|
-
}
|
|
130
119
|
// summaries are newest-first; bound the full-file reads (see doc above).
|
|
131
120
|
for (const summary of summaries.slice(0, maxSessionsPerHarness)) {
|
|
132
121
|
try {
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
*
|
|
7
7
|
* Holds only what the concrete providers genuinely share: safe stat'ing,
|
|
8
8
|
* recursive directory walking, the mtime-filtered file→summary listing loop,
|
|
9
|
-
*
|
|
9
|
+
* and conditional-spread assembly of
|
|
10
10
|
* {@link SessionSummary} refs stamped with the provider's runtime name.
|
|
11
11
|
* Everything platform-specific — file layouts, metadata peeking, SQLite
|
|
12
12
|
* stores, message flattening — stays in the subclasses.
|
|
@@ -82,32 +82,4 @@ export class AbstractSessionLogProvider {
|
|
|
82
82
|
}
|
|
83
83
|
return summaries.sort((a, b) => (b.endedAt ?? 0) - (a.endedAt ?? 0));
|
|
84
84
|
}
|
|
85
|
-
/**
|
|
86
|
-
* The legacy flat line scan shared by both providers' `readEvents`: parse
|
|
87
|
-
* each line as JSON, pull text/session-id through the per-provider
|
|
88
|
-
* selectors, skip entries whose text is missing or under 10 chars, and
|
|
89
|
-
* fall back to the file mtime when the entry carries no numeric timestamp.
|
|
90
|
-
*/
|
|
91
|
-
*logLineEvents(input) {
|
|
92
|
-
for (const line of input.lines) {
|
|
93
|
-
try {
|
|
94
|
-
const entry = JSON.parse(line);
|
|
95
|
-
const text = input.selectText(entry);
|
|
96
|
-
if (typeof text !== "string" || text.length < 10)
|
|
97
|
-
continue;
|
|
98
|
-
const sessionId = input.selectSessionId(entry);
|
|
99
|
-
yield {
|
|
100
|
-
harness: this.name,
|
|
101
|
-
text,
|
|
102
|
-
ts: typeof entry?.timestamp === "number" ? entry.timestamp : input.fallbackTsMs,
|
|
103
|
-
sessionId: typeof sessionId === "string" ? sessionId : undefined,
|
|
104
|
-
role: typeof entry?.role === "string" ? entry.role : "unknown",
|
|
105
|
-
filePath: input.filePath,
|
|
106
|
-
};
|
|
107
|
-
}
|
|
108
|
-
catch {
|
|
109
|
-
// skip malformed lines
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
85
|
}
|
package/dist/llm/client.js
CHANGED
|
@@ -13,7 +13,7 @@ import { ENV_REFERENCE_PATTERN } from "../core/config/schema/primitives.js";
|
|
|
13
13
|
import { formatExtraParamsIssue, validateExtraParams } from "../core/extra-params.js";
|
|
14
14
|
import { parseJsonResponse } from "../core/parse.js";
|
|
15
15
|
import { redactErrorBody, redactSensitiveText } from "../core/redaction.js";
|
|
16
|
-
import { warnVerbose } from "../core/warn.js";
|
|
16
|
+
import { warn, warnVerbose } from "../core/warn.js";
|
|
17
17
|
import { DEFAULT_LLM_TIMEOUT_MS } from "../integrations/agent/config.js";
|
|
18
18
|
import { emitLlmUsage, extractUsageTokens, } from "./usage-telemetry.js";
|
|
19
19
|
/** Maximum length of an upstream response excerpt included in thrown errors. */
|
|
@@ -211,9 +211,8 @@ async function chatCompletionAttempt(config, messages, options, timeoutMs) {
|
|
|
211
211
|
throw new Error(formatExtraParamsIssue("LLM extraParams", issue));
|
|
212
212
|
}
|
|
213
213
|
const headers = { "Content-Type": "application/json" };
|
|
214
|
-
// Resolve ONLY a whole-string env reference.
|
|
215
|
-
// us
|
|
216
|
-
// resolve `$VAR` upstream, and engine config REQUIRES the symbolic form), so
|
|
214
|
+
// Resolve ONLY a whole-string env reference. The execution boundary normally
|
|
215
|
+
// hands us a materialized credential after resolving `$VAR` upstream, so
|
|
217
216
|
// re-running the substitution over a literal key mangled any credential
|
|
218
217
|
// containing `$` — `sk-live$ecret` lost everything from the `$` onward, and
|
|
219
218
|
// the request failed with an opaque 401. The narrow check keeps the symbolic
|
|
@@ -240,14 +239,16 @@ async function chatCompletionAttempt(config, messages, options, timeoutMs) {
|
|
|
240
239
|
: config.provider === "vllm"
|
|
241
240
|
? { chat_template_kwargs: { enable_thinking: resolvedEnableThinking } }
|
|
242
241
|
: { enable_thinking: resolvedEnableThinking };
|
|
242
|
+
const reasoningEffortParams = config.reasoningEffort === undefined ? {} : { reasoning_effort: config.reasoningEffort };
|
|
243
243
|
const requestBody = JSON.stringify({
|
|
244
244
|
model: config.model,
|
|
245
245
|
messages,
|
|
246
246
|
temperature: options?.temperature ?? config.temperature ?? 0.3,
|
|
247
247
|
...(resolvedMaxTokens !== undefined ? { max_tokens: resolvedMaxTokens } : {}),
|
|
248
|
+
...config.extraParams,
|
|
248
249
|
...responseFormat,
|
|
249
250
|
...thinkingParams,
|
|
250
|
-
...
|
|
251
|
+
...reasoningEffortParams,
|
|
251
252
|
});
|
|
252
253
|
// Wall-clock start for per-attempt usage telemetry (#576). Captured here so the
|
|
253
254
|
// emitted duration covers the full request/response/parse cycle of a single
|
|
@@ -344,6 +345,10 @@ async function chatCompletionAttempt(config, messages, options, timeoutMs) {
|
|
|
344
345
|
finishReason: typeof json.choices?.[0]?.finish_reason === "string" ? json.choices[0].finish_reason : undefined,
|
|
345
346
|
...extractUsageTokens(json.usage),
|
|
346
347
|
};
|
|
348
|
+
if (resolvedEnableThinking === false && (terminalFields.reasoningTokens ?? 0) > 0) {
|
|
349
|
+
warn(`[akm] LLM returned ${terminalFields.reasoningTokens} reasoning tokens despite enableThinking: false; ` +
|
|
350
|
+
'the provider may not honor that control. Configure reasoningEffort: "none" when the provider supports it.');
|
|
351
|
+
}
|
|
347
352
|
const content = (json.choices?.[0]?.message?.content ?? "").trim();
|
|
348
353
|
const reasoning = (json.choices?.[0]?.message?.reasoning_content ?? "").trim();
|
|
349
354
|
const result = redactSensitiveText(content || reasoning, resolvedKey ? [resolvedKey] : []);
|
package/dist/llm/embedder.js
CHANGED
|
@@ -5,7 +5,7 @@ import { embedCacheKey, getCachedEmbedding, setCachedEmbedding } from "./embedde
|
|
|
5
5
|
import { DETERMINISTIC_EMBED_MODEL_ID, deterministicEmbed, isDeterministicEmbedEnabled, } from "./embedders/deterministic.js";
|
|
6
6
|
import { DEFAULT_LOCAL_MODEL, isTransformersAvailable as isTransformersAvailableReal, LocalEmbedder, } from "./embedders/local.js";
|
|
7
7
|
import { hasRemoteEndpoint, RemoteEmbedder } from "./embedders/remote.js";
|
|
8
|
-
// ──
|
|
8
|
+
// ── Shared exports ──────────────────────────────────────────────────────────
|
|
9
9
|
export { clearEmbeddingCache } from "./embedders/cache.js";
|
|
10
10
|
export { _setTransformersLoaderForTests, DEFAULT_LOCAL_MODEL } from "./embedders/local.js";
|
|
11
11
|
let embedderOverrides;
|
|
@@ -14,7 +14,7 @@ export function _setEmbedderForTests(fakes) {
|
|
|
14
14
|
embedderOverrides = fakes;
|
|
15
15
|
}
|
|
16
16
|
/**
|
|
17
|
-
* Check whether the
|
|
17
|
+
* Check whether the external Transformers dependency is available.
|
|
18
18
|
* Delegating wrapper around `./embedders/local`'s probe so tests can swap it
|
|
19
19
|
* via {@link _setEmbedderForTests}.
|
|
20
20
|
*/
|
|
@@ -25,7 +25,7 @@ export function isTransformersAvailable() {
|
|
|
25
25
|
}
|
|
26
26
|
// ── Singleton local embedder ────────────────────────────────────────────────
|
|
27
27
|
// `_localEmbedder` is an intentional module-level singleton but constructed
|
|
28
|
-
// lazily on first use. The underlying
|
|
28
|
+
// lazily on first use. The underlying Transformers.js pipeline is
|
|
29
29
|
// expensive to initialise (model download + WASM compilation) and is safe to
|
|
30
30
|
// share across calls because it is stateless once created. Deferring
|
|
31
31
|
// construction to first call keeps the module side-effect-free at import time,
|
|
@@ -48,7 +48,7 @@ export function resetLocalEmbedder() {
|
|
|
48
48
|
/**
|
|
49
49
|
* Generate an embedding for the given text.
|
|
50
50
|
* If embeddingConfig has a remote endpoint, uses the configured OpenAI-compatible endpoint.
|
|
51
|
-
* Otherwise falls back to local
|
|
51
|
+
* Otherwise falls back to local Transformers.js using the model from
|
|
52
52
|
* `embeddingConfig.localModel` or `DEFAULT_LOCAL_MODEL`.
|
|
53
53
|
*
|
|
54
54
|
* Results are cached in an LRU cache (max ~100 entries) keyed by query text
|
|
@@ -128,8 +128,7 @@ export async function embedBatch(texts, embeddingConfig, signal) {
|
|
|
128
128
|
// ── Similarity ──────────────────────────────────────────────────────────────
|
|
129
129
|
// `cosineSimilarity` was moved to `./embedders/types.ts` so importers
|
|
130
130
|
// (notably `db.ts`) can pull the math function without dragging in this
|
|
131
|
-
//
|
|
132
|
-
// preserves the existing public API.
|
|
131
|
+
// module and its Transformers.js import chain.
|
|
133
132
|
export { cosineSimilarity } from "./embedders/types.js";
|
|
134
133
|
// ── Model ID resolution ─────────────────────────────────────────────────────
|
|
135
134
|
/**
|
|
@@ -183,7 +182,7 @@ export async function checkEmbeddingAvailability(embeddingConfig) {
|
|
|
183
182
|
return {
|
|
184
183
|
available: false,
|
|
185
184
|
reason: "missing-package",
|
|
186
|
-
message: "@huggingface/transformers is
|
|
185
|
+
message: "The @huggingface/transformers dependency is unavailable.",
|
|
187
186
|
};
|
|
188
187
|
}
|
|
189
188
|
try {
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
4
|
/**
|
|
5
|
-
* Local @huggingface/transformers
|
|
5
|
+
* Local embedder backed by the external @huggingface/transformers package.
|
|
6
6
|
*
|
|
7
7
|
* Encapsulates the transformer pipeline lifecycle as instance state on a
|
|
8
8
|
* `LocalEmbedder` so tests can construct fresh instances without leaking
|
|
@@ -12,7 +12,6 @@
|
|
|
12
12
|
import path from "node:path";
|
|
13
13
|
import { getCacheDir } from "../../core/paths.js";
|
|
14
14
|
import { warn } from "../../core/warn.js";
|
|
15
|
-
import { getDirname, resolveModule } from "../../runtime.js";
|
|
16
15
|
/**
|
|
17
16
|
* Default local transformer model for embeddings.
|
|
18
17
|
* `bge-small-en-v1.5` scores higher on MTEB benchmarks than the previous
|
|
@@ -23,10 +22,14 @@ export const DEFAULT_LOCAL_MODEL = "Xenova/bge-small-en-v1.5";
|
|
|
23
22
|
function isBatchTensor(v) {
|
|
24
23
|
return (v !== null &&
|
|
25
24
|
typeof v === "object" &&
|
|
26
|
-
|
|
25
|
+
v.data instanceof Float32Array &&
|
|
27
26
|
"dims" in v &&
|
|
28
27
|
Array.isArray(v.dims) &&
|
|
29
|
-
v.dims.length
|
|
28
|
+
v.dims.length === 2 &&
|
|
29
|
+
v.dims.every((dim) => Number.isInteger(dim) && dim > 0));
|
|
30
|
+
}
|
|
31
|
+
function isEnabledEnvironmentFlag(value) {
|
|
32
|
+
return /^(?:1|true|yes|on)$/i.test(value?.trim() ?? "");
|
|
30
33
|
}
|
|
31
34
|
const realTransformersLoader = () => import("@huggingface/transformers");
|
|
32
35
|
let transformersLoader = realTransformersLoader;
|
|
@@ -51,31 +54,6 @@ const LOCAL_BATCH_SIZE = 32;
|
|
|
51
54
|
function resolveLocalModelName(overrideModel) {
|
|
52
55
|
return overrideModel || DEFAULT_LOCAL_MODEL;
|
|
53
56
|
}
|
|
54
|
-
/**
|
|
55
|
-
* Detect whether the current process is running from a Bun-compiled binary
|
|
56
|
-
* (i.e. `bun build --compile` produced a single executable). Bun marks the
|
|
57
|
-
* compiled binary with a synthesized `process.execPath` that ends in the
|
|
58
|
-
* binary name rather than `bun`, AND sets a flag we can probe.
|
|
59
|
-
*
|
|
60
|
-
* Used to gate the "install @huggingface/transformers" hint — that advice
|
|
61
|
-
* is impossible to follow from a single-binary install, so we replace it
|
|
62
|
-
* with the only working remediation (switch to npm/Bun install, or turn
|
|
63
|
-
* semantic search off). See #482.
|
|
64
|
-
*/
|
|
65
|
-
function isCompiledBinary() {
|
|
66
|
-
try {
|
|
67
|
-
const flag = Bun.embeddedFiles;
|
|
68
|
-
if (flag !== undefined)
|
|
69
|
-
return true;
|
|
70
|
-
}
|
|
71
|
-
catch {
|
|
72
|
-
// Bun not available (under Node tests, for example) — treat as not-binary.
|
|
73
|
-
}
|
|
74
|
-
const exec = (process.execPath || "").toLowerCase();
|
|
75
|
-
if (exec.endsWith("/akm") || exec.endsWith("\\akm.exe"))
|
|
76
|
-
return true;
|
|
77
|
-
return false;
|
|
78
|
-
}
|
|
79
57
|
export class LocalEmbedder {
|
|
80
58
|
defaultModel;
|
|
81
59
|
/**
|
|
@@ -102,9 +80,7 @@ export class LocalEmbedder {
|
|
|
102
80
|
/**
|
|
103
81
|
* Embed a batch of texts. Processes in chunks of `LOCAL_BATCH_SIZE` (32) so
|
|
104
82
|
* the transformers pipeline can run genuine batched inference rather than one
|
|
105
|
-
* call per text.
|
|
106
|
-
* array input (older versions of @huggingface/transformers). Each chunk is
|
|
107
|
-
* checked against the AbortSignal between calls.
|
|
83
|
+
* call per text. Each chunk is checked against the AbortSignal between calls.
|
|
108
84
|
*/
|
|
109
85
|
async embedBatch(texts, signal) {
|
|
110
86
|
if (texts.length === 0)
|
|
@@ -119,42 +95,22 @@ export class LocalEmbedder {
|
|
|
119
95
|
throw signal.reason instanceof Error ? signal.reason : new Error("embedding interrupted");
|
|
120
96
|
}
|
|
121
97
|
const chunk = texts.slice(i, i + LOCAL_BATCH_SIZE);
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
if (isBatchTensor(batchResult)) {
|
|
132
|
-
const dim = batchResult.dims[1];
|
|
133
|
-
for (let row = 0; row < chunk.length; row++) {
|
|
134
|
-
results.push(Array.from(batchResult.data.subarray(row * dim, (row + 1) * dim)));
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
|
-
else if (Array.isArray(batchResult)) {
|
|
138
|
-
// Older versions of @huggingface/transformers returned Array<{data}>.
|
|
139
|
-
for (const r of batchResult) {
|
|
140
|
-
results.push(Array.from(r.data));
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
else {
|
|
144
|
-
// Single-text result returned for a chunk — should not happen for
|
|
145
|
-
// string[] input, but handle defensively.
|
|
146
|
-
throw new Error("unexpected pipeline return shape for batch input");
|
|
147
|
-
}
|
|
98
|
+
// @huggingface/transformers 4.2 returns a single batch Tensor. Validate
|
|
99
|
+
// that exact shape and propagate inference failures without re-executing
|
|
100
|
+
// the same inputs through a second runtime path.
|
|
101
|
+
const batchResult = await pipeline(chunk, {
|
|
102
|
+
pooling: "mean",
|
|
103
|
+
normalize: true,
|
|
104
|
+
});
|
|
105
|
+
if (!isBatchTensor(batchResult)) {
|
|
106
|
+
throw new Error("unexpected pipeline return shape for batch input");
|
|
148
107
|
}
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
}
|
|
156
|
-
results.push(await this.embedWithModel(text, this.defaultModel));
|
|
157
|
-
}
|
|
108
|
+
const [batch, dim] = batchResult.dims;
|
|
109
|
+
if (batch !== chunk.length || batchResult.data.length !== batch * dim) {
|
|
110
|
+
throw new Error("unexpected pipeline return shape for batch input");
|
|
111
|
+
}
|
|
112
|
+
for (let row = 0; row < chunk.length; row++) {
|
|
113
|
+
results.push(Array.from(batchResult.data.subarray(row * dim, (row + 1) * dim)));
|
|
158
114
|
}
|
|
159
115
|
}
|
|
160
116
|
return results;
|
|
@@ -187,25 +143,21 @@ export class LocalEmbedder {
|
|
|
187
143
|
let pipeline;
|
|
188
144
|
try {
|
|
189
145
|
const mod = await transformersLoader();
|
|
146
|
+
// Transformers.js 4.x defaults its filesystem cache underneath the
|
|
147
|
+
// installed package and no longer derives it from HF_HOME. Point the
|
|
148
|
+
// public runtime setting at our stable cache explicitly so package
|
|
149
|
+
// reinstalls and test-sandbox HOME rotation do not re-download the
|
|
150
|
+
// model. The exact pinned 4.2 module owns this public environment.
|
|
151
|
+
mod.env.cacheDir = process.env.HF_HOME;
|
|
152
|
+
if (isEnabledEnvironmentFlag(process.env.HF_HUB_OFFLINE)) {
|
|
153
|
+
mod.env.allowRemoteModels = false;
|
|
154
|
+
}
|
|
190
155
|
pipeline = mod.pipeline;
|
|
191
156
|
}
|
|
192
157
|
catch (importError) {
|
|
193
158
|
const msg = importError instanceof Error ? importError.message : String(importError);
|
|
194
|
-
if (/Cannot find module|MODULE_NOT_FOUND|Cannot resolve/i.test(msg)) {
|
|
195
|
-
|
|
196
|
-
// `bun install --omit optional` (release.yml), so binary users
|
|
197
|
-
// can NEVER load @huggingface/transformers. Telling them to
|
|
198
|
-
// `bun add` it is a dead-end — there is no install target.
|
|
199
|
-
// Detect the binary execution path and give the only working
|
|
200
|
-
// remediation: switch to the npm/Bun install of akm-cli, or
|
|
201
|
-
// turn off semantic search.
|
|
202
|
-
const isBinary = isCompiledBinary();
|
|
203
|
-
const hint = isBinary
|
|
204
|
-
? "You are running the prebuilt akm binary, which cannot load optional native dependencies. " +
|
|
205
|
-
"To enable semantic search, install akm-cli via Bun: `curl -fsSL https://bun.sh/install | bash && bun install -g akm-cli`. " +
|
|
206
|
-
"To keep using the binary, set `semanticSearchMode: off` in your config and use keyword-only FTS."
|
|
207
|
-
: "Install it with: `bun add @huggingface/transformers` (or `npm install @huggingface/transformers`).";
|
|
208
|
-
throw new Error(`Semantic search requires @huggingface/transformers. ${hint}`);
|
|
159
|
+
if (/Cannot find (?:module|package)|MODULE_NOT_FOUND|ERR_MODULE_NOT_FOUND|Cannot resolve/i.test(msg)) {
|
|
160
|
+
throw new Error("Semantic search requires @huggingface/transformers. Reinstall akm-cli.");
|
|
209
161
|
}
|
|
210
162
|
throw new Error(`Failed to load embedding runtime: ${msg}. Check platform compatibility.`);
|
|
211
163
|
}
|
|
@@ -239,15 +191,12 @@ function shouldRetryWithoutExplicitDtype(error) {
|
|
|
239
191
|
return /dtype|fp32|precision|quant/i.test(message);
|
|
240
192
|
}
|
|
241
193
|
/**
|
|
242
|
-
* Check whether the
|
|
243
|
-
*
|
|
244
|
-
* (which would trigger heavy WASM/model side-effects) just to test
|
|
245
|
-
* availability. `resolveModule` uses `Bun.resolveSync` on Bun and
|
|
246
|
-
* `require.resolve` on Node.
|
|
194
|
+
* Check whether the declared Transformers dependency can be resolved without
|
|
195
|
+
* loading a model.
|
|
247
196
|
*/
|
|
248
197
|
export function isTransformersAvailable() {
|
|
249
198
|
try {
|
|
250
|
-
|
|
199
|
+
import.meta.resolve("@huggingface/transformers");
|
|
251
200
|
return true;
|
|
252
201
|
}
|
|
253
202
|
catch {
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
*
|
|
7
7
|
* Lives next to {@link EmbeddingVector} so importers (notably `db.ts`)
|
|
8
8
|
* can pull just the math without dragging in the embedder facade and its
|
|
9
|
-
* transitive
|
|
9
|
+
* transitive Transformers.js runtime import chain.
|
|
10
10
|
*
|
|
11
11
|
* Returns 0 when the vectors have different dimensions — silently
|
|
12
12
|
* computing on a truncated view would produce meaningless scores.
|