akm-cli 0.10.261016-beta → 0.10.261018-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +144 -0
- package/README.md +8 -10
- package/SECURITY.md +7 -99
- package/STABILITY.md +57 -98
- package/dist/assets/opencode/akm-plugin.js +103 -1
- package/dist/assets/stash-skeleton/facts/conventions/backlinks.md +2 -2
- package/dist/assets/stash-skeleton/facts/conventions/organization.md +11 -17
- package/dist/cli.js +32 -41
- package/dist/commands/feedback-cli.js +5 -4
- package/dist/commands/health.js +15 -16
- package/dist/commands/proposal/proposal-cli.js +1 -1
- package/dist/commands/read/registry-search.js +4 -1
- package/dist/commands/read/search.js +6 -6
- package/dist/commands/sources/self-update.js +9 -5
- package/dist/commands/tasks/tasks.js +12 -13
- package/dist/core/adapter/adapters/agent-skills-adapter.js +7 -7
- package/dist/core/adapter/adapters/akm-task-adapter.js +1 -1
- package/dist/core/adapter/adapters/akm-workflow-adapter.js +1 -1
- package/dist/core/adapter/adapters/claude-adapter.js +1 -1
- package/dist/core/adapter/adapters/dotenv-adapter.js +1 -1
- package/dist/core/adapter/adapters/generic-files-adapter.js +1 -1
- package/dist/core/adapter/adapters/llm-wiki-adapter.js +1 -1
- package/dist/core/adapter/adapters/opencode-adapter.js +1 -1
- package/dist/core/adapter/adapters/website-snapshot-adapter.js +1 -1
- package/dist/core/bundle-rename.js +17 -7
- package/dist/core/config/convert-engine-references.js +23 -7
- package/dist/core/config/convert-engines.js +94 -28
- package/dist/core/config/convert-models-overlay.js +14 -2
- package/dist/indexer/bundle-identity-guard.js +6 -6
- package/dist/indexer/index-written-assets.js +3 -3
- package/dist/indexer/indexer.js +12 -8
- package/dist/indexer/materialize-embeddings.js +12 -2
- package/dist/indexer/passes/metadata.js +103 -46
- package/dist/indexer/search/ranking.js +7 -3
- package/dist/indexer/search/search-fields.js +1 -1
- package/dist/indexer/usage/usage-events.js +30 -8
- package/dist/indexer/walk/walker.js +1 -1
- package/dist/integrations/agent/spawn.js +7 -2
- package/dist/integrations/harnesses/opencode/agent-builder.js +10 -1
- package/dist/integrations/harnesses/opencode/runtime/binary.js +6 -3
- package/dist/integrations/harnesses/opencode/runtime/index.js +18 -9
- package/dist/integrations/harnesses/opencode/session-log-v2.js +11 -1
- package/dist/output/shapes/helpers.js +3 -3
- package/dist/output/text/command-format.js +5 -6
- package/dist/scripts/akm-migrate-node.js +122 -36
- package/dist/scripts/akm-migrate.js +122 -36
- package/dist/sources/providers/git-provider.js +2 -4
- package/dist/sources/providers/provider-utils.js +8 -13
- package/dist/sources/snapshot-fetchers/website-ingest.js +6 -7
- package/dist/storage/repositories/events-repository.js +15 -0
- package/dist/storage/repositories/improve-ledger-repository.js +13 -0
- package/dist/storage/repositories/index-entries-repository.js +43 -5
- package/dist/storage/repositories/outcome-repository.js +12 -0
- package/dist/storage/repositories/proposals-repository.js +5 -5
- package/dist/storage/repositories/salience-repository.js +13 -0
- package/dist/storage/repositories/task-history-repository.js +8 -7
- package/dist/storage/repositories/workflow-runs-repository.js +10 -8
- package/docs/README.md +24 -60
- package/docs/agents/README.md +3 -8
- package/docs/architecture/README.md +18 -20
- package/docs/architecture/adapters.md +40 -43
- package/docs/architecture/architecture.md +180 -154
- package/docs/architecture/improvement.md +95 -91
- package/docs/architecture/internals/classification.md +101 -119
- package/docs/architecture/internals/fresh-host-rebuild-runbook.md +33 -35
- package/docs/architecture/internals/health-advisories.md +59 -46
- package/docs/architecture/internals/improve-workflow.md +166 -693
- package/docs/architecture/internals/indexing.md +65 -465
- package/docs/architecture/internals/storage-locations.md +301 -848
- package/docs/architecture/persisted-data-compat.md +6 -6
- package/docs/architecture/specs/okf-support.md +67 -95
- package/docs/architecture/specs/ref.md +67 -125
- package/docs/architecture/testing/testing-workflow.md +108 -168
- package/docs/architecture/workflow-engine.md +34 -33
- package/docs/guides/README.md +5 -5
- package/docs/guides/author-bundles.md +72 -98
- package/docs/guides/author-workflows.md +11 -18
- package/docs/guides/bundles.md +91 -123
- package/docs/guides/capture-knowledge.md +10 -9
- package/docs/guides/concepts.md +45 -30
- package/docs/guides/discover-and-load.md +39 -32
- package/docs/guides/getting-started.md +24 -24
- package/docs/guides/improve-the-library.md +98 -80
- package/docs/guides/recipes/headless-install.md +4 -4
- package/docs/guides/recipes/website-source.md +9 -5
- package/docs/guides/run-workflows.md +10 -11
- package/docs/guides/scheduling.md +11 -17
- package/docs/guides/use-with-any-agent.md +15 -14
- package/docs/guides/wikis.md +6 -9
- package/docs/integration/bundling-akm.md +21 -26
- package/docs/maintainers/README.md +2 -2
- package/docs/maintainers/local-development.md +3 -16
- package/docs/migration/README.md +2 -1
- package/docs/migration/release-notes/0.10.0.md +196 -23
- package/docs/migration/release-notes/README.md +2 -2
- package/docs/migration/v0.9-to-v0.10.md +103 -12
- package/docs/product-surface.md +10 -8
- package/docs/reference/README.md +26 -15
- package/docs/reference/asset-types.md +11 -12
- package/docs/reference/bundle-types.md +6 -5
- package/docs/reference/cli.md +52 -28
- package/docs/reference/configuration.md +91 -73
- package/docs/reference/data-and-telemetry.md +204 -421
- package/docs/reference/env-and-secrets.md +80 -83
- package/docs/reference/memory.md +1 -1
- package/docs/reference/refs.md +23 -36
- package/docs/reference/registry.md +138 -348
- package/docs/reference/supported-formats.md +8 -8
- package/docs/reference/tasks.md +13 -19
- package/docs/reference/website-sources.md +23 -15
- package/docs/reference/workflow-schema.md +28 -47
- package/docs/reference/workflows.md +33 -37
- package/package.json +1 -1
|
@@ -30475,6 +30475,7 @@ function modelWorkConfig() {
|
|
|
30475
30475
|
}
|
|
30476
30476
|
const config2 = {
|
|
30477
30477
|
...user,
|
|
30478
|
+
plugins: [...Array.isArray(user?.plugins) ? user.plugins : [], WITHOUT_BROWSER_PLUGIN],
|
|
30478
30479
|
agents: { ...asRecord(user?.agents), [MODEL_WORK_AGENT]: MODEL_WORK_AGENT_DEFINITION }
|
|
30479
30480
|
};
|
|
30480
30481
|
const legacy = asRecord(user?.agent);
|
|
@@ -30507,7 +30508,7 @@ function withEnvReferences(content) {
|
|
|
30507
30508
|
return replaced(true);
|
|
30508
30509
|
}
|
|
30509
30510
|
}
|
|
30510
|
-
var MODEL_WORK_AGENT = "akm-model-work", MODEL_WORK_AGENT_DEFINITION, CONFIG_CONTENT_ENV = "OPENCODE_CONFIG_CONTENT", NO_FREE_MODELS, opencodeBuilder;
|
|
30511
|
+
var MODEL_WORK_AGENT = "akm-model-work", MODEL_WORK_AGENT_DEFINITION, CONFIG_CONTENT_ENV = "OPENCODE_CONFIG_CONTENT", WITHOUT_BROWSER_PLUGIN = "-opencode.browser", NO_FREE_MODELS, opencodeBuilder;
|
|
30511
30512
|
var init_agent_builder = __esm(() => {
|
|
30512
30513
|
init_common();
|
|
30513
30514
|
init_warn();
|
|
@@ -32113,13 +32114,22 @@ function improveStrategyDefault(out, defaults, improve) {
|
|
|
32113
32114
|
out.improve = { ...isRecord(improve) ? improve : {} };
|
|
32114
32115
|
out.improve.strategy ??= defaults.improveStrategy;
|
|
32115
32116
|
}
|
|
32116
|
-
function dropJudgeEngine(out, conversion) {
|
|
32117
|
+
function dropJudgeEngine(out, conversion, target, defaultEngine, judge) {
|
|
32117
32118
|
const workflow = out.workflow;
|
|
32118
|
-
|
|
32119
|
-
|
|
32120
|
-
|
|
32121
|
-
|
|
32122
|
-
|
|
32119
|
+
const named = isRecord(workflow) && "judgeEngine" in workflow;
|
|
32120
|
+
const engine = named ? str(workflow.judgeEngine) : defaultEngine;
|
|
32121
|
+
if (named) {
|
|
32122
|
+
delete workflow.judgeEngine;
|
|
32123
|
+
conversion.changed = true;
|
|
32124
|
+
}
|
|
32125
|
+
const became = engine === undefined ? undefined : target(engine);
|
|
32126
|
+
const stays = became !== undefined && judge !== undefined && isDeepStrictEqual(became, judge);
|
|
32127
|
+
const kind = judge === undefined ? undefined : ("connection" in judge) ? "connection" : "harness";
|
|
32128
|
+
if (named) {
|
|
32129
|
+
conversion.notes.push(stays ? `workflow.judgeEngine: "${engine}" is the default ${kind}, where a gate's judge runs now; the setting is dropped.` : `workflow.judgeEngine${engine === undefined ? "" : ` "${engine}"`}: dropped. A gate's judge runs on the default connection, else the default harness, so a judge that ran on another engine has moved; to keep it, make that engine the default.`);
|
|
32130
|
+
} else if (became !== undefined && judge !== undefined && !stays) {
|
|
32131
|
+
conversion.notes.push(`workflow: with no workflow.judgeEngine, a gate's judge ran on the default engine "${engine}"; it is model work now and runs on the default ${kind}, "${"connection" in judge ? judge.connection : judge.harness}".`);
|
|
32132
|
+
}
|
|
32123
32133
|
}
|
|
32124
32134
|
function convertIndexEngines(out, conversion, retarget) {
|
|
32125
32135
|
const index = out.index;
|
|
@@ -32186,6 +32196,8 @@ function foldModelsOverlay(modelsOverlay, fold) {
|
|
|
32186
32196
|
const writtenHarnesses = written.harnesses;
|
|
32187
32197
|
const configNames = (own2, base, name, alias) => hasModel(own2[name], alias) || hasModel(base[name], alias);
|
|
32188
32198
|
const platformOf = (name) => String((mergedHarnesses[name] ?? inheritedHarnesses[name])?.platform ?? name);
|
|
32199
|
+
const columnOf = (name) => renamed.sdkHarness.includes(name) ? "opencode-sdk" : platformOf(name);
|
|
32200
|
+
const notedOwn = new Set;
|
|
32189
32201
|
let folded = false;
|
|
32190
32202
|
for (const { alias, column, profile } of overlayRows(modelsOverlay)) {
|
|
32191
32203
|
const p = typeof profile === "string" ? { model: profile } : isRecord(profile) ? profile : {};
|
|
@@ -32197,13 +32209,18 @@ function foldModelsOverlay(modelsOverlay, fold) {
|
|
|
32197
32209
|
continue;
|
|
32198
32210
|
}
|
|
32199
32211
|
const candidates = column === "*" ? [...new Set([...Object.keys(mergedConnections), ...Object.keys(inheritedConnections)])] : column === "opencode-sdk" ? [...renamed.runtime] : renamed.connection(column) !== undefined ? [renamed.connection(column)] : (column in writtenConnections) || (column in inheritedConnections) ? [column] : [];
|
|
32200
|
-
const harnessNamed = [...new Set([...Object.keys(mergedHarnesses), ...Object.keys(inheritedHarnesses)])].filter((name) =>
|
|
32212
|
+
const harnessNamed = [...new Set([...Object.keys(mergedHarnesses), ...Object.keys(inheritedHarnesses)])].filter((name) => columnOf(name) === column);
|
|
32201
32213
|
if (candidates.length === 0 && harnessNamed.length === 0) {
|
|
32202
32214
|
notes2.push(`${where}: names no configured connection or harness, so it is not carried over.`);
|
|
32203
32215
|
continue;
|
|
32204
32216
|
}
|
|
32205
32217
|
const connectionNames = candidates.filter((name) => !configNames(writtenConnections, inheritedConnections, name, alias) && !(column === "*" && hasModel(mergedConnections[name], alias)));
|
|
32206
|
-
const
|
|
32218
|
+
const unnamed = harnessNamed.filter((name) => !configNames(writtenHarnesses, inheritedHarnesses, name, alias));
|
|
32219
|
+
const harnessNames = unnamed.filter((name) => !ownModels.harnesses.has(name));
|
|
32220
|
+
for (const name of unnamed.filter((n) => ownModels.harnesses.has(n) && !notedOwn.has(`${column}:${n}`))) {
|
|
32221
|
+
notedOwn.add(`${column}:${name}`);
|
|
32222
|
+
notes2.push(`models.json (${column === "*" ? "all" : column}): harness "${name}" has a model of its own, which names every alias, so the column is not carried over.`);
|
|
32223
|
+
}
|
|
32207
32224
|
if (connectionNames.length === 0 && harnessNames.length === 0) {
|
|
32208
32225
|
const keptByConfig = candidates.some((name) => configNames(writtenConnections, inheritedConnections, name, alias)) || harnessNamed.some((name) => configNames(writtenHarnesses, inheritedHarnesses, name, alias));
|
|
32209
32226
|
if (keptByConfig)
|
|
@@ -32263,15 +32280,23 @@ function implicitKeyVariables(name, defaults) {
|
|
|
32263
32280
|
const own2 = `AKM_ENGINE_${name.toUpperCase().replaceAll("-", "_")}_API_KEY`;
|
|
32264
32281
|
return name === str(defaults.llmEngine) ? [own2, "AKM_LLM_API_KEY"] : [own2];
|
|
32265
32282
|
}
|
|
32266
|
-
function runtimeEngine(name, engine, converted, conversion) {
|
|
32267
|
-
converted.runtimes.set(name,
|
|
32268
|
-
|
|
32269
|
-
}
|
|
32270
|
-
|
|
32271
|
-
|
|
32283
|
+
function runtimeEngine(name, engine, llmEngine, converted, conversion) {
|
|
32284
|
+
converted.runtimes.set(name, llmEngine);
|
|
32285
|
+
const dropped = ["model", "timeoutMs", "workspace", "bin", "agent"].filter((key) => engine[key] !== undefined);
|
|
32286
|
+
const gone = dropped.length === 0 ? "" : `; its ${dropped.join(", ")} ${dropped.length === 1 ? "is" : "are"} dropped`;
|
|
32287
|
+
conversion.notes.push(`engines.${name}: opencode-sdk over llmEngine "${llmEngine}" is akm's own runtime over connection "${llmEngine}" now, with no entry of its own${gone}.`);
|
|
32288
|
+
}
|
|
32289
|
+
function agentEngineToHarness(name, engine, converted, conversion) {
|
|
32290
|
+
const sdk = engine.platform === "opencode-sdk";
|
|
32291
|
+
const harness = { platform: sdk ? "opencode" : engine.platform };
|
|
32292
|
+
if (sdk)
|
|
32293
|
+
converted.sdkHarnesses.add(name);
|
|
32272
32294
|
for (const key of ["bin", "args", "workspace", "agent", "timeoutMs"])
|
|
32273
|
-
if (engine[key] !== undefined)
|
|
32295
|
+
if (engine[key] !== undefined && !(sdk && key === "args"))
|
|
32274
32296
|
harness[key] = engine[key];
|
|
32297
|
+
if (sdk && engine.args !== undefined) {
|
|
32298
|
+
conversion.notes.push(`engines.${name}: args is dropped. 0.9 did not pass it to OpenCode, and the opencode harness would pass it in place of \`run\`.`);
|
|
32299
|
+
}
|
|
32275
32300
|
const model = str(engine.model);
|
|
32276
32301
|
if (model !== undefined)
|
|
32277
32302
|
converted.ownModelOf.push({ kind: "harnesses", engine: name, model });
|
|
@@ -32322,6 +32347,7 @@ function convertEngines(engines, defaults, env, conversion) {
|
|
|
32322
32347
|
connections: {},
|
|
32323
32348
|
harnesses: {},
|
|
32324
32349
|
runtimes: new Map,
|
|
32350
|
+
sdkHarnesses: new Set,
|
|
32325
32351
|
keyHolders: [],
|
|
32326
32352
|
ownModelOf: []
|
|
32327
32353
|
};
|
|
@@ -32331,12 +32357,13 @@ function convertEngines(engines, defaults, env, conversion) {
|
|
|
32331
32357
|
const unknown2 = Object.keys(engine).filter((k) => !(engine.kind === "agent" ? AGENT_ENGINE_KEYS : LLM_ENGINE_KEYS).has(k));
|
|
32332
32358
|
if (unknown2.length > 0)
|
|
32333
32359
|
conversion.notes.push(`engines.${name}: dropped ${unknown2.join(", ")}, which akm does not read.`);
|
|
32360
|
+
const llmEngine = str(engine.llmEngine);
|
|
32334
32361
|
if (engine.kind !== "agent")
|
|
32335
32362
|
llmEngineToConnection(name, engine, defaults, env, converted, conversion);
|
|
32336
|
-
else if (engine.platform === "opencode-sdk")
|
|
32337
|
-
runtimeEngine(name, engine, converted, conversion);
|
|
32363
|
+
else if (engine.platform === "opencode-sdk" && llmEngine !== undefined)
|
|
32364
|
+
runtimeEngine(name, engine, llmEngine, converted, conversion);
|
|
32338
32365
|
else
|
|
32339
|
-
agentEngineToHarness(name, engine, converted);
|
|
32366
|
+
agentEngineToHarness(name, engine, converted, conversion);
|
|
32340
32367
|
}
|
|
32341
32368
|
return converted;
|
|
32342
32369
|
}
|
|
@@ -32354,11 +32381,21 @@ function renameMap(names, taken, from) {
|
|
|
32354
32381
|
}
|
|
32355
32382
|
return (name) => moved.get(name) ?? name;
|
|
32356
32383
|
}
|
|
32384
|
+
function agentWorkMoved(defaultEngine, runtimes, harness) {
|
|
32385
|
+
if (defaultEngine !== undefined) {
|
|
32386
|
+
return runtimes.has(defaultEngine) && harness !== undefined ? `Agent work ran on akm's own runtime (engines.${defaultEngine}); with the harness "${harness}" configured it now runs there. Remove that harness to keep agent work on akm's runtime.` : undefined;
|
|
32387
|
+
}
|
|
32388
|
+
const ran = "Without defaults.engine, agent work ran on your own OpenCode (the implicit fallback), when it was installed";
|
|
32389
|
+
return harness !== undefined ? `${ran}; with the harness "${harness}" configured it now runs there.` : `${ran}; it now runs on akm's own runtime. To keep it on your OpenCode, enable the harness: \`akm setup --harness opencode\`.`;
|
|
32390
|
+
}
|
|
32357
32391
|
function nameDefaults(defaults, converted, conversion, written) {
|
|
32358
|
-
const { connections, harnesses, runtimes, keyHolders } = converted;
|
|
32359
|
-
const
|
|
32392
|
+
const { connections, harnesses, runtimes, sdkHarnesses, keyHolders } = converted;
|
|
32393
|
+
const implicit = sdkHarnesses.has(IMPLICIT_AGENT_ENGINE) || runtimes.has(IMPLICIT_AGENT_ENGINE);
|
|
32394
|
+
const defaultEngine = str(defaults.engine) ?? (implicit ? IMPLICIT_AGENT_ENGINE : undefined);
|
|
32395
|
+
const modelWorkEngine = str(defaults.llmEngine);
|
|
32360
32396
|
const runtimeLlm = defaultEngine === undefined ? undefined : runtimes.get(defaultEngine);
|
|
32361
|
-
const
|
|
32397
|
+
const modelWorkRuntimeLlm = modelWorkEngine === undefined ? undefined : runtimes.get(modelWorkEngine);
|
|
32398
|
+
const defaultConnection = [modelWorkEngine, modelWorkRuntimeLlm, defaultEngine, runtimeLlm].find((n) => n !== undefined && (n in connections));
|
|
32362
32399
|
const defaultHarness = defaultEngine !== undefined && defaultEngine in harnesses && Object.keys(harnesses).length > 1 ? defaultEngine : undefined;
|
|
32363
32400
|
const connectionName = renameMap(Object.keys(connections), Object.keys(written.connections), defaultConnection);
|
|
32364
32401
|
const harnessName = renameMap(Object.keys(harnesses), Object.keys(written.harnesses), defaultHarness);
|
|
@@ -32367,9 +32404,16 @@ function nameDefaults(defaults, converted, conversion, written) {
|
|
|
32367
32404
|
connections: renamed(connections, connectionName),
|
|
32368
32405
|
harnesses: renamed(harnesses, harnessName)
|
|
32369
32406
|
};
|
|
32370
|
-
const defaultHarnessName =
|
|
32371
|
-
|
|
32372
|
-
|
|
32407
|
+
const defaultHarnessName = defaultName(named.harnesses);
|
|
32408
|
+
const hasEngines = Object.keys(connections).length + Object.keys(harnesses).length + runtimes.size > 0;
|
|
32409
|
+
const moved = hasEngines ? agentWorkMoved(defaultEngine, runtimes, defaultHarnessName) : undefined;
|
|
32410
|
+
if (moved !== undefined)
|
|
32411
|
+
conversion.notes.push(moved);
|
|
32412
|
+
if (modelWorkEngine !== undefined && modelWorkEngine in harnesses) {
|
|
32413
|
+
const now = harnessName(modelWorkEngine);
|
|
32414
|
+
if (defaultName({ ...named.connections, ...written.connections }) !== undefined || defaultHarnessName !== now) {
|
|
32415
|
+
conversion.notes.push(`defaults.llmEngine "${modelWorkEngine}" named an agent engine, which ran akm's model work. Model work now runs on the default connection, else the default harness; to keep it on the harness "${now}", set improve.harness to it.`);
|
|
32416
|
+
}
|
|
32373
32417
|
}
|
|
32374
32418
|
for (const [kind, rename] of [
|
|
32375
32419
|
["connections", connectionName],
|
|
@@ -32381,10 +32425,9 @@ function nameDefaults(defaults, converted, conversion, written) {
|
|
|
32381
32425
|
conversion.notes.push(`engines.default is now ${kind}.${aside}, and "default" names another engine: an engine: default key is rewritten by \`akm migrate apply\` where it can be, but a script or scheduled command that passes --engine default now reaches the new "default"; change it to the name ${aside}.`);
|
|
32382
32426
|
}
|
|
32383
32427
|
const target = (engine) => {
|
|
32384
|
-
|
|
32385
|
-
|
|
32386
|
-
return runtime
|
|
32387
|
-
}
|
|
32428
|
+
const runtime = runtimes.get(engine);
|
|
32429
|
+
if (runtime !== undefined)
|
|
32430
|
+
return runtime in connections ? { connection: connectionName(runtime) } : undefined;
|
|
32388
32431
|
if (engine in harnesses)
|
|
32389
32432
|
return { harness: harnessName(engine) };
|
|
32390
32433
|
if (engine in connections)
|
|
@@ -32396,7 +32439,7 @@ function nameDefaults(defaults, converted, conversion, written) {
|
|
|
32396
32439
|
connection: connectionName(engine),
|
|
32397
32440
|
field
|
|
32398
32441
|
}));
|
|
32399
|
-
return { defaultConnection, defaultHarness, connectionName, harnessName, ...named, keys, target };
|
|
32442
|
+
return { defaultEngine, defaultConnection, defaultHarness, connectionName, harnessName, ...named, keys, target };
|
|
32400
32443
|
}
|
|
32401
32444
|
function mergeWritten(out, naming) {
|
|
32402
32445
|
for (const [name, harness] of Object.entries(naming.harnesses)) {
|
|
@@ -32415,6 +32458,19 @@ function ownModelsOf(converted, naming) {
|
|
|
32415
32458
|
}
|
|
32416
32459
|
return ownModels;
|
|
32417
32460
|
}
|
|
32461
|
+
function modelWorkTarget(defaults, naming, merged, inherited, conversion) {
|
|
32462
|
+
const connection = defaultName({ ...inherited?.connections, ...merged.connections });
|
|
32463
|
+
if (connection !== undefined)
|
|
32464
|
+
return { connection };
|
|
32465
|
+
const harness = defaultName({ ...inherited?.harnesses, ...merged.harnesses });
|
|
32466
|
+
const entry = harness === undefined ? undefined : merged.harnesses[harness];
|
|
32467
|
+
if (harness === undefined || !isRecord(entry))
|
|
32468
|
+
return;
|
|
32469
|
+
if (str(defaults.llmEngine) === undefined && harness in naming.harnesses && entry.modelWork === undefined) {
|
|
32470
|
+
conversion.notes.push(`harnesses.${harness}: akm's own model work (improve, index passes, extraction, judges) now runs on this harness, the default for it with no connection. Without defaults.llmEngine, 0.9 ran none of it unless a setting named an engine (a workflow gate's judge ran on defaults.engine). To keep the 0.9 behaviour, set \`modelWork: false\` on this harness: a call that names it still runs there, but a workflow gate's judge then needs a connection (\`akm setup --endpoint URL\`).`);
|
|
32471
|
+
}
|
|
32472
|
+
return entry.modelWork === false ? undefined : { harness };
|
|
32473
|
+
}
|
|
32418
32474
|
function convertEngineConfig(raw, options = {}) {
|
|
32419
32475
|
const out = structuredClone(raw);
|
|
32420
32476
|
const engines = asDict(out.engines);
|
|
@@ -32437,7 +32493,8 @@ function convertEngineConfig(raw, options = {}) {
|
|
|
32437
32493
|
written: { connections: asDict(out.connections), harnesses: asDict(out.harnesses) },
|
|
32438
32494
|
inherited: { connections: options.inherited?.connections ?? {}, harnesses: options.inherited?.harnesses ?? {} },
|
|
32439
32495
|
renamed: {
|
|
32440
|
-
runtime: [...converted.runtimes.values()].filter((name) =>
|
|
32496
|
+
runtime: [...converted.runtimes.values()].filter((name) => (name in converted.connections)).map((name) => naming.connectionName(name)),
|
|
32497
|
+
sdkHarness: [...converted.sdkHarnesses].map((name) => naming.harnessName(name)),
|
|
32441
32498
|
connection: (engine) => (engine in converted.connections) ? naming.connectionName(engine) : undefined
|
|
32442
32499
|
}
|
|
32443
32500
|
});
|
|
@@ -32447,7 +32504,8 @@ function convertEngineConfig(raw, options = {}) {
|
|
|
32447
32504
|
if (isRecord(improve))
|
|
32448
32505
|
moveStrategyExecution(improve, conversion, retarget);
|
|
32449
32506
|
improveStrategyDefault(out, defaults, improve);
|
|
32450
|
-
|
|
32507
|
+
const judge = modelWorkTarget(defaults, naming, merged, options.inherited, conversion);
|
|
32508
|
+
dropJudgeEngine(out, conversion, naming.target, naming.defaultEngine, judge);
|
|
32451
32509
|
convertIndexEngines(out, conversion, retarget);
|
|
32452
32510
|
const config2 = {};
|
|
32453
32511
|
let placed = false;
|
|
@@ -32478,7 +32536,7 @@ function convertEngineConfig(raw, options = {}) {
|
|
|
32478
32536
|
targets
|
|
32479
32537
|
};
|
|
32480
32538
|
}
|
|
32481
|
-
var LLM_ENGINE_KEYS, AGENT_ENGINE_KEYS, CARRIED;
|
|
32539
|
+
var LLM_ENGINE_KEYS, AGENT_ENGINE_KEYS, CARRIED, defaultName = (entries) => ("default" in entries) ? "default" : Object.keys(entries).length === 1 ? Object.keys(entries)[0] : undefined, IMPLICIT_AGENT_ENGINE = "opencode-sdk";
|
|
32482
32540
|
var init_convert_engines = __esm(() => {
|
|
32483
32541
|
init_common();
|
|
32484
32542
|
init_convert_engine_references();
|
|
@@ -34567,6 +34625,27 @@ function isMarkdownFenceClosing(line, fence) {
|
|
|
34567
34625
|
return false;
|
|
34568
34626
|
return /^[\t ]*$/.test(line.slice(cursor));
|
|
34569
34627
|
}
|
|
34628
|
+
function fencedCodeIdentifiers(line) {
|
|
34629
|
+
const cleaned = line.replace(/(["'`])((?:\\.|(?!\1).)*)\1(\s*:)?/g, (match, _quote, body, colon, offset) => {
|
|
34630
|
+
if (colon)
|
|
34631
|
+
return /^\s*["'`]/.test(line.slice(offset + match.length)) ? ` ${body} ` : ` ${body}: `;
|
|
34632
|
+
return /^\/[\w./-]+$/.test(body) ? ` ${body} ` : " ";
|
|
34633
|
+
}).replace(/([A-Za-z_][A-Za-z0-9_.-]*\s*[:=]\s*)((?:[Bb]earer|[Bb]asic|[Tt]oken|[Dd]igest)\s+\S+|\S+)/g, (_match, name, value) => /^[!(]*[A-Za-z_][\w$]*(?:\.[A-Za-z_][\w$]*)+[()\]}]*[,;]?$/.test(value) ? `${name} ${value}` : name).replace(/(?<![\w.-])[A-Za-z0-9+/=]{20,}(?![\w.-])/g, (run) => /[a-z]/.test(run) && /[A-Z]/.test(run) && /\d/.test(run) ? " " : run);
|
|
34634
|
+
const words = [];
|
|
34635
|
+
for (let token of cleaned.match(/--?[A-Za-z][A-Za-z0-9-]*|[A-Za-z_][A-Za-z0-9_.-]*/g) ?? []) {
|
|
34636
|
+
token = token.replace(/[.-]+$/, "");
|
|
34637
|
+
const bare = token.replace(/^-+/, "");
|
|
34638
|
+
if (bare.length < 2)
|
|
34639
|
+
continue;
|
|
34640
|
+
if (bare.length >= TOKEN_LIKE_MIN_LENGTH && /\d/.test(bare))
|
|
34641
|
+
continue;
|
|
34642
|
+
words.push(token);
|
|
34643
|
+
const parts = bare.split(/[_.-]|(?<=[a-z0-9])(?=[A-Z])/).filter((part) => part.length > 1);
|
|
34644
|
+
if (parts.length > 1)
|
|
34645
|
+
words.push(...parts);
|
|
34646
|
+
}
|
|
34647
|
+
return words.join(" ");
|
|
34648
|
+
}
|
|
34570
34649
|
function stripMarkdownHtmlComments(line, state) {
|
|
34571
34650
|
let cursor = 0;
|
|
34572
34651
|
let visible = "";
|
|
@@ -34732,6 +34811,7 @@ function projectMarkdownContent(body, truncationInfo) {
|
|
|
34732
34811
|
const innerBlock = findInnerFrontmatterBlock(lines);
|
|
34733
34812
|
const start = innerBlock && isFrontmatterShaped(lines, innerBlock) ? innerBlock.close + 1 : 0;
|
|
34734
34813
|
const projected = [];
|
|
34814
|
+
const fenced = [];
|
|
34735
34815
|
forEachVisibleMarkdownLine(lines, start, (_index, visible) => {
|
|
34736
34816
|
let trimmed = visible.trim();
|
|
34737
34817
|
if (!trimmed || /^(-{3,}|\*{3,}|_{3,}|=+)$/.test(trimmed))
|
|
@@ -34749,15 +34829,19 @@ function projectMarkdownContent(body, truncationInfo) {
|
|
|
34749
34829
|
trimmed = trimmed.replace(/\s+/g, " ").trim();
|
|
34750
34830
|
if (trimmed)
|
|
34751
34831
|
projected.push(trimmed);
|
|
34832
|
+
}, (_index, fencedLine) => {
|
|
34833
|
+
const identifiers = fencedCodeIdentifiers(fencedLine);
|
|
34834
|
+
if (identifiers)
|
|
34835
|
+
fenced.push(identifiers);
|
|
34752
34836
|
});
|
|
34753
|
-
const text2 = projected.join(" ").replace(/\s+/g, " ").trim();
|
|
34837
|
+
const text2 = [...projected, ...fenced].join(" ").replace(/\s+/g, " ").trim();
|
|
34754
34838
|
if (!text2)
|
|
34755
34839
|
return;
|
|
34756
34840
|
if (truncationInfo)
|
|
34757
34841
|
truncationInfo.truncated = text2.length > MARKDOWN_CONTENT_MAX_CHARS;
|
|
34758
34842
|
return truncateUnicodeSafe(text2, MARKDOWN_CONTENT_MAX_CHARS);
|
|
34759
34843
|
}
|
|
34760
|
-
function forEachVisibleMarkdownLine(lines, start, visit3) {
|
|
34844
|
+
function forEachVisibleMarkdownLine(lines, start, visit3, visitFenced) {
|
|
34761
34845
|
let fence;
|
|
34762
34846
|
const htmlComment = { inComment: false };
|
|
34763
34847
|
for (let index = start;index < lines.length; index += 1) {
|
|
@@ -34765,6 +34849,8 @@ function forEachVisibleMarkdownLine(lines, start, visit3) {
|
|
|
34765
34849
|
if (fence) {
|
|
34766
34850
|
if (isMarkdownFenceClosing(rawLine, fence))
|
|
34767
34851
|
fence = undefined;
|
|
34852
|
+
else
|
|
34853
|
+
visitFenced?.(index, rawLine);
|
|
34768
34854
|
continue;
|
|
34769
34855
|
}
|
|
34770
34856
|
const leadingFence = htmlComment.inComment ? undefined : parseMarkdownFenceOpening(rawLine);
|
|
@@ -34935,7 +35021,7 @@ function extractDirTagsFromName(name) {
|
|
|
34935
35021
|
}
|
|
34936
35022
|
return Array.from(tags);
|
|
34937
35023
|
}
|
|
34938
|
-
var SCOPE_KEYS, KNOWN_QUALITY_VALUES, warnedUnknownQualityValues, COMMENT_TAGS, MARKDOWN_CONTENT_MAX_CHARS = 1e6, MAX_MARKDOWN_LINK_NESTING = 32, markdownFragmentContentByEntry, markdownFragmentProjectionEntries;
|
|
35024
|
+
var SCOPE_KEYS, KNOWN_QUALITY_VALUES, warnedUnknownQualityValues, COMMENT_TAGS, MARKDOWN_CONTENT_MAX_CHARS = 1e6, TOKEN_LIKE_MIN_LENGTH = 20, MAX_MARKDOWN_LINK_NESTING = 32, markdownFragmentContentByEntry, markdownFragmentProjectionEntries;
|
|
34939
35025
|
var init_metadata = __esm(() => {
|
|
34940
35026
|
init_asset_ref();
|
|
34941
35027
|
init_frontmatter();
|
|
@@ -154,10 +154,8 @@ function hasExtractedRepo(repoDir) {
|
|
|
154
154
|
try {
|
|
155
155
|
if (!fs.statSync(repoDir).isDirectory())
|
|
156
156
|
return false;
|
|
157
|
-
//
|
|
158
|
-
// adapter's directoryList()
|
|
159
|
-
// stash-subdir names, with `stashDirNames()` kept live as the fallback. The
|
|
160
|
-
// content/ subdir check above is preserved verbatim.
|
|
157
|
+
// The "any type dir present" test is sourced from the `akm`
|
|
158
|
+
// adapter's directoryList(), with `stashDirNames()` as the fallback.
|
|
161
159
|
const ownedDirs = akmAdapter.directoryList?.({ id: "akm", adapter: "akm", root: repoDir, writable: false }) ?? stashDirNames();
|
|
162
160
|
return ownedDirs.some((dirName) => fs.existsSync(path.join(repoDir, dirName)));
|
|
163
161
|
}
|
|
@@ -37,22 +37,16 @@ function looksLikeAnyBundleRoot(root) {
|
|
|
37
37
|
}
|
|
38
38
|
/**
|
|
39
39
|
* Find the directory inside `extractedDir` that should be treated as the
|
|
40
|
-
* bundle/stash root. Probes the top-level dir via the ordered
|
|
40
|
+
* bundle/stash root. Probes the top-level dir via the ordered registry
|
|
41
41
|
* probe; if nothing claims it, BFS for the shallowest nested candidate.
|
|
42
42
|
*/
|
|
43
43
|
export function detectStashRoot(extractedDir) {
|
|
44
44
|
const root = path.resolve(extractedDir);
|
|
45
|
-
//
|
|
46
|
-
//
|
|
47
|
-
//
|
|
48
|
-
//
|
|
49
|
-
//
|
|
50
|
-
// (akm-adapter.looksLikeRoot) and `akm` is in the probe set — so those two
|
|
51
|
-
// local checks are now subsumed and were removed. It additionally recognizes
|
|
52
|
-
// NON-akm bundle roots (okf root index / llm-wiki / `.claude` / …) at the top
|
|
53
|
-
// level, which the single-adapter probe missed but the old final `return root`
|
|
54
|
-
// fallback also happened to yield. The shallowest-BFS nested-layout discovery
|
|
55
|
-
// is unchanged.
|
|
45
|
+
// The top-level root check routes through the ordered `looksLikeRoot` probe
|
|
46
|
+
// over ALL built-in adapters (`registry.getAdapters()`). `akm.looksLikeRoot`
|
|
47
|
+
// covers the `.stash` marker and immediate stash subdirs; the other adapters
|
|
48
|
+
// recognize NON-akm bundle roots (okf root index / llm-wiki / `.claude` / …).
|
|
49
|
+
// Otherwise fall through to the shallowest-BFS nested-layout discovery.
|
|
56
50
|
if (looksLikeAnyBundleRoot(root)) {
|
|
57
51
|
return root;
|
|
58
52
|
}
|
|
@@ -161,7 +155,8 @@ function countStashDirs(dirPath) {
|
|
|
161
155
|
* (scripts/, skills/, etc.), so nested layouts like `project/my-stash/scripts/`
|
|
162
156
|
* are discovered even without a `.stash` marker.
|
|
163
157
|
*
|
|
164
|
-
* Skips `root` itself since the caller already checked it via
|
|
158
|
+
* Skips `root` itself since the caller already checked it via
|
|
159
|
+
* `looksLikeAnyBundleRoot`.
|
|
165
160
|
*/
|
|
166
161
|
const BFS_MAX_DEPTH = 5;
|
|
167
162
|
function findShallowestStashRoot(root) {
|
|
@@ -90,7 +90,7 @@ export async function ensureWebsiteMirror(config, options) {
|
|
|
90
90
|
crawlTimeoutMs: resolveCrawlTimeoutMs(config.options?.crawlTimeoutMs),
|
|
91
91
|
resolveSecret: options?.resolveSecret,
|
|
92
92
|
// As-supplied, pre-normalization start URL (see crawlWebsite's
|
|
93
|
-
// `rawStartUrl` doc comment): threaded through purely for the
|
|
93
|
+
// `rawStartUrl` doc comment): threaded through purely for the start-URL
|
|
94
94
|
// robots.txt check, which must see the trailing slash the user
|
|
95
95
|
// actually typed before `normalizeSiteUrl` strips it.
|
|
96
96
|
rawStartUrl: rawUrl,
|
|
@@ -461,7 +461,7 @@ async function resolveCrawlRobotsDecision(robots, normalizedUrl, rawUrl) {
|
|
|
461
461
|
/**
|
|
462
462
|
* Fail fast, before any page fetch, when the start URL itself is
|
|
463
463
|
* off-limits per an actually-parsed robots.txt rule. A 5xx on robots.txt
|
|
464
|
-
* itself
|
|
464
|
+
* itself does not reach this as `disallowAll`: `loadRobotsTxt` reports it the same way a
|
|
465
465
|
* 4xx is reported (`ALLOW_ALL_RULES`), with its own warning, rather than
|
|
466
466
|
* treating the site's own transient error as a Disallow. The check below
|
|
467
467
|
* stays as defense-in-depth for a `RobotsPolicy` built over some other
|
|
@@ -594,7 +594,7 @@ async function crawlWebsite(startUrl, options) {
|
|
|
594
594
|
}
|
|
595
595
|
// Counts actual `fetchWebsitePage` invocations (regardless of outcome) so
|
|
596
596
|
// Crawl-delay pacing skips the first fetch and never charges a delay slot
|
|
597
|
-
// to a URL that robots.txt skipped without ever being fetched
|
|
597
|
+
// to a URL that robots.txt skipped without ever being fetched.
|
|
598
598
|
let fetchAttempts = 0;
|
|
599
599
|
// URLs pushed back because their Crawl-delay would not fit in the remaining
|
|
600
600
|
// budget, and URLs that ran out of retries entirely. Reported at the end so
|
|
@@ -827,8 +827,7 @@ async function fetchWebsiteResponse(pageUrl, redirectCount = 0, options) {
|
|
|
827
827
|
// origin is the literal string "null", and re-resolving against it
|
|
828
828
|
// throws an unhandled TypeError that aborts the whole crawl. Refuse
|
|
829
829
|
// the hop outright instead of ever handing such a URL to the policy —
|
|
830
|
-
// `fetchWebsitePage` maps this to a graceful skip.
|
|
831
|
-
// introduced by a67412c, which fell back to the raw `nextUrl` here.
|
|
830
|
+
// `fetchWebsitePage` maps this to a graceful skip.
|
|
832
831
|
warnVerbose("[akm] website crawl: skipping redirect to %s (not an http(s) URL)", nextUrl);
|
|
833
832
|
throw new RobotsDisallowedRedirectError(nextUrl);
|
|
834
833
|
}
|
|
@@ -990,7 +989,7 @@ export function parseLlmsTxtLinks(text, baseUrl) {
|
|
|
990
989
|
* content legitimately contains `##` headings too — so recovered page
|
|
991
990
|
* boundaries can't be trusted, whereas per-page fetches through the existing
|
|
992
991
|
* pipeline are cheap, bounded by this author-curated list, and produce
|
|
993
|
-
* cleanly addressable assets.
|
|
992
|
+
* cleanly addressable assets.
|
|
994
993
|
*/
|
|
995
994
|
async function fetchLlmsManifest(start, robots, options) {
|
|
996
995
|
const manifestUrl = new URL("/llms.txt", start.origin).toString();
|
|
@@ -1043,7 +1042,7 @@ function normalizeCrawlUrl(rawUrl) {
|
|
|
1043
1042
|
}
|
|
1044
1043
|
}
|
|
1045
1044
|
/**
|
|
1046
|
-
*
|
|
1045
|
+
* `index.md`/`log.md` are OKF reserved structural filenames at every
|
|
1047
1046
|
* depth — no adapter indexes them, so a crawled page must never land on one.
|
|
1048
1047
|
* Same remap convention as the content migration (`index.md` →
|
|
1049
1048
|
* `index-content.md`). Segments are already lowercased by slugifySegment.
|
|
@@ -147,3 +147,18 @@ export function purgeOldEvents(db, retentionDays = 90) {
|
|
|
147
147
|
const changes = result.changes ?? 0;
|
|
148
148
|
return typeof changes === "bigint" ? Number(changes) : changes;
|
|
149
149
|
}
|
|
150
|
+
/**
|
|
151
|
+
* Rewrite every `events.ref` naming `oldBundleId` to `newBundleId` (`akm bundle
|
|
152
|
+
* rename`). Improve reads an asset's feedback events by this ref, and
|
|
153
|
+
* `insertEventOnce` finds a journal event by it under the proposal or workflow
|
|
154
|
+
* ref the rename also rewrites, so an event left under the old prefix stops
|
|
155
|
+
* counting for the renamed bundle. A legacy unqualified ref is not matched by
|
|
156
|
+
* the `<old>//` prefix and is left alone; refs inside `metadata_json` are
|
|
157
|
+
* history and stay as written. Returns the number of rows rewritten.
|
|
158
|
+
*/
|
|
159
|
+
export function renameEventRefs(db, oldBundleId, newBundleId) {
|
|
160
|
+
const prefix = `${oldBundleId}//`;
|
|
161
|
+
return Number(db
|
|
162
|
+
.prepare(`UPDATE events SET ref = ? || substr(ref, length(?) + 1) WHERE substr(ref, 1, length(?)) = ?`)
|
|
163
|
+
.run(newBundleId, oldBundleId, prefix, prefix).changes);
|
|
164
|
+
}
|
|
@@ -269,3 +269,16 @@ export function listImproveLedgerRows(db, stashDir, sources) {
|
|
|
269
269
|
.all(stashDir, ...(sources && sources.length > 0 ? sources : []));
|
|
270
270
|
return rows.map((row) => toRow(withHash ? row : { ...row, content_hash: null }));
|
|
271
271
|
}
|
|
272
|
+
/**
|
|
273
|
+
* Rewrite every `improve_ledger.ref` naming `oldBundleId` to `newBundleId`
|
|
274
|
+
* (`akm bundle rename`), so improve keeps the renamed bundle's attempts and
|
|
275
|
+
* back-off windows next to its feedback events. A row already at the new key
|
|
276
|
+
* can only be an orphan of a removed bundle by that name; the renamed bundle's
|
|
277
|
+
* row replaces it. Returns the number of rows rewritten.
|
|
278
|
+
*/
|
|
279
|
+
export function renameImproveLedgerRefs(db, oldBundleId, newBundleId) {
|
|
280
|
+
const prefix = `${oldBundleId}//`;
|
|
281
|
+
return Number(db
|
|
282
|
+
.prepare(`UPDATE OR REPLACE improve_ledger SET ref = ? || substr(ref, length(?) + 1) WHERE substr(ref, 1, length(?)) = ?`)
|
|
283
|
+
.run(newBundleId, oldBundleId, prefix, prefix).changes);
|
|
284
|
+
}
|
|
@@ -18,6 +18,7 @@ import { getStateDbPath, withStateDb } from "../../core/state-db.js";
|
|
|
18
18
|
import { warn } from "../../core/warn.js";
|
|
19
19
|
import { getMarkdownFragmentContent, hasMarkdownFragmentContent, } from "../../indexer/passes/metadata.js";
|
|
20
20
|
import { buildSearchText } from "../../indexer/search/search-fields.js";
|
|
21
|
+
import { capEmbeddingText, DEFAULT_MAX_INPUT_TOKENS } from "../../llm/embedders/remote.js";
|
|
21
22
|
import { sha256Hex } from "../../runtime.js";
|
|
22
23
|
import { ENTRY_COLUMNS, rowToIndexedEntry } from "./index-entry-mapper.js";
|
|
23
24
|
import { deleteFtsEntries, replaceFtsEntry } from "./index-fts-repository.js";
|
|
@@ -26,13 +27,21 @@ import { SQLITE_CHUNK_SIZE } from "./index-sql.js";
|
|
|
26
27
|
import { deleteEntryVectors } from "./index-vec-repository.js";
|
|
27
28
|
// ── Entry operations ────────────────────────────────────────────────────────
|
|
28
29
|
/**
|
|
29
|
-
* Hash of
|
|
30
|
-
*
|
|
31
|
-
*
|
|
30
|
+
* Hash of an entry's whole search text (`buildSearchText`), stored as
|
|
31
|
+
* `entries.embed_hash`: text that hashes the same cannot change what the
|
|
32
|
+
* entry's vector is embedded from.
|
|
32
33
|
*/
|
|
33
34
|
function embedHash(entry) {
|
|
34
35
|
return sha256Hex(buildSearchText(entry));
|
|
35
36
|
}
|
|
37
|
+
/**
|
|
38
|
+
* The text an entry's vector is embedded from: its search text cut to the
|
|
39
|
+
* embedding cap (`embedding.maxInputTokens`). Text past the cut never reaches
|
|
40
|
+
* the embedder, so a change there leaves the vector as it is.
|
|
41
|
+
*/
|
|
42
|
+
function embeddedInput(entry, maxInputTokens) {
|
|
43
|
+
return capEmbeddingText(buildSearchText(entry), maxInputTokens).text;
|
|
44
|
+
}
|
|
36
45
|
/**
|
|
37
46
|
* Insert or update one canonical entry and all synchronously derived search
|
|
38
47
|
* state. Returns the stable row id.
|
|
@@ -40,8 +49,13 @@ function embedHash(entry) {
|
|
|
40
49
|
* The entries row, FTS projection, declared links, and stale-vector
|
|
41
50
|
* invalidation commit as one SQLite transaction. Callers therefore cannot
|
|
42
51
|
* publish an entry and forget a second FTS or links maintenance step.
|
|
52
|
+
*
|
|
53
|
+
* The entry's vector is deleted when the text it is embedded from changes,
|
|
54
|
+
* that is when the search text cut to `maxInputTokens` (the embedding cap,
|
|
55
|
+
* `embedding.maxInputTokens`) differs from the cut of the stored document. A
|
|
56
|
+
* change past the cut rewrites the keyword rows and keeps the vector.
|
|
43
57
|
*/
|
|
44
|
-
export function upsertEntry(db, filePath, entry, provenance, contentHash) {
|
|
58
|
+
export function upsertEntry(db, filePath, entry, provenance, contentHash, maxInputTokens = DEFAULT_MAX_INPUT_TOKENS) {
|
|
45
59
|
// Hot path during indexing — cache prepared statements per database
|
|
46
60
|
// connection so we don't pay the SQL parse/compile cost on every call.
|
|
47
61
|
const stmts = getUpsertStmts(db);
|
|
@@ -54,10 +68,14 @@ export function upsertEntry(db, filePath, entry, provenance, contentHash) {
|
|
|
54
68
|
// writer's current value.
|
|
55
69
|
const apply = () => {
|
|
56
70
|
const previous = stmts.findByItemRef.get(provenance.itemRef);
|
|
71
|
+
// Decided before the upsert overwrites the stored document it compares against.
|
|
72
|
+
const vectorStale = previous != null &&
|
|
73
|
+
previous.embed_hash !== hash &&
|
|
74
|
+
!storedDocumentEmbedsAs(stmts, previous, embeddedInput(entry, maxInputTokens), maxInputTokens);
|
|
57
75
|
const result = stmts.upsert.get(provenance.itemRef, provenance.bundleId, provenance.componentId, provenance.conceptId, provenance.adapterId, entry.type, filePath, contentHash ?? null, JSON.stringify(entry), derivedFrom, hash);
|
|
58
76
|
if (!result)
|
|
59
77
|
throw new Error("upsertEntry: item_ref not found after upsert");
|
|
60
|
-
if (previous?.id === result.id &&
|
|
78
|
+
if (previous?.id === result.id && vectorStale)
|
|
61
79
|
deleteEntryVectors(db, result.id);
|
|
62
80
|
replaceFtsEntry(db, result.id, entry, hasMarkdownFragmentContent(entry) ? (getMarkdownFragmentContent(entry) ?? null) : undefined);
|
|
63
81
|
replaceEntryLinks(db, result.id, entry, provenance);
|
|
@@ -69,6 +87,25 @@ export function upsertEntry(db, filePath, entry, provenance, contentHash) {
|
|
|
69
87
|
// partial entries row through its outer transaction.
|
|
70
88
|
return db.transaction(apply)();
|
|
71
89
|
}
|
|
90
|
+
/**
|
|
91
|
+
* Whether the stored document of `previous` embeds as `input`. The stored
|
|
92
|
+
* document is what the entry's current vector was embedded from, so the
|
|
93
|
+
* previous input is derived from it and needs no column of its own. The stored
|
|
94
|
+
* hash vouches for that: a document that cannot be read, or whose search text
|
|
95
|
+
* no longer hashes to `embed_hash`, has no known input, and its vector goes.
|
|
96
|
+
*/
|
|
97
|
+
function storedDocumentEmbedsAs(stmts, previous, input, maxInputTokens) {
|
|
98
|
+
const row = stmts.documentById.get(previous.id);
|
|
99
|
+
if (!row)
|
|
100
|
+
return false;
|
|
101
|
+
try {
|
|
102
|
+
const text = buildSearchText(JSON.parse(row.document_json));
|
|
103
|
+
return sha256Hex(text) === previous.embed_hash && capEmbeddingText(text, maxInputTokens).text === input;
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
return false;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
72
109
|
const upsertStmtsByDb = new WeakMap();
|
|
73
110
|
// item_ref is the sole durable conflict target. `content_hash` COALESCEs so
|
|
74
111
|
// an upsert that omits it (`contentHash` is optional) cannot wipe a scan hash.
|
|
@@ -101,6 +138,7 @@ function getUpsertStmts(db) {
|
|
|
101
138
|
RETURNING id
|
|
102
139
|
`),
|
|
103
140
|
findByItemRef: db.prepare("SELECT id, embed_hash FROM entries WHERE item_ref = ?"),
|
|
141
|
+
documentById: db.prepare("SELECT document_json FROM entries WHERE id = ?"),
|
|
104
142
|
};
|
|
105
143
|
upsertStmtsByDb.set(db, stmts);
|
|
106
144
|
return stmts;
|
|
@@ -118,3 +118,15 @@ export function countAssetOutcomeMissing(db) {
|
|
|
118
118
|
const row = db.prepare(`SELECT COUNT(*) AS n FROM asset_outcome WHERE missing_since IS NOT NULL`).get();
|
|
119
119
|
return row.n;
|
|
120
120
|
}
|
|
121
|
+
/**
|
|
122
|
+
* Rewrite every `asset_outcome.asset_ref` naming `oldBundleId` to
|
|
123
|
+
* `newBundleId` (`akm bundle rename`). Same prefix match and orphan-replacing
|
|
124
|
+
* collision policy as `renameAssetSalienceRefs`. Returns the number of rows
|
|
125
|
+
* rewritten.
|
|
126
|
+
*/
|
|
127
|
+
export function renameAssetOutcomeRefs(db, oldBundleId, newBundleId) {
|
|
128
|
+
const prefix = `${oldBundleId}//`;
|
|
129
|
+
return Number(db
|
|
130
|
+
.prepare(`UPDATE OR REPLACE asset_outcome SET asset_ref = ? || substr(asset_ref, length(?) + 1) WHERE substr(asset_ref, 1, length(?)) = ?`)
|
|
131
|
+
.run(newBundleId, oldBundleId, prefix, prefix).changes);
|
|
132
|
+
}
|
|
@@ -504,8 +504,8 @@ export function listStateProposalIdsByPrefix(db, stashDir, idPrefix) {
|
|
|
504
504
|
* rename --dry-run`: the same two `WHERE` predicates, read-only.
|
|
505
505
|
*/
|
|
506
506
|
export function countProposalsForBundleRename(db, oldBundleId) {
|
|
507
|
-
const prefix = `${
|
|
508
|
-
const refs = db.prepare(`SELECT COUNT(*) AS n FROM proposals WHERE ref
|
|
507
|
+
const prefix = `${oldBundleId}//`;
|
|
508
|
+
const refs = db.prepare(`SELECT COUNT(*) AS n FROM proposals WHERE substr(ref, 1, length(?)) = ?`).get(prefix, prefix).n;
|
|
509
509
|
const targets = db
|
|
510
510
|
.prepare(`SELECT COUNT(*) AS n FROM proposals WHERE json_extract(metadata_json, '$.proposedTarget.source') = ?`)
|
|
511
511
|
.get(oldBundleId).n;
|
|
@@ -522,11 +522,11 @@ export function countProposalsForBundleRename(db, oldBundleId) {
|
|
|
522
522
|
* rewritten.
|
|
523
523
|
*/
|
|
524
524
|
export function renameProposalsBundleRef(db, oldBundleId, newBundleId) {
|
|
525
|
-
const prefix = `${
|
|
525
|
+
const prefix = `${oldBundleId}//`;
|
|
526
526
|
return db.transaction(() => {
|
|
527
527
|
const refs = Number(db
|
|
528
|
-
.prepare(`UPDATE proposals SET ref = ? || substr(ref, length(?) + 1) WHERE ref
|
|
529
|
-
.run(newBundleId, oldBundleId,
|
|
528
|
+
.prepare(`UPDATE proposals SET ref = ? || substr(ref, length(?) + 1) WHERE substr(ref, 1, length(?)) = ?`)
|
|
529
|
+
.run(newBundleId, oldBundleId, prefix, prefix).changes);
|
|
530
530
|
const targets = Number(db
|
|
531
531
|
.prepare(`UPDATE proposals SET metadata_json = json_set(metadata_json, '$.proposedTarget.source', ?)
|
|
532
532
|
WHERE json_extract(metadata_json, '$.proposedTarget.source') = ?`)
|
|
@@ -153,3 +153,16 @@ export function countAssetSalienceMissing(db) {
|
|
|
153
153
|
const row = db.prepare(`SELECT COUNT(*) AS n FROM asset_salience WHERE missing_since IS NOT NULL`).get();
|
|
154
154
|
return row.n;
|
|
155
155
|
}
|
|
156
|
+
/**
|
|
157
|
+
* Rewrite every `asset_salience.asset_ref` naming `oldBundleId` to
|
|
158
|
+
* `newBundleId` (`akm bundle rename`). A legacy unqualified ref is not matched
|
|
159
|
+
* by the `<old>//` prefix and is left alone. A row already at the new ref can only
|
|
160
|
+
* be an orphan of a removed bundle by that name; the renamed bundle's row
|
|
161
|
+
* replaces it. Returns the number of rows rewritten.
|
|
162
|
+
*/
|
|
163
|
+
export function renameAssetSalienceRefs(db, oldBundleId, newBundleId) {
|
|
164
|
+
const prefix = `${oldBundleId}//`;
|
|
165
|
+
return Number(db
|
|
166
|
+
.prepare(`UPDATE OR REPLACE asset_salience SET asset_ref = ? || substr(asset_ref, length(?) + 1) WHERE substr(asset_ref, 1, length(?)) = ?`)
|
|
167
|
+
.run(newBundleId, oldBundleId, prefix, prefix).changes);
|
|
168
|
+
}
|