akm-cli 0.9.0-rc.2 → 0.9.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -8
- package/dist/assets/improve-strategies/catchup.json +3 -1
- package/dist/assets/improve-strategies/consolidate.json +3 -1
- package/dist/assets/improve-strategies/frequent.json +3 -1
- package/dist/assets/improve-strategies/graph-refresh.json +3 -1
- package/dist/assets/improve-strategies/memory-focus.json +4 -1
- package/dist/assets/improve-strategies/proactive-maintenance.json +1 -0
- package/dist/assets/improve-strategies/quick.json +3 -1
- package/dist/assets/improve-strategies/recombine-only.json +2 -0
- package/dist/assets/improve-strategies/reflect-distill.json +1 -0
- package/dist/assets/improve-strategies/synthesize.json +2 -0
- package/dist/assets/tasks/core/backup.yml +2 -2
- package/dist/cli/config-migrate.js +554 -26
- package/dist/cli.js +3 -1
- package/dist/commands/backup-cli.js +6 -4
- package/dist/commands/config-cli.js +10 -3
- package/dist/commands/feedback-cli.js +24 -7
- package/dist/commands/health/checks.js +94 -15
- package/dist/commands/health/surfaces.js +2 -1
- package/dist/commands/improve/anti-collapse.js +3 -3
- package/dist/commands/improve/collapse-detector.js +5 -5
- package/dist/commands/improve/consolidate.js +123 -66
- package/dist/commands/improve/distill/promote-memory.js +8 -6
- package/dist/commands/improve/distill/quality-gate.js +1 -1
- package/dist/commands/improve/distill-guards.js +1 -1
- package/dist/commands/improve/distill-promotion-policy.js +32 -26
- package/dist/commands/improve/distill.js +52 -36
- package/dist/commands/improve/eligibility.js +9 -8
- package/dist/commands/improve/extract-prompt.js +2 -2
- package/dist/commands/improve/extract.js +43 -11
- package/dist/commands/improve/improve-auto-accept.js +14 -28
- package/dist/commands/improve/improve-cli.js +4 -2
- package/dist/commands/improve/improve-strategies.js +2 -1
- package/dist/commands/improve/improve.js +49 -8
- package/dist/commands/improve/loop-stages.js +11 -6
- package/dist/commands/improve/preparation.js +51 -17
- package/dist/commands/improve/procedural.js +10 -6
- package/dist/commands/improve/recombine.js +31 -7
- package/dist/commands/improve/reflect.js +24 -14
- package/dist/commands/improve/salience.js +5 -1
- package/dist/commands/improve/source-identity.js +43 -0
- package/dist/commands/migrate-cli.js +36 -0
- package/dist/commands/mv-cli.js +647 -258
- package/dist/commands/proposal/drain.js +16 -4
- package/dist/commands/proposal/proposal-cli.js +3 -2
- package/dist/commands/proposal/proposal.js +16 -55
- package/dist/commands/proposal/repository.js +664 -58
- package/dist/commands/read/curate.js +4 -2
- package/dist/commands/read/knowledge.js +4 -1
- package/dist/commands/read/search.js +6 -2
- package/dist/commands/read/show.js +8 -7
- package/dist/commands/sources/self-update.js +156 -112
- package/dist/commands/sources/sources-cli.js +7 -2
- package/dist/core/common.js +36 -3
- package/dist/core/config/config-io.js +2 -2
- package/dist/core/config/config-schema.js +15 -3
- package/dist/core/config/config.js +14 -13
- package/dist/core/file-lock.js +2 -2
- package/dist/core/migration-backup.js +816 -171
- package/dist/core/migration-operation.js +44 -0
- package/dist/core/state/migrations.js +4 -7
- package/dist/core/state-db.js +22 -7
- package/dist/core/write-source.js +86 -18
- package/dist/indexer/db/db.js +109 -53
- package/dist/indexer/index-writer-lock.js +38 -37
- package/dist/indexer/index-written-assets.js +73 -56
- package/dist/indexer/indexer.js +5 -1
- package/dist/indexer/search/search-source.js +2 -2
- package/dist/indexer/usage/usage-events.js +8 -2
- package/dist/integrations/agent/engine-resolution.js +14 -7
- package/dist/scripts/migrate-storage.js +867 -990
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +405 -566
- package/dist/setup/detected-engines.js +7 -13
- package/dist/setup/engine-config.js +14 -4
- package/dist/setup/setup.js +1 -1
- package/dist/setup/steps/connection.js +1 -1
- package/dist/setup/steps/tasks.js +1 -1
- package/dist/sources/providers/git-stash.js +58 -21
- package/dist/sources/providers/git.js +1 -1
- package/dist/storage/engines/sqlite-migrations.js +138 -3
- package/dist/storage/repositories/events-repository.js +24 -0
- package/dist/storage/repositories/proposals-repository.js +14 -0
- package/dist/tasks/embedded.js +2 -0
- package/dist/workflows/db.js +24 -13
- package/dist/workflows/validator.js +24 -2
- package/docs/migration/release-notes/0.9.0.md +24 -7
- package/docs/migration/v0.8-to-v0.9.md +124 -19
- package/package.json +1 -1
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
import crypto from "node:crypto";
|
|
5
|
+
import fs from "node:fs";
|
|
6
|
+
import path from "node:path";
|
|
7
|
+
import { ConfigError } from "./errors.js";
|
|
8
|
+
import { getConfigPath, getDataDir } from "./paths.js";
|
|
9
|
+
let afterPendingCheckHook;
|
|
10
|
+
/** TEST-ONLY: run once after a clear pending-operation check. */
|
|
11
|
+
export function _setAfterPendingOperationCheckHookForTests(hook) {
|
|
12
|
+
afterPendingCheckHook = hook;
|
|
13
|
+
}
|
|
14
|
+
function installationId() {
|
|
15
|
+
return crypto
|
|
16
|
+
.createHash("sha256")
|
|
17
|
+
.update(path.resolve(path.dirname(getConfigPath())))
|
|
18
|
+
.update("\0")
|
|
19
|
+
.update(path.resolve(getDataDir()))
|
|
20
|
+
.digest("hex")
|
|
21
|
+
.slice(0, 24);
|
|
22
|
+
}
|
|
23
|
+
export function getMigrationOperationRoot() {
|
|
24
|
+
return path.join(getDataDir(), "backups", "migrations", installationId());
|
|
25
|
+
}
|
|
26
|
+
export function getMigrationRestoreJournalPath() {
|
|
27
|
+
return path.join(getMigrationOperationRoot(), "restore-active.json");
|
|
28
|
+
}
|
|
29
|
+
export function getMigrationApplyJournalPath() {
|
|
30
|
+
return path.join(getMigrationOperationRoot(), "apply-active.json");
|
|
31
|
+
}
|
|
32
|
+
export function assertNoPendingMigrationOperation() {
|
|
33
|
+
for (const [kind, journalPath] of [
|
|
34
|
+
["restore", getMigrationRestoreJournalPath()],
|
|
35
|
+
["migration apply", getMigrationApplyJournalPath()],
|
|
36
|
+
]) {
|
|
37
|
+
if (fs.existsSync(journalPath)) {
|
|
38
|
+
throw new ConfigError(`AKM ${kind} recovery is pending at ${journalPath}; refusing canonical config/database access until recovery completes.`, "INVALID_CONFIG_FILE");
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
const hook = afterPendingCheckHook;
|
|
42
|
+
afterPendingCheckHook = undefined;
|
|
43
|
+
hook?.();
|
|
44
|
+
}
|
|
@@ -2,8 +2,7 @@
|
|
|
2
2
|
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
4
|
import { runMigrations as runSqliteMigrations } from "../../storage/engines/sqlite-migrations.js";
|
|
5
|
-
|
|
6
|
-
const MIGRATIONS = [
|
|
5
|
+
export const STATE_MIGRATIONS = [
|
|
7
6
|
// ── Migration 001 — initial schema ──────────────────────────────────────────
|
|
8
7
|
{
|
|
9
8
|
id: "001-initial-schema",
|
|
@@ -776,10 +775,8 @@ const MIGRATIONS = [
|
|
|
776
775
|
* Called automatically by `openStateDatabase()`.
|
|
777
776
|
*/
|
|
778
777
|
export function runMigrations(db, options) {
|
|
779
|
-
runSqliteMigrations(db,
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
ensureMigrationBackup();
|
|
783
|
-
},
|
|
778
|
+
runSqliteMigrations(db, STATE_MIGRATIONS, {
|
|
779
|
+
applyPending: options?.applyPending,
|
|
780
|
+
generationMarker: options?.generationMarker,
|
|
784
781
|
});
|
|
785
782
|
}
|
package/dist/core/state-db.js
CHANGED
|
@@ -57,10 +57,13 @@
|
|
|
57
57
|
*/
|
|
58
58
|
import fs from "node:fs";
|
|
59
59
|
import path from "node:path";
|
|
60
|
+
import { openDatabase } from "../storage/database.js";
|
|
61
|
+
import { assertCurrentMigrationLedger, assertMigrationLedger } from "../storage/engines/sqlite-migrations.js";
|
|
60
62
|
import { openManagedDatabase, withManagedDb, withManagedDbAsync } from "../storage/managed-db.js";
|
|
61
63
|
import { acquireMaintenanceActivitySync } from "./maintenance-barrier.js";
|
|
62
|
-
import {
|
|
64
|
+
import { assertNoPendingMigrationOperation } from "./migration-operation.js";
|
|
63
65
|
import { getDataDir } from "./paths.js";
|
|
66
|
+
import { runMigrations, STATE_MIGRATIONS } from "./state/migrations.js";
|
|
64
67
|
// ── Path helper ──────────────────────────────────────────────────────────────
|
|
65
68
|
/**
|
|
66
69
|
* Default path: `<dataDir>/state.db`.
|
|
@@ -102,15 +105,28 @@ export function openStateDatabase(dbPath) {
|
|
|
102
105
|
const canonicalPath = getStateDbPath();
|
|
103
106
|
const resolvedPath = dbPath ?? canonicalPath;
|
|
104
107
|
const isCanonical = path.resolve(resolvedPath) === path.resolve(canonicalPath);
|
|
108
|
+
if (isCanonical)
|
|
109
|
+
assertNoPendingMigrationOperation();
|
|
105
110
|
const releaseActivity = isCanonical ? acquireMaintenanceActivitySync("state-db") : undefined;
|
|
106
111
|
try {
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
112
|
+
if (isCanonical)
|
|
113
|
+
assertNoPendingMigrationOperation();
|
|
114
|
+
const existed = fs.existsSync(resolvedPath);
|
|
115
|
+
if (existed) {
|
|
116
|
+
const preflight = openDatabase(resolvedPath, { readonly: true });
|
|
117
|
+
try {
|
|
118
|
+
if (isCanonical)
|
|
119
|
+
assertCurrentMigrationLedger(preflight, STATE_MIGRATIONS);
|
|
120
|
+
else
|
|
121
|
+
assertMigrationLedger(preflight, STATE_MIGRATIONS);
|
|
122
|
+
}
|
|
123
|
+
finally {
|
|
124
|
+
preflight.close();
|
|
125
|
+
}
|
|
126
|
+
}
|
|
111
127
|
const db = openManagedDatabase({
|
|
112
128
|
path: resolvedPath,
|
|
113
|
-
init: (db) => runMigrations(db, {
|
|
129
|
+
init: (db) => runMigrations(db, { applyPending: !(isCanonical && existed) }),
|
|
114
130
|
});
|
|
115
131
|
if (!releaseActivity)
|
|
116
132
|
return db;
|
|
@@ -164,7 +180,6 @@ export function withStateDbAsync(fn, opts) {
|
|
|
164
180
|
// The MIGRATIONS registry + runMigrations live in ./state/migrations (the single
|
|
165
181
|
// append-only ordered source of truth). Imported for internal use by
|
|
166
182
|
// openStateDatabase.
|
|
167
|
-
import { runMigrations } from "./state/migrations.js";
|
|
168
183
|
// ── BEGIN IMMEDIATE transaction helper ───────────────────────────────────────
|
|
169
184
|
/**
|
|
170
185
|
* Run `fn` inside a `BEGIN IMMEDIATE` transaction.
|
|
@@ -13,8 +13,8 @@
|
|
|
13
13
|
* commit behaviour — they only ever touch the filesystem. Git-backed targets
|
|
14
14
|
* are committed in a SINGLE batch at the operation boundary via
|
|
15
15
|
* {@link commitWriteTargetBoundary} (which delegates to `saveGitStash`). This
|
|
16
|
-
* stages
|
|
17
|
-
* one noisy, incomplete commit per asset.
|
|
16
|
+
* stages only operation-owned paths that still have a Git status entry as one
|
|
17
|
+
* complete commit instead of one noisy, incomplete commit per asset.
|
|
18
18
|
*
|
|
19
19
|
* This module is still the **single dispatch point** for write/delete: callers
|
|
20
20
|
* (remember, import, source-add, etc.) MUST go through `writeAssetToSource` /
|
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
*/
|
|
25
25
|
import fs from "node:fs";
|
|
26
26
|
import path from "node:path";
|
|
27
|
-
import { getCachePaths, parseGitRepoUrl, saveGitStash } from "../sources/providers/git.js";
|
|
27
|
+
import { getCachePaths, listGitChangedPaths, parseGitRepoUrl, saveGitStash } from "../sources/providers/git.js";
|
|
28
28
|
import { makeAssetRef } from "./asset/asset-ref.js";
|
|
29
29
|
import { resolveAssetPathFromName, TYPE_DIRS } from "./asset/asset-spec.js";
|
|
30
30
|
import { isWithin, resolveStashDir } from "./common.js";
|
|
@@ -45,6 +45,30 @@ const REJECTED_WRITABLE_KINDS = new Set(["website", "npm"]);
|
|
|
45
45
|
* stream a downstream consumer parses.
|
|
46
46
|
*/
|
|
47
47
|
const COMMIT_MESSAGE_MAX_LENGTH = 4096;
|
|
48
|
+
const pendingGitPaths = new Map();
|
|
49
|
+
function gitTargetKey(source) {
|
|
50
|
+
return `${path.resolve(source.repoPath ?? source.path)}\0${path.resolve(source.path)}`;
|
|
51
|
+
}
|
|
52
|
+
/** Register a successful direct mutation for the target's exact-path boundary commit. */
|
|
53
|
+
export function recordWriteTargetPath(source, filePath) {
|
|
54
|
+
if (source.kind !== "git")
|
|
55
|
+
return;
|
|
56
|
+
const key = gitTargetKey(source);
|
|
57
|
+
const paths = pendingGitPaths.get(key) ?? new Set();
|
|
58
|
+
paths.add(path.resolve(filePath));
|
|
59
|
+
pendingGitPaths.set(key, paths);
|
|
60
|
+
}
|
|
61
|
+
function takeGitTargetPaths(source) {
|
|
62
|
+
const key = gitTargetKey(source);
|
|
63
|
+
const absolutePaths = pendingGitPaths.get(key);
|
|
64
|
+
pendingGitPaths.delete(key);
|
|
65
|
+
if (!absolutePaths)
|
|
66
|
+
return [];
|
|
67
|
+
const repoDir = path.resolve(source.repoPath ?? source.path);
|
|
68
|
+
return [...absolutePaths]
|
|
69
|
+
.map((filePath) => path.relative(repoDir, filePath).replaceAll(path.sep, "/"))
|
|
70
|
+
.filter((filePath) => filePath && filePath !== ".." && !filePath.startsWith("../"));
|
|
71
|
+
}
|
|
48
72
|
/**
|
|
49
73
|
* Sanitize a string before passing it as `git commit -m <message>`.
|
|
50
74
|
*
|
|
@@ -161,6 +185,7 @@ export async function writeAssetToSource(source, config, ref, content) {
|
|
|
161
185
|
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
162
186
|
const normalized = content.endsWith("\n") ? content : `${content}\n`;
|
|
163
187
|
fs.writeFileSync(filePath, normalized, "utf8");
|
|
188
|
+
recordWriteTargetPath(source, filePath);
|
|
164
189
|
// Non-fatal portability advisory (review 13, D1): flag absolute host home
|
|
165
190
|
// paths in the written content. These make the stash non-portable and leak
|
|
166
191
|
// the local username. We warn AFTER the write so the advisory never blocks it.
|
|
@@ -185,6 +210,7 @@ export async function deleteAssetFromSource(source, config, ref) {
|
|
|
185
210
|
throw new UsageError(`Asset "${formatRefForMessage(ref)}" not found in source "${source.name}" (expected at ${filePath}).`, "MISSING_REQUIRED_ARGUMENT");
|
|
186
211
|
}
|
|
187
212
|
fs.unlinkSync(filePath);
|
|
213
|
+
recordWriteTargetPath(source, filePath);
|
|
188
214
|
return { path: filePath, ref: makeAssetRef(ref.type, ref.name, ref.origin) };
|
|
189
215
|
}
|
|
190
216
|
/**
|
|
@@ -196,9 +222,10 @@ export async function deleteAssetFromSource(source, config, ref) {
|
|
|
196
222
|
* primary stash stay non-committing here — the primary stash is committed by
|
|
197
223
|
* the existing improve auto-sync boundary).
|
|
198
224
|
*
|
|
199
|
-
* For a git target it delegates to `saveGitStash(name, message, writable, …)
|
|
200
|
-
*
|
|
201
|
-
* and pushes when the target is writable, has a
|
|
225
|
+
* For a git target it delegates to `saveGitStash(name, message, writable, …)`
|
|
226
|
+
* with the exact paths recorded by the write/delete helpers (plus any explicit
|
|
227
|
+
* caller paths), commits once, and pushes when the target is writable, has a
|
|
228
|
+
* remote, and `push !== false`.
|
|
202
229
|
*
|
|
203
230
|
* The push intent honours a deprecated `options.pushOnCommit` on the source
|
|
204
231
|
* config (mapped onto the batch push gate) when `push` is not explicitly set.
|
|
@@ -212,12 +239,16 @@ export function commitWriteTargetBoundary(target, message, options) {
|
|
|
212
239
|
// gates the actual push on writable + remote, so this only ever opts *in*.
|
|
213
240
|
const push = options?.push ?? (target.config.options?.pushOnCommit === true ? true : undefined);
|
|
214
241
|
const writable = resolveWritable(target.config);
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
242
|
+
const repoDir = target.source.repoPath ?? target.source.path;
|
|
243
|
+
const changedPaths = new Set(listGitChangedPaths(repoDir));
|
|
244
|
+
const paths = [...new Set([...(options?.paths ?? []), ...takeGitTargetPaths(target.source)])]
|
|
245
|
+
.map((filePath) => filePath.replaceAll(path.sep, "/"))
|
|
246
|
+
.filter((filePath) => changedPaths.has(filePath));
|
|
247
|
+
// Assets may live under <repo>/content, but git synchronization always runs
|
|
248
|
+
// against the repository root.
|
|
219
249
|
saveGitStash(undefined, message, writable, {
|
|
220
|
-
repoDir
|
|
250
|
+
repoDir,
|
|
251
|
+
paths,
|
|
221
252
|
...(push === undefined ? {} : { push }),
|
|
222
253
|
});
|
|
223
254
|
}
|
|
@@ -239,6 +270,32 @@ function warnIfPushOnCommit(config) {
|
|
|
239
270
|
"akm now commits writes in a single batch at the operation boundary and pushes when the target is " +
|
|
240
271
|
"writable with a remote. Remove the option or rely on sync push instead.\n");
|
|
241
272
|
}
|
|
273
|
+
/** Enumerate enabled writable targets, deduplicated by materialized content root. */
|
|
274
|
+
export function resolveWritableTargets(akmConfig) {
|
|
275
|
+
const byRoot = new Map();
|
|
276
|
+
for (const runtime of resolveConfiguredSources(akmConfig)) {
|
|
277
|
+
if (runtime.enabled === false || !resolveWritable({ type: runtime.type, writable: runtime.writable }))
|
|
278
|
+
continue;
|
|
279
|
+
const target = adaptConfiguredSource(runtime);
|
|
280
|
+
const root = path.resolve(target.source.path);
|
|
281
|
+
const existing = byRoot.get(root);
|
|
282
|
+
if (!existing || target.source.name === akmConfig.defaultWriteTarget)
|
|
283
|
+
byRoot.set(root, target);
|
|
284
|
+
}
|
|
285
|
+
if (byRoot.size === 0) {
|
|
286
|
+
try {
|
|
287
|
+
const stashDir = resolveStashDir({ readOnly: true });
|
|
288
|
+
byRoot.set(path.resolve(stashDir), {
|
|
289
|
+
source: { kind: "filesystem", name: "stash", path: stashDir },
|
|
290
|
+
config: { type: "filesystem", path: stashDir, name: "stash", writable: true },
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
catch {
|
|
294
|
+
// No active working stash; configured writable targets above are complete.
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
return [...byRoot.values()];
|
|
298
|
+
}
|
|
242
299
|
/**
|
|
243
300
|
* Resolve the destination for a write per locked decision 3:
|
|
244
301
|
*
|
|
@@ -250,8 +307,9 @@ function warnIfPushOnCommit(config) {
|
|
|
250
307
|
* The legacy `first-writable-in-source-array-order` fallback is *not* used —
|
|
251
308
|
* see plan §6 decision 3 for the rationale.
|
|
252
309
|
*/
|
|
253
|
-
export function resolveWriteTarget(akmConfig, explicitTarget) {
|
|
310
|
+
export function resolveWriteTarget(akmConfig, explicitTarget, options = {}) {
|
|
254
311
|
const configuredSources = resolveConfiguredSources(akmConfig);
|
|
312
|
+
const requireWritable = options.requireWritable !== false;
|
|
255
313
|
// 1. Explicit --target wins.
|
|
256
314
|
if (explicitTarget) {
|
|
257
315
|
const match = configuredSources.find((s) => s.name === explicitTarget);
|
|
@@ -264,7 +322,7 @@ export function resolveWriteTarget(akmConfig, explicitTarget) {
|
|
|
264
322
|
// effective writable flag (filesystem defaults to true; everything else
|
|
265
323
|
// defaults to false) so unset values are interpreted correctly.
|
|
266
324
|
const effectiveWritable = resolveWritable({ type: match.type, writable: match.writable });
|
|
267
|
-
if (!effectiveWritable) {
|
|
325
|
+
if (requireWritable && !effectiveWritable) {
|
|
268
326
|
throw new ConfigError(`source ${explicitTarget} is not writable`, "INVALID_CONFIG_FILE", `Set \`writable: true\` on the "${explicitTarget}" source in your config, or pass --target to a different source.`);
|
|
269
327
|
}
|
|
270
328
|
return adaptConfiguredSource(match);
|
|
@@ -279,7 +337,7 @@ export function resolveWriteTarget(akmConfig, explicitTarget) {
|
|
|
279
337
|
// ConfigError, rather than surfacing as a generic UsageError after
|
|
280
338
|
// path-building has already begun.
|
|
281
339
|
const effectiveWritable = resolveWritable({ type: match.type, writable: match.writable });
|
|
282
|
-
if (!effectiveWritable) {
|
|
340
|
+
if (requireWritable && !effectiveWritable) {
|
|
283
341
|
throw new ConfigError(`defaultWriteTarget "${akmConfig.defaultWriteTarget}" is not writable`, "INVALID_CONFIG_FILE", `Set \`writable: true\` on the "${akmConfig.defaultWriteTarget}" source in your config, or change \`defaultWriteTarget\` to a writable source.`);
|
|
284
342
|
}
|
|
285
343
|
return adaptConfiguredSource(match);
|
|
@@ -366,8 +424,8 @@ export function formatRefForMessage(ref) {
|
|
|
366
424
|
* `git` by the config loader, so this mapping is straightforward.
|
|
367
425
|
*/
|
|
368
426
|
function adaptConfiguredSource(runtime) {
|
|
369
|
-
const
|
|
370
|
-
if (!
|
|
427
|
+
const repoPath = pathFromConfiguredSource(runtime);
|
|
428
|
+
if (!repoPath) {
|
|
371
429
|
throw new ConfigError(`Source "${runtime.name}" has no resolvable on-disk path; writes are unsupported for this entry.`, "INVALID_CONFIG_FILE");
|
|
372
430
|
}
|
|
373
431
|
// Map the runtime kind to the write helper's `kind` discriminator. Only
|
|
@@ -383,15 +441,25 @@ function adaptConfiguredSource(runtime) {
|
|
|
383
441
|
const config = {
|
|
384
442
|
type: runtime.type,
|
|
385
443
|
name: runtime.name,
|
|
386
|
-
...(
|
|
444
|
+
...(repoPath !== undefined ? { path: repoPath } : {}),
|
|
387
445
|
...(runtime.writable !== undefined ? { writable: runtime.writable } : {}),
|
|
388
446
|
...(runtime.options ? { options: runtime.options } : {}),
|
|
389
447
|
};
|
|
390
448
|
return {
|
|
391
|
-
source: {
|
|
449
|
+
source: {
|
|
450
|
+
kind,
|
|
451
|
+
name: runtime.name,
|
|
452
|
+
path: kind === "git" ? resolveGitContentRoot(repoPath) : repoPath,
|
|
453
|
+
...(kind === "git" ? { repoPath } : {}),
|
|
454
|
+
},
|
|
392
455
|
config,
|
|
393
456
|
};
|
|
394
457
|
}
|
|
458
|
+
/** Resolve the asset root inside a git checkout while preserving root-layout repos. */
|
|
459
|
+
export function resolveGitContentRoot(repoPath) {
|
|
460
|
+
const contentPath = path.join(repoPath, "content");
|
|
461
|
+
return fs.existsSync(contentPath) && fs.statSync(contentPath).isDirectory() ? contentPath : repoPath;
|
|
462
|
+
}
|
|
395
463
|
function pathFromConfiguredSource(runtime) {
|
|
396
464
|
// ConfiguredSource.source is the parsed SourceSpec (filesystem|git|website|npm).
|
|
397
465
|
// For writable kinds we only ever care about a local on-disk path: filesystem
|
package/dist/indexer/db/db.js
CHANGED
|
@@ -351,9 +351,14 @@ export function getBaseBeliefStatesForDerivedTwins(db, twinIds) {
|
|
|
351
351
|
* full `akm index` picks the file up as a fresh entry).
|
|
352
352
|
*/
|
|
353
353
|
export function rekeyEntryInPlace(db, opts) {
|
|
354
|
-
const row = db
|
|
354
|
+
const row = db
|
|
355
|
+
.prepare("SELECT id, stash_dir, entry_json, search_text FROM entries WHERE entry_key = ?")
|
|
356
|
+
.get(opts.oldEntryKey);
|
|
355
357
|
if (!row)
|
|
356
358
|
return null;
|
|
359
|
+
if (opts.sourceRoot && path.resolve(row.stash_dir) !== path.resolve(opts.sourceRoot)) {
|
|
360
|
+
throw new Error(`Refusing to re-key entry ${opts.oldEntryKey}: source root does not match.`);
|
|
361
|
+
}
|
|
357
362
|
// Patch the JSON payload. On corrupt entry_json still re-key key + paths so
|
|
358
363
|
// the utility history survives; the next full index heals the JSON.
|
|
359
364
|
let entryJson = row.entry_json;
|
|
@@ -391,7 +396,7 @@ export function rekeyEntryInPlace(db, opts) {
|
|
|
391
396
|
// the part after the last `//` (stored origins never contain `//`, so a
|
|
392
397
|
// qualified ref has exactly one — same normalization as
|
|
393
398
|
// getRetrievalCounts). Legacy DBs may predate usage_events.
|
|
394
|
-
|
|
399
|
+
try {
|
|
395
400
|
// Live-asset-wins collision policy, mirroring the stale-entries eviction
|
|
396
401
|
// above and mv's state.db re-key: DETACHED orphan events (entry_id NULL
|
|
397
402
|
// — a deleted asset's history retained by a full rebuild) already
|
|
@@ -401,17 +406,30 @@ export function rekeyEntryInPlace(db, opts) {
|
|
|
401
406
|
// moved asset would adopt the stranger's history: getRetrievalCounts
|
|
402
407
|
// reads by entry_ref immediately, and the next full rebuild's
|
|
403
408
|
// relinkUsageEvents would attach every stranger event by ref.
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
409
|
+
if (opts.sourceName) {
|
|
410
|
+
const origins = new Set([opts.sourceName]);
|
|
411
|
+
if (opts.sourceName === "stash" && opts.includeLegacyBare)
|
|
412
|
+
origins.add("local");
|
|
413
|
+
for (const origin of origins) {
|
|
414
|
+
const oldQualifiedRef = `${origin}//${opts.oldRef}`;
|
|
415
|
+
const newQualifiedRef = `${origin}//${opts.newRef}`;
|
|
416
|
+
db.prepare("DELETE FROM usage_events WHERE entry_id IS NULL AND entry_ref = ?").run(newQualifiedRef);
|
|
417
|
+
db.prepare("UPDATE usage_events SET entry_ref = ? WHERE entry_ref = ?").run(newQualifiedRef, oldQualifiedRef);
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
if (opts.includeLegacyBare || !opts.sourceName) {
|
|
421
|
+
db.prepare("DELETE FROM usage_events WHERE entry_id IS NULL AND entry_ref = ?").run(opts.newRef);
|
|
422
|
+
db.prepare("UPDATE usage_events SET entry_ref = ? WHERE entry_ref = ?").run(opts.newRef, opts.oldRef);
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
catch (error) {
|
|
426
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
427
|
+
const missingLegacyUsageSchema = /no such table:\s*(?:main\.)?usage_events\b/i.test(message) ||
|
|
428
|
+
/no such column:\s*(?:usage_events\.)?(?:entry_id|entry_ref)\b/i.test(message) ||
|
|
429
|
+
/table\s+usage_events\s+has no column named\s+(?:entry_id|entry_ref)\b/i.test(message);
|
|
430
|
+
if (!missingLegacyUsageSchema)
|
|
431
|
+
throw error;
|
|
432
|
+
}
|
|
415
433
|
db.prepare("INSERT OR IGNORE INTO entries_fts_dirty (entry_id) VALUES (?)").run(row.id);
|
|
416
434
|
})();
|
|
417
435
|
return row.id;
|
|
@@ -884,7 +902,7 @@ export function getEntitiesByEntryIds(db, entryIds) {
|
|
|
884
902
|
}
|
|
885
903
|
return result;
|
|
886
904
|
}
|
|
887
|
-
export function findEntryIdByRef(db, ref) {
|
|
905
|
+
export function findEntryIdByRef(db, ref, stashDir) {
|
|
888
906
|
const parsed = parseAssetRef(ref);
|
|
889
907
|
const nameVariants = [parsed.name];
|
|
890
908
|
if (parsed.name.endsWith(".md")) {
|
|
@@ -893,10 +911,14 @@ export function findEntryIdByRef(db, ref) {
|
|
|
893
911
|
else {
|
|
894
912
|
nameVariants.push(`${parsed.name}.md`);
|
|
895
913
|
}
|
|
896
|
-
const stmt = db.prepare(
|
|
914
|
+
const stmt = db.prepare(`SELECT id FROM entries
|
|
915
|
+
WHERE entry_type = ?
|
|
916
|
+
AND substr(entry_key, length(entry_key) - length(?) + 1) = ?
|
|
917
|
+
${stashDir ? "AND stash_dir = ?" : ""}
|
|
918
|
+
LIMIT 1`);
|
|
897
919
|
for (const name of nameVariants) {
|
|
898
920
|
const suffix = `${parsed.type}:${name}`;
|
|
899
|
-
const row = stmt.get(parsed.type, suffix, suffix);
|
|
921
|
+
const row = stmt.get(parsed.type, suffix, suffix, ...(stashDir ? [stashDir] : []));
|
|
900
922
|
if (row)
|
|
901
923
|
return row.id;
|
|
902
924
|
}
|
|
@@ -914,7 +936,7 @@ export function getEmbeddingCount(db) {
|
|
|
914
936
|
return row.cnt;
|
|
915
937
|
}
|
|
916
938
|
export function getEntryById(db, id) {
|
|
917
|
-
const row = db.prepare("SELECT file_path, entry_json FROM entries WHERE id = ?").get(id);
|
|
939
|
+
const row = db.prepare("SELECT file_path, stash_dir, entry_json FROM entries WHERE id = ?").get(id);
|
|
918
940
|
if (!row)
|
|
919
941
|
return undefined;
|
|
920
942
|
// Guard against corrupt JSON
|
|
@@ -926,7 +948,7 @@ export function getEntryById(db, id) {
|
|
|
926
948
|
warn(`[db] getEntryById: skipping entry id=${id} — corrupt entry_json`);
|
|
927
949
|
return undefined;
|
|
928
950
|
}
|
|
929
|
-
return { filePath: row.file_path, entry };
|
|
951
|
+
return { filePath: row.file_path, stashDir: row.stash_dir, entry };
|
|
930
952
|
}
|
|
931
953
|
export function getEntriesByDir(db, dirPath) {
|
|
932
954
|
const rows = db.prepare(`SELECT ${ENTRY_COLUMNS} FROM entries WHERE dir_path = ?`).all(dirPath);
|
|
@@ -1192,33 +1214,12 @@ function bareRef(ref) {
|
|
|
1192
1214
|
return ref;
|
|
1193
1215
|
}
|
|
1194
1216
|
}
|
|
1195
|
-
|
|
1196
|
-
* Count retrieval events for the given entry refs.
|
|
1197
|
-
*
|
|
1198
|
-
* Counts `search`, `show`, and `curate` usage events. Returns a
|
|
1199
|
-
* Map<inputRef, count> keyed by the *input* ref strings (only those with at
|
|
1200
|
-
* least one matching event appear). Used by the improve loop to find
|
|
1201
|
-
* high-retrieval assets without feedback.
|
|
1202
|
-
*
|
|
1203
|
-
* Matching is normalization-aware: each stored `entry_ref` is reduced to its
|
|
1204
|
-
* bare `type:name` form before comparison, so a stash-prefixed stored ref
|
|
1205
|
-
* (`origin//type:name`) still matches a bare input ref (`type:name`) and vice
|
|
1206
|
-
* versa. Previously the raw `entry_ref IN (...)` comparison silently dropped
|
|
1207
|
-
* roughly half the signal whenever the two spellings disagreed.
|
|
1208
|
-
*
|
|
1209
|
-
* `curate` events are included: their per-item rows are written with
|
|
1210
|
-
* entry_ref populated (see logCurateEvent), so curation is a real retrieval
|
|
1211
|
-
* signal here. Legacy summary-only curate rows with a NULL entry_ref simply
|
|
1212
|
-
* contribute nothing.
|
|
1213
|
-
*
|
|
1214
|
-
* Machine-sourced events (`source` = 'improve' or 'task') are EXCLUDED: this
|
|
1215
|
-
* count feeds salience/ranking, and pipeline probe traffic counting as demand
|
|
1216
|
-
* creates a self-reinforcing loop (meta-review 05 DRIFT-6). NULL sources
|
|
1217
|
-
* (pre-column rows) count as user demand.
|
|
1218
|
-
*/
|
|
1219
|
-
export function getRetrievalCounts(db, refs) {
|
|
1217
|
+
export function getRetrievalCounts(db, refs, options = {}) {
|
|
1220
1218
|
if (refs.length === 0)
|
|
1221
1219
|
return new Map();
|
|
1220
|
+
if (options.sourceName || options.stashDir) {
|
|
1221
|
+
return getSourceScopedRetrievalCounts(db, refs, options);
|
|
1222
|
+
}
|
|
1222
1223
|
// Map each distinct bare form back to the input ref(s) that produced it so we
|
|
1223
1224
|
// can re-key DB results (grouped by bare form) onto the caller's ref strings.
|
|
1224
1225
|
const bareToInputs = new Map();
|
|
@@ -1274,6 +1275,55 @@ export function getRetrievalCounts(db, refs) {
|
|
|
1274
1275
|
}
|
|
1275
1276
|
return result;
|
|
1276
1277
|
}
|
|
1278
|
+
function getSourceScopedRetrievalCounts(db, refs, options) {
|
|
1279
|
+
const bareToInputs = new Map();
|
|
1280
|
+
for (const ref of refs) {
|
|
1281
|
+
const bare = bareRef(ref);
|
|
1282
|
+
const inputs = bareToInputs.get(bare);
|
|
1283
|
+
if (inputs)
|
|
1284
|
+
inputs.push(ref);
|
|
1285
|
+
else
|
|
1286
|
+
bareToInputs.set(bare, [ref]);
|
|
1287
|
+
}
|
|
1288
|
+
const countsByBare = new Map();
|
|
1289
|
+
const bareForms = [...bareToInputs.keys()];
|
|
1290
|
+
const selectedRoot = options.stashDir ? path.resolve(options.stashDir) : undefined;
|
|
1291
|
+
for (let i = 0; i < bareForms.length; i += SQLITE_CHUNK_SIZE) {
|
|
1292
|
+
const chunk = bareForms.slice(i, i + SQLITE_CHUNK_SIZE);
|
|
1293
|
+
const placeholders = chunk.map(() => "?").join(", ");
|
|
1294
|
+
const rows = db
|
|
1295
|
+
.prepare(`SELECT ue.entry_ref, ue.entry_id, e.stash_dir
|
|
1296
|
+
FROM usage_events ue
|
|
1297
|
+
LEFT JOIN entries e ON e.id = ue.entry_id
|
|
1298
|
+
WHERE ue.event_type IN ('search','show','curate')
|
|
1299
|
+
AND ue.entry_ref IS NOT NULL
|
|
1300
|
+
AND (ue.source IS NULL OR ue.source NOT IN ('improve','task'))
|
|
1301
|
+
AND CASE
|
|
1302
|
+
WHEN instr(ue.entry_ref, '//') > 0
|
|
1303
|
+
THEN substr(ue.entry_ref, instr(ue.entry_ref, '//') + 2)
|
|
1304
|
+
ELSE ue.entry_ref
|
|
1305
|
+
END IN (${placeholders})`)
|
|
1306
|
+
.all(...chunk);
|
|
1307
|
+
for (const row of rows) {
|
|
1308
|
+
const bare = bareRef(row.entry_ref);
|
|
1309
|
+
const linkedToSelectedRoot = row.entry_id !== null && selectedRoot !== undefined && row.stash_dir !== null
|
|
1310
|
+
? path.resolve(row.stash_dir) === selectedRoot
|
|
1311
|
+
: false;
|
|
1312
|
+
const detached = row.entry_id === null || selectedRoot === undefined;
|
|
1313
|
+
const qualifiedForSource = detached && options.sourceName !== undefined && row.entry_ref === `${options.sourceName}//${bare}`;
|
|
1314
|
+
const acceptedLegacyBare = detached && options.includeLegacyBare === true && row.entry_ref === bare;
|
|
1315
|
+
if (!linkedToSelectedRoot && !qualifiedForSource && !acceptedLegacyBare)
|
|
1316
|
+
continue;
|
|
1317
|
+
countsByBare.set(bare, (countsByBare.get(bare) ?? 0) + 1);
|
|
1318
|
+
}
|
|
1319
|
+
}
|
|
1320
|
+
const result = new Map();
|
|
1321
|
+
for (const [bare, count] of countsByBare) {
|
|
1322
|
+
for (const input of bareToInputs.get(bare) ?? [])
|
|
1323
|
+
result.set(input, count);
|
|
1324
|
+
}
|
|
1325
|
+
return result;
|
|
1326
|
+
}
|
|
1277
1327
|
/**
|
|
1278
1328
|
* Apply a MemRL reward signal to a batch of entries via exponential moving
|
|
1279
1329
|
* average (EMA): next = clamp(current + lr * (reward - current), 0, 1).
|
|
@@ -1434,13 +1484,23 @@ export function applyFeedbackToUtilityScore(db, entryId, positiveCount, negative
|
|
|
1434
1484
|
`).run(entryId, result.nextUtility, now, now, result.nextUtility, now);
|
|
1435
1485
|
return result;
|
|
1436
1486
|
}
|
|
1487
|
+
function resolveUsageEventEntryId(db, ref, options) {
|
|
1488
|
+
const parsed = parseAssetRef(ref);
|
|
1489
|
+
if (!parsed.origin) {
|
|
1490
|
+
return options.defaultStashDir ? findEntryIdByRef(db, ref, options.defaultStashDir) : undefined;
|
|
1491
|
+
}
|
|
1492
|
+
const source = parsed.origin === "local" || parsed.origin === "stash"
|
|
1493
|
+
? options.sources?.[0]
|
|
1494
|
+
: options.sources?.find((candidate) => candidate.registryId === parsed.origin);
|
|
1495
|
+
return source ? findEntryIdByRef(db, ref, source.path) : undefined;
|
|
1496
|
+
}
|
|
1437
1497
|
/**
|
|
1438
1498
|
* Re-link detached usage_events to their current entry_ids via entry_ref.
|
|
1439
1499
|
*
|
|
1440
1500
|
* After a full rebuild, entry IDs change. This restores each event's link
|
|
1441
1501
|
* using the stable `entry_ref` column so usage history survives a reindex.
|
|
1442
1502
|
*/
|
|
1443
|
-
export function relinkUsageEvents(db) {
|
|
1503
|
+
export function relinkUsageEvents(db, options = {}) {
|
|
1444
1504
|
bestEffort(() => {
|
|
1445
1505
|
// Step 1: null out stale entry_ids (entry was deleted, re-keyed, etc).
|
|
1446
1506
|
// Leaving them in place would let `recomputeUtilityScores` aggregate
|
|
@@ -1455,14 +1515,10 @@ export function relinkUsageEvents(db) {
|
|
|
1455
1515
|
WHERE entry_id IS NOT NULL
|
|
1456
1516
|
AND entry_id NOT IN (SELECT id FROM entries)
|
|
1457
1517
|
`);
|
|
1458
|
-
// Step 2: re-resolve
|
|
1459
|
-
//
|
|
1460
|
-
//
|
|
1461
|
-
//
|
|
1462
|
-
// O(events × entries) non-indexable `substr(entry_key, …)` scan. It also
|
|
1463
|
-
// fixes a silent correctness bug: the old suffix match compared the RAW
|
|
1464
|
-
// `entry_ref`, so origin-qualified refs ("source//type:name") never matched
|
|
1465
|
-
// an `entry_key` and lost their usage history on every full rebuild.
|
|
1518
|
+
// Step 2: re-resolve each distinct ref inside its source boundary. Qualified
|
|
1519
|
+
// refs require an origin→root mapping; bare legacy refs require the explicit
|
|
1520
|
+
// historical/default root. This keeps duplicate refs from adopting whichever
|
|
1521
|
+
// entries row SQLite happens to return first while retaining indexed lookups.
|
|
1466
1522
|
const refs = db
|
|
1467
1523
|
.prepare("SELECT DISTINCT entry_ref AS ref FROM usage_events WHERE entry_id IS NULL AND entry_ref IS NOT NULL")
|
|
1468
1524
|
.all();
|
|
@@ -1471,7 +1527,7 @@ export function relinkUsageEvents(db) {
|
|
|
1471
1527
|
for (const { ref } of refs) {
|
|
1472
1528
|
let id;
|
|
1473
1529
|
try {
|
|
1474
|
-
id =
|
|
1530
|
+
id = resolveUsageEventEntryId(db, ref, options);
|
|
1475
1531
|
}
|
|
1476
1532
|
catch (err) {
|
|
1477
1533
|
if (err instanceof Error && err.name === "UsageError")
|