@ngockhoale/ukit 3.0.11 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -1
- package/README.md +1 -0
- package/manifests/documentation.yaml +12 -0
- package/package.json +1 -1
- package/scripts/bench/data-foundation.mjs +368 -50
- package/src/cli/commands/doctor.js +232 -3
- package/src/cli/commands/feedback.js +64 -1
- package/src/cli/commands/install.js +18 -0
- package/src/cli/commands/memory.js +42 -37
- package/src/cli/commands/telemetry.js +460 -0
- package/src/cli/index.js +7 -0
- package/src/core/agentRuntime/adapters.js +83 -2
- package/src/core/agentRuntime/diagnostics.js +104 -0
- package/src/core/agentRuntime/supervisor.js +137 -0
- package/src/core/agentRuntime/telemetry.js +204 -0
- package/src/core/memory/memoryEmit.js +131 -0
- package/src/core/memory/memoryHit.js +1 -1
- package/src/core/memory/migrate.js +18 -11
- package/src/core/memory/migrateMapping.js +15 -7
- package/src/core/memory/mutateMemory.js +22 -4
- package/src/core/memory/recordIndex.js +10 -3
- package/src/core/memory/recordStore.js +28 -3
- package/src/core/memory/retrieval.js +79 -38
- package/src/core/memory/store.js +37 -37
- package/src/core/memory/storeV2.js +28 -25
- package/src/core/memory/storeV2Loader.js +2 -2
- package/src/core/observability/adapters/ingest.js +576 -0
- package/src/core/observability/analytics/anomalies.js +415 -0
- package/src/core/observability/analytics/summary.js +16 -1
- package/src/core/observability/emit/config.js +69 -1
- package/src/core/observability/emit/crash.js +434 -0
- package/src/core/observability/emit/lifecycle.js +349 -0
- package/src/core/observability/emit/recorder.js +135 -9
- package/src/core/observability/evaluation/aiPacket.js +52 -10
- package/src/core/observability/evaluation/outcomes.js +95 -0
- package/src/core/observability/evaluation/runner.js +225 -0
- package/src/core/observability/privacy/allowlist.js +23 -3
- package/src/core/observability/schema/compatibility.js +48 -3
- package/src/core/observability/schema/constants.js +5 -0
- package/src/core/observability/schema/registry.js +57 -0
- package/src/core/observability/schema/validate.js +68 -6
- package/src/core/observability/segments/internal.js +42 -8
- package/src/core/observability/segments/readSegments.js +35 -1
- package/src/core/observability/segments/recovery.js +3 -2
- package/src/core/observability/segments/retention.js +137 -33
- package/src/core/observability/support/projector.js +88 -18
- package/src/core/observability/support/provision.js +160 -0
- package/src/core/observability/support/renderer.js +2 -2
- package/src/core/observability/support/schedule.js +174 -0
- package/template_project/.claude/hooks/auto-allow-bash.sh +7 -1
- package/template_project/.claude/hooks/auto-prune-bash.sh +16 -7
- package/template_project/.claude/hooks/verification-guard.sh +13 -4
- package/template_project/.claude/ukit/runtime/async-lock.mjs +26 -0
|
@@ -13,7 +13,7 @@ import { readJsonIfExists, writeJson } from '../fileOps.js';
|
|
|
13
13
|
import { loadRuntimeConfig } from '../runtimeConfig.js';
|
|
14
14
|
import { mapLegacySources } from './migrateMapping.js';
|
|
15
15
|
import { readRecordStore } from './recordStore.js';
|
|
16
|
-
import { lookupRegisteredProjectId } from './projectIdentity.js';
|
|
16
|
+
import { lookupRegisteredProjectId, resolveProjectIdentity } from './projectIdentity.js';
|
|
17
17
|
|
|
18
18
|
const MARKER_NAME = 'migrated-from-v1.json';
|
|
19
19
|
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
@@ -123,8 +123,8 @@ function hasLegacyContent(sources) {
|
|
|
123
123
|
|| sources.sessions.length > 0 || sources.corruptFiles > 0;
|
|
124
124
|
}
|
|
125
125
|
|
|
126
|
-
async function episodeTtlMsFor(projectRoot) {
|
|
127
|
-
const config = await loadRuntimeConfig(projectRoot);
|
|
126
|
+
async function episodeTtlMsFor(projectRoot, { homeDir } = {}) {
|
|
127
|
+
const config = await loadRuntimeConfig(projectRoot, { homeDir });
|
|
128
128
|
const ttlDays = Number.isFinite(config?.memoryV2?.episodeTtlDays) && config.memoryV2.episodeTtlDays > 0
|
|
129
129
|
? config.memoryV2.episodeTtlDays
|
|
130
130
|
: 90;
|
|
@@ -144,12 +144,12 @@ export async function needsMigration(projectRoot) {
|
|
|
144
144
|
}
|
|
145
145
|
|
|
146
146
|
/**
|
|
147
|
-
* runMigration(projectRoot, { dryRun? } = {})
|
|
147
|
+
* runMigration(projectRoot, { dryRun?, homeDir? } = {})
|
|
148
148
|
* → { migrated, skipped, plan, markerPath, backupDir }
|
|
149
149
|
* No-op when the marker exists or no legacy files exist. dryRun returns the
|
|
150
150
|
* full per-record plan ({id, source, action, reason}) and writes nothing.
|
|
151
151
|
*/
|
|
152
|
-
export async function runMigration(projectRoot, { dryRun = false } = {}) {
|
|
152
|
+
export async function runMigration(projectRoot, { dryRun = false, homeDir } = {}) {
|
|
153
153
|
const paths = buildRuntimePaths(projectRoot);
|
|
154
154
|
const markerPath = path.join(paths.memoryV2Dir, MARKER_NAME);
|
|
155
155
|
const empty = { migrated: 0, skipped: 0, plan: [], markerPath, backupDir: null };
|
|
@@ -161,19 +161,26 @@ export async function runMigration(projectRoot, { dryRun = false } = {}) {
|
|
|
161
161
|
|
|
162
162
|
// Tombstones + alias resolution feed the pure mapper — tombstone
|
|
163
163
|
// precedence is planned here and enforced again inside mutateMemory.
|
|
164
|
-
// Registry lookups are async, so legacy ids are
|
|
165
|
-
// to the mapper as a synchronous closure.
|
|
164
|
+
// Registry lookups are async and scoped to THIS home, so legacy ids are
|
|
165
|
+
// pre-resolved and handed to the mapper as a synchronous closure.
|
|
166
|
+
// `projectId` is the migrating project's own verified identity
|
|
167
|
+
// (read-only: mint:false) — the mapper only marks an alias conflict when a
|
|
168
|
+
// legacy name resolves away from this VERIFIED identity; an unbound or
|
|
169
|
+
// foreign-registered name keeps the legacy id rather than losing data.
|
|
166
170
|
const prior = await readRecordStore(paths.memoryV2RecordsPath);
|
|
171
|
+
const ownIdentity = await resolveProjectIdentity(projectRoot, { homeDir, mint: false })
|
|
172
|
+
.catch(() => null);
|
|
173
|
+
const ownProjectId = ownIdentity?.projectId ?? null;
|
|
167
174
|
const resolvedIds = new Map();
|
|
168
175
|
for (const p of sources.projects) {
|
|
169
176
|
if (p.id && !resolvedIds.has(p.id)) {
|
|
170
|
-
resolvedIds.set(p.id, await lookupRegisteredProjectId(p.id));
|
|
177
|
+
resolvedIds.set(p.id, await lookupRegisteredProjectId(p.id, { homeDir }));
|
|
171
178
|
}
|
|
172
179
|
}
|
|
173
180
|
const { records, plan, skipped } = mapLegacySources(sources, {
|
|
174
|
-
projectId:
|
|
181
|
+
projectId: ownProjectId,
|
|
175
182
|
now: Date.now(),
|
|
176
|
-
episodeTtlMs: await episodeTtlMsFor(projectRoot),
|
|
183
|
+
episodeTtlMs: await episodeTtlMsFor(projectRoot, { homeDir }),
|
|
177
184
|
tombstones: prior.tombstones,
|
|
178
185
|
resolveProjectId: (legacyId) => resolvedIds.get(legacyId) ?? null,
|
|
179
186
|
});
|
|
@@ -206,7 +213,7 @@ export async function runMigration(projectRoot, { dryRun = false } = {}) {
|
|
|
206
213
|
// records.
|
|
207
214
|
const { mutateMemory } = await import('./mutateMemory.js');
|
|
208
215
|
const restore = await mutateMemory(
|
|
209
|
-
{ kind: 'project', projectRoot },
|
|
216
|
+
{ kind: 'project', projectRoot, homeDir },
|
|
210
217
|
{ op: 'restore', payload: { records } },
|
|
211
218
|
);
|
|
212
219
|
if (restore.status !== 'ok') {
|
|
@@ -6,10 +6,12 @@
|
|
|
6
6
|
// opts: { projectId, now, episodeTtlMs?, tombstones?, resolveProjectId? }
|
|
7
7
|
// Deterministic: record ids are `v1-<sha1(sourceKind:type:normalizedText)>` —
|
|
8
8
|
// stable across runs and legacy file renames. No I/O; `now` is injected.
|
|
9
|
-
// Rules (FR-003): tombstone precedence → plan 'tombstoned'; alias
|
|
10
|
-
// (
|
|
11
|
-
//
|
|
12
|
-
//
|
|
9
|
+
// Rules (FR-003): tombstone precedence → plan 'tombstoned'; alias conflict
|
|
10
|
+
// (a contested 'ambiguous' name, or a name whose registered identity
|
|
11
|
+
// provably differs from the migrating project's own verified id) → plan
|
|
12
|
+
// 'conflict'; pending candidates → trust_tier 'candidate' + meta.legacyStatus
|
|
13
|
+
// 'pending', never status 'active'; everything else → trust_tier
|
|
14
|
+
// 'legacy-unverified'.
|
|
13
15
|
|
|
14
16
|
import crypto from 'node:crypto';
|
|
15
17
|
import { createRecord } from './records.js';
|
|
@@ -119,11 +121,17 @@ export function mapLegacySources(sources, {
|
|
|
119
121
|
const file = entry?.file ?? `${legacyId ?? 'project'}.json`;
|
|
120
122
|
if (!content || typeof content !== 'object') continue;
|
|
121
123
|
|
|
122
|
-
// Alias
|
|
123
|
-
//
|
|
124
|
+
// Alias conflict: only a VERIFIED mismatch is a conflict — a contested
|
|
125
|
+
// ('ambiguous') name, or a name resolving to a registered identity that
|
|
126
|
+
// provably differs from this project's own verified id. When the
|
|
127
|
+
// migrating project has no verified binding (projectId null) the legacy
|
|
128
|
+
// name is kept as-is: repo-scoped reads still reach it through the
|
|
129
|
+
// registry's attested alias list, and keeping the id loses no data —
|
|
130
|
+
// rebinding to a foreign id is what must never happen silently.
|
|
124
131
|
if (typeof resolveProjectId === 'function' && legacyId) {
|
|
125
132
|
const resolved = resolveProjectId(legacyId);
|
|
126
|
-
if (resolved
|
|
133
|
+
if (resolved === 'ambiguous'
|
|
134
|
+
|| (resolved && resolved !== legacyId && projectId != null && resolved !== projectId)) {
|
|
127
135
|
plan.push({
|
|
128
136
|
id: `v1-${crypto.createHash('sha1').update(`project:${legacyId}`).digest('hex')}`,
|
|
129
137
|
source: file,
|
|
@@ -105,7 +105,7 @@ export async function mutateMemory(target, envelope = {}, { classifyAdd, guard }
|
|
|
105
105
|
let projectId = null;
|
|
106
106
|
if (target.kind === 'project') {
|
|
107
107
|
try {
|
|
108
|
-
config = await loadRuntimeConfig(target.projectRoot);
|
|
108
|
+
config = await loadRuntimeConfig(target.projectRoot, { homeDir: target.homeDir });
|
|
109
109
|
if (config?.memoryV2?.enabled === false) return { status: 'disabled' };
|
|
110
110
|
} catch {
|
|
111
111
|
// unreadable config → default enabled, keep going
|
|
@@ -291,9 +291,21 @@ export async function mutateMemory(target, envelope = {}, { classifyAdd, guard }
|
|
|
291
291
|
additions.push(record);
|
|
292
292
|
}
|
|
293
293
|
appliedRecordId = additions[0]?.id;
|
|
294
|
-
// No-op restore: nothing to add → skip the write entirely so
|
|
295
|
-
// re-run leaves the doc byte-identical (no generation bump).
|
|
294
|
+
// No-op restore: nothing to add → normally skip the write entirely so
|
|
295
|
+
// a re-run leaves the doc byte-identical (no generation bump). The
|
|
296
|
+
// exception is a missing store: a completed restore — and lazy v1→v2
|
|
297
|
+
// migration through this path — must still materialize an
|
|
298
|
+
// authoritative records.json, or a fully-skipped/empty legacy store
|
|
299
|
+
// silently strands the project on the pre-v2 facade. A corrupt doc is
|
|
300
|
+
// never overwritten by an empty restore — corrupt bytes stay
|
|
301
|
+
// byte-identical and reads keep reporting typed 'corrupt'.
|
|
296
302
|
if (additions.length === 0) {
|
|
303
|
+
if (prior.state === 'missing') {
|
|
304
|
+
return {
|
|
305
|
+
result: { status: 'ok', added: 0, records: [] },
|
|
306
|
+
records,
|
|
307
|
+
};
|
|
308
|
+
}
|
|
297
309
|
return { result: { status: 'ok', added: 0, records: [] } };
|
|
298
310
|
}
|
|
299
311
|
return {
|
|
@@ -301,7 +313,13 @@ export async function mutateMemory(target, envelope = {}, { classifyAdd, guard }
|
|
|
301
313
|
records: [...records, ...additions],
|
|
302
314
|
postWrite,
|
|
303
315
|
};
|
|
304
|
-
}, {
|
|
316
|
+
}, {
|
|
317
|
+
onCorrupt: op === 'restore' ? 'quarantine-empty' : undefined,
|
|
318
|
+
// TASK-007 (DF2-FR08): binds the memoized recorder to this project —
|
|
319
|
+
// recordStore emits one memory.write per persisted doc write. User
|
|
320
|
+
// targets carry no projectRoot → the disabled recorder no-ops.
|
|
321
|
+
telemetry: { op, projectRoot: target.projectRoot, config },
|
|
322
|
+
});
|
|
305
323
|
if (receiptStage) {
|
|
306
324
|
// Shadow receipt (FR-004): outcome/code/latency band only — never the
|
|
307
325
|
// record body. Swallowed internally; cannot break the write.
|
|
@@ -45,7 +45,9 @@ function recordTokens(record) {
|
|
|
45
45
|
}
|
|
46
46
|
|
|
47
47
|
// Identical scoring to retrieval.js scoreRecord — keep in sync (parity test).
|
|
48
|
-
|
|
48
|
+
// `now` is injectable so callers that pin a clock (parity checks, rankHits)
|
|
49
|
+
// get byte-identical lexical scores regardless of wall-clock drift.
|
|
50
|
+
export function scoreRecordTokens(record, queryTokens, now) {
|
|
49
51
|
const typeWeight = RECORD_TYPE_WEIGHTS[record.type] ?? 1;
|
|
50
52
|
if (queryTokens.length === 0) {
|
|
51
53
|
return typeWeight * Math.max(0.1, record.confidence ?? 1);
|
|
@@ -56,8 +58,9 @@ export function scoreRecordTokens(record, queryTokens) {
|
|
|
56
58
|
if (tokens.has(token)) matched += 1;
|
|
57
59
|
}
|
|
58
60
|
if (matched === 0) return 0;
|
|
61
|
+
const nowMs = typeof now === 'number' ? now : Date.now();
|
|
59
62
|
const recencyBonus = record.created_at > 0
|
|
60
|
-
? Math.min(1, record.created_at /
|
|
63
|
+
? Math.min(1, record.created_at / nowMs)
|
|
61
64
|
: 0;
|
|
62
65
|
return typeWeight * matched * Math.max(0.1, record.confidence ?? 1) + recencyBonus;
|
|
63
66
|
}
|
|
@@ -74,7 +77,11 @@ export function createIndex({ maxRecords = 10000, maxPostings = 50000 } = {}) {
|
|
|
74
77
|
let records = [];
|
|
75
78
|
let postings = new Map(); // token → number[] (indexes into records)
|
|
76
79
|
let postingsCount = 0;
|
|
77
|
-
|
|
80
|
+
// -1 = "no build observed yet". A fresh index must NOT report generation 0:
|
|
81
|
+
// generation-0 stores (new docs, marker-only fixtures) would compare equal
|
|
82
|
+
// in consumers' "needs rebuild" check and the first query would never build
|
|
83
|
+
// — query() then sees builtGeneration(-1) !== generation(0) and returns [].
|
|
84
|
+
let generation = -1;
|
|
78
85
|
let builtGeneration = -1; // generation the postings actually reflect
|
|
79
86
|
let evicted = 0;
|
|
80
87
|
|
|
@@ -10,6 +10,7 @@ import path from 'node:path';
|
|
|
10
10
|
|
|
11
11
|
import { copyFileRawExclusive, readJsonIfExists, writeJson, withFileLock } from '../fileOps.js';
|
|
12
12
|
import { normalizeRecord } from './records.js';
|
|
13
|
+
import { emitMemoryWrite } from './memoryEmit.js';
|
|
13
14
|
|
|
14
15
|
const SCHEMA_VERSION = 2;
|
|
15
16
|
const TOMBSTONE_MAX = 512;
|
|
@@ -72,7 +73,10 @@ export async function readRecordStore(recordsPath) {
|
|
|
72
73
|
}
|
|
73
74
|
|
|
74
75
|
/**
|
|
75
|
-
* writeRecordStore(recordsPath, records
|
|
76
|
+
* writeRecordStore(recordsPath, records, {generation?, tombstones?,
|
|
77
|
+
* telemetry?}) → {generation, recordCount} of the persisted doc.
|
|
78
|
+
* `telemetry` = {projectRoot, config, op} → one metadata-only memory.write
|
|
79
|
+
* record after the rename lands (DF2-FR08); absent → no emit.
|
|
76
80
|
* Atomic write; drops unrecoverable entries.
|
|
77
81
|
*
|
|
78
82
|
* Generation is strictly monotonic (SPEC §5 FR-005): the persisted value is
|
|
@@ -81,7 +85,7 @@ export async function readRecordStore(recordsPath) {
|
|
|
81
85
|
* (one stat+parse) and keeps this function safe even when invoked outside
|
|
82
86
|
* mutateRecordStore's lock.
|
|
83
87
|
*/
|
|
84
|
-
export async function writeRecordStore(recordsPath, records, { generation, tombstones } = {}) {
|
|
88
|
+
export async function writeRecordStore(recordsPath, records, { generation, tombstones, telemetry } = {}) {
|
|
85
89
|
const normalized = (Array.isArray(records) ? records : [])
|
|
86
90
|
.map((raw) => normalizeRecord(raw))
|
|
87
91
|
.filter(Boolean);
|
|
@@ -90,12 +94,25 @@ export async function writeRecordStore(recordsPath, records, { generation, tombs
|
|
|
90
94
|
const baseGeneration = Math.max(supplied, prior.generation);
|
|
91
95
|
const baseTombstones = tombstones === undefined ? prior.tombstones : tombstones;
|
|
92
96
|
const nextTombstones = (Array.isArray(baseTombstones) ? baseTombstones : []).slice(-TOMBSTONE_MAX);
|
|
97
|
+
const nextGeneration = baseGeneration + 1;
|
|
93
98
|
await writeJson(recordsPath, {
|
|
94
99
|
schemaVersion: SCHEMA_VERSION,
|
|
95
|
-
generation:
|
|
100
|
+
generation: nextGeneration,
|
|
96
101
|
tombstones: nextTombstones,
|
|
97
102
|
records: normalized,
|
|
98
103
|
}, { fsync: true });
|
|
104
|
+
// TASK-007 (DF2-FR08): one memory.write record per persisted doc write,
|
|
105
|
+
// emitted only after the tmp+rename lands. Metadata-only (op, counts,
|
|
106
|
+
// kind buckets, sha256 refs); O(1) no-op when the stage is off and
|
|
107
|
+
// never-throw — telemetry can never fail the write.
|
|
108
|
+
if (telemetry != null) {
|
|
109
|
+
emitMemoryWrite({
|
|
110
|
+
...telemetry,
|
|
111
|
+
recordCount: normalized.length,
|
|
112
|
+
tombstoneCount: nextTombstones.length,
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
return { generation: nextGeneration, recordCount: normalized.length };
|
|
99
116
|
}
|
|
100
117
|
|
|
101
118
|
/**
|
|
@@ -179,6 +196,14 @@ export async function mutateRecordStore(recordsPath, mutate, opts = {}) {
|
|
|
179
196
|
await writeRecordStore(recordsPath, mutation.records, {
|
|
180
197
|
generation: prior.generation,
|
|
181
198
|
tombstones: mutation.tombstones ?? prior.tombstones,
|
|
199
|
+
// Emit contract (TASK-007): the caller supplies the observability
|
|
200
|
+
// binding once ({projectRoot, config}); the subject — the record(s)
|
|
201
|
+
// this write changed — is derived from the mutation's own result so
|
|
202
|
+
// every op emits exactly one record for one persisted write.
|
|
203
|
+
telemetry: opts.telemetry == null ? undefined : {
|
|
204
|
+
...opts.telemetry,
|
|
205
|
+
subject: mutation.result?.record ?? mutation.result?.records,
|
|
206
|
+
},
|
|
182
207
|
});
|
|
183
208
|
// Test-only crash-injection hook (TASK-002): UKIT_TEST_CRASH_AT=
|
|
184
209
|
// 'post-doc-write' kills the process after the doc rename but before
|
|
@@ -14,6 +14,7 @@ import { createIndex, normalize, scoreRecordTokens, tokenize } from './recordInd
|
|
|
14
14
|
import { buildMemoryHit, rankHits } from './memoryHit.js';
|
|
15
15
|
import { resolveRecordFreshness } from './memoryFreshness.js';
|
|
16
16
|
import { getHeadSha } from '../codeintel/freshness.js';
|
|
17
|
+
import { emitRetrievalQuery, emitMemoryRetrieved } from './memoryEmit.js';
|
|
17
18
|
|
|
18
19
|
function compactPhraseList(values, { limit = 1, fallback = 'n/a' } = {}) {
|
|
19
20
|
const normalized = [...new Set(
|
|
@@ -173,13 +174,13 @@ export const RECORD_TYPE_WEIGHTS = Object.freeze({
|
|
|
173
174
|
// 'empty' — 'corrupt'|'disabled'|'migrating' mean the v2 lane is authoritative
|
|
174
175
|
// and empty (FR-008). queryRecords itself triggers ensureMigrated on the real
|
|
175
176
|
// store — no explicit migration here.
|
|
176
|
-
async function queryV2Records(projectRoot, { projectId } = {}) {
|
|
177
|
+
async function queryV2Records(projectRoot, { projectId, homeDir } = {}) {
|
|
177
178
|
let policy = {};
|
|
178
179
|
let config = null;
|
|
179
180
|
try {
|
|
180
|
-
config = await loadRuntimeConfig(projectRoot);
|
|
181
|
+
config = await loadRuntimeConfig(projectRoot, { homeDir });
|
|
181
182
|
if (config?.memoryV2?.enabled === false) {
|
|
182
|
-
|
|
183
|
+
return { state: 'disabled', records: [], policy, generation: 0, config };
|
|
183
184
|
}
|
|
184
185
|
if (config?.memoryV2?.policy && typeof config.memoryV2.policy === 'object') {
|
|
185
186
|
policy = config.memoryV2.policy;
|
|
@@ -195,11 +196,11 @@ async function queryV2Records(projectRoot, { projectId } = {}) {
|
|
|
195
196
|
? resolveMemoryStage(config, 'eligibility', { projectId })
|
|
196
197
|
: 'default';
|
|
197
198
|
if (eligibilityStage === 'off') {
|
|
198
|
-
return { state: 'missing', records: [], policy, generation: 0 };
|
|
199
|
+
return { state: 'missing', records: [], policy, generation: 0, config };
|
|
199
200
|
}
|
|
200
201
|
if (eligibilityStage === 'shadow') {
|
|
201
202
|
const startedAt = Date.now();
|
|
202
|
-
const probe = await loadV2Records(projectRoot, {});
|
|
203
|
+
const probe = await loadV2Records(projectRoot, {}, { config, projectId, homeDir });
|
|
203
204
|
await appendRolloutReceipt(projectRoot, {
|
|
204
205
|
plane: 'eligibility',
|
|
205
206
|
stage: 'shadow',
|
|
@@ -207,15 +208,15 @@ async function queryV2Records(projectRoot, { projectId } = {}) {
|
|
|
207
208
|
code: probe.state,
|
|
208
209
|
latencyBand: latencyBandForMs(Date.now() - startedAt),
|
|
209
210
|
});
|
|
210
|
-
return { state: 'missing', records: [], policy, generation: 0 };
|
|
211
|
+
return { state: 'missing', records: [], policy, generation: 0, config };
|
|
211
212
|
}
|
|
212
|
-
const { state, records, generation } = await loadV2Records(projectRoot, {}, { config, projectId });
|
|
213
|
+
const { state, records, generation } = await loadV2Records(projectRoot, {}, { config, projectId, homeDir });
|
|
213
214
|
// Index plane (FR-003): 'off' → linear scan; 'shadow' → indexed path runs
|
|
214
215
|
// for measurement only, linear result is served; live → indexed path.
|
|
215
216
|
const indexStage = config
|
|
216
217
|
? resolveMemoryStage(config, 'index', { projectId })
|
|
217
218
|
: 'default';
|
|
218
|
-
return { state, records, policy, generation: generation ?? 0, indexStage };
|
|
219
|
+
return { state, records, policy, generation: generation ?? 0, indexStage, config };
|
|
219
220
|
}
|
|
220
221
|
|
|
221
222
|
// 'missing'|'empty' → legacy compat lane; every other state keeps the v2 lane
|
|
@@ -236,10 +237,15 @@ export const LEGACY_SCOPE_MATCH = {
|
|
|
236
237
|
// rankHits includes bonus-only (lexical=0) records; the index path can only
|
|
237
238
|
// surface token-matching candidates, so the scan path pre-filters to the
|
|
238
239
|
// same candidate pool (scoreRecordTokens > 0) — identical hits either way.
|
|
239
|
-
|
|
240
|
+
// Returns {hits, candidateCount} — the count feeds retrieval.query telemetry
|
|
241
|
+
// (DF2-FR08) without recomputing the pool.
|
|
242
|
+
function searchRecords(records, query, scope, limit, eligibleCtx, policy, now) {
|
|
240
243
|
const queryTokens = tokenize(query);
|
|
241
|
-
const candidates = records.filter((record) => scoreRecordTokens(record, queryTokens) > 0);
|
|
242
|
-
return
|
|
244
|
+
const candidates = records.filter((record) => scoreRecordTokens(record, queryTokens, now) > 0);
|
|
245
|
+
return {
|
|
246
|
+
hits: rankHits(candidates, { queryTokens, eligibleCtx, policy, scope, limit, now }),
|
|
247
|
+
candidateCount: candidates.length,
|
|
248
|
+
};
|
|
243
249
|
}
|
|
244
250
|
|
|
245
251
|
// Freshness is resolved post-limit only (≤ limit × MAX_EVIDENCE stats per
|
|
@@ -247,7 +253,7 @@ function searchRecords(records, query, scope, limit, eligibleCtx, policy) {
|
|
|
247
253
|
// non-file evidence, `command` is filled lazily via getHeadSha (one bounded
|
|
248
254
|
// git spawn per call, never per record). Resolver errors degrade to
|
|
249
255
|
// 'unknown' and never break search.
|
|
250
|
-
async function attachFreshness(hits, records, { projectRoot, watermarks } = {}) {
|
|
256
|
+
async function attachFreshness(hits, records, { projectRoot, watermarks, now } = {}) {
|
|
251
257
|
if (!Array.isArray(hits) || hits.length === 0) return hits;
|
|
252
258
|
const recordById = new Map((records ?? []).map((record) => [record.id, record]));
|
|
253
259
|
const needsCommand = watermarks?.command == null && hits.some((hit) => {
|
|
@@ -266,6 +272,7 @@ async function attachFreshness(hits, records, { projectRoot, watermarks } = {})
|
|
|
266
272
|
hit.freshness = await resolveRecordFreshness(record, {
|
|
267
273
|
projectRoot,
|
|
268
274
|
watermarks: resolvedWatermarks,
|
|
275
|
+
now,
|
|
269
276
|
});
|
|
270
277
|
}));
|
|
271
278
|
return hits;
|
|
@@ -287,10 +294,10 @@ export function resetRecordIndexForTests() {
|
|
|
287
294
|
recordIndexes.clear();
|
|
288
295
|
}
|
|
289
296
|
|
|
290
|
-
function searchRecordsIndexed(records, query, scope, limit, eligibleCtx, policy, indexKey, generation) {
|
|
297
|
+
function searchRecordsIndexed(records, query, scope, limit, eligibleCtx, policy, indexKey, generation, now) {
|
|
291
298
|
const queryTokens = tokenize(query);
|
|
292
299
|
if (queryTokens.length === 0) {
|
|
293
|
-
return searchRecords(records, query, scope, limit, eligibleCtx, policy);
|
|
300
|
+
return searchRecords(records, query, scope, limit, eligibleCtx, policy, now);
|
|
294
301
|
}
|
|
295
302
|
try {
|
|
296
303
|
let index = recordIndexes.get(indexKey);
|
|
@@ -303,19 +310,48 @@ function searchRecordsIndexed(records, query, scope, limit, eligibleCtx, policy,
|
|
|
303
310
|
}
|
|
304
311
|
const candidates = index.query(queryTokens, { limit: records.length })
|
|
305
312
|
.map((hit) => hit.record);
|
|
306
|
-
return
|
|
313
|
+
return {
|
|
314
|
+
hits: rankHits(candidates, { queryTokens, eligibleCtx, policy, scope, limit, now }),
|
|
315
|
+
candidateCount: candidates.length,
|
|
316
|
+
};
|
|
307
317
|
} catch {
|
|
308
|
-
return searchRecords(records, query, scope, limit, eligibleCtx, policy);
|
|
318
|
+
return searchRecords(records, query, scope, limit, eligibleCtx, policy, now);
|
|
309
319
|
}
|
|
310
320
|
}
|
|
311
321
|
|
|
312
|
-
export async function search(query, scope = 'all', limit = 10, { projectRoot, projectId, projectAliases, index = true, watermarks } = {}) {
|
|
313
|
-
const
|
|
322
|
+
export async function search(query, scope = 'all', limit = 10, { projectRoot, projectId, projectAliases, index = true, watermarks, homeDir, now } = {}) {
|
|
323
|
+
const startedAt = Date.now();
|
|
324
|
+
const { state, records, policy, generation, indexStage, config } = await queryV2Records(projectRoot, { projectId, homeDir });
|
|
314
325
|
const effective = resolveEffectiveScope(
|
|
315
326
|
{ projectId: projectId ?? null },
|
|
316
327
|
REQUESTED_SCOPE_TO_POLICY[scope] ?? scope,
|
|
317
328
|
);
|
|
329
|
+
|
|
330
|
+
// TASK-007 (DF2-FR08): every search() exits through emitQuery once —
|
|
331
|
+
// one retrieval.query + one memory.retrieved per returned hit, metadata
|
|
332
|
+
// only. Stage-off short-circuits inside the helpers before any hashing.
|
|
333
|
+
const emitQuery = ({ lane, indexed, candidateCount, results }) => {
|
|
334
|
+
const freshness = {};
|
|
335
|
+
for (const hit of results) {
|
|
336
|
+
const bucket = hit?.freshness?.state;
|
|
337
|
+
if (typeof bucket === 'string') freshness[bucket] = (freshness[bucket] ?? 0) + 1;
|
|
338
|
+
}
|
|
339
|
+
emitRetrievalQuery({
|
|
340
|
+
projectRoot,
|
|
341
|
+
config,
|
|
342
|
+
lane,
|
|
343
|
+
scope,
|
|
344
|
+
indexed,
|
|
345
|
+
candidateCount,
|
|
346
|
+
returnedCount: results.length,
|
|
347
|
+
durationMs: Date.now() - startedAt,
|
|
348
|
+
freshness,
|
|
349
|
+
});
|
|
350
|
+
emitMemoryRetrieved({ projectRoot, config, hits: results });
|
|
351
|
+
};
|
|
352
|
+
|
|
318
353
|
if (effective.denied) {
|
|
354
|
+
emitQuery({ lane: 'v2', indexed: false, candidateCount: 0, results: [] });
|
|
319
355
|
return [];
|
|
320
356
|
}
|
|
321
357
|
const eligibleCtx = {
|
|
@@ -330,36 +366,38 @@ export async function search(query, scope = 'all', limit = 10, { projectRoot, pr
|
|
|
330
366
|
// indexed path for measurement, discards it, and serves the scan result
|
|
331
367
|
// (zero output change) plus a bounded receipt.
|
|
332
368
|
if (index !== false && indexStage === 'shadow') {
|
|
333
|
-
const
|
|
369
|
+
const shadowStarted = Date.now();
|
|
334
370
|
try {
|
|
335
|
-
searchRecordsIndexed(records, query, scope, limit, eligibleCtx, policy, indexKey, generation);
|
|
371
|
+
searchRecordsIndexed(records, query, scope, limit, eligibleCtx, policy, indexKey, generation, now);
|
|
336
372
|
await appendRolloutReceipt(projectRoot, {
|
|
337
373
|
plane: 'index', stage: 'shadow', outcome: 'ok', code: 'search',
|
|
338
|
-
latencyBand: latencyBandForMs(Date.now() -
|
|
374
|
+
latencyBand: latencyBandForMs(Date.now() - shadowStarted),
|
|
339
375
|
});
|
|
340
376
|
} catch {
|
|
341
377
|
await appendRolloutReceipt(projectRoot, {
|
|
342
378
|
plane: 'index', stage: 'shadow', outcome: 'error', code: 'search',
|
|
343
|
-
latencyBand: latencyBandForMs(Date.now() -
|
|
379
|
+
latencyBand: latencyBandForMs(Date.now() - shadowStarted),
|
|
344
380
|
});
|
|
345
381
|
}
|
|
346
382
|
}
|
|
347
|
-
const
|
|
348
|
-
|
|
349
|
-
|
|
383
|
+
const useIndex = index !== false && indexStage !== 'off' && indexStage !== 'shadow';
|
|
384
|
+
const { hits, candidateCount } = useIndex
|
|
385
|
+
? searchRecordsIndexed(
|
|
350
386
|
records, query, scope, limit, eligibleCtx, policy,
|
|
351
387
|
indexKey,
|
|
352
388
|
generation,
|
|
353
|
-
|
|
354
|
-
|
|
389
|
+
now,
|
|
390
|
+
)
|
|
391
|
+
: searchRecords(records, query, scope, limit, eligibleCtx, policy, now);
|
|
392
|
+
const results = await attachFreshness(hits, records, { projectRoot, watermarks, now });
|
|
393
|
+
emitQuery({ lane: 'v2', indexed: useIndex, candidateCount, results });
|
|
394
|
+
return results;
|
|
355
395
|
}
|
|
356
396
|
|
|
357
|
-
const items = await listMemoryItems(projectRoot);
|
|
397
|
+
const items = await listMemoryItems(projectRoot, { homeDir });
|
|
358
398
|
const queryTokens = tokenize(query);
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
return items
|
|
362
|
-
.filter((item) => withinScope(item, scope, projectId, projectAliases))
|
|
399
|
+
const inScope = items.filter((item) => withinScope(item, scope, projectId, projectAliases));
|
|
400
|
+
const results = inScope
|
|
363
401
|
.map((item) => ({
|
|
364
402
|
item,
|
|
365
403
|
score: scoreItem(item, queryTokens),
|
|
@@ -372,6 +410,8 @@ export async function search(query, scope = 'all', limit = 10, { projectRoot, pr
|
|
|
372
410
|
))
|
|
373
411
|
.slice(0, limit)
|
|
374
412
|
.map((entry) => buildSearchResult(entry.item, entry.score));
|
|
413
|
+
emitQuery({ lane: 'legacy', indexed: false, candidateCount: inScope.length, results });
|
|
414
|
+
return results;
|
|
375
415
|
}
|
|
376
416
|
|
|
377
417
|
function findRelatedItemIds(targetItem, allItems) {
|
|
@@ -402,8 +442,8 @@ function findRelatedItemIds(targetItem, allItems) {
|
|
|
402
442
|
.slice(0, 5);
|
|
403
443
|
}
|
|
404
444
|
|
|
405
|
-
export async function expand(id, { projectRoot, projectId, projectAliases } = {}) {
|
|
406
|
-
const { state, records, policy } = await queryV2Records(projectRoot, { projectId });
|
|
445
|
+
export async function expand(id, { projectRoot, projectId, projectAliases, homeDir } = {}) {
|
|
446
|
+
const { state, records, policy } = await queryV2Records(projectRoot, { projectId, homeDir });
|
|
407
447
|
if (!v2AllowsLegacyFallback(state)) {
|
|
408
448
|
const effective = resolveEffectiveScope({ projectId: projectId ?? null }, 'all');
|
|
409
449
|
const eligibleCtx = {
|
|
@@ -439,7 +479,7 @@ export async function expand(id, { projectRoot, projectId, projectAliases } = {}
|
|
|
439
479
|
};
|
|
440
480
|
}
|
|
441
481
|
|
|
442
|
-
const items = await listMemoryItems(projectRoot);
|
|
482
|
+
const items = await listMemoryItems(projectRoot, { homeDir });
|
|
443
483
|
const target = items.find((item) => item.id === id);
|
|
444
484
|
if (!target) {
|
|
445
485
|
return { id, fullContent: '', relatedItems: [] };
|
|
@@ -622,18 +662,19 @@ export async function getContextInjection(
|
|
|
622
662
|
promptCache = false,
|
|
623
663
|
inputCompression = true,
|
|
624
664
|
projectAliases = [],
|
|
665
|
+
homeDir,
|
|
625
666
|
} = {},
|
|
626
667
|
) {
|
|
627
|
-
const { state, records, policy } = await queryV2Records(projectRoot, { projectId });
|
|
668
|
+
const { state, records, policy } = await queryV2Records(projectRoot, { projectId, homeDir });
|
|
628
669
|
const v2Active = !v2AllowsLegacyFallback(state);
|
|
629
|
-
const items = v2Active ? [] : await listMemoryItems(projectRoot);
|
|
670
|
+
const items = v2Active ? [] : await listMemoryItems(projectRoot, { homeDir });
|
|
630
671
|
const pressureState = await readCompactPressureState(projectRoot);
|
|
631
672
|
const recallPlan = resolveRecallPressurePlan({
|
|
632
673
|
maxTokens,
|
|
633
674
|
limit,
|
|
634
675
|
pressureState,
|
|
635
676
|
});
|
|
636
|
-
const rankedItems = await search(currentTask, 'all', Math.max(limit * 3, 8), { projectRoot, projectId, projectAliases });
|
|
677
|
+
const rankedItems = await search(currentTask, 'all', Math.max(limit * 3, 8), { projectRoot, projectId, projectAliases, homeDir });
|
|
637
678
|
const eligibleCtx = { projectId: projectId ?? null, projectAliases, includeUser: recallPlan.includeUserMemory };
|
|
638
679
|
const selectedItems = v2Active
|
|
639
680
|
? dedupeItemsById(rankedItems
|