@ngockhoale/ukit 3.5.4 → 3.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/manifests/engineConformance.yaml +0 -13
- package/manifests/platform.full.yaml +25 -0
- package/package.json +3 -9
- package/src/cli/commands/indexTools.js +1 -0
- package/src/cli/commands/memory.js +44 -4
- package/src/core/applyPlan.js +25 -72
- package/src/core/diffPlan.js +38 -9
- package/src/core/managedTemplateMerge.js +146 -0
- package/src/core/memory/memoryCheckRun.js +13 -1
- package/src/core/memory/memoryFlags.js +5 -6
- package/src/core/memory/memoryHit.js +31 -15
- package/src/core/memory/policy.js +5 -0
- package/src/core/memory/recallEngine.js +39 -3
- package/src/core/memory/recallLanes.js +182 -0
- package/src/core/memory/retrieval.js +13 -1
- package/src/core/memory/store.js +17 -0
- package/src/core/memory/topicCommand.js +75 -0
- package/src/core/memory/topicPages.js +545 -0
- package/src/core/metadata.js +12 -8
- package/src/core/runInstallPipeline.js +16 -10
- package/src/core/runtimeConfig.js +59 -83
- package/src/core/uninstall.js +8 -1
- package/src/decision/registry.js +33 -16
- package/src/decision/runtimeDecide.js +5 -5
- package/src/index/buildIndex.js +287 -6
- package/src/index/languageTools.js +100 -0
- package/src/index/paths.js +2 -1
- package/src/index/queryIndex.js +174 -10
- package/src/index/routeResolver.js +2 -2
- package/src/index/semanticLane.js +269 -0
- package/src/learning/lessonClass.js +110 -0
- package/src/learning/patternProposals.js +111 -7
- package/src/learning/reeditMiner.js +104 -0
- package/src/learning/selfImprove.js +9 -1
- package/template_project/.claude/skills/canvas-design/SKILL.md +2 -2
- package/template_project/.claude/skills/canvas-design/canvas-fonts/{InstrumentSans-OFL.txt → InstrumentSerif-OFL.txt} +3 -3
- package/template_project/.claude/skills/canvas-design/scripts/fetch-fonts.mjs +148 -0
- package/template_project/.claude/ukit/index/embed-index.mjs +26 -0
- package/template_project/.claude/ukit/index/lib/index-core.mjs +561 -14
- package/template_project/.claude/ukit/index/lib/semantic-lane.mjs +269 -0
- package/template_project/.claude/ukit/index/query-index.mjs +9 -1
- package/template_project/.claude/ukit/index/refresh-index.mjs +11 -0
- package/template_project/.claude/ukit/index/route-resolver.mjs +2 -2
- package/template_project/.claude/ukit/runtime/memory-lanes.mjs +9 -2
- package/template_project/.claude/ukit/runtime/memory-policy.mjs +5 -0
- package/template_project/.claude/ukit/runtime/memory-recall.mjs +39 -3
- package/template_project/.claude/ukit/runtime/memory-topics.mjs +545 -0
- package/template_project/ukit/storage/config.json +19 -34
- package/manifests/instructionRules.yaml +0 -442
- package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-OFL.txt +0 -93
- package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-Regular.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-BoldItalic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Italic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Regular.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSerif-Italic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/JetBrainsMono-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-BoldItalic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Italic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-OFL.txt +0 -93
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Regular.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Outfit-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Tektur-Medium.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/YoungSerif-OFL.txt +0 -93
- package/template_project/.claude/skills/canvas-design/canvas-fonts/YoungSerif-Regular.ttf +0 -0
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import { loadRuntimeConfig } from '../runtimeConfig.js';
|
|
4
4
|
import { loadSensitiveAllowlist, scanText } from '../sensitiveValueScanner.js';
|
|
5
5
|
import { checkMemorySources } from './memoryCheck.js';
|
|
6
|
+
import { checkTopics } from './topicPages.js';
|
|
6
7
|
import { expandMemorySources, normalizeMemorySources, parseMarkdownBullets, tokenize } from './recallEngine.js';
|
|
7
8
|
|
|
8
9
|
export async function runMemoryCheck(projectRoot, args = []) {
|
|
@@ -17,11 +18,22 @@ export async function runMemoryCheck(projectRoot, args = []) {
|
|
|
17
18
|
scan: (text) => scanText(text, { allowlistHashes }),
|
|
18
19
|
files: files.length > 0 ? files : null,
|
|
19
20
|
});
|
|
21
|
+
// 3.5.5: topic pages (docs/memory/topics) share the check: size cap, line cap, secrets are
|
|
22
|
+
// violations, a stale fingerprint is a warning. Skipped when the run is scoped with --file.
|
|
23
|
+
if (files.length === 0) {
|
|
24
|
+
const topics = await checkTopics(projectRoot, { scan: (text) => scanText(text, { allowlistHashes }) });
|
|
25
|
+
res.violations.push(...topics.violations);
|
|
26
|
+
res.warnings.push(...topics.warnings);
|
|
27
|
+
res.stats.topicPages = topics.pages;
|
|
28
|
+
if (topics.violations.length > 0) res.ok = false;
|
|
29
|
+
}
|
|
20
30
|
if (json) return { ok: res.ok, output: JSON.stringify(res, null, 2) };
|
|
21
31
|
const rows = [
|
|
22
32
|
`[UKit] memory check: ${res.ok ? 'OK' : 'VIOLATIONS'} — ${res.stats.files} file(s), ${res.stats.bullets} note(s), core ${res.stats.coreLines} lines / ${res.stats.coreBytes} bytes`,
|
|
23
33
|
...res.violations.map((v) => ` violation ${v.file}:${v.line} ${v.code} — ${v.message}`),
|
|
24
34
|
];
|
|
25
|
-
|
|
35
|
+
for (const w of res.warnings.filter((x) => String(x.code).startsWith('topic-'))) rows.push(` warning ${w.file} ${w.code} - ${w.message}`);
|
|
36
|
+
const noteWarnings = res.warnings.filter((x) => !String(x.code).startsWith('topic-'));
|
|
37
|
+
if (noteWarnings.length > 0) rows.push(` ${noteWarnings.length} warning(s) (notes naming no existing file); \`--json\` lists them`);
|
|
26
38
|
return { ok: res.ok, output: rows.join('\n') };
|
|
27
39
|
}
|
|
@@ -4,12 +4,11 @@
|
|
|
4
4
|
// resolveConfigStage machinery: off → shadow → canary → default. Flags only
|
|
5
5
|
// ever REDUCE capability — malformed/missing resolves 'off', killSwitch is
|
|
6
6
|
// absolute, and 'canary' without an opted-in projectId resolves 'off'.
|
|
7
|
-
// The `decision` plane is reserved for the memoryV2
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
// remove; user configs may pin its stage key.
|
|
7
|
+
// The `decision` plane is reserved for the memoryV2 decision lane and has no
|
|
8
|
+
// consumer today — do not remove; user configs may pin its stage key. Lesson
|
|
9
|
+
// classification (registry key `learn.candidate-class.v1`, family 'learn',
|
|
10
|
+
// owner src/learning/lessonClass.js) is a separate namespace: it is gated by
|
|
11
|
+
// `decisionPlane.families.learn.stage`, NOT `memoryV2.decision.stage`.
|
|
13
12
|
//
|
|
14
13
|
// Shadow receipts are the only telemetry: bounded JSONL of counts/codes/
|
|
15
14
|
// latency bands — never record text, prompt bodies, or secrets.
|
|
@@ -126,15 +126,11 @@ export function buildMarkdownHit(doc, { score = 0 } = {}) {
|
|
|
126
126
|
// and an explicit empty result. Type weights, the evidence bonus and the
|
|
127
127
|
// half-life recency bonus were removed from the score: they only ever added a
|
|
128
128
|
// query-independent constant that surfaced content-free episodes.
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
policy = {},
|
|
134
|
-
scope = 'all',
|
|
135
|
-
limit = 10,
|
|
136
|
-
now,
|
|
137
|
-
extraDocs = [],
|
|
129
|
+
// The eligible document set (records through the policy gate and requested
|
|
130
|
+
// scope, plus `extraDocs`): the exact corpus BM25 ranks, reused by the optional
|
|
131
|
+
// vector lane so both lanes see the same candidates.
|
|
132
|
+
export function collectEligibleDocs(records, {
|
|
133
|
+
eligibleCtx = {}, policy = {}, scope = 'all', now, extraDocs = [],
|
|
138
134
|
} = {}) {
|
|
139
135
|
const nowMs = typeof now === 'number' ? now : Date.now();
|
|
140
136
|
const ctx = { ...eligibleCtx, now: nowMs };
|
|
@@ -148,14 +144,34 @@ export function rankHitsDetailed(records, {
|
|
|
148
144
|
docs.push(record);
|
|
149
145
|
}
|
|
150
146
|
for (const doc of extraDocs) docs.push(doc);
|
|
147
|
+
return docs;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// Hit for one ranked doc (markdown bullet or record).
|
|
151
|
+
export function hitForDoc(doc, score, extraReasons = []) {
|
|
152
|
+
return doc.type === 'markdown'
|
|
153
|
+
? buildMarkdownHit(doc, { score })
|
|
154
|
+
: buildMemoryHit(doc, {
|
|
155
|
+
score,
|
|
156
|
+
reasons: ['lexical', ...extraReasons, `type:${doc.type}`, ...(hasLocatorEvidence(doc) ? ['evidence'] : [])],
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
export function rankHitsDetailed(records, {
|
|
161
|
+
query = '',
|
|
162
|
+
queryTokens,
|
|
163
|
+
eligibleCtx = {},
|
|
164
|
+
policy = {},
|
|
165
|
+
scope = 'all',
|
|
166
|
+
limit = 10,
|
|
167
|
+
now,
|
|
168
|
+
extraDocs = [],
|
|
169
|
+
} = {}) {
|
|
170
|
+
const docs = collectEligibleDocs(records, { eligibleCtx, policy, scope, now, extraDocs });
|
|
151
171
|
const result = rankDocs(docs, Array.isArray(queryTokens) ? queryTokens : query, { limit });
|
|
152
|
-
const hits = result.ranked.map((entry) => (entry.doc.
|
|
153
|
-
? buildMarkdownHit(entry.doc, { score: entry.score })
|
|
154
|
-
: buildMemoryHit(entry.doc, {
|
|
155
|
-
score: entry.score,
|
|
156
|
-
reasons: ['lexical', `type:${entry.doc.type}`, ...(hasLocatorEvidence(entry.doc) ? ['evidence'] : [])],
|
|
157
|
-
})));
|
|
172
|
+
const hits = result.ranked.map((entry) => hitForDoc(entry.doc, entry.score));
|
|
158
173
|
return {
|
|
174
|
+
docs,
|
|
159
175
|
hits,
|
|
160
176
|
candidateCount: result.candidateCount,
|
|
161
177
|
queryTokenCount: result.queryTokenCount,
|
|
@@ -70,6 +70,11 @@ export function eligible(record, context = {}, policy = {}) {
|
|
|
70
70
|
if (NON_AUTHORITY_TRUST_TIERS.includes(record.trust_tier)) {
|
|
71
71
|
return deny('non-authority-trust-tier');
|
|
72
72
|
}
|
|
73
|
+
// Records written before candidates got trust_tier 'candidate' sit as
|
|
74
|
+
// active/verified with meta.legacyStatus 'pending' — still unreviewed.
|
|
75
|
+
if (record.meta?.legacyStatus === 'pending') {
|
|
76
|
+
return deny('pending-approval');
|
|
77
|
+
}
|
|
73
78
|
|
|
74
79
|
if (record.scope === 'user') {
|
|
75
80
|
if (context.includeUser !== true) {
|
|
@@ -155,9 +155,17 @@ function stem(token) {
|
|
|
155
155
|
|
|
156
156
|
// Tokens of `text`: whole tokens plus their separator-split parts
|
|
157
157
|
// (`async-lock` -> async-lock, async, lock), stemmed, stopwords removed.
|
|
158
|
-
|
|
158
|
+
// A camelCase identifier also yields its words (`deriveDeterministicBucket`
|
|
159
|
+
// -> derive, deterministic, bucket); the whole identifier stays a token too.
|
|
160
|
+
const CAMEL_RE = /\b[a-z][a-z0-9]*(?:[A-Z][a-z0-9]+)+\b/g;
|
|
161
|
+
const splitCamel = (word) => `${word} ${word.replace(/([a-z0-9])([A-Z])/g, '$1 $2')}`;
|
|
162
|
+
|
|
163
|
+
// Query side only (`{ camel: true }`): in a long note the split words of its many
|
|
164
|
+
// identifiers (`postEditReviewPolicy`) would add common-word matches.
|
|
165
|
+
export function tokenize(text, { camel = false } = {}) {
|
|
159
166
|
const out = [];
|
|
160
|
-
|
|
167
|
+
const source = camel ? String(text ?? '').replace(CAMEL_RE, splitCamel) : text;
|
|
168
|
+
for (const raw of normalize(source).split(/\s+/)) {
|
|
161
169
|
if (!raw) continue;
|
|
162
170
|
const parts = raw.split(/[./:_-]+/).filter((part) => part.length > 1 && !/^\d+$/.test(part));
|
|
163
171
|
const candidates = parts.length > 1 || (parts.length === 1 && parts[0] !== raw) ? [raw, ...parts] : [raw];
|
|
@@ -169,6 +177,21 @@ export function tokenize(text) {
|
|
|
169
177
|
return out;
|
|
170
178
|
}
|
|
171
179
|
|
|
180
|
+
// File names in a query (`name.ext`): a query that names a file is only
|
|
181
|
+
// answered by a note that names it too and covers the rest of the question.
|
|
182
|
+
const QUERY_FILE_RE = /[\w-]+\.(?:test\.)?(?:m?js|cjs|tsx?|jsx|sh|md|json|py|ya?ml)\b/gi;
|
|
183
|
+
const FILE_EXT_TOKENS = new Set(['js', 'mjs', 'cjs', 'ts', 'tsx', 'jsx', 'sh', 'md', 'json', 'py', 'yml', 'yaml', 'test']);
|
|
184
|
+
const FILE_REST_COVERAGE = 0.3;
|
|
185
|
+
const FILE_REST_MIN_TOKENS = 2;
|
|
186
|
+
function queryFileSpec(query) {
|
|
187
|
+
if (typeof query !== 'string') return null;
|
|
188
|
+
const files = query.match(QUERY_FILE_RE);
|
|
189
|
+
if (!files) return null;
|
|
190
|
+
const fileTokens = new Set(files.flatMap((f) => tokenize(f.toLowerCase())).filter((t) => !FILE_EXT_TOKENS.has(t)));
|
|
191
|
+
const restTokens = [...new Set(tokenize(query.replace(QUERY_FILE_RE, ' '), { camel: true }))];
|
|
192
|
+
return fileTokens.size > 0 ? { fileTokens, restTokens } : null;
|
|
193
|
+
}
|
|
194
|
+
|
|
172
195
|
function episodeBody(text) {
|
|
173
196
|
return String(text ?? '')
|
|
174
197
|
.replace(/^\s*Session\s+\S+?:\s*/i, '')
|
|
@@ -272,7 +295,7 @@ function idf(totalDocs, docFreq) {
|
|
|
272
295
|
// `docs` is the ELIGIBLE set (IDF is computed over exactly this set);
|
|
273
296
|
// `query` is a string or a pre-tokenized array. Never throws.
|
|
274
297
|
export function rankDocs(docs, query, { limit = 10, twins = 'detail' } = {}) {
|
|
275
|
-
const queryTokens = [...new Set(Array.isArray(query) ? query : tokenize(query))];
|
|
298
|
+
const queryTokens = [...new Set(Array.isArray(query) ? query : tokenize(query, { camel: true }))];
|
|
276
299
|
const empty = {
|
|
277
300
|
ranked: [], candidateCount: 0, queryTokenCount: queryTokens.length, topScore: 0, abstained: true,
|
|
278
301
|
};
|
|
@@ -302,6 +325,10 @@ export function rankDocs(docs, query, { limit = 10, twins = 'detail' } = {}) {
|
|
|
302
325
|
let queryMass = 0;
|
|
303
326
|
for (const token of queryTokens) queryMass += idf(total, docFreq.get(token) ?? 0);
|
|
304
327
|
|
|
328
|
+
const fileSpec = total >= SMALL_CORPUS_DOCS ? queryFileSpec(query) : null;
|
|
329
|
+
let restMass = 0;
|
|
330
|
+
if (fileSpec) for (const token of fileSpec.restTokens) restMass += idf(total, docFreq.get(token) ?? 0);
|
|
331
|
+
|
|
305
332
|
const scored = [];
|
|
306
333
|
let candidateCount = 0;
|
|
307
334
|
for (const { doc, tf, length } of stats) {
|
|
@@ -324,6 +351,15 @@ export function rankDocs(docs, query, { limit = 10, twins = 'detail' } = {}) {
|
|
|
324
351
|
// clear the absolute floor above while the query's real subject is absent.
|
|
325
352
|
// It must also cover a fraction of the query's own IDF mass.
|
|
326
353
|
if (total >= SMALL_CORPUS_DOCS && doc?.type === 'episode' && idfMass < EPISODE_MIN_COVERAGE * queryMass) continue;
|
|
354
|
+
if (fileSpec) {
|
|
355
|
+
let named = false;
|
|
356
|
+
for (const token of fileSpec.fileTokens) if (tf.has(token)) { named = true; break; }
|
|
357
|
+
let restHit = 0;
|
|
358
|
+
for (const token of fileSpec.restTokens) if (tf.has(token)) restHit += idf(total, docFreq.get(token));
|
|
359
|
+
// One leftover word ("what does X check") is too little to judge coverage by.
|
|
360
|
+
const judgeRest = fileSpec.restTokens.length >= FILE_REST_MIN_TOKENS && restMass > 0;
|
|
361
|
+
if (!named || (judgeRest && restHit < FILE_REST_COVERAGE * restMass)) continue;
|
|
362
|
+
}
|
|
327
363
|
scored.push({ doc, score: score * (TYPE_PRIOR[doc.type] ?? 1), matched });
|
|
328
364
|
}
|
|
329
365
|
// Ties break on the stable logical id, never on a timestamp (CTX-01): a re-saved
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
// Optional recall lanes (3.5.5), CLI path only, both off unless enabled.
|
|
2
|
+
//
|
|
3
|
+
// vector - embedding cosine over the eligible docs, fused with the BM25
|
|
4
|
+
// order by RRF (recallEngine.fuseRanked). Provider = the
|
|
5
|
+
// codeIntel.embedding seam (hashed by default, `auto` takes a loader).
|
|
6
|
+
// rerank - the top BM25 candidates are put to `unic-decision` (the owner's
|
|
7
|
+
// local decision model, reached through the decision-plane client)
|
|
8
|
+
// as one yes/no relevance question each; confirmed candidates move
|
|
9
|
+
// up, nothing is dropped. Hard timeout, any failure keeps the BM25
|
|
10
|
+
// order.
|
|
11
|
+
//
|
|
12
|
+
// Neither lane can turn an abstention into a result: both run only on a
|
|
13
|
+
// non-empty BM25 result. Neither runs on a hook path: the hook engine
|
|
14
|
+
// (recallEngine.js / memory-recall.mjs) never imports this file, so no network
|
|
15
|
+
// code can sit inside an armed hook deadline. Never throws.
|
|
16
|
+
|
|
17
|
+
import { cosineSimilarity, createEmbeddingProvider } from '../codeintel/vectorProvider.js';
|
|
18
|
+
import { createDecisionClient } from '../../decision/client.js';
|
|
19
|
+
import { resolveConfigStage } from '../runtimeConfig.js';
|
|
20
|
+
import { fuseRanked } from './recallEngine.js';
|
|
21
|
+
import { hitForDoc } from './memoryHit.js';
|
|
22
|
+
|
|
23
|
+
export const VECTOR_WEIGHT = 0.6;
|
|
24
|
+
export const VECTOR_TOP_K = 10;
|
|
25
|
+
export const VECTOR_MIN_COSINE = 0.2;
|
|
26
|
+
export const RERANK_TOP_N = 6;
|
|
27
|
+
export const RERANK_TIMEOUT_MS = 1500;
|
|
28
|
+
export const RERANK_DECISION_KEY = 'memory.relevance.v1';
|
|
29
|
+
const SNIPPET_CHARS = 200;
|
|
30
|
+
const QUERY_CHARS = 200;
|
|
31
|
+
|
|
32
|
+
const envFlag = (name) => {
|
|
33
|
+
const raw = process.env[name];
|
|
34
|
+
return raw === '1' ? true : raw === '0' ? false : null;
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
export function resolveLaneFlags(config) {
|
|
38
|
+
const recall = config?.memory?.recall ?? {};
|
|
39
|
+
const vectorEnv = envFlag('UKIT_MEMORY_VECTOR');
|
|
40
|
+
const rerankEnv = envFlag('UKIT_MEMORY_RERANK');
|
|
41
|
+
const rerankStage = resolveConfigStage(config, 'memory.recall.rerank.stage');
|
|
42
|
+
return {
|
|
43
|
+
vector: vectorEnv ?? recall.vector?.enabled === true,
|
|
44
|
+
rerank: rerankEnv ?? (rerankStage === 'canary' || rerankStage === 'default'),
|
|
45
|
+
rerankStage,
|
|
46
|
+
vectorWeight: Number.isFinite(recall.vector?.weight) ? recall.vector.weight : VECTOR_WEIGHT,
|
|
47
|
+
vectorMinCosine: Number.isFinite(recall.vector?.minCosine) ? recall.vector.minCosine : VECTOR_MIN_COSINE,
|
|
48
|
+
rerankTopN: Number.isInteger(recall.rerank?.topN) && recall.rerank.topN > 0
|
|
49
|
+
? Math.min(recall.rerank.topN, 8) : RERANK_TOP_N,
|
|
50
|
+
rerankTimeoutMs: Number.isFinite(recall.rerank?.timeoutMs) && recall.rerank.timeoutMs > 0
|
|
51
|
+
? recall.rerank.timeoutMs : RERANK_TIMEOUT_MS,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const providers = new Map();
|
|
56
|
+
function providerFor(projectRoot, config) {
|
|
57
|
+
const embedding = config?.codeIntel?.embedding;
|
|
58
|
+
const key = `${projectRoot}\n${embedding?.provider ?? 'hashed'}\n${embedding?.dimensions ?? ''}`;
|
|
59
|
+
let provider = providers.get(key);
|
|
60
|
+
if (!provider) {
|
|
61
|
+
provider = createEmbeddingProvider({ projectRoot, config });
|
|
62
|
+
providers.set(key, provider);
|
|
63
|
+
}
|
|
64
|
+
return provider;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const docText = (doc) => String(doc?.text ?? doc?.summary ?? '');
|
|
68
|
+
|
|
69
|
+
// Ids of the docs nearest the query by cosine, best first (floor + top-K).
|
|
70
|
+
export async function vectorRanking(docs, query, { provider, minCosine = VECTOR_MIN_COSINE, topK = VECTOR_TOP_K } = {}) {
|
|
71
|
+
try {
|
|
72
|
+
if (!provider || typeof provider.embed !== 'function') return [];
|
|
73
|
+
const queryVec = await provider.embed(String(query ?? ''));
|
|
74
|
+
const scored = [];
|
|
75
|
+
for (const doc of docs) {
|
|
76
|
+
const cos = cosineSimilarity(queryVec, await provider.embed(docText(doc)));
|
|
77
|
+
if (cos >= minCosine) scored.push({ id: doc.id, cos });
|
|
78
|
+
}
|
|
79
|
+
scored.sort((a, b) => b.cos - a.cos || String(a.id).localeCompare(String(b.id)));
|
|
80
|
+
return scored.slice(0, topK).map((entry) => entry.id);
|
|
81
|
+
} catch {
|
|
82
|
+
return [];
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// RRF fusion of the BM25 hit order with the vector order. Vector-only docs
|
|
87
|
+
// enter as hits; BM25 hits keep their own score.
|
|
88
|
+
export function fuseVector(hits, vectorIds, docsById, { weight = VECTOR_WEIGHT, limit = 10 } = {}) {
|
|
89
|
+
if (!Array.isArray(vectorIds) || vectorIds.length === 0) return hits;
|
|
90
|
+
const byId = new Map(hits.map((hit) => [hit.id, hit]));
|
|
91
|
+
const fused = fuseRanked([hits.map((h) => h.id), vectorIds], { weights: [1, weight] });
|
|
92
|
+
const out = [];
|
|
93
|
+
for (const { id } of fused) {
|
|
94
|
+
const hit = byId.get(id) ?? (docsById.get(id) ? hitForDoc(docsById.get(id), 0, ['vector']) : null);
|
|
95
|
+
if (hit) out.push(hit);
|
|
96
|
+
}
|
|
97
|
+
return out.slice(0, Math.max(limit, 1));
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const oneLine = (text, max) => String(text ?? '').replace(/\s+/g, ' ').trim().slice(0, max);
|
|
101
|
+
|
|
102
|
+
export function buildRerankBatch(query, hits) {
|
|
103
|
+
const questions = hits.map((hit, i) => ({
|
|
104
|
+
decisionKey: `${RERANK_DECISION_KEY}#c${i + 1}`,
|
|
105
|
+
schemaVersion: 1,
|
|
106
|
+
family: 'memory',
|
|
107
|
+
owner: 'memoryRecall',
|
|
108
|
+
kind: 'noul',
|
|
109
|
+
instruction: `Does this project note answer the question? Question: ${oneLine(query, QUERY_CHARS)} | Note: ${oneLine(hit.snippet ?? hit.summary, SNIPPET_CHARS)}`,
|
|
110
|
+
candidates: [],
|
|
111
|
+
}));
|
|
112
|
+
return { statePacket: { task: 'memory-relevance', candidateCount: hits.length }, questions };
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function withTimeout(promise, ms) {
|
|
116
|
+
let timer;
|
|
117
|
+
const timeout = new Promise((resolve) => { timer = setTimeout(() => resolve({ status: 'timeout', answers: [] }), ms); });
|
|
118
|
+
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// Stable partition: candidates the model confirms first, the rest in their
|
|
122
|
+
// BM25 order, everything past topN untouched. `meta.status` is the client's
|
|
123
|
+
// batch status; `meta.applied` is true only when a valid answer moved something.
|
|
124
|
+
export async function rerankHits(hits, query, {
|
|
125
|
+
client, topN = RERANK_TOP_N, timeoutMs = RERANK_TIMEOUT_MS,
|
|
126
|
+
} = {}) {
|
|
127
|
+
const meta = { status: 'skipped', applied: false };
|
|
128
|
+
try {
|
|
129
|
+
if (!client || !Array.isArray(hits) || hits.length < 2) return { hits, meta };
|
|
130
|
+
const head = hits.slice(0, topN);
|
|
131
|
+
const batch = buildRerankBatch(query, head);
|
|
132
|
+
const result = await withTimeout(
|
|
133
|
+
client.requestBatch({ ...batch, deadlineMs: timeoutMs }),
|
|
134
|
+
timeoutMs,
|
|
135
|
+
);
|
|
136
|
+
meta.status = result?.status ?? 'invalid';
|
|
137
|
+
const verdict = new Map();
|
|
138
|
+
for (const answer of result?.answers ?? []) {
|
|
139
|
+
// noul answers decode to P(true) in [0,1]; >= 0.5 confirms the candidate.
|
|
140
|
+
if (answer?.validationStatus !== 'valid' || !Number.isFinite(answer.value)) continue;
|
|
141
|
+
const idx = Number(String(answer.decisionKey).split('#c')[1]);
|
|
142
|
+
if (Number.isInteger(idx) && idx >= 1 && idx <= head.length) verdict.set(idx - 1, answer.value >= 0.5);
|
|
143
|
+
}
|
|
144
|
+
if (verdict.size === 0) return { hits, meta };
|
|
145
|
+
const yes = head.filter((_, i) => verdict.get(i) === true);
|
|
146
|
+
const rest = head.filter((_, i) => verdict.get(i) !== true);
|
|
147
|
+
const reordered = [...yes, ...rest, ...hits.slice(topN)];
|
|
148
|
+
meta.applied = reordered.some((hit, i) => hit !== hits[i]);
|
|
149
|
+
return { hits: reordered, meta };
|
|
150
|
+
} catch {
|
|
151
|
+
meta.status = 'error';
|
|
152
|
+
return { hits, meta };
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// Entry point for retrieval.search(). `hits` is the BM25 result; empty stays empty.
|
|
157
|
+
export async function applyRecallLanes({
|
|
158
|
+
config, query, docs, hits, limit = 10, projectRoot, deps = {},
|
|
159
|
+
} = {}) {
|
|
160
|
+
const flags = resolveLaneFlags(config);
|
|
161
|
+
if ((!flags.vector && !flags.rerank) || !Array.isArray(hits) || hits.length === 0) {
|
|
162
|
+
return { hits, lanes: {} };
|
|
163
|
+
}
|
|
164
|
+
const lanes = {};
|
|
165
|
+
let out = hits;
|
|
166
|
+
if (flags.vector) {
|
|
167
|
+
const provider = deps.provider ?? providerFor(projectRoot, config);
|
|
168
|
+
const ids = await vectorRanking(docs ?? [], query, { provider, minCosine: flags.vectorMinCosine });
|
|
169
|
+
const docsById = new Map((docs ?? []).map((doc) => [doc.id, doc]));
|
|
170
|
+
out = fuseVector(out, ids, docsById, { weight: flags.vectorWeight, limit });
|
|
171
|
+
lanes.vector = { candidates: ids.length };
|
|
172
|
+
}
|
|
173
|
+
if (flags.rerank) {
|
|
174
|
+
const client = deps.client ?? createDecisionClient({ config, projectRoot, transport: deps.transport });
|
|
175
|
+
const reranked = await rerankHits(out, query, {
|
|
176
|
+
client, topN: flags.rerankTopN, timeoutMs: flags.rerankTimeoutMs,
|
|
177
|
+
});
|
|
178
|
+
out = reranked.hits;
|
|
179
|
+
lanes.rerank = reranked.meta;
|
|
180
|
+
}
|
|
181
|
+
return { hits: out, lanes };
|
|
182
|
+
}
|
|
@@ -13,6 +13,8 @@ import { resolveMemoryStage, appendRolloutReceipt, latencyBandForMs } from './me
|
|
|
13
13
|
import { normalize, tokenize } from './recordIndex.js';
|
|
14
14
|
import { buildMemoryHit, rankHitsDetailed } from './memoryHit.js';
|
|
15
15
|
import { loadMarkdownDocs, normalizeMemorySources, renderMarkdownDoc } from './recallEngine.js';
|
|
16
|
+
import { loadTopicDocs, topicsRecallEnabled } from './topicPages.js';
|
|
17
|
+
import { applyRecallLanes } from './recallLanes.js';
|
|
16
18
|
import { resolveRecordFreshness } from './memoryFreshness.js';
|
|
17
19
|
import { getHeadSha } from '../codeintel/freshness.js';
|
|
18
20
|
import { emitRetrievalQuery, emitMemoryRetrieved } from './memoryEmit.js';
|
|
@@ -327,6 +329,11 @@ export async function search(query, scope = 'all', limit = 10, { projectRoot, pr
|
|
|
327
329
|
const markdownDocs = scope === 'all' || scope === 'project' || scope === 'repo'
|
|
328
330
|
? await loadMarkdownDocs(projectRoot, normalizeMemorySources(config)).catch(() => [])
|
|
329
331
|
: [];
|
|
332
|
+
// 3.5.5 topic pages: recall documents of their own kind, only when enabled
|
|
333
|
+
// (memory.topics.recall / UKIT_MEMORY_TOPICS); a stale page carries validity.
|
|
334
|
+
if (markdownDocs.length > 0 && topicsRecallEnabled(config)) {
|
|
335
|
+
markdownDocs.push(...await loadTopicDocs(projectRoot).catch(() => []));
|
|
336
|
+
}
|
|
330
337
|
|
|
331
338
|
if (!v2AllowsLegacyFallback(state)) {
|
|
332
339
|
// One engine (recallEngine.js BM25): eligibility first, then IDF over the
|
|
@@ -337,7 +344,12 @@ export async function search(query, scope = 'all', limit = 10, { projectRoot, pr
|
|
|
337
344
|
const ranked = rankHitsDetailed(anchored.slice(0, records.length), {
|
|
338
345
|
query, eligibleCtx, policy, scope, limit, now, extraDocs: anchored.slice(records.length),
|
|
339
346
|
});
|
|
340
|
-
|
|
347
|
+
// Optional vector / rerank lanes (3.5.5, off by default, CLI path only; an
|
|
348
|
+
// empty BM25 result stays empty so abstention is untouched).
|
|
349
|
+
const laned = await applyRecallLanes({
|
|
350
|
+
config, query, docs: ranked.docs, hits: ranked.hits, limit, projectRoot,
|
|
351
|
+
});
|
|
352
|
+
const results = await attachFreshness(laned.hits, records, { projectRoot, watermarks, now });
|
|
341
353
|
emitQuery({ lane: 'v2', indexed: false, candidateCount: ranked.candidateCount, results });
|
|
342
354
|
return results;
|
|
343
355
|
}
|
package/src/core/memory/store.js
CHANGED
|
@@ -608,6 +608,8 @@ function patternCandidateFromRecord(record) {
|
|
|
608
608
|
};
|
|
609
609
|
}
|
|
610
610
|
|
|
611
|
+
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
612
|
+
|
|
611
613
|
async function proposePatternCandidateV2(projectRoot, projectId, candidate, { homeDir } = {}) {
|
|
612
614
|
const v2 = await storeV2();
|
|
613
615
|
const normalizedText = normalizePatternText(candidate?.text);
|
|
@@ -643,6 +645,19 @@ async function proposePatternCandidateV2(projectRoot, projectId, candidate, { ho
|
|
|
643
645
|
confidence: 0.4,
|
|
644
646
|
createdBy: 'propose',
|
|
645
647
|
projectId,
|
|
648
|
+
// Unreviewed until `memory approve`: candidate tier keeps it out of every
|
|
649
|
+
// recall path; approval flips it to verified and clears the expiry.
|
|
650
|
+
// Only mined lessons ask for an expiry (candidate.ttlDays); a hand-written
|
|
651
|
+
// `memory propose` candidate stays pending until a person decides.
|
|
652
|
+
trustTier: 'candidate',
|
|
653
|
+
validUntil: Number(candidate?.ttlDays) > 0 ? Date.now() + Number(candidate.ttlDays) * DAY_MS : null,
|
|
654
|
+
evidence: (Array.isArray(candidate?.evidence) ? candidate.evidence : []).map((ev) => ({
|
|
655
|
+
kind: ev.kind,
|
|
656
|
+
locator: ev.locator,
|
|
657
|
+
observed_at: Date.now(),
|
|
658
|
+
fingerprint: null,
|
|
659
|
+
trust_tier: 'verified',
|
|
660
|
+
})),
|
|
646
661
|
meta: { ...entry, legacyStatus: 'pending' },
|
|
647
662
|
}, { homeDir });
|
|
648
663
|
return entry;
|
|
@@ -684,6 +699,8 @@ async function resolvePatternCandidateV2(projectRoot, projectId, candidateId, de
|
|
|
684
699
|
provenance: `${target.provenance ?? 'pattern-candidate'};promoted-from:${target.id}`,
|
|
685
700
|
// Approval is a trust transition — the promoted record is authoritative.
|
|
686
701
|
trust_tier: 'verified',
|
|
702
|
+
// A promoted rule must not inherit a mined lesson's expiry.
|
|
703
|
+
valid_until: null,
|
|
687
704
|
meta: {
|
|
688
705
|
...target.meta,
|
|
689
706
|
legacyStatus: 'approved',
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
// `ukit memory topic build <dir>... | list | show <dir> | check [--json]` (3.5.5).
|
|
2
|
+
// Thin CLI runner over topicPages.js: gathers the memory notes that name paths
|
|
3
|
+
// (markdown bullets + active records) and writes/reads docs/memory/topics pages.
|
|
4
|
+
import { loadRuntimeConfig } from '../runtimeConfig.js';
|
|
5
|
+
import { loadSensitiveAllowlist, scanText } from '../sensitiveValueScanner.js';
|
|
6
|
+
import { loadRecords } from './storeV2.js';
|
|
7
|
+
import { loadMarkdownDocs, normalizeMemorySources } from './recallEngine.js';
|
|
8
|
+
import {
|
|
9
|
+
checkTopics, listTopics, readTopicPage, topicRel, validTopicDir, writeTopicPage,
|
|
10
|
+
} from './topicPages.js';
|
|
11
|
+
|
|
12
|
+
async function gatherNotes(projectRoot, config) {
|
|
13
|
+
const notes = [];
|
|
14
|
+
try {
|
|
15
|
+
for (const d of await loadMarkdownDocs(projectRoot, normalizeMemorySources(config))) notes.push({ ref: d.provenance, text: d.text });
|
|
16
|
+
} catch { /* no markdown notes */ }
|
|
17
|
+
try {
|
|
18
|
+
for (const r of await loadRecords(projectRoot)) {
|
|
19
|
+
if (r?.superseded_by || r?.status === 'archived' || r?.type === 'episode') continue;
|
|
20
|
+
notes.push({ ref: `record:${r.id}`, text: String(r.text ?? ''), type: r.type });
|
|
21
|
+
}
|
|
22
|
+
} catch { /* no records */ }
|
|
23
|
+
return notes;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export async function runTopicCommand(projectRoot, args = []) {
|
|
27
|
+
const [action, ...rest] = args;
|
|
28
|
+
const dirs = rest.filter((a) => !a.startsWith('--'));
|
|
29
|
+
const json = rest.includes('--json');
|
|
30
|
+
if (action === 'build') {
|
|
31
|
+
if (dirs.length === 0) return { ok: false, output: 'Usage: ukit memory topic build <dir> [<dir>...]' };
|
|
32
|
+
const config = await loadRuntimeConfig(projectRoot).catch(() => ({}));
|
|
33
|
+
const notes = await gatherNotes(projectRoot, config);
|
|
34
|
+
const rows = [];
|
|
35
|
+
let ok = true;
|
|
36
|
+
for (const dir of dirs) {
|
|
37
|
+
if (!validTopicDir(dir)) { ok = false; rows.push(`[UKit] rejected: ${dir} is not a repo-relative directory`); continue; }
|
|
38
|
+
try {
|
|
39
|
+
const built = await writeTopicPage(projectRoot, dir, notes);
|
|
40
|
+
rows.push(`[UKit] topic built: ${built.rel} (${built.count} files, ${built.bytes} bytes)`);
|
|
41
|
+
} catch (error) {
|
|
42
|
+
ok = false;
|
|
43
|
+
rows.push(`[UKit] topic not built: ${error.message}`);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return { ok, output: rows.join('\n') };
|
|
47
|
+
}
|
|
48
|
+
if (action === 'list') {
|
|
49
|
+
const topics = await listTopics(projectRoot);
|
|
50
|
+
if (json) return { ok: true, output: JSON.stringify(topics, null, 2) };
|
|
51
|
+
if (topics.length === 0) return { ok: true, output: '[UKit] no topic pages. Build one with `ukit memory topic build <dir>`.' };
|
|
52
|
+
return { ok: true, output: topics.map((t) => `${t.rel} ${t.dir} ${t.missing ? 'MISSING-DIR' : t.stale ? 'STALE' : 'current'}${t.filled ? ' agent-notes' : ''} ${t.bytes}B`).join('\n') };
|
|
53
|
+
}
|
|
54
|
+
if (action === 'show') {
|
|
55
|
+
if (dirs.length !== 1) return { ok: false, output: 'Usage: ukit memory topic show <dir>' };
|
|
56
|
+
const page = await readTopicPage(projectRoot, topicRel(dirs[0]));
|
|
57
|
+
if (!page) return { ok: false, output: `[UKit] no topic page for ${dirs[0]} (expected ${topicRel(dirs[0])})` };
|
|
58
|
+
const label = page.missing
|
|
59
|
+
? '[UKit] STALE: the described directory no longer exists. Do not rely on this page.\n'
|
|
60
|
+
: page.stale ? '[UKit] STALE: the directory changed since this page was built. Verify before relying on it; rebuild with `ukit memory topic build`.\n' : '';
|
|
61
|
+
return { ok: true, output: `${label}${page.text}` };
|
|
62
|
+
}
|
|
63
|
+
if (action === 'check') {
|
|
64
|
+
const config = await loadRuntimeConfig(projectRoot).catch(() => ({}));
|
|
65
|
+
const allowlistHashes = loadSensitiveAllowlist(config);
|
|
66
|
+
const res = await checkTopics(projectRoot, { scan: (text) => scanText(text, { allowlistHashes }) });
|
|
67
|
+
const ok = res.violations.length === 0;
|
|
68
|
+
if (json) return { ok, output: JSON.stringify({ ok, ...res }, null, 2) };
|
|
69
|
+
const rows = [`[UKit] topic check: ${ok ? 'OK' : 'VIOLATIONS'} - ${res.pages} page(s), ${res.warnings.length} stale`,
|
|
70
|
+
...res.violations.map((v) => ` violation ${v.file}:${v.line} ${v.code} - ${v.message}`),
|
|
71
|
+
...res.warnings.map((v) => ` warning ${v.file} ${v.code} - ${v.message}`)];
|
|
72
|
+
return { ok, output: rows.join('\n') };
|
|
73
|
+
}
|
|
74
|
+
return { ok: false, output: 'Usage: ukit memory topic build <dir>... | list [--json] | show <dir> | check [--json]' };
|
|
75
|
+
}
|