@lorekit/cli 1.41.0 → 1.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -8
- package/package.json +1 -1
- package/src/adapters/claude.mjs +10 -0
- package/src/config.mjs +53 -5
- package/src/control.mjs +50 -2
- package/src/core/lessons.mjs +236 -7
- package/src/core/state.mjs +112 -0
- package/src/doctor.mjs +20 -5
- package/src/hook.mjs +64 -1
- package/src/install.mjs +18 -1
- package/src/lessons-pure.mjs +167 -0
- package/src/store/local.mjs +43 -7
- package/src/store/remote.mjs +6 -2
package/README.md
CHANGED
|
@@ -51,7 +51,9 @@ without needing a marketplace:
|
|
|
51
51
|
2. **MCP server** (`lorekit`) — the connection to your lessons, merged into the
|
|
52
52
|
MCP config (preserving any other servers).
|
|
53
53
|
3. **Hooks** — the *deterministic* layer: lessons injected on every
|
|
54
|
-
`SessionStart`,
|
|
54
|
+
`SessionStart`, the few that match what you just typed on every substantive
|
|
55
|
+
prompt (`UserPromptSubmit`, `hooks.userPrompt`), and on a tool failure
|
|
56
|
+
(`PostToolUseFailure`) any lessons that
|
|
55
57
|
look **relevant to that failure** ("you've hit this before") plus a nudge to
|
|
56
58
|
record the fix, and a retrospective nudge on `Stop` — by default only when the
|
|
57
59
|
session actually hit friction (`hooks.stop`). These fire the shared
|
|
@@ -133,8 +135,8 @@ and the write is still the model calling `memory.write`.
|
|
|
133
135
|
|
|
134
136
|
| Mode | Wires | What you get |
|
|
135
137
|
|------|-------|--------------|
|
|
136
|
-
| `all` | `SessionStart`, `PostToolUseFailure`, `Stop` | Lessons injected at session start, plus a nudge on a tool failure and a friction-gated one at end of turn |
|
|
137
|
-
| `read-only` | `SessionStart` | Lessons injected; nothing
|
|
138
|
+
| `all` | `SessionStart`, `UserPromptSubmit`, `PostToolUseFailure`, `Stop` | Lessons injected at session start, the ones matching each substantive prompt injected as you go, plus a nudge on a tool failure and a friction-gated one at end of turn |
|
|
139
|
+
| `read-only` | `SessionStart` | Lessons injected ONCE at session start; nothing nudges and nothing runs per turn |
|
|
138
140
|
| `none` | — | Skills + MCP only; memory stays model-invoked |
|
|
139
141
|
|
|
140
142
|
```bash
|
|
@@ -489,9 +491,9 @@ links now back the hooks' write-confirmation and retrospective nudges.)
|
|
|
489
491
|
The **shared hook engine** behind the Claude Code / Cursor / Codex plugins.
|
|
490
492
|
It is not run by hand — the plugins wire it into their hook config. It reads
|
|
491
493
|
the host framework's JSON on stdin and prints that host's injection format on
|
|
492
|
-
stdout (lessons at session start;
|
|
493
|
-
|
|
494
|
-
block the host agent.
|
|
494
|
+
stdout (lessons at session start; the ones matching each substantive prompt as
|
|
495
|
+
you go; relevant lessons plus a write-nudge on a tool failure; a retrospective
|
|
496
|
+
nudge at end of turn), always exiting 0 so it can never block the host agent.
|
|
495
497
|
|
|
496
498
|
```bash
|
|
497
499
|
lorekit hook --adapter <claude|cursor|codex> --event <SessionStart|Stop|…>
|
|
@@ -659,7 +661,25 @@ Both files share this schema — all fields optional:
|
|
|
659
661
|
// ── Hook behaviour ─────────────────────────────────────────────────────────
|
|
660
662
|
"hooks.disabled": ["Stop"],
|
|
661
663
|
// suppress specific hook events; union across layers
|
|
662
|
-
// values: "SessionStart" | "PostToolUseFailure" | "Stop"
|
|
664
|
+
// values: "SessionStart" | "UserPromptSubmit" | "PostToolUseFailure" | "Stop"
|
|
665
|
+
|
|
666
|
+
"hooks.userPrompt": "on",
|
|
667
|
+
// the per-turn relevance pull (UserPromptSubmit):
|
|
668
|
+
// "on" (default) — on each substantive prompt, query the
|
|
669
|
+
// store for memories matching what you typed and inject
|
|
670
|
+
// at most 3 you have NOT already been shown this session
|
|
671
|
+
// "off" — keep the rest of hook mode "all", drop
|
|
672
|
+
// just this event's output
|
|
673
|
+
// a switch, not a mode: the knobs a mode would expose (how
|
|
674
|
+
// many, how strict) are the two things that must not be
|
|
675
|
+
// turned up on a hook that fires every single turn.
|
|
676
|
+
// two install paths reach it: hook mode "all" ("read-only"
|
|
677
|
+
// and "none" never wire the event), and the Claude
|
|
678
|
+
// marketplace plugin, whose hooks.json wires it
|
|
679
|
+
// unconditionally — no mode involved, so hooks.userPrompt is
|
|
680
|
+
// the mode-independent opt-out there. (hooks.disabled:
|
|
681
|
+
// ["UserPromptSubmit"] also switches it off, one gate
|
|
682
|
+
// earlier, so it is not the only opt-out.) repo wins over user
|
|
663
683
|
|
|
664
684
|
"hooks.stop": "friction",
|
|
665
685
|
// gate the end-of-turn retrospective nudge:
|
|
@@ -704,13 +724,14 @@ Both files share this schema — all fields optional:
|
|
|
704
724
|
|
|
705
725
|
"hooks.instructions": {
|
|
706
726
|
"SessionStart": "Focus on migration safety. Treat any lesson tagged 'migration' as high-priority.",
|
|
727
|
+
"UserPromptSubmit": "Prefer a memory that names the file you are editing.",
|
|
707
728
|
"PostToolUseFailure": "When recording a failure, always include the exact command and exit code.",
|
|
708
729
|
"Stop": null
|
|
709
730
|
},
|
|
710
731
|
// per-event custom text appended to the hook output.
|
|
711
732
|
// both layers merged: repo instructions first, then user.
|
|
712
733
|
// null (or absent key) means no extra instruction for that event.
|
|
713
|
-
// values: string | null (keys: "SessionStart" | "PostToolUseFailure" | "Stop")
|
|
734
|
+
// values: string | null (keys: "SessionStart" | "UserPromptSubmit" | "PostToolUseFailure" | "Stop")
|
|
714
735
|
|
|
715
736
|
// ── Telemetry ──────────────────────────────────────────────────────────────
|
|
716
737
|
"telemetry.disabled": true,
|
package/package.json
CHANGED
package/src/adapters/claude.mjs
CHANGED
|
@@ -8,6 +8,12 @@ export const claude = {
|
|
|
8
8
|
switch (event) {
|
|
9
9
|
case 'SessionStart':
|
|
10
10
|
return 'read';
|
|
11
|
+
// The per-turn relevance pull. Distinct from 'read': that one fires once
|
|
12
|
+
// and injects a ranked slice of everything applicable; this one fires on
|
|
13
|
+
// every prompt and injects only what the prompt itself points at, minus
|
|
14
|
+
// whatever has already been shown.
|
|
15
|
+
case 'UserPromptSubmit':
|
|
16
|
+
return 'relevant-read';
|
|
11
17
|
case 'PostToolUse':
|
|
12
18
|
return 'confirm';
|
|
13
19
|
case 'PostToolUseFailure':
|
|
@@ -41,6 +47,10 @@ export const claude = {
|
|
|
41
47
|
toolName: input.tool_name || 'tool',
|
|
42
48
|
toolInput: input.tool_input || null,
|
|
43
49
|
toolResponse: input.tool_response || null,
|
|
50
|
+
// The user's message, present on UserPromptSubmit. Claude Code sends it
|
|
51
|
+
// as `prompt`; it is the only field that event carries beyond the
|
|
52
|
+
// session envelope.
|
|
53
|
+
prompt: typeof input.prompt === 'string' ? input.prompt : null,
|
|
44
54
|
event: input.hook_event_name || null,
|
|
45
55
|
// Path to the session JSONL (present on Stop/SubagentStop) — read by the
|
|
46
56
|
// friction-gated retrospective to decide whether the session is worth a nudge.
|
package/src/config.mjs
CHANGED
|
@@ -87,10 +87,33 @@ export function settingsPath(root, scope = 'project') {
|
|
|
87
87
|
return path.join(base, '.claude', 'settings.json');
|
|
88
88
|
}
|
|
89
89
|
|
|
90
|
-
// The lifecycle events the memory loop wires: read lessons on start,
|
|
90
|
+
// The lifecycle events the memory loop wires: read lessons on start, pull the
|
|
91
|
+
// ones matching each substantive prompt as the turn is submitted, nudge on a
|
|
91
92
|
// tool failure, nudge a retrospective at end of turn. Mirrors the plugin's
|
|
92
|
-
// hooks.json so `install` delivers the same deterministic layer
|
|
93
|
-
|
|
93
|
+
// hooks.json so `install` delivers the same deterministic layer — a parity test
|
|
94
|
+
// in `test/frameworks.test.mjs` holds the two lists to that claim.
|
|
95
|
+
export const CLAUDE_HOOK_EVENTS = ['SessionStart', 'UserPromptSubmit', 'PostToolUseFailure', 'Stop'];
|
|
96
|
+
|
|
97
|
+
// The event set `all` meant BEFORE `UserPromptSubmit` was wired.
|
|
98
|
+
//
|
|
99
|
+
// It exists for one job: `hookModeFromEvents` must still answer 'all' for an
|
|
100
|
+
// install done before that event existed. Without this, every pre-existing
|
|
101
|
+
// installation would read as 'custom' the next time `install` inspected it —
|
|
102
|
+
// and 'custom' is the answer that means "a human hand-wired this, do not touch
|
|
103
|
+
// it", so the upgrade prompt would default to leaving them behind on the old
|
|
104
|
+
// three events forever. Recognising the legacy set is what lets `install`
|
|
105
|
+
// UPGRADE such a wiring instead of preserving it verbatim.
|
|
106
|
+
//
|
|
107
|
+
// It does NOT make a bare `install` re-run an upgrade. A fully-installed scope
|
|
108
|
+
// short-circuits in `install.mjs` before the hook step, so reaching this
|
|
109
|
+
// recognition still needs `--hooks <mode>` or `--force`; the short-circuit
|
|
110
|
+
// summary names that command when an upgrade is available.
|
|
111
|
+
//
|
|
112
|
+
// Add to this list, never edit it: each entry is a historical fact about a
|
|
113
|
+
// version that shipped, not a configuration.
|
|
114
|
+
const LEGACY_ALL_EVENT_SETS = [
|
|
115
|
+
['SessionStart', 'PostToolUseFailure', 'Stop'],
|
|
116
|
+
];
|
|
94
117
|
|
|
95
118
|
// Matches a hook command that fires the lorekit engine, whether wired as a
|
|
96
119
|
// global `lorekit hook …` or `npx -y @lorekit/cli hook …`. Shared by the
|
|
@@ -181,13 +204,38 @@ export function hookEventsForMode(mode) {
|
|
|
181
204
|
// wired only `Stop`) — the caller must not silently rewrite such a setup.
|
|
182
205
|
export function hookModeFromEvents(events) {
|
|
183
206
|
const set = new Set(events || []);
|
|
207
|
+
const matches = (want) => want.length === set.size && want.every((e) => set.has(e));
|
|
184
208
|
for (const mode of HOOK_MODES) {
|
|
185
|
-
|
|
186
|
-
|
|
209
|
+
if (matches(hookEventsForMode(mode))) return mode;
|
|
210
|
+
}
|
|
211
|
+
// An install from before a lifecycle event was added is still 'all' — see
|
|
212
|
+
// LEGACY_ALL_EVENT_SETS for why reading it as 'custom' would strand it.
|
|
213
|
+
for (const legacy of LEGACY_ALL_EVENT_SETS) {
|
|
214
|
+
if (matches(legacy)) return 'all';
|
|
187
215
|
}
|
|
188
216
|
return 'custom';
|
|
189
217
|
}
|
|
190
218
|
|
|
219
|
+
// Which events an existing wiring is MISSING relative to the mode it reads as.
|
|
220
|
+
//
|
|
221
|
+
// An install predating a lifecycle event still reads as its mode (see
|
|
222
|
+
// LEGACY_ALL_EVENT_SETS), so `hookModeFromEvents` alone cannot tell a current
|
|
223
|
+
// `all` from a stale one — and a stale one keeps reporting the mode it no
|
|
224
|
+
// longer delivers. This is the difference, and it is the single derivation
|
|
225
|
+
// BOTH surfaces that report it use: `install`'s already-installed summary and
|
|
226
|
+
// `doctor`'s hooks line. A second copy is how the two would come to disagree
|
|
227
|
+
// about what "up to date" means.
|
|
228
|
+
//
|
|
229
|
+
// `custom` yields [] deliberately: it means a human hand-wired this, and
|
|
230
|
+
// `hookEventsForMode` answers the full set for any unrecognised mode, so
|
|
231
|
+
// anything else would advertise an "upgrade" away from a wiring the user chose.
|
|
232
|
+
export function missingHookEvents(events) {
|
|
233
|
+
const mode = hookModeFromEvents(events);
|
|
234
|
+
if (mode === 'custom') return [];
|
|
235
|
+
const wired = new Set(events || []);
|
|
236
|
+
return hookEventsForMode(mode).filter((e) => !wired.has(e));
|
|
237
|
+
}
|
|
238
|
+
|
|
191
239
|
// Extract the flat list of hook command strings for one event out of the nested
|
|
192
240
|
// group shape Claude Code uses: { [event]: [ { hooks: [ { type, command } ] } ] }.
|
|
193
241
|
export function hookCommandsForEvent(hooksObj, event) {
|
package/src/control.mjs
CHANGED
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
// hooks.stop — Stop-hook gating ("friction" default | "always" | "off")
|
|
11
11
|
// hooks.sessionStart — injected-block shape ("hybrid" default | "index" | "map")
|
|
12
12
|
// hooks.sessionStart.maxChars — character budget for that block (default 1500)
|
|
13
|
+
// hooks.userPrompt — the per-turn relevance pull ("on" default | "off")
|
|
13
14
|
// hooks.adapter — explicit adapter override ("claude" | "cursor" | "codex")
|
|
14
15
|
//
|
|
15
16
|
// Two layers of config, two kinds of statement:
|
|
@@ -57,6 +58,26 @@ export function normalizeStopMode(v) {
|
|
|
57
58
|
return null;
|
|
58
59
|
}
|
|
59
60
|
|
|
61
|
+
// `hooks.userPrompt` — the per-turn relevance pull, on or off.
|
|
62
|
+
//
|
|
63
|
+
// A BOOLEAN, not a mode, and that is a deliberate limit on the surface. The
|
|
64
|
+
// interesting knobs a mode would expose (how many lessons, how strict the
|
|
65
|
+
// match) are the two things this hook must never let a user turn up: it fires
|
|
66
|
+
// on EVERY prompt, so a generous setting is not a preference, it is a way to
|
|
67
|
+
// make the assistant unusable. The gates are fixed in code and reviewed here.
|
|
68
|
+
//
|
|
69
|
+
// Same forgiving vocabulary as `hooks.stop`, so a config that says `false`,
|
|
70
|
+
// `none` or `disabled` means what it looks like.
|
|
71
|
+
export const USER_PROMPT_MODES = ['on', 'off'];
|
|
72
|
+
export function normalizeUserPromptMode(v) {
|
|
73
|
+
if (typeof v === 'boolean') return v ? 'on' : 'off';
|
|
74
|
+
if (typeof v !== 'string') return null;
|
|
75
|
+
const s = v.trim().toLowerCase();
|
|
76
|
+
if (['off', 'none', 'false', 'disabled', 'never', 'no'].includes(s)) return 'off';
|
|
77
|
+
if (['on', 'true', 'enabled', 'always', 'yes'].includes(s)) return 'on';
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
80
|
+
|
|
60
81
|
// SessionStart shape: what the injected block LOOKS like once the budget is
|
|
61
82
|
// spent. All three spend the same character budget; they differ in what they do
|
|
62
83
|
// with the lessons that did not fit.
|
|
@@ -93,6 +114,18 @@ export const DEFAULT_SESSION_START_MAX_CHARS = 1500;
|
|
|
93
114
|
export const MIN_SESSION_START_MAX_CHARS = 200;
|
|
94
115
|
export const MAX_SESSION_START_MAX_CHARS = 20000;
|
|
95
116
|
|
|
117
|
+
// The events `hooks.instructions` can carry text for — every lifecycle event
|
|
118
|
+
// that emits something a project instruction could ride along with. Exported so
|
|
119
|
+
// `doctor` reports the same set the resolver reads: they were separate literals
|
|
120
|
+
// and drifted, which is how `UserPromptSubmit` ended up documented, accepted in
|
|
121
|
+
// config, and silently dropped by both.
|
|
122
|
+
export const HOOK_INSTRUCTION_EVENTS = [
|
|
123
|
+
'SessionStart',
|
|
124
|
+
'UserPromptSubmit',
|
|
125
|
+
'PostToolUseFailure',
|
|
126
|
+
'Stop',
|
|
127
|
+
];
|
|
128
|
+
|
|
96
129
|
// Clamp a configured budget into the supported range, or null when the value is
|
|
97
130
|
// not a usable number at all (absent, a bare string, NaN). Total: the caller
|
|
98
131
|
// substitutes the default for null. Out-of-range CLAMPS rather than rejecting —
|
|
@@ -256,6 +289,21 @@ export function resolveControl({
|
|
|
256
289
|
normalizeStopMode(userConfig['hooks.stop']) ||
|
|
257
290
|
'friction';
|
|
258
291
|
|
|
292
|
+
// `hooks.userPrompt` — repo layer wins over user layer, default `on`.
|
|
293
|
+
//
|
|
294
|
+
// Default-on is safe because for an `install` the WIRING is the real switch:
|
|
295
|
+
// the event is installed by hook mode `all` alone, so a user who chose
|
|
296
|
+
// `read-only` or `none` never reaches this resolution at all, and someone who
|
|
297
|
+
// chose `all` opted into per-turn relevance — making them opt in twice would
|
|
298
|
+
// leave it dark for everyone who never read the config reference. A
|
|
299
|
+
// marketplace-plugin install has no mode and wires the event unconditionally,
|
|
300
|
+
// so there this key is the whole opt-out; that is why it stays a real config
|
|
301
|
+
// key rather than being folded into the hook mode.
|
|
302
|
+
const hooksUserPrompt =
|
|
303
|
+
normalizeUserPromptMode(repoConfig['hooks.userPrompt']) ||
|
|
304
|
+
normalizeUserPromptMode(userConfig['hooks.userPrompt']) ||
|
|
305
|
+
'on';
|
|
306
|
+
|
|
259
307
|
// `hooks.sessionStart` — repo layer wins over user layer, default `hybrid`.
|
|
260
308
|
// Chooses the SHAPE of the injected block (see SESSION_START_MODES). An
|
|
261
309
|
// unrecognised value falls through to the next layer and finally to the
|
|
@@ -291,7 +339,6 @@ export function resolveControl({
|
|
|
291
339
|
// Both layers contribute: repo instructions come first, user instructions follow
|
|
292
340
|
// (same direction as `tags.default` — repo supplements, user personalises).
|
|
293
341
|
// null for a given event means "no custom instruction for that event".
|
|
294
|
-
const HOOK_EVENTS = ['SessionStart', 'PostToolUseFailure', 'Stop'];
|
|
295
342
|
const hooksInstructions = {};
|
|
296
343
|
{
|
|
297
344
|
const repoInstr =
|
|
@@ -300,7 +347,7 @@ export function resolveControl({
|
|
|
300
347
|
const userInstr =
|
|
301
348
|
(userConfig['hooks.instructions'] && typeof userConfig['hooks.instructions'] === 'object')
|
|
302
349
|
? userConfig['hooks.instructions'] : {};
|
|
303
|
-
for (const ev of
|
|
350
|
+
for (const ev of HOOK_INSTRUCTION_EVENTS) {
|
|
304
351
|
const parts = [repoInstr[ev], userInstr[ev]]
|
|
305
352
|
.filter((v) => typeof v === 'string' && v.trim().length > 0);
|
|
306
353
|
hooksInstructions[ev] = parts.length > 0 ? parts.join('\n') : null;
|
|
@@ -318,6 +365,7 @@ export function resolveControl({
|
|
|
318
365
|
scopeDefaults,
|
|
319
366
|
hooksDisabled,
|
|
320
367
|
hooksStop,
|
|
368
|
+
hooksUserPrompt,
|
|
321
369
|
hooksSessionStart,
|
|
322
370
|
hooksSessionStartMaxChars,
|
|
323
371
|
hooksAdapter,
|
package/src/core/lessons.mjs
CHANGED
|
@@ -12,7 +12,7 @@ import { deriveScope } from '../scope.mjs';
|
|
|
12
12
|
// the injected set is chosen by ONE scorer, and a future `memory.relevant` verb
|
|
13
13
|
// must be able to reuse it rather than grow a second ranking with its own idea
|
|
14
14
|
// of what "most useful" means.
|
|
15
|
-
import { resolvePrecedence, rankLessons } from '../lessons-pure.mjs';
|
|
15
|
+
import { resolvePrecedence, rankLessons, diversifyRankedLessons } from '../lessons-pure.mjs';
|
|
16
16
|
// The store's own scope inventory, normalised — the SAME helper `memory.scopes`
|
|
17
17
|
// uses, so the map and the MCP tool cannot disagree about what a scope holds or
|
|
18
18
|
// about what a failed enumeration looks like.
|
|
@@ -167,7 +167,16 @@ export async function fetchLessons(store, cwd, { now = Date.now() } = {}) {
|
|
|
167
167
|
// first-appearance default — they agree today, but the hierarchy is
|
|
168
168
|
// `readOrder`'s to state, not an artefact of how this function happens to
|
|
169
169
|
// build its array.
|
|
170
|
-
|
|
170
|
+
// ONE options object feeds both the ranking and the diversification below, so
|
|
171
|
+
// the two can never drift: `diversifyRankedLessons` recomputes each entry's
|
|
172
|
+
// score to seed the MMR objective, and if its `terms`/`now`/`weights` differed
|
|
173
|
+
// from what `rankLessons` sorted on, those scores would not line up with the
|
|
174
|
+
// order — the near-identical `now` clock especially. Sharing the object makes
|
|
175
|
+
// that agreement structural rather than a thing two call sites have to keep in
|
|
176
|
+
// step by hand. `k` is diversification-only; `scopeOrder` is ranking-only and
|
|
177
|
+
// simply ignored by the diversifier's destructuring.
|
|
178
|
+
const rankOpts = { terms: [], now, scopeOrder: scope.readOrder };
|
|
179
|
+
const ranked = rankLessons(winners, rankOpts);
|
|
171
180
|
|
|
172
181
|
// ── the scope map: EXACT counts when the store can enumerate ───────────────
|
|
173
182
|
//
|
|
@@ -219,12 +228,29 @@ export async function fetchLessons(store, cwd, { now = Date.now() } = {}) {
|
|
|
219
228
|
? scopeInventoryFromStore(inventory.scopes, scope.readOrder, derivedCounts)
|
|
220
229
|
: derivedCounts;
|
|
221
230
|
|
|
231
|
+
// DIVERSIFY before the ceiling, so the budget is not spent on near-identical
|
|
232
|
+
// lessons. Ranking answers "which lessons score highest"; on an active repo
|
|
233
|
+
// the highest cluster is often one task's iteration log — a dozen
|
|
234
|
+
// `review-outcomes::pr395-it{3,4,5}` rows that score alike AND read alike, so
|
|
235
|
+
// a plain top-N hands the reader the same lesson several times and evicts the
|
|
236
|
+
// variety underneath. `diversifyRankedLessons` applies the SAME MMR
|
|
237
|
+
// (`selectDiverse`, λ=0.7 lexical Jaccard) the hosted `order=rank` path
|
|
238
|
+
// already uses, which was defined and exported here but never wired into the
|
|
239
|
+
// session-start read. It seeds with the top-ranked lesson (score is still
|
|
240
|
+
// 0.7 of the objective) and only spends the remaining 0.3 pushing down a
|
|
241
|
+
// lesson that repeats one already shown — so the best lesson stays first and
|
|
242
|
+
// the set stops being a wall of duplicates. `terms: []` matches the
|
|
243
|
+
// `rankLessons` call above (relevance contributes nothing at session start),
|
|
244
|
+
// which the scores MUST agree with. The scope map and `applicable` still read
|
|
245
|
+
// from `ranked` — the map is a pointer to what EXISTS per scope, a question
|
|
246
|
+
// diversification does not change.
|
|
247
|
+
//
|
|
222
248
|
// `applicable` is the honest denominator for the header — how many the reader
|
|
223
249
|
// has, as opposed to how many fitted. It is counted BEFORE the ceiling, so
|
|
224
250
|
// "8 of 50" stays true no matter how the render is bounded.
|
|
225
251
|
return {
|
|
226
252
|
scope,
|
|
227
|
-
lessons: ranked
|
|
253
|
+
lessons: diversifyRankedLessons(ranked, { ...rankOpts, k: HARD_LESSON_CEILING }),
|
|
228
254
|
scopeCounts,
|
|
229
255
|
applicable: ranked.length,
|
|
230
256
|
};
|
|
@@ -327,15 +353,29 @@ function lessonHook(value, max = HOOK_LEN) {
|
|
|
327
353
|
// `hooks.instructions.SessionStart` in the control config. Lets teams inject
|
|
328
354
|
// project-specific guidance (e.g. "focus on migration safety") without touching
|
|
329
355
|
// the hook internals. Visible even when there are no lessons.
|
|
356
|
+
// `onShown` — an optional callback receiving the lessons this call actually
|
|
357
|
+
// RENDERED, which is a subset of `lessons` whenever the budget or the ceiling
|
|
358
|
+
// binds. The selection happens in here and nowhere else, so a caller that needs
|
|
359
|
+
// to know what the reader saw (the shown-set bookkeeping) has to be told rather
|
|
360
|
+
// than re-deriving it — a second copy of the fit maths would drift the moment
|
|
361
|
+
// either bound changes.
|
|
330
362
|
export function formatLessons(lessons, scope, {
|
|
331
363
|
instruction = null,
|
|
332
364
|
mode = 'hybrid',
|
|
333
365
|
maxChars = DEFAULT_SESSION_START_MAX_CHARS,
|
|
334
366
|
scopeCounts = null,
|
|
335
367
|
applicable = null,
|
|
368
|
+
onShown = null,
|
|
336
369
|
} = {}) {
|
|
370
|
+
// Never let bookkeeping break the render: this function's contract is to
|
|
371
|
+
// return a block, and a throwing callback must not cost the reader theirs.
|
|
372
|
+
const report = (rendered) => {
|
|
373
|
+
if (typeof onShown !== 'function') return;
|
|
374
|
+
try { onShown(rendered); } catch { /* best-effort */ }
|
|
375
|
+
};
|
|
337
376
|
const all = Array.isArray(lessons) ? lessons : [];
|
|
338
377
|
if (all.length === 0) {
|
|
378
|
+
report([]);
|
|
339
379
|
// No lessons — only emit if there is a custom instruction to show.
|
|
340
380
|
if (!instruction) return null;
|
|
341
381
|
return (
|
|
@@ -360,6 +400,7 @@ export function formatLessons(lessons, scope, {
|
|
|
360
400
|
|
|
361
401
|
const ceiling = shape === 'map' ? Math.min(MAP_TOP_K, HARD_LESSON_CEILING) : HARD_LESSON_CEILING;
|
|
362
402
|
const { shown } = fitLines(all, budget - reserve, ceiling);
|
|
403
|
+
report(shown.map((s) => s.lesson));
|
|
363
404
|
|
|
364
405
|
// `map` always shows the inventory; `hybrid` shows it only when something was
|
|
365
406
|
// actually left out — otherwise the reader is looking at the complete set and
|
|
@@ -434,10 +475,29 @@ export function renderScopeMap(scopeCounts) {
|
|
|
434
475
|
// stays meaningful, and the count is capped (`MAX_TERMS`) so a huge error blob
|
|
435
476
|
// can't blow up the downstream scan.
|
|
436
477
|
export function failureQuery(toolName, toolResponse) {
|
|
437
|
-
|
|
478
|
+
return distilTerms(`${toolName ? String(toolName) : ''} ${errorText(toolResponse)}`);
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
// The tokenizer both query builders share. Lowercased `[a-z0-9]+` runs of at
|
|
482
|
+
// least `MIN_TERM_LEN` characters, stopword-filtered, de-duplicated, capped at
|
|
483
|
+
// `MAX_TERMS`, over at most `MAX_SCAN_CHARS` of input. The floor is INCLUSIVE:
|
|
484
|
+
// the filter is `raw.length < MIN_TERM_LEN`, so a term exactly `MIN_TERM_LEN`
|
|
485
|
+
// characters long is kept.
|
|
486
|
+
//
|
|
487
|
+
// The bound is applied to the TEXT before splitting, not to the token array
|
|
488
|
+
// after: a multi-megabyte stderr blob (or a pasted file) would otherwise
|
|
489
|
+
// materialise a giant token array on the way to being capped — a CPU and memory
|
|
490
|
+
// spike for a result that was always going to be twelve words.
|
|
491
|
+
//
|
|
492
|
+
// Producing `[a-z0-9]+` runs is also what keeps the terms safe to hand to the
|
|
493
|
+
// remote store, whose `search` joins them into ONE `websearch` FTS query: no
|
|
494
|
+
// FTS metacharacter can survive this filter, so no caller has to escape one.
|
|
495
|
+
// Pure and total.
|
|
496
|
+
export function distilTerms(text) {
|
|
497
|
+
const scanned = String(text ?? '').slice(0, MAX_SCAN_CHARS).toLowerCase();
|
|
438
498
|
const seen = new Set();
|
|
439
499
|
const terms = [];
|
|
440
|
-
for (const raw of
|
|
500
|
+
for (const raw of scanned.split(/[^a-z0-9]+/)) {
|
|
441
501
|
if (raw.length < MIN_TERM_LEN || STOPWORDS.has(raw) || seen.has(raw)) continue;
|
|
442
502
|
seen.add(raw);
|
|
443
503
|
terms.push(raw);
|
|
@@ -484,12 +544,19 @@ export function dedupeRelevant(entries, cap = MAX_RELEVANT) {
|
|
|
484
544
|
// capped by the pure `dedupeRelevant`, keeping the store's own ordering (see its
|
|
485
545
|
// docblock). Best-effort: an unusable/throwing store returns [] so the
|
|
486
546
|
// caller falls back to the write-nudge alone.
|
|
487
|
-
export async function relevantLessonsFromStore(store, scope, terms, { cap = MAX_RELEVANT } = {}) {
|
|
547
|
+
export async function relevantLessonsFromStore(store, scope, terms, { cap = MAX_RELEVANT, timeoutMs, walkLimit } = {}) {
|
|
488
548
|
if (!store || typeof store.search !== 'function') return [];
|
|
489
549
|
if (!scope || !Array.isArray(scope.readOrder) || scope.readOrder.length === 0) return [];
|
|
490
550
|
if (!Array.isArray(terms) || terms.length === 0) return [];
|
|
491
551
|
try {
|
|
492
|
-
|
|
552
|
+
// `timeoutMs` bounds the REMOTE route (a network fetch); `walkLimit` bounds
|
|
553
|
+
// the OFFLINE one (a synchronous file walk `timeoutMs` cannot interrupt).
|
|
554
|
+
// They are deliberately separate names: `RemoteStore.search` reads `limit`
|
|
555
|
+
// (→ `body.limit`), so a shared name would truncate the remote hit set
|
|
556
|
+
// BEFORE `rankLessons` runs — exactly what a hot-path caller must avoid.
|
|
557
|
+
// `walkLimit` is honoured only by the local stores; the remote ignores it
|
|
558
|
+
// and stays bounded by `timeoutMs` alone, as it was before this budget.
|
|
559
|
+
const res = await store.search({ q: terms, scopes: scope.readOrder, timeoutMs, walkLimit });
|
|
493
560
|
if (!res || !res.ok || !Array.isArray(res.entries)) return [];
|
|
494
561
|
return dedupeRelevant(res.entries, cap);
|
|
495
562
|
} catch {
|
|
@@ -673,3 +740,165 @@ const STOPWORDS = new Set([
|
|
|
673
740
|
'response',
|
|
674
741
|
'status',
|
|
675
742
|
]);
|
|
743
|
+
|
|
744
|
+
// ── the per-prompt relevance pull (UserPromptSubmit) ─────────────────────────
|
|
745
|
+
//
|
|
746
|
+
// SessionStart injects once, at the top of a session, before the user has said
|
|
747
|
+
// what they are doing. That set is necessarily a guess: it is ranked on recency
|
|
748
|
+
// and recurrence because there is nothing else to rank on yet. The moment the
|
|
749
|
+
// user types "the migration keeps deadlocking", there IS something to rank on —
|
|
750
|
+
// and until this hook, nothing used it. The only mid-session trigger was a tool
|
|
751
|
+
// FAILURE, which means the loop could only ever tell you about a mistake after
|
|
752
|
+
// you had already made it.
|
|
753
|
+
//
|
|
754
|
+
// The whole design problem is that this fires on EVERY turn, so the cost of
|
|
755
|
+
// being wrong is paid over and over. Three gates keep it quiet, and each one
|
|
756
|
+
// exists because the failure mode without it is worse than showing nothing:
|
|
757
|
+
//
|
|
758
|
+
// LENGTH — "yes", "continue", "go on" carry no terms worth querying, and a
|
|
759
|
+
// store lookup per keystroke-sized prompt is pure overhead.
|
|
760
|
+
// RELEVANCE — no match means silence. An "in case it helps" lesson attached
|
|
761
|
+
// to an unrelated prompt trains the reader to skim past the
|
|
762
|
+
// block, which costs the SessionStart injection its credibility
|
|
763
|
+
// too.
|
|
764
|
+
// DELTA — a lesson already shown this session is not news. Re-injecting
|
|
765
|
+
// it is the specific way a per-turn hook becomes wallpaper.
|
|
766
|
+
|
|
767
|
+
// Shortest prompt worth a store lookup. Tuned to skip the acknowledgements that
|
|
768
|
+
// dominate a real session ("yes", "ok", "continue", "do it", "next") while
|
|
769
|
+
// keeping anything that states an intent. Deliberately generous: the relevance
|
|
770
|
+
// gate is the real filter, and this one only exists to avoid paying for a query
|
|
771
|
+
// whose terms would be discarded anyway.
|
|
772
|
+
const MIN_PROMPT_CHARS = 24;
|
|
773
|
+
|
|
774
|
+
// Cap on lessons injected per prompt. Smaller than the SessionStart budget by an
|
|
775
|
+
// order of magnitude, because this competes with the user's own turn: three
|
|
776
|
+
// index lines is a glance, and anything more is an interruption.
|
|
777
|
+
const MAX_PROMPT_LESSONS = 3;
|
|
778
|
+
|
|
779
|
+
// Fetch budget for the per-prompt pull, deliberately far below `restFetch`'s
|
|
780
|
+
// 10s default. This is the ONE lookup that sits on the user's critical path —
|
|
781
|
+
// it runs before their turn is handed to the assistant — so a slow or wedged
|
|
782
|
+
// store must cost them a fraction of a second, not ten. Timing out is not a
|
|
783
|
+
// failure mode here: the abort surfaces as no hits, and no hits is already this
|
|
784
|
+
// hook's most common and entirely valid answer. Same reasoning as
|
|
785
|
+
// `telemetry.mjs`'s 1500 ms export budget; a touch more generous because a
|
|
786
|
+
// missed lesson is worth slightly more than a missed metric.
|
|
787
|
+
export const PROMPT_FETCH_TIMEOUT_MS = 2000;
|
|
788
|
+
|
|
789
|
+
// The offline-store counterpart to the fetch budget above. `timeoutMs` bounds
|
|
790
|
+
// the remote route, but the local store walks every scope's files synchronously
|
|
791
|
+
// on every prompt — an unbounded walk a wall-clock budget cannot interrupt. The
|
|
792
|
+
// per-prompt pull forwards this as `walkLimit` (NOT `limit`, which the remote
|
|
793
|
+
// store maps to `body.limit` and would truncate its hit set pre-ranking), so it
|
|
794
|
+
// bounds only the offline walk: the block ranks then keeps MAX_PROMPT_LESSONS,
|
|
795
|
+
// so a few hundred nearest-scope hits is far more than the ranker needs to
|
|
796
|
+
// surface the best three. Only the hot-path caller passes it; the failure hook
|
|
797
|
+
// stays unbounded, as it was.
|
|
798
|
+
export const PROMPT_LOCAL_SEARCH_LIMIT = 200;
|
|
799
|
+
|
|
800
|
+
/**
|
|
801
|
+
* Is this prompt worth a relevance lookup?
|
|
802
|
+
*
|
|
803
|
+
* Length is measured AFTER trimming, on the raw prompt. A long prompt made
|
|
804
|
+
* entirely of stopwords still passes here and is caught by the term gate below
|
|
805
|
+
* — two cheap checks in series rather than one clever one.
|
|
806
|
+
*/
|
|
807
|
+
export function isSubstantivePrompt(prompt, min = MIN_PROMPT_CHARS) {
|
|
808
|
+
return String(prompt ?? '').trim().length >= min;
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
/**
|
|
812
|
+
* Distil search terms from a user prompt.
|
|
813
|
+
*
|
|
814
|
+
* The same tokenizer the failure lookup uses, for the same reason: whatever
|
|
815
|
+
* reaches the store must be `[a-z0-9]+` runs, and the two callers must agree on
|
|
816
|
+
* what counts as a term or "why did the failure hook find this and my prompt
|
|
817
|
+
* not?" becomes unanswerable.
|
|
818
|
+
*
|
|
819
|
+
* Returns `[]` for a prompt that is too short or carries nothing but stopwords,
|
|
820
|
+
* which the caller reads as "stay silent".
|
|
821
|
+
*/
|
|
822
|
+
export function promptQuery(prompt) {
|
|
823
|
+
if (!isSubstantivePrompt(prompt)) return [];
|
|
824
|
+
return distilTerms(prompt);
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
/**
|
|
828
|
+
* Render the per-turn block, or null when there is nothing to say.
|
|
829
|
+
*
|
|
830
|
+
* INDEX ONLY, and shorter than the failure block's lines. This arrives while
|
|
831
|
+
* the user is mid-thought, so it has to be scannable in a glance and cost as
|
|
832
|
+
* little context as possible; the body is always one `memory.read` away. The
|
|
833
|
+
* framing is "you have notes on this", never an instruction — the same
|
|
834
|
+
* considerations-not-rules posture as every other injection.
|
|
835
|
+
*/
|
|
836
|
+
export function formatPromptLessons(lessons, { instruction = null } = {}) {
|
|
837
|
+
if (!lessons || lessons.length === 0) return null;
|
|
838
|
+
const noun = lessons.length === 1 ? 'memory' : 'memories';
|
|
839
|
+
const header =
|
|
840
|
+
`LoreKit: ${lessons.length} ${noun} related to this — `
|
|
841
|
+
+ `considerations, not rules; read in full with memory.read:`;
|
|
842
|
+
const body = lessons.map((l) => `- (${l.scope}) ${l.key} — ${lessonHook(l.value)}`).join('\n');
|
|
843
|
+
// `hooks.instructions.UserPromptSubmit`, appended the same way the other
|
|
844
|
+
// events append theirs. It rides an EXISTING block and never creates one:
|
|
845
|
+
// this hook fires on every turn, so an instruction that could emit on its own
|
|
846
|
+
// would be a line on every prompt — the noise the relevance gate exists to
|
|
847
|
+
// prevent.
|
|
848
|
+
const extra = typeof instruction === 'string' && instruction.trim()
|
|
849
|
+
? `\n\nProject instruction: ${instruction}` : '';
|
|
850
|
+
return `${header}\n${body}${extra}`;
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
/**
|
|
854
|
+
* The per-prompt pull: query the store for this prompt's terms, rank, drop
|
|
855
|
+
* anything already shown, cap.
|
|
856
|
+
*
|
|
857
|
+
* QUERYING rather than filtering the injected set is the same call the failure
|
|
858
|
+
* hook makes, and for the same reason: a post-filter can only ever resurface a
|
|
859
|
+
* lesson that was already on screen, so a paraphrased match or one that lost
|
|
860
|
+
* the SessionStart ranking would be permanently unreachable — which is exactly
|
|
861
|
+
* the lore this hook exists to surface.
|
|
862
|
+
*
|
|
863
|
+
* RANKING runs after the store returns, because the store's own ordering is not
|
|
864
|
+
* relevance: the remote route answers `updated_at desc` and the local two-tier
|
|
865
|
+
* store answers project-tier-first. `rankLessons` with the prompt's terms
|
|
866
|
+
* applies the same scorer the SessionStart block uses, so a recurring lesson
|
|
867
|
+
* beats a fresher one-off here too.
|
|
868
|
+
*
|
|
869
|
+
* `alreadyShown` is a Set of `scope::key`. Filtering AFTER ranking rather than
|
|
870
|
+
* before is deliberate: it keeps the cap meaningful. Filtering first would let
|
|
871
|
+
* three weak lessons take the slots a strong-but-already-shown one vacated,
|
|
872
|
+
* which is worse than showing two.
|
|
873
|
+
*
|
|
874
|
+
* Best-effort and total — any failure yields `[]`, and the hook stays silent.
|
|
875
|
+
* That includes the fetch budget: the lookup runs under
|
|
876
|
+
* `PROMPT_FETCH_TIMEOUT_MS` rather than `restFetch`'s 10s default, and an abort
|
|
877
|
+
* arrives here as no hits, the same as a store that simply had nothing.
|
|
878
|
+
*/
|
|
879
|
+
export async function promptLessonsFromStore(store, scope, terms, {
|
|
880
|
+
alreadyShown = new Set(),
|
|
881
|
+
cap = MAX_PROMPT_LESSONS,
|
|
882
|
+
now = Date.now(),
|
|
883
|
+
timeoutMs = PROMPT_FETCH_TIMEOUT_MS,
|
|
884
|
+
} = {}) {
|
|
885
|
+
if (!Array.isArray(terms) || terms.length === 0) return [];
|
|
886
|
+
const hits = await relevantLessonsFromStore(store, scope, terms, {
|
|
887
|
+
cap: Number.MAX_SAFE_INTEGER,
|
|
888
|
+
timeoutMs,
|
|
889
|
+
walkLimit: PROMPT_LOCAL_SEARCH_LIMIT,
|
|
890
|
+
});
|
|
891
|
+
if (hits.length === 0) return [];
|
|
892
|
+
const ranked = rankLessons(hits, {
|
|
893
|
+
terms,
|
|
894
|
+
now,
|
|
895
|
+
scopeOrder: Array.isArray(scope?.readOrder) ? scope.readOrder : null,
|
|
896
|
+
});
|
|
897
|
+
const fresh = ranked.filter((e) => !alreadyShown.has(lessonId(e)));
|
|
898
|
+
return dedupeRelevant(fresh, cap);
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
/** The identity a shown-set is keyed on. One spelling, used by both sides. */
|
|
902
|
+
export function lessonId(entry) {
|
|
903
|
+
return `${entry?.scope ?? ''}::${entry?.key ?? ''}`;
|
|
904
|
+
}
|
package/src/core/state.mjs
CHANGED
|
@@ -41,3 +41,115 @@ export function sessionMarkerExists(sessionId, tag) {
|
|
|
41
41
|
return false;
|
|
42
42
|
}
|
|
43
43
|
}
|
|
44
|
+
|
|
45
|
+
// ── the per-session shown-set ────────────────────────────────────────────────
|
|
46
|
+
//
|
|
47
|
+
// The one-shot markers above answer "has this hook fired yet". The per-prompt
|
|
48
|
+
// relevance hook needs a finer question: "has the reader already been shown
|
|
49
|
+
// THIS lesson this session". Without it the hook re-injects its own best match
|
|
50
|
+
// on every turn of a conversation that stays on one topic — which is the exact
|
|
51
|
+
// shape that turns an injection into wallpaper the reader learns to skip.
|
|
52
|
+
//
|
|
53
|
+
// Deliberately the same directory and the same session key as the markers, so a
|
|
54
|
+
// session's hook state is one thing that appears and disappears together.
|
|
55
|
+
//
|
|
56
|
+
// STORED AS LINES, APPENDED on the hot path. An append is atomic enough for
|
|
57
|
+
// this: two hooks racing can interleave whole lines but cannot corrupt one, and
|
|
58
|
+
// a duplicate line is harmless because the reader is a Set. A read-modify-write
|
|
59
|
+
// on every write would be the version that loses entries under a race — which
|
|
60
|
+
// is why the common write stays a bare append. Left there the file would grow
|
|
61
|
+
// without bound in a long session, so it is instead bounded by an occasional
|
|
62
|
+
// atomic-rename compaction (see COMPACT_AT_BYTES below), never a per-write
|
|
63
|
+
// rewrite.
|
|
64
|
+
|
|
65
|
+
// Cap on how many ids are READ back per session — the newest N. Reading only
|
|
66
|
+
// the newest N is enough for what this guards: the value of an id decays, and
|
|
67
|
+
// re-showing a lesson from 600 injections ago is not the repetition the
|
|
68
|
+
// shown-set exists to prevent.
|
|
69
|
+
const MAX_SHOWN_IDS = 500;
|
|
70
|
+
|
|
71
|
+
// Bound the file ON DISK, not just the read. The append-only writer keeps the
|
|
72
|
+
// hot path race-safe, but on its own a long session would grow the file without
|
|
73
|
+
// limit. Once the file passes this size the writer rewrites it down to the
|
|
74
|
+
// newest MAX_SHOWN_IDS ids via a temp-file + atomic rename (see
|
|
75
|
+
// `compactIfLarge`). The threshold sits well above what `shownLessons` ever
|
|
76
|
+
// reads, so compaction is rare — amortized O(1) per append — and never drops an
|
|
77
|
+
// id still inside the read window. It is session-scoped scratch in a
|
|
78
|
+
// temp/plugin-data directory the host clears with the rest of the session's
|
|
79
|
+
// hook state; compaction just keeps it bounded within one long-lived session.
|
|
80
|
+
// Exported so the test asserts the bound against THIS value rather than a
|
|
81
|
+
// restated literal that would silently stop proving it if the threshold moved.
|
|
82
|
+
export const COMPACT_AT_BYTES = 256 * 1024;
|
|
83
|
+
|
|
84
|
+
function shownPath(sessionId) {
|
|
85
|
+
return `${markerPath(sessionId, 'shown')}.ids`;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// Rewrite the append-only file down to its newest MAX_SHOWN_IDS ids once it has
|
|
89
|
+
// grown past COMPACT_AT_BYTES, via a temp-file + atomic rename. rename(2) is
|
|
90
|
+
// atomic on POSIX and the temp file lives in the same directory (so the same
|
|
91
|
+
// filesystem), so a concurrent reader always sees a complete old-or-new file,
|
|
92
|
+
// never a torn one. A racing append lost to the rename is the same "at worst a
|
|
93
|
+
// repeat" trade the bare append already makes, and it can only happen on the
|
|
94
|
+
// rare compaction — not the hot path. Best-effort: any failure leaves the file
|
|
95
|
+
// as-is and never breaks the host.
|
|
96
|
+
function compactIfLarge(file) {
|
|
97
|
+
try {
|
|
98
|
+
if (fs.statSync(file).size <= COMPACT_AT_BYTES) return;
|
|
99
|
+
const kept = fs
|
|
100
|
+
.readFileSync(file, 'utf8')
|
|
101
|
+
.split('\n')
|
|
102
|
+
.filter(Boolean)
|
|
103
|
+
.slice(-MAX_SHOWN_IDS);
|
|
104
|
+
const tmp = `${file}.${process.pid}.tmp`;
|
|
105
|
+
fs.writeFileSync(tmp, kept.length ? `${kept.join('\n')}\n` : '');
|
|
106
|
+
fs.renameSync(tmp, file);
|
|
107
|
+
} catch {
|
|
108
|
+
// Compaction is best-effort; a failure just leaves the file large.
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* The set of `scope::key` ids already injected this session.
|
|
114
|
+
*
|
|
115
|
+
* Missing session id, missing file, or an unreadable one all yield an EMPTY
|
|
116
|
+
* set, which fails toward showing a lesson again rather than toward silence. A
|
|
117
|
+
* repeated lesson is a small annoyance; a lesson silently withheld because a
|
|
118
|
+
* state file could not be read is the failure nobody would ever diagnose.
|
|
119
|
+
*/
|
|
120
|
+
export function shownLessons(sessionId) {
|
|
121
|
+
if (!sessionId) return new Set();
|
|
122
|
+
try {
|
|
123
|
+
const lines = fs.readFileSync(shownPath(sessionId), 'utf8').split('\n');
|
|
124
|
+
const kept = lines.filter(Boolean).slice(-MAX_SHOWN_IDS);
|
|
125
|
+
return new Set(kept);
|
|
126
|
+
} catch {
|
|
127
|
+
return new Set();
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Record ids as shown. Best-effort: a failed write means at worst a repeat.
|
|
133
|
+
*
|
|
134
|
+
* Called by BOTH injection paths — SessionStart records what it injected, and
|
|
135
|
+
* the per-prompt hook records what it added — because "already shown" has to
|
|
136
|
+
* mean shown by anything, not shown by this hook. A per-prompt hook that only
|
|
137
|
+
* remembered its own output would re-surface the SessionStart set one lesson at
|
|
138
|
+
* a time.
|
|
139
|
+
*/
|
|
140
|
+
export function recordShownLessons(sessionId, ids) {
|
|
141
|
+
if (!sessionId || !Array.isArray(ids) || ids.length === 0) return;
|
|
142
|
+
const clean = ids.filter((id) => typeof id === 'string' && id && !id.includes('\n'));
|
|
143
|
+
if (clean.length === 0) return;
|
|
144
|
+
try {
|
|
145
|
+
// No mkdir here: `shownPath` → `markerPath` → `stateDir`, which creates the
|
|
146
|
+
// directory as part of resolving the path.
|
|
147
|
+
const file = shownPath(sessionId);
|
|
148
|
+
fs.appendFileSync(file, `${clean.join('\n')}\n`);
|
|
149
|
+
// The append above is the race-safe hot path; this bounds the file on disk,
|
|
150
|
+
// rewriting down to the newest ids only once it has grown large.
|
|
151
|
+
compactIfLarge(file);
|
|
152
|
+
} catch {
|
|
153
|
+
// Never break the host over bookkeeping.
|
|
154
|
+
}
|
|
155
|
+
}
|
package/src/doctor.mjs
CHANGED
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
CLAUDE_HOOK_EVENTS,
|
|
12
12
|
installedHookEvents,
|
|
13
13
|
hookModeFromEvents,
|
|
14
|
+
missingHookEvents,
|
|
14
15
|
readLorekitServer,
|
|
15
16
|
readMcpConfig,
|
|
16
17
|
tokenKind,
|
|
@@ -23,7 +24,7 @@ import {
|
|
|
23
24
|
probeTelemetryExport,
|
|
24
25
|
} from './telemetry.mjs';
|
|
25
26
|
import { deriveScope } from './scope.mjs';
|
|
26
|
-
import { loadControl } from './control.mjs';
|
|
27
|
+
import { loadControl, HOOK_INSTRUCTION_EVENTS } from './control.mjs';
|
|
27
28
|
import { createStore } from './store/index.mjs';
|
|
28
29
|
import { log, heading, status, c } from './util.mjs';
|
|
29
30
|
|
|
@@ -101,7 +102,22 @@ export async function doctor(args) {
|
|
|
101
102
|
);
|
|
102
103
|
} else {
|
|
103
104
|
for (const { scope, events } of perScope) {
|
|
104
|
-
|
|
105
|
+
// An install predating a lifecycle event still READS as its mode, so
|
|
106
|
+
// reporting the mode alone tells a legacy wiring it is current — the
|
|
107
|
+
// one state where this line is actively misleading. Stay a `pass` (the
|
|
108
|
+
// wiring works, it is just not the full set any more) and name the
|
|
109
|
+
// upgrade the same way `install`'s already-installed summary does, via
|
|
110
|
+
// the same `missingHookEvents` derivation.
|
|
111
|
+
const mode = hookModeFromEvents(events);
|
|
112
|
+
const missing = missingHookEvents(events);
|
|
113
|
+
// Name the SCOPE in the command. `install` prompts for project vs
|
|
114
|
+
// global when neither flag is given, so a bare command offered against
|
|
115
|
+
// a `hooks global` gap can just as easily rewire the project and leave
|
|
116
|
+
// the gap exactly where it was.
|
|
117
|
+
const upgrade = missing.length > 0
|
|
118
|
+
? ` — missing ${missing.join(', ')}; run \`lorekit install --${scope} --hooks ${mode}\` to wire ${missing.length === 1 ? 'it' : 'them'}`
|
|
119
|
+
: '';
|
|
120
|
+
record('pass', `hooks ${scope}`, `${mode} — ${events.join(', ')}${upgrade}`);
|
|
105
121
|
}
|
|
106
122
|
}
|
|
107
123
|
}
|
|
@@ -157,10 +173,9 @@ export async function doctor(args) {
|
|
|
157
173
|
// 6. Hook instructions — show resolved per-event custom instructions when any are set.
|
|
158
174
|
{
|
|
159
175
|
const instr = control.hooksInstructions || {};
|
|
160
|
-
const
|
|
161
|
-
const configured = EVENTS.filter((ev) => instr[ev]);
|
|
176
|
+
const configured = HOOK_INSTRUCTION_EVENTS.filter((ev) => instr[ev]);
|
|
162
177
|
if (configured.length > 0) {
|
|
163
|
-
for (const ev of
|
|
178
|
+
for (const ev of HOOK_INSTRUCTION_EVENTS) {
|
|
164
179
|
const text = instr[ev];
|
|
165
180
|
if (text) {
|
|
166
181
|
record('info', `hooks.instructions.${ev}`, c.dim(text.length > 80 ? text.slice(0, 77) + '…' : text));
|
package/src/hook.mjs
CHANGED
|
@@ -13,13 +13,22 @@ import {
|
|
|
13
13
|
retrospectiveNudge,
|
|
14
14
|
failureNudge,
|
|
15
15
|
failureQuery,
|
|
16
|
+
promptQuery,
|
|
17
|
+
promptLessonsFromStore,
|
|
18
|
+
formatPromptLessons,
|
|
19
|
+
lessonId,
|
|
16
20
|
relevantLessonsFromStore,
|
|
17
21
|
formatRelevantLessons,
|
|
18
22
|
writeConfirmation,
|
|
19
23
|
} from './core/lessons.mjs';
|
|
20
24
|
import { isFailure } from './core/failure.mjs';
|
|
21
25
|
import { readSessionFriction, shouldRetrospect, FRICTION_FAILURE } from './core/friction.mjs';
|
|
22
|
-
import {
|
|
26
|
+
import {
|
|
27
|
+
firstTimeThisSession,
|
|
28
|
+
sessionMarkerExists,
|
|
29
|
+
shownLessons,
|
|
30
|
+
recordShownLessons,
|
|
31
|
+
} from './core/state.mjs';
|
|
23
32
|
import { recordFixture } from './core/record.mjs';
|
|
24
33
|
import { claude } from './adapters/claude.mjs';
|
|
25
34
|
import { cursor } from './adapters/cursor.mjs';
|
|
@@ -113,10 +122,64 @@ async function run(args) {
|
|
|
113
122
|
maxChars: control.hooksSessionStartMaxChars,
|
|
114
123
|
scopeCounts,
|
|
115
124
|
applicable,
|
|
125
|
+
// Record what this injection RENDERED, so the per-prompt hook treats it as
|
|
126
|
+
// already seen. It must be the rendered subset, not the fetched set: the
|
|
127
|
+
// budget and the hard ceiling routinely drop lessons, and marking those
|
|
128
|
+
// shown would let the delta gate suppress — for the whole session —
|
|
129
|
+
// exactly the lessons the reader never saw. Bookkeeping only:
|
|
130
|
+
// `recordShownLessons` never throws, and a failure costs at most one
|
|
131
|
+
// repeated lesson later in the session.
|
|
132
|
+
onShown: (rendered) => recordShownLessons(parsed.sessionId, rendered.map(lessonId)),
|
|
116
133
|
}));
|
|
117
134
|
return 0;
|
|
118
135
|
}
|
|
119
136
|
|
|
137
|
+
if (intent === 'relevant-read') {
|
|
138
|
+
// The per-turn relevance pull. Fires on EVERY prompt, so every branch below
|
|
139
|
+
// is a reason to stay silent — the hook's default answer is nothing.
|
|
140
|
+
//
|
|
141
|
+
// Config gate, and the ONLY off switch some users have. Via `install` the
|
|
142
|
+
// event is wired by hook mode `all` alone, so reaching here means the user
|
|
143
|
+
// opted into the full lifecycle and `hooks.userPrompt` lets them keep it
|
|
144
|
+
// while switching off just this one. Via the Claude marketplace plugin
|
|
145
|
+
// there is no mode at all — `plugins/lorekit-claude/hooks/hooks.json` wires
|
|
146
|
+
// the event unconditionally — so for a plugin install this setting is the
|
|
147
|
+
// whole opt-out.
|
|
148
|
+
if ((control.hooksUserPrompt || 'on') === 'off') return 0;
|
|
149
|
+
|
|
150
|
+
// Length gate. "yes" / "continue" / "do it" carry nothing worth querying,
|
|
151
|
+
// and a store lookup per acknowledgement is pure overhead on the user's
|
|
152
|
+
// critical path.
|
|
153
|
+
const terms = promptQuery(parsed.prompt);
|
|
154
|
+
if (terms.length === 0) return 0;
|
|
155
|
+
|
|
156
|
+
try {
|
|
157
|
+
const store = createStore(control);
|
|
158
|
+
if (!store) return 0;
|
|
159
|
+
const lessons = await promptLessonsFromStore(store, scope, terms, {
|
|
160
|
+
// Delta only. Includes the SessionStart set, because "already shown"
|
|
161
|
+
// has to mean shown by anything — a hook that only remembered its own
|
|
162
|
+
// output would resurface the session's opening injection one lesson at
|
|
163
|
+
// a time.
|
|
164
|
+
alreadyShown: shownLessons(parsed.sessionId),
|
|
165
|
+
});
|
|
166
|
+
// Relevance gate: nothing matched, or everything that matched was already
|
|
167
|
+
// on screen. Either way there is no news, and an "in case it helps" block
|
|
168
|
+
// attached to an unrelated prompt is how a reader learns to skip the
|
|
169
|
+
// block entirely — which would cost the SessionStart injection its
|
|
170
|
+
// credibility too.
|
|
171
|
+
if (lessons.length === 0) return 0;
|
|
172
|
+
recordShownLessons(parsed.sessionId, lessons.map(lessonId));
|
|
173
|
+
emit(formatPromptLessons(lessons, {
|
|
174
|
+
instruction: (control.hooksInstructions && control.hooksInstructions.UserPromptSubmit) || null,
|
|
175
|
+
}));
|
|
176
|
+
} catch {
|
|
177
|
+
// Best-effort, like every other branch: the user's turn proceeds either
|
|
178
|
+
// way, and a store hiccup must never cost them their prompt.
|
|
179
|
+
}
|
|
180
|
+
return 0;
|
|
181
|
+
}
|
|
182
|
+
|
|
120
183
|
if (intent === 'confirm') {
|
|
121
184
|
// Fire only when a lorekit memory write actually succeeded — the adapter's
|
|
122
185
|
// isLoreWrite() inspects the tool name and the response shape. Any error
|
package/src/install.mjs
CHANGED
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
HOOK_MODES,
|
|
18
18
|
hookEventsForMode,
|
|
19
19
|
hookModeFromEvents,
|
|
20
|
+
missingHookEvents,
|
|
20
21
|
installedHookEvents,
|
|
21
22
|
resolveConnection,
|
|
22
23
|
tokenKind,
|
|
@@ -120,7 +121,7 @@ export const HOOK_PROMPT_OPTIONS = [
|
|
|
120
121
|
{
|
|
121
122
|
label: 'Yes, all of them',
|
|
122
123
|
value: 'all',
|
|
123
|
-
hint: 'inject lessons at session start; nudge on a tool failure and at end of turn',
|
|
124
|
+
hint: 'inject lessons at session start and as each prompt matches them; nudge on a tool failure and at end of turn',
|
|
124
125
|
},
|
|
125
126
|
{
|
|
126
127
|
label: 'Read-only',
|
|
@@ -275,6 +276,22 @@ export async function install(args) {
|
|
|
275
276
|
: ` ${c.dim('Hooks: none wired — the skills work, but only when the model invokes them')}`,
|
|
276
277
|
);
|
|
277
278
|
|
|
279
|
+
// An install predating a lifecycle event still reads as its mode (see
|
|
280
|
+
// LEGACY_ALL_EVENT_SETS in config.mjs), but this branch returns before the
|
|
281
|
+
// hook step — so the upgrade is available and will not happen on its own.
|
|
282
|
+
// Say so here rather than leaving the user to infer it: silently keeping a
|
|
283
|
+
// stale event set is how a wired install stops firing the hooks it says it
|
|
284
|
+
// has. Only ever a HINT — rewiring stays an explicit `--hooks` / `--force`.
|
|
285
|
+
const wiredMode = hookModeFromEvents(wiredEvents);
|
|
286
|
+
// `missingHookEvents` owns the derivation (incl. why `custom` is excluded)
|
|
287
|
+
// so `doctor`'s hooks line reports the same upgrade this summary does.
|
|
288
|
+
const missingEvents = missingHookEvents(wiredEvents);
|
|
289
|
+
if (missingEvents.length > 0) {
|
|
290
|
+
log(
|
|
291
|
+
` ${c.yellow(`Hook upgrade available: ${missingEvents.join(', ')} — run lorekit install --${scope} --hooks ${wiredMode} to wire ${missingEvents.length === 1 ? 'it' : 'them'}.`)}`,
|
|
292
|
+
);
|
|
293
|
+
}
|
|
294
|
+
|
|
278
295
|
log('');
|
|
279
296
|
log(` Run ${c.cyan('npx @lorekit/cli doctor')} to verify the connection.`);
|
|
280
297
|
log(` Change the hooks with ${c.cyan(`--hooks ${HOOK_MODES.join('|')}`)}.`);
|
package/src/lessons-pure.mjs
CHANGED
|
@@ -487,6 +487,134 @@ function seenCountFrom(entry) {
|
|
|
487
487
|
return typeof n === 'number' && Number.isFinite(n) && n > 0 ? n : 0;
|
|
488
488
|
}
|
|
489
489
|
|
|
490
|
+
/**
|
|
491
|
+
* MMR λ (lambda) — weight given to relevance vs diversity in the MMR objective.
|
|
492
|
+
* At 0.7 the selector favours relevance, with 0.3 of the budget for diversity.
|
|
493
|
+
* Anchored to Carbonell & Goldstein (1998), the original MMR paper.
|
|
494
|
+
* Mirrored byte-identically in the edge twin and `packages/mcp-core/src/lesson-rank.ts`.
|
|
495
|
+
*/
|
|
496
|
+
export const MMR_LAMBDA = 0.7;
|
|
497
|
+
|
|
498
|
+
/**
|
|
499
|
+
* Tokenise a lesson `value` into a deduplicated Set: case-fold, split on
|
|
500
|
+
* non-alphanumeric characters, drop empties. Dependency-free and deterministic
|
|
501
|
+
* so it mirrors the TS/Deno twin's `tokenizeValue`.
|
|
502
|
+
*
|
|
503
|
+
* PERF: `selectDiverse` calls this ONCE per candidate before the MMR loop and
|
|
504
|
+
* caches the result. `jaccardSimilarity` takes the cached Sets, so the O(k²)
|
|
505
|
+
* pairwise comparisons inside the loop cost no tokenisation. Do NOT re-introduce
|
|
506
|
+
* a `tokenize(value)` call inside the loop: that regresses the hot path to
|
|
507
|
+
* O(n·k²) tokenisations (measured 34s CPU at n=200/k=100 with 1.5KB values).
|
|
508
|
+
*/
|
|
509
|
+
function tokenizeValue(v) {
|
|
510
|
+
const tokens = String(v ?? '').toLowerCase().split(/[^a-z0-9]+/).filter(Boolean);
|
|
511
|
+
return new Set(tokens);
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
/**
|
|
515
|
+
* Word/token Jaccard similarity between two PRE-TOKENISED value Sets:
|
|
516
|
+
* |A∩B| / |A∪B|. Both-empty → 0.
|
|
517
|
+
*
|
|
518
|
+
* Takes Sets rather than raw values on purpose — the caller tokenises each
|
|
519
|
+
* candidate once (see `tokenizeValue`) so this stays allocation-free on the
|
|
520
|
+
* O(k²) hot path.
|
|
521
|
+
*/
|
|
522
|
+
function jaccardSimilarity(setA, setB) {
|
|
523
|
+
if (setA.size === 0 && setB.size === 0) return 0;
|
|
524
|
+
let intersection = 0;
|
|
525
|
+
for (const t of setA) if (setB.has(t)) intersection += 1;
|
|
526
|
+
const union = setA.size + setB.size - intersection;
|
|
527
|
+
return union === 0 ? 0 : intersection / union;
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
/**
|
|
531
|
+
* Select the top-K lessons from a ranked list using Maximal Marginal Relevance
|
|
532
|
+
* (Carbonell & Goldstein, 1998).
|
|
533
|
+
*
|
|
534
|
+
* Accepts bare entries (the shape `rankLessons` returns in the `.mjs` twin)
|
|
535
|
+
* plus a parallel `scores` array — REQUIRED, one score per entry. This is the
|
|
536
|
+
* MMR relevance term; there is no meaningful default. A missing `scores`, a
|
|
537
|
+
* non-array, or a length that does not match `entries` throws rather than
|
|
538
|
+
* silently defaulting to 0 for every entry (which would degrade selection to
|
|
539
|
+
* pure diversity and quietly discard the ranking). The TS twin cannot hit this
|
|
540
|
+
* footgun — it reads the score off each `{ entry, score }` input.
|
|
541
|
+
*
|
|
542
|
+
* Greedy MMR: seed with index 0 (highest-ranked), then at each step pick
|
|
543
|
+
* the unselected candidate that maximises
|
|
544
|
+
* λ·quantise(score(i)) − (1−λ)·max_{j∈selected} jaccardSimilarity(value_i, value_j)
|
|
545
|
+
* Ties break by input order (first-wins) for determinism.
|
|
546
|
+
*
|
|
547
|
+
* SCORE QUANTISATION: the relevance term uses the score SNAPPED onto the
|
|
548
|
+
* `SCORE_EPSILON` grid — the exact grid `rankLessons` buckets on for its
|
|
549
|
+
* scope-precedence tie-break. Two scores within `SCORE_EPSILON` land in the same
|
|
550
|
+
* bucket, so the MMR objective sees them as equal and the input order (which
|
|
551
|
+
* `rankLessons` already sorted by scope precedence, then key) decides. Comparing
|
|
552
|
+
* the RAW score would let a 1e-10 float difference override scope precedence.
|
|
553
|
+
*
|
|
554
|
+
* PERF — the algorithm is O(n·k), and BOTH factors that could regress it to
|
|
555
|
+
* O(n·k²) are held down explicitly:
|
|
556
|
+
* 1. TOKENISE axis: each candidate's `value` is tokenised ONCE up front (the
|
|
557
|
+
* `tokens` field) so the pairwise Jaccard comparisons never tokenise —
|
|
558
|
+
* O(n) tokenisations total. See `tokenizeValue`.
|
|
559
|
+
* 2. INTERSECTION axis: each remaining candidate carries a RUNNING `maxSim`
|
|
560
|
+
* (its greatest Jaccard similarity to anything selected so far). The
|
|
561
|
+
* objective reads that cached scalar — it does NOT loop over `selected`.
|
|
562
|
+
* After each pick we update `maxSim` for the still-remaining candidates
|
|
563
|
+
* against the ONE just-selected entry only. That is k picks × n candidates
|
|
564
|
+
* = O(n·k) Jaccard intersections total, not O(n·k²).
|
|
565
|
+
* Re-introducing either an in-loop `tokenizeValue` call OR an inner
|
|
566
|
+
* `for (… of selected)` maxSim recomputation silently restores O(n·k²) —
|
|
567
|
+
* measured 2.6s CPU at n=200/k=100 vs 58ms for the running form. Don't.
|
|
568
|
+
*
|
|
569
|
+
* Always-on for ranked mode — no optional param needed. The recency wire path is
|
|
570
|
+
* never affected.
|
|
571
|
+
*/
|
|
572
|
+
export function selectDiverse(entries, k, { lambda: lambdaOpt, scores } = {}) {
|
|
573
|
+
// Match the TS twin: fall back to MMR_LAMBDA unless `lambda` is a FINITE
|
|
574
|
+
// number. A bare destructuring default only catches `undefined`, so
|
|
575
|
+
// `{ lambda: NaN }` would otherwise poison every MMR objective here while the
|
|
576
|
+
// TS twin quietly used MMR_LAMBDA — a silent cross-twin divergence.
|
|
577
|
+
const lambda = typeof lambdaOpt === 'number' && Number.isFinite(lambdaOpt) ? lambdaOpt : MMR_LAMBDA;
|
|
578
|
+
if (!Array.isArray(entries) || entries.length === 0 || k <= 0) return [];
|
|
579
|
+
if (!Array.isArray(scores) || scores.length !== entries.length) {
|
|
580
|
+
throw new TypeError(
|
|
581
|
+
`selectDiverse: \`scores\` must be an array with one score per entry `
|
|
582
|
+
+ `(got ${Array.isArray(scores) ? `length ${scores.length}` : typeof scores} `
|
|
583
|
+
+ `for ${entries.length} entries)`,
|
|
584
|
+
);
|
|
585
|
+
}
|
|
586
|
+
const selected = [];
|
|
587
|
+
// Pre-tokenise every candidate ONCE and snap its score onto the SCORE_EPSILON
|
|
588
|
+
// grid up front. `maxSim` is the running greatest Jaccard similarity to any
|
|
589
|
+
// already-selected entry — 0 while nothing is selected. The loop below reads
|
|
590
|
+
// these caches; it never tokenises, re-quantises, or rescans `selected`.
|
|
591
|
+
const remaining = entries.map((e, i) => ({
|
|
592
|
+
entry: e,
|
|
593
|
+
tokens: tokenizeValue(e?.value),
|
|
594
|
+
qScore: Math.round((scores[i] ?? 0) / SCORE_EPSILON) * SCORE_EPSILON,
|
|
595
|
+
maxSim: 0,
|
|
596
|
+
}));
|
|
597
|
+
while (selected.length < k && remaining.length > 0) {
|
|
598
|
+
let bestIdx = 0;
|
|
599
|
+
let bestMmr = -Infinity;
|
|
600
|
+
for (let i = 0; i < remaining.length; i++) {
|
|
601
|
+
const candidate = remaining[i];
|
|
602
|
+
// Reads the cached running maxSim — no inner scan over `selected`.
|
|
603
|
+
const mmr = lambda * candidate.qScore - (1 - lambda) * candidate.maxSim;
|
|
604
|
+
if (mmr > bestMmr) { bestMmr = mmr; bestIdx = i; }
|
|
605
|
+
}
|
|
606
|
+
const [justSelected] = remaining.splice(bestIdx, 1);
|
|
607
|
+
selected.push(justSelected.entry);
|
|
608
|
+
// Fold the just-selected entry into every remaining candidate's running
|
|
609
|
+
// maxSim — one Jaccard per remaining candidate per pick, so O(n·k) total.
|
|
610
|
+
for (const candidate of remaining) {
|
|
611
|
+
const sim = jaccardSimilarity(candidate.tokens, justSelected.tokens);
|
|
612
|
+
if (sim > candidate.maxSim) candidate.maxSim = sim;
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
return selected;
|
|
616
|
+
}
|
|
617
|
+
|
|
490
618
|
/**
|
|
491
619
|
* Rank lessons best-first, returning a NEW array — the input is never reordered
|
|
492
620
|
* in place, because callers hold it (`fetchLessons` builds it from the
|
|
@@ -578,3 +706,42 @@ export function rankLessons(entries = [], {
|
|
|
578
706
|
return scored.map((s) => s.entry);
|
|
579
707
|
}
|
|
580
708
|
|
|
709
|
+
/**
|
|
710
|
+
* Rank-then-diversify in one call — the `.mjs` twin's convenience for what the
|
|
711
|
+
* TS twin gets for free by carrying `{ entry, score }` pairs into `selectDiverse`.
|
|
712
|
+
*
|
|
713
|
+
* `selectDiverse` needs a parallel `scores` array, and those scores MUST be the
|
|
714
|
+
* same set-relative values the list was sorted on: the salience factor is
|
|
715
|
+
* normalised against the max `seen_count` in the candidate SET, so a score
|
|
716
|
+
* recomputed over a different population would not line up with the order. This
|
|
717
|
+
* helper recomputes the scores over exactly the list it diversifies, beside the
|
|
718
|
+
* unexported `seenCountFrom`/`scoreWithTerms`, so callers can apply MMR without
|
|
719
|
+
* reconstructing that alignment (and without `seenCountFrom` leaking out).
|
|
720
|
+
*
|
|
721
|
+
* `entries` is expected to be `rankLessons` output (best-first) so the seed of
|
|
722
|
+
* the greedy MMR is the top-ranked lesson; the pass-through
|
|
723
|
+
* `terms`/`weights`/`halfLifeDays`/`now` MUST match the `rankLessons` call that
|
|
724
|
+
* produced it, or the recomputed scores diverge from the sort. `k` caps the
|
|
725
|
+
* returned count (default: all). Empty/degenerate input returns `[]`.
|
|
726
|
+
*/
|
|
727
|
+
export function diversifyRankedLessons(entries = [], {
|
|
728
|
+
terms = [],
|
|
729
|
+
now = Date.now(),
|
|
730
|
+
weights = DEFAULT_RANK_WEIGHTS,
|
|
731
|
+
halfLifeDays = RECENCY_HALF_LIFE_DAYS,
|
|
732
|
+
k = Infinity,
|
|
733
|
+
lambda,
|
|
734
|
+
} = {}) {
|
|
735
|
+
const list = Array.isArray(entries) ? entries.filter((e) => e && typeof e === 'object') : [];
|
|
736
|
+
if (list.length === 0) return [];
|
|
737
|
+
const termSet = distinctTerms(terms);
|
|
738
|
+
let maxSeenCount = 0;
|
|
739
|
+
for (const e of list) maxSeenCount = Math.max(maxSeenCount, seenCountFrom(e));
|
|
740
|
+
const scores = list.map((e) => scoreWithTerms(e, termSet, { now, weights, maxSeenCount, halfLifeDays }));
|
|
741
|
+
// `numberOr` is the module's coercion convention: a non-finite `k` (the
|
|
742
|
+
// `Infinity` default, `null`, or a stringy `'40'`) resolves to a real cap —
|
|
743
|
+
// the default falls through to the whole list, `'40'` becomes 40 — rather than
|
|
744
|
+
// silently returning everything on a shape a caller plausibly passes.
|
|
745
|
+
const limit = numberOr(k, list.length);
|
|
746
|
+
return selectDiverse(list, limit, { scores, lambda });
|
|
747
|
+
}
|
package/src/store/local.mjs
CHANGED
|
@@ -235,17 +235,32 @@ class LocalStore {
|
|
|
235
235
|
// this walks each scope EXACTLY ONCE — the failure hook passes all its terms
|
|
236
236
|
// in one call rather than one call per term, so N terms no longer re-read the
|
|
237
237
|
// store N times. An empty query (or empty list) returns everything, unchanged.
|
|
238
|
-
|
|
238
|
+
// `walkLimit` bounds the walk for hot-path callers (the per-prompt hook): a
|
|
239
|
+
// local search re-reads every scope's files synchronously, which the remote
|
|
240
|
+
// `timeoutMs` cannot bound, so an unbounded store would stall the prompt.
|
|
241
|
+
// Once `walkLimit` matches are collected the walk stops — later scopes in
|
|
242
|
+
// `scopes` are skipped entirely. Scopes are visited most-specific-first, so
|
|
243
|
+
// the retained matches are the nearest-scope ones, consistent with this
|
|
244
|
+
// store's precedence. Omitted → unbounded, exactly as before. Named
|
|
245
|
+
// `walkLimit`, NOT `limit`, so it is local-only: `RemoteStore.search` reads
|
|
246
|
+
// `limit` (→ `body.limit`) and a shared name would truncate the remote hit
|
|
247
|
+
// set pre-ranking, which the per-prompt caller must not do.
|
|
248
|
+
async search({ q, scopes, tags, walkLimit } = {}) {
|
|
239
249
|
const needles = (Array.isArray(q) ? q : [q])
|
|
240
250
|
.map((n) => String(n || '').toLowerCase())
|
|
241
251
|
.filter(Boolean);
|
|
242
252
|
const matchAll = needles.length === 0;
|
|
253
|
+
const cap = Number.isInteger(walkLimit) && walkLimit > 0 ? walkLimit : Infinity;
|
|
243
254
|
const out = [];
|
|
244
255
|
for (const scope of scopes || []) {
|
|
256
|
+
if (out.length >= cap) break;
|
|
245
257
|
const { entries } = await this.list({ scope, tags });
|
|
246
258
|
for (const e of entries) {
|
|
247
259
|
const hay = `${e.key}\n${(e.tags || []).join(' ')}\n${e.value || ''}`.toLowerCase();
|
|
248
|
-
if (matchAll || needles.some((n) => hay.includes(n)))
|
|
260
|
+
if (matchAll || needles.some((n) => hay.includes(n))) {
|
|
261
|
+
out.push(e);
|
|
262
|
+
if (out.length >= cap) break;
|
|
263
|
+
}
|
|
249
264
|
}
|
|
250
265
|
}
|
|
251
266
|
return { ok: true, entries: out };
|
|
@@ -412,13 +427,34 @@ class TwoTierStore {
|
|
|
412
427
|
return this.home.restore({ scope, key });
|
|
413
428
|
}
|
|
414
429
|
|
|
415
|
-
async search({ q, scopes, tags } = {}) {
|
|
416
|
-
const
|
|
430
|
+
async search({ q, scopes, tags, walkLimit } = {}) {
|
|
431
|
+
const id = (e) => `${e.scope}\x00${e.key}`;
|
|
432
|
+
const homeRes = await this.home.search({ q, scopes, tags, walkLimit });
|
|
417
433
|
const projRes = this.projectActive()
|
|
418
|
-
? await this.project.search({ q, scopes, tags })
|
|
434
|
+
? await this.project.search({ q, scopes, tags, walkLimit })
|
|
419
435
|
: { entries: [] };
|
|
420
|
-
const merged = mergeByKey(projRes.entries, homeRes.entries,
|
|
421
|
-
return { ok: true, entries: merged };
|
|
436
|
+
const merged = mergeByKey(projRes.entries, homeRes.entries, id);
|
|
437
|
+
if (!Number.isInteger(walkLimit) || walkLimit <= 0) return { ok: true, entries: merged };
|
|
438
|
+
|
|
439
|
+
// Each tier is walked under its OWN `walkLimit` (that bound exists to stop
|
|
440
|
+
// an unbounded synchronous file walk on the per-prompt hot path, so it has
|
|
441
|
+
// to stay per-tier). Slicing the project-first merge to `walkLimit` would
|
|
442
|
+
// then starve the home tier outright: a project tier that fills its own
|
|
443
|
+
// budget occupies every slot, and no home-tier lesson survives to reach
|
|
444
|
+
// `rankLessons`. Split the budget instead — each tier is guaranteed its
|
|
445
|
+
// half, and whatever the other tier leaves unused is handed straight back,
|
|
446
|
+
// so a single populated tier still fills the cap exactly as before.
|
|
447
|
+
const projectIds = new Set(projRes.entries.map(id));
|
|
448
|
+
const fromProject = merged.filter((e) => projectIds.has(id(e)));
|
|
449
|
+
const fromHome = merged.filter((e) => !projectIds.has(id(e)));
|
|
450
|
+
const projectTake = Math.min(
|
|
451
|
+
fromProject.length,
|
|
452
|
+
Math.max(Math.ceil(walkLimit / 2), walkLimit - fromHome.length),
|
|
453
|
+
);
|
|
454
|
+
return {
|
|
455
|
+
ok: true,
|
|
456
|
+
entries: [...fromProject.slice(0, projectTake), ...fromHome.slice(0, walkLimit - projectTake)],
|
|
457
|
+
};
|
|
422
458
|
}
|
|
423
459
|
|
|
424
460
|
// Merged, de-duplicated non-archived count across the given scopes.
|
package/src/store/remote.mjs
CHANGED
|
@@ -84,7 +84,11 @@ class RemoteStore {
|
|
|
84
84
|
};
|
|
85
85
|
}
|
|
86
86
|
|
|
87
|
-
|
|
87
|
+
// `timeoutMs` is optional and defaults to `restFetch`'s own budget, so every
|
|
88
|
+
// existing caller is unaffected. It exists for the callers on a user's
|
|
89
|
+
// critical path, which would rather answer nothing than stall (see
|
|
90
|
+
// `PROMPT_FETCH_TIMEOUT_MS` in `core/lessons.mjs`).
|
|
91
|
+
async search({ q, scopes, tags, limit, cursor, timeoutMs } = {}) {
|
|
88
92
|
// A list of terms collapses into ONE `websearch` query joined by `OR`, so a
|
|
89
93
|
// multi-term failure lookup is a single round-trip (the server FTS ORs them
|
|
90
94
|
// and stems each). `failureQuery` distils terms to `[a-z0-9]+` tokens, so no
|
|
@@ -96,7 +100,7 @@ class RemoteStore {
|
|
|
96
100
|
if (tags?.length) body.tags = tags;
|
|
97
101
|
if (limit) body.limit = limit;
|
|
98
102
|
if (cursor) body.cursor = cursor;
|
|
99
|
-
const res = await this._rest('/memories/search', { method: 'POST', body });
|
|
103
|
+
const res = await this._rest('/memories/search', { method: 'POST', body, timeoutMs });
|
|
100
104
|
if (!res.ok) return { ok: false, error: res.error, networkError: res.networkError };
|
|
101
105
|
const data = res.data ?? {};
|
|
102
106
|
return {
|