dsh-loci 2.0.3 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -0
- package/README.md +1 -1
- package/README.zh.md +1 -1
- package/client.js +42 -4
- package/docs/SECURITY.md +1 -1
- package/package.json +1 -1
- package/src/host/api/routes.js +6 -0
- package/src/host/config.js +9 -2
- package/src/host/index.js +1 -1
- package/src/host/inject/index-section.js +46 -10
- package/src/host/store/migrate.js +13 -1
- package/src/host/store/schema.js +24 -1
- package/src/host/store/store.js +98 -0
- package/src/host/tools/tools.js +11 -1
- package/src/mcp/store-facade.js +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,51 @@ All notable changes to this project are documented here. The format follows
|
|
|
4
4
|
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/); the project follows
|
|
5
5
|
[Semantic Versioning](https://semver.org/spec/v2.0.0.html) from 1.0.0.
|
|
6
6
|
|
|
7
|
+
## [2.1.0] — 2026-11-15
|
|
8
|
+
|
|
9
|
+
Two findings from an independent blind review, where seven reviewers compared this
|
|
10
|
+
plugin against the rest of the field. Two of them caught the same false claim; the
|
|
11
|
+
other had the sharpest product objection, and both are addressed here.
|
|
12
|
+
|
|
13
|
+
### Fixed
|
|
14
|
+
|
|
15
|
+
- **The "no network" claim was false.** `README` and the threat-model row both said
|
|
16
|
+
"no network unless `vectorMode` is http", while `updateCheck` defaulted to `true`
|
|
17
|
+
and the panel's update route fetched the registry. Rather than reword the claim,
|
|
18
|
+
the default is now **false**, so the sentence is true: turning the check on is one
|
|
19
|
+
click, and a disabled check reads "Update check off" instead of looking like a
|
|
20
|
+
dead button. `test/unit/network-claims.test.mjs` also greps the source, so a new
|
|
21
|
+
`fetch` outside the two opt-in modules fails CI.
|
|
22
|
+
|
|
23
|
+
### Added
|
|
24
|
+
|
|
25
|
+
- **A pending-review line in the injected section.** Verified-only injection has an
|
|
26
|
+
invisible failure mode: a fact written at midnight is absent at 9 a.m., and "never
|
|
27
|
+
saved" looks exactly like "saved but gated". When — and only when — something is
|
|
28
|
+
waiting, the footer now carries one line with the count and says those items are
|
|
29
|
+
deliberately absent. An empty queue still costs exactly 0 bytes, and the count is
|
|
30
|
+
part of the render cache key so a new candidate appears on the next request.
|
|
31
|
+
|
|
32
|
+
- **Per-entry usage counters (schema v2).** `memory_usage` records `inject_count`,
|
|
33
|
+
`read_count`, `last_injected_at`, `last_read_at` per memory, so "which line spends
|
|
34
|
+
prompt bytes and is never opened" becomes answerable. The panel's Stats tab lists
|
|
35
|
+
that dead weight; `loci_get` records a read.
|
|
36
|
+
|
|
37
|
+
### Migration
|
|
38
|
+
|
|
39
|
+
Schema v1 → v2 adds one table. It runs through the existing migration runner: a
|
|
40
|
+
`VACUUM INTO` snapshot first, then one immediate transaction, then a row-count guard
|
|
41
|
+
over every pre-existing table. Rollback stays "restore the snapshot". This is the
|
|
42
|
+
first real migration this project has shipped, so `test/integration/migrate.test.mjs`
|
|
43
|
+
now exercises both the rehearsal step and the shipped one.
|
|
44
|
+
|
|
45
|
+
### Note
|
|
46
|
+
|
|
47
|
+
The same review found two things worth stating openly rather than burying: the
|
|
48
|
+
plugin is **not yet in the DSH plugin catalog** (the catalog snapshot predates its
|
|
49
|
+
publication), and it has been verified on `0.2.2-rc.1` only, while `engines.dsh`
|
|
50
|
+
admits the whole `0.2.2-rc.1 <0.3.0-0` range. Both are tracked as open work.
|
|
51
|
+
|
|
7
52
|
## [2.0.3] — 2026-11-14
|
|
8
53
|
|
|
9
54
|
### Fixed
|
package/README.md
CHANGED
|
@@ -119,7 +119,7 @@ The settings page writes the fields marked *live* without a restart.
|
|
|
119
119
|
|
|
120
120
|
- No model call in any automatic path in v1.0 — capture and summaries are
|
|
121
121
|
deterministic, so the cost is visible and zero.
|
|
122
|
-
- No cloud, no telemetry
|
|
122
|
+
- No cloud, no telemetry. Two optional network paths, both off by default: the update check (`updateCheck`) and the HTTP embedding channel (`vectorMode: 'http'`).
|
|
123
123
|
- No cross-project leakage: reads use an explicit scope-key set, never a wildcard.
|
|
124
124
|
- No physical deletion by default; `purge` needs a config switch, a confirmation,
|
|
125
125
|
and leaves an audit line.
|
package/README.zh.md
CHANGED
|
@@ -88,7 +88,7 @@ node "$HOME/.dsh/profiles/<profile>/node_modules/dsh-loci/scripts/cli.mjs" docto
|
|
|
88
88
|
## 它明确不做
|
|
89
89
|
|
|
90
90
|
- v1.0 的自动路径里没有模型调用 —— 捕获与摘要是确定性的,成本可见且为零。
|
|
91
|
-
-
|
|
91
|
+
- 无云、无遥测。只有两条可选出网路径,**都默认关闭**:更新检查(`updateCheck`)与 HTTP 向量通道(`vectorMode: 'http'`)。
|
|
92
92
|
- 不跨项目串味:读取用显式 scope-key 集合,从不通配。
|
|
93
93
|
- 默认不物理删除;`purge` 需要配置开关 + 确认 + 留审计。
|
|
94
94
|
|
package/client.js
CHANGED
|
@@ -43,6 +43,7 @@ window.__ModuleLoader__.load({
|
|
|
43
43
|
'card.check': 'Check for updates',
|
|
44
44
|
'card.upToDate': 'Up to date',
|
|
45
45
|
'card.updateTo': 'Update available',
|
|
46
|
+
'card.checkOff': 'Update check off',
|
|
46
47
|
'card.checking': 'Checking…',
|
|
47
48
|
'card.unknown': 'Version check unavailable',
|
|
48
49
|
'card.unpublished': 'Not on npm yet',
|
|
@@ -138,6 +139,9 @@ window.__ModuleLoader__.load({
|
|
|
138
139
|
'panel.verifyAll': 'Verify all {n}',
|
|
139
140
|
'panel.actionFailed': 'Action failed',
|
|
140
141
|
'panel.injections': 'Recent injections',
|
|
142
|
+
'panel.deadWeight': 'Injected but never opened',
|
|
143
|
+
'panel.injected': 'injected',
|
|
144
|
+
'panel.opened': 'opened',
|
|
141
145
|
'panel.store': 'Store',
|
|
142
146
|
'panel.confirmDelete': 'Soft-delete this entry?',
|
|
143
147
|
'panel.error': 'Request failed',
|
|
@@ -160,6 +164,7 @@ window.__ModuleLoader__.load({
|
|
|
160
164
|
'card.check': '检查更新',
|
|
161
165
|
'card.upToDate': '已是最新',
|
|
162
166
|
'card.updateTo': '有新版本',
|
|
167
|
+
'card.checkOff': '更新检查已关闭',
|
|
163
168
|
'card.checking': '检查中…',
|
|
164
169
|
'card.unknown': '无法检查版本',
|
|
165
170
|
'card.unpublished': '尚未发布',
|
|
@@ -255,6 +260,9 @@ window.__ModuleLoader__.load({
|
|
|
255
260
|
'panel.verifyAll': '全部核实({n})',
|
|
256
261
|
'panel.actionFailed': '操作失败',
|
|
257
262
|
'panel.injections': '最近注入',
|
|
263
|
+
'panel.deadWeight': '注入了但从未被打开',
|
|
264
|
+
'panel.injected': '次注入',
|
|
265
|
+
'panel.opened': '次打开',
|
|
258
266
|
'panel.store': '存储',
|
|
259
267
|
'panel.confirmDelete': '软删除这条记忆?',
|
|
260
268
|
'panel.error': '请求失败',
|
|
@@ -483,13 +491,18 @@ window.__ModuleLoader__.load({
|
|
|
483
491
|
const version = typeof update?.version === 'string' ? update.version.replace(/^v/i, '') : null;
|
|
484
492
|
const hasUpdate = update?.updateAvailable === true;
|
|
485
493
|
const unknown = update !== undefined && update.status === 'unavailable';
|
|
494
|
+
// The check is opt-in, so "off" is a normal state, not a failure: it must read
|
|
495
|
+
// as switched off rather than as a button that does nothing.
|
|
496
|
+
const checkDisabled = update?.checkEnabled === false;
|
|
486
497
|
const text = hasUpdate
|
|
487
498
|
? `${t('card.updateTo')}${update.latestVersion ? ` · v${update.latestVersion}` : ''}`
|
|
488
499
|
: unknown
|
|
489
500
|
? t('card.unknown')
|
|
490
|
-
:
|
|
491
|
-
? t('card.
|
|
492
|
-
:
|
|
501
|
+
: checkDisabled
|
|
502
|
+
? t('card.checkOff')
|
|
503
|
+
: update?.status === 'ok'
|
|
504
|
+
? t('card.upToDate')
|
|
505
|
+
: t('card.check');
|
|
493
506
|
return h(
|
|
494
507
|
'span',
|
|
495
508
|
{ className: 'loci-card-line', style: { gap: '5px' } },
|
|
@@ -500,7 +513,7 @@ window.__ModuleLoader__.load({
|
|
|
500
513
|
type: 'button',
|
|
501
514
|
className: 'loci-btn is-quiet',
|
|
502
515
|
title: text,
|
|
503
|
-
disabled: checking === true,
|
|
516
|
+
disabled: checking === true || checkDisabled,
|
|
504
517
|
onClick: (event) => {
|
|
505
518
|
event.stopPropagation();
|
|
506
519
|
onCheck?.();
|
|
@@ -1077,6 +1090,31 @@ window.__ModuleLoader__.load({
|
|
|
1077
1090
|
),
|
|
1078
1091
|
),
|
|
1079
1092
|
h('div', { className: 'loci-meta', style: { marginTop: '14px' } }, `${t('panel.version')} v${data.pluginVersion ?? '—'}`),
|
|
1093
|
+
// Which lines spend prompt bytes and are never opened. Sorted host-side,
|
|
1094
|
+
// so the first row is the most expensive and least used.
|
|
1095
|
+
(data.usage ?? []).some((row) => row.injections > 0 && row.reads === 0)
|
|
1096
|
+
? h(
|
|
1097
|
+
'div',
|
|
1098
|
+
null,
|
|
1099
|
+
h('div', { className: 'loci-section' }, t('panel.deadWeight')),
|
|
1100
|
+
(data.usage ?? [])
|
|
1101
|
+
.filter((row) => row.injections > 0 && row.reads === 0)
|
|
1102
|
+
.slice(0, 8)
|
|
1103
|
+
.map((row) =>
|
|
1104
|
+
h(
|
|
1105
|
+
'div',
|
|
1106
|
+
{ key: row.id, className: 'loci-row' },
|
|
1107
|
+
h('span', { className: 'loci-id' }, row.id),
|
|
1108
|
+
h(
|
|
1109
|
+
'div',
|
|
1110
|
+
{ className: 'loci-main' },
|
|
1111
|
+
h('div', { className: 'loci-name' }, row.title),
|
|
1112
|
+
h('div', { className: 'loci-meta' }, `${row.injections}×${t('panel.injected')} · 0×${t('panel.opened')}`),
|
|
1113
|
+
),
|
|
1114
|
+
),
|
|
1115
|
+
),
|
|
1116
|
+
)
|
|
1117
|
+
: null,
|
|
1080
1118
|
)
|
|
1081
1119
|
: null,
|
|
1082
1120
|
);
|
package/docs/SECURITY.md
CHANGED
|
@@ -17,7 +17,7 @@ file sandbox, approval policy) are the Harness's and are assumed to work.
|
|
|
17
17
|
| T6 | The panel API is reachable from another origin | Local admin surface exposed to a web page | exact `connection.fetch` route (owner Peer, same-origin fence), JSON-only writes, 256 KB body cap, no CORS headers |
|
|
18
18
|
| T7 | Data written outside the Harness home | Un-backed-up, un-scoped secrets | single root `$DSH_HOME/memory/loci/`, every path asserted with `assertInside` |
|
|
19
19
|
| T8 | A write blocks the agent loop | Turn hangs | WAL + bounded busy timeout + six retries; transactions are single-digit milliseconds; capture is debounced and never awaited by the turn |
|
|
20
|
-
| T9 | Memory exfiltration over the network | Privacy breach |
|
|
20
|
+
| T9 | Memory exfiltration over the network | Privacy breach | two opt-in paths only, both off by default: the update check (`updateCheck`, one `GET` of the package metadata, carries nothing about memory) and the HTTP embedding channel (`vectorMode: 'http'`, endpoint user-configured, failure parks the channel). No other request exists — `docs/PRIVACY.md` §3 lists both |
|
|
21
21
|
| T10 | Unbounded growth | Token cost creep | hard byte budget, pinned/rule caps, soft per-scope cap option |
|
|
22
22
|
|
|
23
23
|
## 2. The trust gate, precisely
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-loci",
|
|
3
|
-
"version": "2.0
|
|
3
|
+
"version": "2.1.0",
|
|
4
4
|
"description": "Layered long-term memory for DeepSeek Harness: the prompt sees only a stable index, full text is one lookup away. Bi-temporal facts, strict scope isolation, verified-only injection.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/src/host/api/routes.js
CHANGED
|
@@ -101,6 +101,9 @@ export function registerApiRoutes(ctx, deps) {
|
|
|
101
101
|
pluginVersion: deps.version ?? '0.0.0',
|
|
102
102
|
injections: store.recentInjections(50),
|
|
103
103
|
audit: store.auditTail(50),
|
|
104
|
+
// Which entries spend prompt bytes and are never opened. Aggregate request
|
|
105
|
+
// stats cannot answer this, which is why schema v2 added per-entry counters.
|
|
106
|
+
usage: store.usageReport({ scopeKeys, limit: 50 }),
|
|
104
107
|
});
|
|
105
108
|
});
|
|
106
109
|
|
|
@@ -347,6 +350,9 @@ function installedOnly(version) {
|
|
|
347
350
|
updateAvailable: false,
|
|
348
351
|
checkedAt: null,
|
|
349
352
|
status: 'idle',
|
|
353
|
+
// The card shows a "check" button; without this flag a disabled check looks
|
|
354
|
+
// broken ("nothing happens") instead of switched off.
|
|
355
|
+
checkEnabled: false,
|
|
350
356
|
};
|
|
351
357
|
}
|
|
352
358
|
|
package/src/host/config.js
CHANGED
|
@@ -100,8 +100,15 @@ export const DEFAULTS = Object.freeze({
|
|
|
100
100
|
workingTtlDays: 7,
|
|
101
101
|
/** Allow `op:'purge'` to physically delete revisions. */
|
|
102
102
|
allowHardDelete: false,
|
|
103
|
-
/**
|
|
104
|
-
|
|
103
|
+
/**
|
|
104
|
+
* Ask the registry for a newer version when the card is opened.
|
|
105
|
+
*
|
|
106
|
+
* Off by default: the whole product claims "no network unless you opt in", and a
|
|
107
|
+
* default-on version check made that sentence false. A reviewer caught it, and the
|
|
108
|
+
* claim is worth more than the convenience — turning this on is one click in
|
|
109
|
+
* Settings, and the request carries nothing about memory.
|
|
110
|
+
*/
|
|
111
|
+
updateCheck: false,
|
|
105
112
|
/** Registry used by that check; point it at a mirror when npmjs.org is slow. */
|
|
106
113
|
updateRegistry: 'https://registry.npmjs.org',
|
|
107
114
|
/** 'error' | 'info' | 'debug'. */
|
package/src/host/index.js
CHANGED
|
@@ -22,7 +22,7 @@ import { registerCommands } from './commands.js';
|
|
|
22
22
|
import { runDoctor } from './doctor.js';
|
|
23
23
|
|
|
24
24
|
/** Version stamped into the database and diagnostics. */
|
|
25
|
-
export const VERSION = '2.0
|
|
25
|
+
export const VERSION = '2.1.0';
|
|
26
26
|
|
|
27
27
|
/** Optional collaborators; without them the plugin still serves tools. */
|
|
28
28
|
export const inject = ['systemPrompt', 'tools'];
|
|
@@ -24,8 +24,14 @@ export const INDEX_SECTION_NAME = 'loci:index';
|
|
|
24
24
|
/** Placement: after MCP servers (3100), before the tools SDK section (5000). */
|
|
25
25
|
export const INDEX_SECTION_ORDER = 4000;
|
|
26
26
|
|
|
27
|
-
/**
|
|
28
|
-
|
|
27
|
+
/**
|
|
28
|
+
* Bytes reserved for the closing footer before index lines are admitted.
|
|
29
|
+
*
|
|
30
|
+
* The footer is not fixed: "… +N more" and the pending-review notice grow with the
|
|
31
|
+
* store, and the shrink loop below would otherwise admit a line and immediately
|
|
32
|
+
* take it back on every render. 400 bytes covers the worst realistic footer.
|
|
33
|
+
*/
|
|
34
|
+
const FOOTER_RESERVE = 400;
|
|
29
35
|
|
|
30
36
|
/** Days in milliseconds. */
|
|
31
37
|
export const DAY = 86_400_000;
|
|
@@ -64,14 +70,17 @@ export function renderIndexBody(options) {
|
|
|
64
70
|
const rulesMaxBytes = Math.max(0, Math.min(maxBytes, cfg('rulesMaxBytes')));
|
|
65
71
|
const maxEntries = Math.max(1, Math.min(2000, cfg('indexMaxEntries')));
|
|
66
72
|
const lineChars = Math.max(40, Math.min(400, cfg('indexLineMaxChars')));
|
|
67
|
-
const empty = { text: '', bytes: 0, listed: 0, total: 0, hidden: 0 };
|
|
73
|
+
const empty = { text: '', bytes: 0, listed: 0, total: 0, hidden: 0, ids: [] };
|
|
68
74
|
|
|
69
75
|
const rules = store.ruleEntries({ scopeKeys, now, limit: 24 });
|
|
70
76
|
const entries = store.indexEntries({ scopeKeys, now, limit: maxEntries + 1 });
|
|
71
77
|
const truncatedByCount = entries.length > maxEntries;
|
|
72
78
|
const visible = truncatedByCount ? entries.slice(0, maxEntries) : entries;
|
|
73
79
|
const total = truncatedByCount ? store.countIndex(scopeKeys, now) : visible.length;
|
|
74
|
-
|
|
80
|
+
// Counted always, rendered only when it is not zero: an empty queue must cost
|
|
81
|
+
// exactly zero bytes, or the "0 bytes when empty" promise is a lie.
|
|
82
|
+
const pending = store.countPending(scopeKeys);
|
|
83
|
+
if (rules.length === 0 && visible.length === 0 && pending === 0) return empty;
|
|
75
84
|
|
|
76
85
|
const headLines = [FENCE_OPEN];
|
|
77
86
|
const ruleLines = [];
|
|
@@ -95,19 +104,30 @@ export function renderIndexBody(options) {
|
|
|
95
104
|
// A scope label comes from a directory name, which a user can make contain a
|
|
96
105
|
// newline. Flattening it here is what stops a crafted folder name from writing
|
|
97
106
|
// extra lines into the fence.
|
|
98
|
-
|
|
107
|
+
// Only when there is an index to label. A store holding nothing but pending
|
|
108
|
+
// candidates used to render an empty "Index —" heading, which reads as "memory is
|
|
109
|
+
// empty" right next to the line saying something is waiting.
|
|
110
|
+
const indexHeader = visible.length > 0 ? `Index — ${renderSafe(indexTitle(scopeLabel, 120))}` : '';
|
|
99
111
|
const entryLine = (entry) => `- ${entry.id} · ${entry.kind} · ${formatDate(entry.validFrom)} · ${renderSafe(indexTitle(entry.title, lineChars))}`;
|
|
100
112
|
|
|
101
113
|
const footerFor = (listed) => {
|
|
102
114
|
const hidden = Math.max(0, total - listed);
|
|
103
115
|
const parts = [];
|
|
104
116
|
if (hidden > 0) parts.push(`… +${hidden} more (loci_search finds the rest; this section shows ${listed} of ${total})`);
|
|
117
|
+
// The one line that makes verified-only injection legible from inside the
|
|
118
|
+
// conversation. Without it "you never saved that" and "I saved it but it is
|
|
119
|
+
// waiting for you" are indistinguishable to the model and to the user.
|
|
120
|
+
if (pending > 0) {
|
|
121
|
+
parts.push(
|
|
122
|
+
`${pending} item(s) await review and are deliberately absent here — a person approves them in Memory → Review (/loci review); say so if the user expects them.`,
|
|
123
|
+
);
|
|
124
|
+
}
|
|
105
125
|
parts.push(`as of ${formatDate(now)} · ${FULL_TEXT_HINT}`);
|
|
106
126
|
parts.push(FENCE_CLOSE);
|
|
107
127
|
return parts.join('\n');
|
|
108
128
|
};
|
|
109
129
|
|
|
110
|
-
const baseBytes = bytes([...headLines, ...ruleLines, indexHeader].join('\n'));
|
|
130
|
+
const baseBytes = bytes([...headLines, ...ruleLines, ...(indexHeader === '' ? [] : [indexHeader])].join('\n'));
|
|
111
131
|
|
|
112
132
|
// Admit entries while the reserved footer still fits.
|
|
113
133
|
let admitted = 0;
|
|
@@ -123,7 +143,7 @@ export function renderIndexBody(options) {
|
|
|
123
143
|
let listed = admitted;
|
|
124
144
|
let text = '';
|
|
125
145
|
for (let n = admitted; ; n -= 1) {
|
|
126
|
-
const lines = [...headLines, ...ruleLines, indexHeader, ...visible.slice(0, n).map(entryLine), footerFor(n)];
|
|
146
|
+
const lines = [...headLines, ...ruleLines, ...(indexHeader === '' ? [] : [indexHeader]), ...visible.slice(0, n).map(entryLine), footerFor(n)];
|
|
127
147
|
text = lines.join('\n');
|
|
128
148
|
if (bytes(text) <= maxBytes || n === 0) {
|
|
129
149
|
listed = n;
|
|
@@ -135,12 +155,23 @@ export function renderIndexBody(options) {
|
|
|
135
155
|
// lines from the tail rather than ever exceeding the budget.
|
|
136
156
|
while (bytes(text) > maxBytes && ruleLines.length > 0) {
|
|
137
157
|
ruleLines.pop();
|
|
138
|
-
const lines = [...headLines, ...ruleLines, indexHeader, footerFor(0)];
|
|
158
|
+
const lines = [...headLines, ...ruleLines, ...(indexHeader === '' ? [] : [indexHeader]), footerFor(0)];
|
|
139
159
|
text = lines.join('\n');
|
|
140
160
|
listed = 0;
|
|
141
161
|
}
|
|
142
162
|
|
|
143
|
-
return {
|
|
163
|
+
return {
|
|
164
|
+
text,
|
|
165
|
+
bytes: bytes(text),
|
|
166
|
+
listed,
|
|
167
|
+
total,
|
|
168
|
+
hidden: Math.max(0, total - listed),
|
|
169
|
+
// Rule lines are injected too, and they are the ones a person most wants usage
|
|
170
|
+
// data for, so both blocks report their ids.
|
|
171
|
+
// Rule lines are injected too, and they are the ones a person most wants usage
|
|
172
|
+
// data for, so both blocks report their ids. `ruleEntries` is already verified-only.
|
|
173
|
+
ids: [...rules.slice(0, 12).map((rule) => rule.id), ...visible.slice(0, listed).map((entry) => entry.id)],
|
|
174
|
+
};
|
|
144
175
|
}
|
|
145
176
|
|
|
146
177
|
/**
|
|
@@ -168,6 +199,7 @@ export function createIndexSection(options) {
|
|
|
168
199
|
bytes: rendered.bytes,
|
|
169
200
|
listed: rendered.listed,
|
|
170
201
|
hidden: rendered.hidden,
|
|
202
|
+
ids: rendered.ids ?? [],
|
|
171
203
|
});
|
|
172
204
|
} catch (error) {
|
|
173
205
|
log?.warn?.(`loci: cannot record injection: ${String(error?.message ?? error)}`);
|
|
@@ -203,7 +235,11 @@ export function createIndexSection(options) {
|
|
|
203
235
|
].join(':');
|
|
204
236
|
const now = Date.now();
|
|
205
237
|
const day = new Date(now).toISOString().slice(0, 10);
|
|
206
|
-
|
|
238
|
+
// The pending count is part of the key: if it is not, a newly queued candidate
|
|
239
|
+
// stays invisible until something else invalidates the cache, which defeats
|
|
240
|
+
// the whole point of the notice.
|
|
241
|
+
const pending = store.countPending(context.scopeKeys);
|
|
242
|
+
const key = `${context.scopeKeys.join(',')}|${store.indexRev()}|${configKey}|${day}|${pending}`;
|
|
207
243
|
const cached = cache.get(key);
|
|
208
244
|
if (cached !== undefined) {
|
|
209
245
|
record(context, cached);
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
import { copyFileSync, existsSync, readdirSync, rmSync, statSync } from 'node:fs';
|
|
13
13
|
import { join } from 'node:path';
|
|
14
14
|
import { backupTo, tableCounts, withTransaction } from './db.js';
|
|
15
|
+
import { MEMORY_USAGE_DDL } from './schema.js';
|
|
15
16
|
import {
|
|
16
17
|
META_KEYS,
|
|
17
18
|
SCHEMA_VERSION,
|
|
@@ -28,7 +29,18 @@ import {
|
|
|
28
29
|
* drives it with a rehearsal step so the path is never untested.
|
|
29
30
|
* @type {Array<{from:number,to:number,up:(db:import('node:sqlite').DatabaseSync)=>void,verify?:(db:import('node:sqlite').DatabaseSync)=>void}>}
|
|
30
31
|
*/
|
|
31
|
-
export const SHIPPED_MIGRATIONS = [
|
|
32
|
+
export const SHIPPED_MIGRATIONS = [
|
|
33
|
+
{
|
|
34
|
+
from: 1,
|
|
35
|
+
to: 2,
|
|
36
|
+
// v2 adds per-entry usage counters. The table is new and empty, so the
|
|
37
|
+
// migration cannot lose a row; the row-count guard in \`migrate()\` still runs
|
|
38
|
+
// over every existing table and would catch a mistake here.
|
|
39
|
+
up(db) {
|
|
40
|
+
db.exec(MEMORY_USAGE_DDL);
|
|
41
|
+
},
|
|
42
|
+
},
|
|
43
|
+
];
|
|
32
44
|
|
|
33
45
|
/**
|
|
34
46
|
* Whether the file already holds the Loci core table.
|
package/src/host/store/schema.js
CHANGED
|
@@ -8,7 +8,27 @@
|
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
/** Schema version this build writes and expects. */
|
|
11
|
-
export const SCHEMA_VERSION =
|
|
11
|
+
export const SCHEMA_VERSION = 2;
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* DDL for the v2 per-entry usage table.
|
|
15
|
+
*
|
|
16
|
+
* Kept as its own constant because two paths must agree on it: a fresh database
|
|
17
|
+
* (via {@link SCHEMA_SQL}) and an existing one (via the v1→v2 migration). A table
|
|
18
|
+
* that only exists on new installs is the classic way a feature "works on my
|
|
19
|
+
* machine".
|
|
20
|
+
*/
|
|
21
|
+
export const MEMORY_USAGE_DDL = `
|
|
22
|
+
CREATE TABLE IF NOT EXISTS memory_usage (
|
|
23
|
+
id TEXT PRIMARY KEY,
|
|
24
|
+
inject_count INTEGER NOT NULL DEFAULT 0,
|
|
25
|
+
read_count INTEGER NOT NULL DEFAULT 0,
|
|
26
|
+
last_injected_at INTEGER,
|
|
27
|
+
last_read_at INTEGER
|
|
28
|
+
);
|
|
29
|
+
CREATE INDEX IF NOT EXISTS ix_memory_usage_inject ON memory_usage (inject_count DESC);
|
|
30
|
+
CREATE INDEX IF NOT EXISTS ix_memory_usage_read ON memory_usage (read_count DESC);
|
|
31
|
+
`;
|
|
12
32
|
|
|
13
33
|
/** DDL for a fresh database. Order matters: FTS and the head table follow `memory`. */
|
|
14
34
|
export const SCHEMA_SQL = `
|
|
@@ -113,6 +133,9 @@ CREATE TABLE IF NOT EXISTS token_stat (
|
|
|
113
133
|
df INTEGER NOT NULL DEFAULT 0
|
|
114
134
|
);
|
|
115
135
|
|
|
136
|
+
${MEMORY_USAGE_DDL}
|
|
137
|
+
|
|
138
|
+
|
|
116
139
|
CREATE TABLE IF NOT EXISTS request_stat (
|
|
117
140
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
118
141
|
ts INTEGER NOT NULL,
|
package/src/host/store/store.js
CHANGED
|
@@ -829,6 +829,32 @@ export class Store {
|
|
|
829
829
|
);
|
|
830
830
|
}
|
|
831
831
|
|
|
832
|
+
/**
|
|
833
|
+
* How many entries in these scopes are waiting for a human.
|
|
834
|
+
*
|
|
835
|
+
* This is what the injected section reports. Verified-only injection has a
|
|
836
|
+
* failure mode the user cannot see: a fact the model wrote at midnight is simply
|
|
837
|
+
* absent at 9 a.m., and "never saved" looks exactly like "saved but gated". One
|
|
838
|
+
* number in the prompt is enough to tell those apart, and it costs nothing when
|
|
839
|
+
* the queue is empty.
|
|
840
|
+
*
|
|
841
|
+
* @param scopeKeys - explicit scope keys (never a wildcard).
|
|
842
|
+
* @returns the count of candidate and disputed entries.
|
|
843
|
+
*/
|
|
844
|
+
countPending(scopeKeys) {
|
|
845
|
+
if (scopeKeys.length === 0) return 0;
|
|
846
|
+
const marks = scopeKeys.map(() => '?').join(', ');
|
|
847
|
+
return Number(
|
|
848
|
+
this.db
|
|
849
|
+
.prepare(
|
|
850
|
+
`SELECT COUNT(*) AS n FROM memory m
|
|
851
|
+
WHERE m.status IN ('candidate', 'disputed') AND m.deleted_at IS NULL
|
|
852
|
+
AND m.scope_key IN (${marks})`,
|
|
853
|
+
)
|
|
854
|
+
.get(...scopeKeys)?.n ?? 0,
|
|
855
|
+
);
|
|
856
|
+
}
|
|
857
|
+
|
|
832
858
|
/**
|
|
833
859
|
* Verified rules for the rules block, newest last.
|
|
834
860
|
* @param options - `scopeKeys`, `now`, `limit`.
|
|
@@ -1062,6 +1088,20 @@ export class Store {
|
|
|
1062
1088
|
'INSERT INTO request_stat (ts, session_id, scope_key, section_bytes, entries_listed, entries_hidden) VALUES (?, ?, ?, ?, ?, ?)',
|
|
1063
1089
|
)
|
|
1064
1090
|
.run(this.now(), row.sessionId ?? null, row.scopeKey ?? null, row.bytes, row.listed, row.hidden);
|
|
1091
|
+
// Per-entry counters, so "which line earns its bytes" stays answerable after the
|
|
1092
|
+
// aggregate rows rotate out. A listed entry never fetched by id is the dead
|
|
1093
|
+
// weight this product exists to find.
|
|
1094
|
+
if (Array.isArray(row.ids) && row.ids.length > 0) {
|
|
1095
|
+
const bump = this.db.prepare(
|
|
1096
|
+
`INSERT INTO memory_usage (id, inject_count, read_count, last_injected_at)
|
|
1097
|
+
VALUES (?, 1, 0, ?)
|
|
1098
|
+
ON CONFLICT(id) DO UPDATE SET inject_count = inject_count + 1, last_injected_at = excluded.last_injected_at`,
|
|
1099
|
+
);
|
|
1100
|
+
const now = this.now();
|
|
1101
|
+
withTransaction(this.db, () => {
|
|
1102
|
+
for (const id of row.ids) bump.run(id, now);
|
|
1103
|
+
});
|
|
1104
|
+
}
|
|
1065
1105
|
this.db.prepare('DELETE FROM request_stat WHERE ts < ?').run(this.now() - 90 * 86_400_000);
|
|
1066
1106
|
this.audit.append('inject', {
|
|
1067
1107
|
sessionId: row.sessionId ?? null,
|
|
@@ -1072,6 +1112,64 @@ export class Store {
|
|
|
1072
1112
|
});
|
|
1073
1113
|
}
|
|
1074
1114
|
|
|
1115
|
+
/**
|
|
1116
|
+
* Record that one entry was fetched by id.
|
|
1117
|
+
*
|
|
1118
|
+
* Injection counts alone cannot tell "the index line was enough" from "the model
|
|
1119
|
+
* opened it": only this counter can.
|
|
1120
|
+
* @param id - memory id.
|
|
1121
|
+
*/
|
|
1122
|
+
recordRead(id) {
|
|
1123
|
+
if (typeof id !== 'string' || id.length === 0) return;
|
|
1124
|
+
try {
|
|
1125
|
+
this.db
|
|
1126
|
+
.prepare(
|
|
1127
|
+
`INSERT INTO memory_usage (id, inject_count, read_count, last_read_at)
|
|
1128
|
+
VALUES (?, 0, 1, ?)
|
|
1129
|
+
ON CONFLICT(id) DO UPDATE SET read_count = read_count + 1, last_read_at = excluded.last_read_at`,
|
|
1130
|
+
)
|
|
1131
|
+
.run(id, this.now());
|
|
1132
|
+
} catch {
|
|
1133
|
+
// A counter must never fail a read.
|
|
1134
|
+
}
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1137
|
+
/**
|
|
1138
|
+
* Which entries spend prompt budget without being opened.
|
|
1139
|
+
*
|
|
1140
|
+
* @param options - `scopeKeys`, `limit`.
|
|
1141
|
+
* @returns rows sorted by injections desc, reads asc.
|
|
1142
|
+
*/
|
|
1143
|
+
usageReport(options = {}) {
|
|
1144
|
+
const keys = options.scopeKeys ?? [];
|
|
1145
|
+
const limit = Math.max(1, Math.min(200, options.limit ?? 20));
|
|
1146
|
+
const marks = keys.length > 0 ? ` AND m.scope_key IN (${keys.map(() => '?').join(', ')})` : '';
|
|
1147
|
+
const rows = this.db
|
|
1148
|
+
.prepare(
|
|
1149
|
+
`SELECT m.id, m.title, m.kind, h.status,
|
|
1150
|
+
COALESCE(u.inject_count, 0) AS injections,
|
|
1151
|
+
COALESCE(u.read_count, 0) AS reads,
|
|
1152
|
+
u.last_injected_at, u.last_read_at
|
|
1153
|
+
FROM memory m
|
|
1154
|
+
JOIN memory_head h ON h.ord = m.ord
|
|
1155
|
+
LEFT JOIN memory_usage u ON u.id = m.id
|
|
1156
|
+
WHERE m.deleted_at IS NULL${marks}
|
|
1157
|
+
ORDER BY COALESCE(u.inject_count, 0) DESC, COALESCE(u.read_count, 0) ASC
|
|
1158
|
+
LIMIT ?`,
|
|
1159
|
+
)
|
|
1160
|
+
.all(...keys, limit);
|
|
1161
|
+
return rows.map((row) => ({
|
|
1162
|
+
id: row.id,
|
|
1163
|
+
title: row.title,
|
|
1164
|
+
kind: row.kind,
|
|
1165
|
+
status: row.status,
|
|
1166
|
+
injections: Number(row.injections),
|
|
1167
|
+
reads: Number(row.reads),
|
|
1168
|
+
lastInjectedAt: row.last_injected_at === null ? null : Number(row.last_injected_at),
|
|
1169
|
+
lastReadAt: row.last_read_at === null ? null : Number(row.last_read_at),
|
|
1170
|
+
}));
|
|
1171
|
+
}
|
|
1172
|
+
|
|
1075
1173
|
/**
|
|
1076
1174
|
* Record one recall.
|
|
1077
1175
|
* @param row - `query` length, counts, timing, channels.
|
package/src/host/tools/tools.js
CHANGED
|
@@ -333,7 +333,17 @@ export function createTools(deps) {
|
|
|
333
333
|
const id = str(args, 'id', { required: true, maxChars: 64 });
|
|
334
334
|
const revision = int(args, 'revision', { min: 1 });
|
|
335
335
|
const includeHistory = bool(args, 'includeHistory', true);
|
|
336
|
-
|
|
336
|
+
const result = store.get(id, { revision, includeHistory });
|
|
337
|
+
// A read counter is what separates "the index line was enough" from "the model
|
|
338
|
+
// had to open it". It must never fail the read itself.
|
|
339
|
+
if (result?.entry !== undefined) {
|
|
340
|
+
try {
|
|
341
|
+
store.recordRead(result.entry.id);
|
|
342
|
+
} catch {
|
|
343
|
+
/* counting is best effort */
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
return result;
|
|
337
347
|
},
|
|
338
348
|
};
|
|
339
349
|
|