@elvisw/dsh-magic-context 0.0.0-stage → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +23 -0
- package/NOTICE +93 -0
- package/README.md +501 -2
- package/README.zh-CN.md +398 -0
- package/cordis.patch.yml +20 -0
- package/lib/client.js +2753 -0
- package/lib/core/compartments.js +424 -0
- package/lib/core/corpus-store.js +176 -0
- package/lib/core/directive-detect.js +147 -0
- package/lib/core/domain-reconcile.js +394 -0
- package/lib/core/domains.js +127 -0
- package/lib/core/embedding-local.js +247 -0
- package/lib/core/global-autostage.js +223 -0
- package/lib/core/global-domains.js +302 -0
- package/lib/core/journal.js +248 -0
- package/lib/core/llm-deadline.js +53 -0
- package/lib/core/machine-domain.js +106 -0
- package/lib/core/maintenance.js +380 -0
- package/lib/core/memory-bridge.js +818 -0
- package/lib/core/memory-categories.js +84 -0
- package/lib/core/memory-scoring.js +65 -0
- package/lib/core/memory-search.js +414 -0
- package/lib/core/memory-write.js +1176 -0
- package/lib/core/policy.js +111 -0
- package/lib/core/profile-confirmation.js +55 -0
- package/lib/core/profile-review.js +375 -0
- package/lib/core/profile-write.js +283 -0
- package/lib/core/reference-seeds.js +63 -0
- package/lib/core/route-health.js +106 -0
- package/lib/core/stores.js +1021 -0
- package/lib/core/tagger.js +7 -0
- package/lib/core/types.js +15 -0
- package/lib/core/vector-backfill.js +140 -0
- package/lib/core/vector-lane.js +184 -0
- package/lib/dsh/command-surface.js +376 -0
- package/lib/dsh/compaction-engine.js +231 -0
- package/lib/dsh/config-switches.js +56 -0
- package/lib/dsh/config.js +13 -0
- package/lib/dsh/context-pressure.js +36 -0
- package/lib/dsh/drop-landing.js +42 -0
- package/lib/dsh/emergency.js +89 -0
- package/lib/dsh/flush-command.js +148 -0
- package/lib/dsh/global-review.js +412 -0
- package/lib/dsh/historian.js +907 -0
- package/lib/dsh/live-journal.js +81 -0
- package/lib/dsh/maintenance-wiring.js +229 -0
- package/lib/dsh/management-pass.js +564 -0
- package/lib/dsh/memory-lane.js +535 -0
- package/lib/dsh/pre-step.js +675 -0
- package/lib/dsh/project-drift.js +116 -0
- package/lib/dsh/recomp.js +192 -0
- package/lib/dsh/reduce-guidance.js +188 -0
- package/lib/dsh/seams.js +588 -0
- package/lib/dsh/search-hint.js +186 -0
- package/lib/dsh/session-notice.js +100 -0
- package/lib/dsh/status-command.js +75 -0
- package/lib/dsh/telemetry.js +349 -0
- package/lib/dsh/tool-bridge.js +1009 -0
- package/lib/dsh/volatile.js +89 -0
- package/lib/index.js +931 -0
- package/package.json +108 -4
- package/tools/migrate-profile-package-name.mjs +317 -0
- package/vendor/onnx/LICENSE +27 -0
- package/vendor/onnx/ort-wasm-simd-threaded.mjs +59 -0
- package/vendor/onnx/ort-wasm-simd-threaded.wasm +0 -0
- package/vendor/onnx/ort.node.min.mjs +8 -0
|
@@ -0,0 +1,424 @@
|
|
|
1
|
+
import { errText } from "./types.js";
|
|
2
|
+
/** The `[compacted §C<n>§]` placeholder (Magic Context's dropped-then-compacted marker). */
|
|
3
|
+
export function compactedPlaceholder(compartmentId) {
|
|
4
|
+
return `[compacted §C${compartmentId}§]`;
|
|
5
|
+
}
|
|
6
|
+
/** Rough token estimate used for triggering and budgets (chars/4, min 1). */
|
|
7
|
+
export function estimateTokens(text) {
|
|
8
|
+
return estimateTokensFromLength(text.length);
|
|
9
|
+
}
|
|
10
|
+
/** Length-only form for a surface node that already carries `textLength`. */
|
|
11
|
+
export function estimateTokensFromLength(length) {
|
|
12
|
+
return Math.max(1, Math.ceil(length / 4));
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Pick the detail tier for one compartment from its importance and age.
|
|
16
|
+
* Adapter-side approximation of Magic Context's decay math (documented
|
|
17
|
+
* divergence): four fixed age bands, promoted/demoted one step by importance.
|
|
18
|
+
*/
|
|
19
|
+
export function selectTier(importance, ageMs) {
|
|
20
|
+
const bands = ['p1', 'p2', 'p3', 'p4'];
|
|
21
|
+
const age = Math.max(0, ageMs);
|
|
22
|
+
let index = age < 10 * 60_000 ? 0 : age < 2 * 60 * 60_000 ? 1 : age < 24 * 60 * 60_000 ? 2 : 3;
|
|
23
|
+
if (importance >= 80)
|
|
24
|
+
index -= 1;
|
|
25
|
+
else if (importance <= 20)
|
|
26
|
+
index += 1;
|
|
27
|
+
return bands[Math.min(bands.length - 1, Math.max(0, index))];
|
|
28
|
+
}
|
|
29
|
+
/** The text of one compartment at its natural tier. */
|
|
30
|
+
export function tierText(compartment, now) {
|
|
31
|
+
const tier = selectTier(compartment.importance, now - compartment.createdAt);
|
|
32
|
+
return { tier, text: compartment.tiers[tier] };
|
|
33
|
+
}
|
|
34
|
+
/** The header line identifying a compartment in a rendered baseline. */
|
|
35
|
+
export function compartmentHeader(compartment, now) {
|
|
36
|
+
const { tier } = tierText(compartment, now);
|
|
37
|
+
const minutes = Math.max(0, Math.round((now - compartment.createdAt) / 60_000));
|
|
38
|
+
return `§C${compartment.compartmentId}§ [${tier}, importance ${String(compartment.importance)}, ${String(minutes)}m ago]`;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Render one compartment as a baseline block: header, digest, and the
|
|
42
|
+
* extracted facts/events. `mostCompressedTier` bounds fidelity downward (the
|
|
43
|
+
* budget path forces p4 on a compartment whose natural tier is more detailed).
|
|
44
|
+
*/
|
|
45
|
+
export function renderCompartment(compartment, now, mostCompressedTier) {
|
|
46
|
+
const natural = tierText(compartment, now);
|
|
47
|
+
const order = ['p1', 'p2', 'p3', 'p4'];
|
|
48
|
+
const forced = mostCompressedTier !== undefined
|
|
49
|
+
&& order.indexOf(mostCompressedTier) > order.indexOf(natural.tier);
|
|
50
|
+
const tier = forced ? mostCompressedTier : natural.tier;
|
|
51
|
+
const lines = [`${compartmentHeader(compartment, now)}`, compartment.tiers[tier]];
|
|
52
|
+
if (compartment.facts.length > 0) {
|
|
53
|
+
lines.push(`facts: ${compartment.facts.map(fact => fact.category.length === 0 ? fact.content : `[${fact.category}] ${fact.content}`).join(' | ')}`);
|
|
54
|
+
}
|
|
55
|
+
if (compartment.events.length > 0)
|
|
56
|
+
lines.push(`events: ${compartment.events.join(' | ')}`);
|
|
57
|
+
return lines.join('\n');
|
|
58
|
+
}
|
|
59
|
+
/** The in-band sentinels wrapping the injected baseline inside its node. */
|
|
60
|
+
export const BASELINE_OPEN = '<magic-context>';
|
|
61
|
+
export const BASELINE_CLOSE = '</magic-context>';
|
|
62
|
+
/**
|
|
63
|
+
* Magic Context's visibility sentinel: the mutation only flipped the row's
|
|
64
|
+
* cross-project visibility, the content is unchanged (§6.2).
|
|
65
|
+
*/
|
|
66
|
+
export const MEMORY_VISIBILITY_SENTINEL = '__mc_visibility__';
|
|
67
|
+
/** MC inject-compartments.ts:1142 — forced rows per delta render, hard cap. */
|
|
68
|
+
export const MAX_FORCED_MEMORIES_PER_DELTA = 10;
|
|
69
|
+
export const MEMORY_UPDATES_OPEN = '<memory-updates>';
|
|
70
|
+
export const MEMORY_UPDATES_CLOSE = '</memory-updates>';
|
|
71
|
+
export const MEMORY_UPDATES_HEADER = 'These memories changed since the snapshot below — trust these:';
|
|
72
|
+
/** Attribute escaping (MC `memory_render.rs:36-47`): `& < > "`. */
|
|
73
|
+
function escapeAttribute(value) {
|
|
74
|
+
return value.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"');
|
|
75
|
+
}
|
|
76
|
+
/** Text escaping (MC `memory_render.rs:36-47`): `& < >`. */
|
|
77
|
+
function escapeText(value) {
|
|
78
|
+
return value.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Render Magic Context's `<memory-updates>` block, or `''` when there is
|
|
82
|
+
* nothing to say. The header alone means "empty" (§6.4), so an empty delta
|
|
83
|
+
* contributes zero bytes — the cache-prefix invariant depends on it.
|
|
84
|
+
*
|
|
85
|
+
* @param options.renderedIds - ids whose content is already rendered in the
|
|
86
|
+
* sibling `<new-memories>` block of the same m[1] region. An `<updated>`
|
|
87
|
+
* line for one of them is dropped so no memory's content is rendered twice;
|
|
88
|
+
* pointer lines (`<superseded>`/`<removed>`) are still emitted.
|
|
89
|
+
* @param options.baselineIds - ids rendered in the m[0] baseline. When given,
|
|
90
|
+
* mutations for any other target are dropped entirely (MC
|
|
91
|
+
* inject-compartments.ts:2843-2848): the model never saw that id, so an
|
|
92
|
+
* `<updated>`/`<removed>` pointer for it would be phantom.
|
|
93
|
+
*/
|
|
94
|
+
export function renderMemoryUpdates(updates, options = {}) {
|
|
95
|
+
const lines = [MEMORY_UPDATES_HEADER];
|
|
96
|
+
const emitted = new Set();
|
|
97
|
+
// MC inject-compartments.ts:2822-2869 — rows the model never saw in m[0]
|
|
98
|
+
// can still be FORCED into `<new-memories>`: a `superseded` replacement and
|
|
99
|
+
// a visibility-flipped target are eligible even when out of the baseline.
|
|
100
|
+
const forcedIds = new Set();
|
|
101
|
+
for (const update of [...updates].sort((left, right) => left.id - right.id)) {
|
|
102
|
+
const target = update.targetMemoryId;
|
|
103
|
+
// When the caller supplies the eligible set it is authoritative (MC);
|
|
104
|
+
// without it a legacy caller cannot be checked — stay permissive.
|
|
105
|
+
const eligible = options.eligibleIds === undefined || options.eligibleIds.has(target);
|
|
106
|
+
if (update.mutationType === 'superseded') {
|
|
107
|
+
// MC order (:2825-2841): the replacement is forced FIRST (even when the
|
|
108
|
+
// target itself was never in the baseline), then the pointer line only
|
|
109
|
+
// renders when the target WAS in the baseline — otherwise it is phantom.
|
|
110
|
+
const replacement = update.supersededById;
|
|
111
|
+
const replacementVisible = options.eligibleIds === undefined
|
|
112
|
+
|| (replacement !== undefined && options.eligibleIds.has(replacement));
|
|
113
|
+
const replacementResolvable = update.replacementResolvable === true && replacementVisible;
|
|
114
|
+
if (replacement !== undefined && replacementResolvable
|
|
115
|
+
&& options.baselineIds?.has(replacement) !== true) {
|
|
116
|
+
forcedIds.add(replacement);
|
|
117
|
+
}
|
|
118
|
+
if (options.baselineIds !== undefined && !options.baselineIds.has(target))
|
|
119
|
+
continue;
|
|
120
|
+
emitted.add(target);
|
|
121
|
+
lines.push(replacement !== undefined && replacementResolvable
|
|
122
|
+
? ` <superseded id="${String(target)}" by="${String(replacement)}"/>`
|
|
123
|
+
: ` <removed id="${String(target)}"/>`);
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
if (options.baselineIds !== undefined && !options.baselineIds.has(target)) {
|
|
127
|
+
// Out-of-baseline target: a visibility flip still surfaces the row
|
|
128
|
+
// (forced), anything else stays silent (MC :2843-2848).
|
|
129
|
+
if (update.mutationType === 'update'
|
|
130
|
+
&& update.category === MEMORY_VISIBILITY_SENTINEL
|
|
131
|
+
&& eligible) {
|
|
132
|
+
forcedIds.add(target);
|
|
133
|
+
}
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
// Coalescing already keeps one row per target upstream; this is the
|
|
137
|
+
// defensive second gate ("the same change never appears twice").
|
|
138
|
+
if (emitted.has(target))
|
|
139
|
+
continue;
|
|
140
|
+
if (update.mutationType !== 'update') {
|
|
141
|
+
emitted.add(target);
|
|
142
|
+
lines.push(` <removed id="${String(target)}"/>`);
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
if (options.renderedIds?.has(target) === true)
|
|
146
|
+
continue;
|
|
147
|
+
const content = update.content;
|
|
148
|
+
if (content === undefined || content.length === 0) {
|
|
149
|
+
emitted.add(target);
|
|
150
|
+
lines.push(` <removed id="${String(target)}"/>`);
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
emitted.add(target);
|
|
154
|
+
const category = update.category;
|
|
155
|
+
// MC omits the attribute for an empty category and for the visibility
|
|
156
|
+
// sentinel (it is not a real category) — §6.4.
|
|
157
|
+
const hasCategory = category !== undefined
|
|
158
|
+
&& category.length > 0
|
|
159
|
+
&& category !== MEMORY_VISIBILITY_SENTINEL;
|
|
160
|
+
lines.push(hasCategory
|
|
161
|
+
? ` <updated id="${String(target)}" category="${escapeAttribute(category)}">${escapeText(content)}</updated>`
|
|
162
|
+
: ` <updated id="${String(target)}">${escapeText(content)}</updated>`);
|
|
163
|
+
}
|
|
164
|
+
const forcedMemoryIds = [...forcedIds].sort((left, right) => left - right)
|
|
165
|
+
.slice(0, MAX_FORCED_MEMORIES_PER_DELTA);
|
|
166
|
+
if (lines.length === 1)
|
|
167
|
+
return { block: '', count: 0, forcedMemoryIds };
|
|
168
|
+
return {
|
|
169
|
+
block: [MEMORY_UPDATES_OPEN, ...lines, MEMORY_UPDATES_CLOSE].join('\n'),
|
|
170
|
+
count: lines.length - 1,
|
|
171
|
+
forcedMemoryIds,
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Compose the m[0] baseline / m[1] delta block. m[0] is frozen history
|
|
176
|
+
* (compartments created at or before the last hard rebuild, plus project
|
|
177
|
+
* memories); m[1] is the delta accumulated since. Newest first inside m[1],
|
|
178
|
+
* oldest first inside m[0] so the model reads a stable, chronological spine.
|
|
179
|
+
*
|
|
180
|
+
* @param budgetTokens - soft cap; older m[0] compartments degrade to their
|
|
181
|
+
* compressed tier and then drop out when the budget is exhausted. m[1] is
|
|
182
|
+
* never truncated (it is the part that must not be lost).
|
|
183
|
+
*/
|
|
184
|
+
export function renderBaseline(input) {
|
|
185
|
+
const sections = [];
|
|
186
|
+
const profileLines = input.profileLines ?? [];
|
|
187
|
+
if (profileLines.length > 0) {
|
|
188
|
+
sections.push(['<user-profile>', ...profileLines, '</user-profile>'].join('\n'));
|
|
189
|
+
}
|
|
190
|
+
if (input.memories.length > 0) {
|
|
191
|
+
sections.push(['<project-memories>', ...input.memories, '</project-memories>'].join('\n'));
|
|
192
|
+
}
|
|
193
|
+
if (input.baseline.length > 0) {
|
|
194
|
+
const ordered = [...input.baseline].sort((left, right) => left.createdAt - right.createdAt);
|
|
195
|
+
const rendered = [];
|
|
196
|
+
let used = 0;
|
|
197
|
+
for (const compartment of ordered) {
|
|
198
|
+
const full = renderCompartment(compartment, input.now);
|
|
199
|
+
const fullCost = estimateTokens(full);
|
|
200
|
+
if (used + fullCost <= input.budgetTokens) {
|
|
201
|
+
rendered.push(full);
|
|
202
|
+
used += fullCost;
|
|
203
|
+
continue;
|
|
204
|
+
}
|
|
205
|
+
const compressed = renderCompartment(compartment, input.now, 'p4');
|
|
206
|
+
const compressedCost = estimateTokens(compressed);
|
|
207
|
+
if (used + compressedCost <= input.budgetTokens) {
|
|
208
|
+
rendered.push(compressed);
|
|
209
|
+
used += compressedCost;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
if (rendered.length > 0)
|
|
213
|
+
sections.push(['<compartment-baseline>', ...rendered, '</compartment-baseline>'].join('\n'));
|
|
214
|
+
}
|
|
215
|
+
// m[1] begins here: every section below is delta-shaped and may change without
|
|
216
|
+
// touching a single byte of the sections above (the cache-prefix invariant).
|
|
217
|
+
const memoryUpdates = renderMemoryUpdates(input.memoryUpdates ?? [], {
|
|
218
|
+
...input.newMemoryIds === undefined ? {} : { renderedIds: new Set(input.newMemoryIds) },
|
|
219
|
+
...input.baselineIds === undefined ? {} : { baselineIds: new Set(input.baselineIds) },
|
|
220
|
+
...input.eligibleIds === undefined ? {} : { eligibleIds: new Set(input.eligibleIds) },
|
|
221
|
+
});
|
|
222
|
+
if (memoryUpdates.block.length > 0)
|
|
223
|
+
sections.push(memoryUpdates.block);
|
|
224
|
+
const forcedMemoryLines = input.forcedMemoryLines ?? [];
|
|
225
|
+
const budgetedNewMemories = input.newMemories ?? [];
|
|
226
|
+
const newMemories = budgetedNewMemories.length > 0 || forcedMemoryLines.length > 0
|
|
227
|
+
? [...budgetedNewMemories, ...forcedMemoryLines]
|
|
228
|
+
: [];
|
|
229
|
+
if (newMemories.length > 0) {
|
|
230
|
+
sections.push(['<new-memories>', ...newMemories, '</new-memories>'].join('\n'));
|
|
231
|
+
}
|
|
232
|
+
const newProfileLines = input.newProfileLines ?? [];
|
|
233
|
+
if (newProfileLines.length > 0) {
|
|
234
|
+
sections.push(['<new-user-profile>', ...newProfileLines, '</new-user-profile>'].join('\n'));
|
|
235
|
+
}
|
|
236
|
+
if (input.delta.length > 0) {
|
|
237
|
+
const ordered = [...input.delta].sort((left, right) => right.createdAt - left.createdAt);
|
|
238
|
+
sections.push([
|
|
239
|
+
'<compartment-delta>',
|
|
240
|
+
...ordered.map(compartment => renderCompartment(compartment, input.now)),
|
|
241
|
+
'</compartment-delta>',
|
|
242
|
+
].join('\n'));
|
|
243
|
+
}
|
|
244
|
+
if (sections.length === 0)
|
|
245
|
+
return '';
|
|
246
|
+
return [BASELINE_OPEN, ...sections, BASELINE_CLOSE].join('\n');
|
|
247
|
+
}
|
|
248
|
+
/** Index of the baseline block inside a message's text, if it carries one. */
|
|
249
|
+
export function findBaselineRange(text) {
|
|
250
|
+
const start = text.indexOf(BASELINE_OPEN);
|
|
251
|
+
if (start === -1)
|
|
252
|
+
return undefined;
|
|
253
|
+
const close = text.indexOf(BASELINE_CLOSE, start + BASELINE_OPEN.length);
|
|
254
|
+
if (close === -1)
|
|
255
|
+
return undefined;
|
|
256
|
+
return { start, end: close + BASELINE_CLOSE.length };
|
|
257
|
+
}
|
|
258
|
+
/** Strip any previously injected baseline block from a message text. */
|
|
259
|
+
export function stripBaseline(text) {
|
|
260
|
+
const range = findBaselineRange(text);
|
|
261
|
+
if (range === undefined)
|
|
262
|
+
return text;
|
|
263
|
+
return (text.slice(0, range.start) + text.slice(range.end)).trim();
|
|
264
|
+
}
|
|
265
|
+
export class HistorianOutputError extends Error {
|
|
266
|
+
constructor(message) {
|
|
267
|
+
super(`magic-context historian output rejected: ${message}`);
|
|
268
|
+
this.name = 'HistorianOutputError';
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
function asTier(value, key) {
|
|
272
|
+
if (typeof value !== 'string' || value.trim().length === 0) {
|
|
273
|
+
throw new HistorianOutputError(`summary.${key} must be a non-empty string`);
|
|
274
|
+
}
|
|
275
|
+
return value.trim();
|
|
276
|
+
}
|
|
277
|
+
function asStringArray(value) {
|
|
278
|
+
if (value === undefined || value === null)
|
|
279
|
+
return [];
|
|
280
|
+
if (!Array.isArray(value))
|
|
281
|
+
throw new HistorianOutputError('facts/events must be arrays');
|
|
282
|
+
return value.filter((item) => typeof item === 'string' && item.trim().length > 0)
|
|
283
|
+
.map(item => item.trim());
|
|
284
|
+
}
|
|
285
|
+
/**
|
|
286
|
+
* Facts as `{category, content}` (Magic Context's promotable shape).
|
|
287
|
+
*
|
|
288
|
+
* Legacy plain strings are still accepted by the HISTORIAN contract — a model
|
|
289
|
+
* that returns `["..."]` must not fail the whole run — but they carry no
|
|
290
|
+
* category and are therefore not promotable; they are kept as `category: ''`
|
|
291
|
+
* so the compartment still renders them and the journal can explain why they
|
|
292
|
+
* were skipped. Unknown categories are kept verbatim for the same reason:
|
|
293
|
+
* Magic Context drops them at promotion time, not at parse time.
|
|
294
|
+
*/
|
|
295
|
+
function asFactArray(value) {
|
|
296
|
+
if (value === undefined || value === null)
|
|
297
|
+
return [];
|
|
298
|
+
if (!Array.isArray(value))
|
|
299
|
+
throw new HistorianOutputError('facts must be an array');
|
|
300
|
+
const facts = [];
|
|
301
|
+
for (const item of value) {
|
|
302
|
+
if (typeof item === 'string') {
|
|
303
|
+
const content = item.trim();
|
|
304
|
+
if (content.length > 0)
|
|
305
|
+
facts.push({ category: '', content });
|
|
306
|
+
continue;
|
|
307
|
+
}
|
|
308
|
+
if (item !== null && typeof item === 'object') {
|
|
309
|
+
const record = item;
|
|
310
|
+
const content = typeof record.content === 'string' ? record.content.trim() : '';
|
|
311
|
+
if (content.length === 0)
|
|
312
|
+
continue;
|
|
313
|
+
const category = typeof record.category === 'string' ? record.category.trim() : '';
|
|
314
|
+
facts.push({ category, content });
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
return facts;
|
|
318
|
+
}
|
|
319
|
+
/**
|
|
320
|
+
* A one-line description of why a JSON-ish text does not balance, for the
|
|
321
|
+
* failure journal.
|
|
322
|
+
*
|
|
323
|
+
* All four live `invalid-output` failures (2026-10-08..10-10) reported a JSON
|
|
324
|
+
* syntax error at the LAST character of the output — position = `outputChars - 1`
|
|
325
|
+
* — while the tail the journal kept was a well-formed ending (`...".` then `}`).
|
|
326
|
+
* A parser only trips on the final `}` when something earlier never closed, so
|
|
327
|
+
* the text looked intact at both ends and the defect sat in the middle, exactly
|
|
328
|
+
* where the journal kept nothing. Counting openers against closers, ignoring
|
|
329
|
+
* both inside string literals, turns that into a recorded fact instead of a
|
|
330
|
+
* guess about the model.
|
|
331
|
+
*/
|
|
332
|
+
export function describeJsonImbalance(text) {
|
|
333
|
+
const stack = [];
|
|
334
|
+
let inString = false;
|
|
335
|
+
let escaped = false;
|
|
336
|
+
for (const char of text) {
|
|
337
|
+
if (inString) {
|
|
338
|
+
if (escaped)
|
|
339
|
+
escaped = false;
|
|
340
|
+
else if (char === '\\')
|
|
341
|
+
escaped = true;
|
|
342
|
+
else if (char === '"')
|
|
343
|
+
inString = false;
|
|
344
|
+
continue;
|
|
345
|
+
}
|
|
346
|
+
if (char === '"') {
|
|
347
|
+
inString = true;
|
|
348
|
+
continue;
|
|
349
|
+
}
|
|
350
|
+
if (char === '{' || char === '[') {
|
|
351
|
+
stack.push(char);
|
|
352
|
+
continue;
|
|
353
|
+
}
|
|
354
|
+
if (char === '}' || char === ']') {
|
|
355
|
+
const want = char === '}' ? '{' : '[';
|
|
356
|
+
if (stack.length === 0)
|
|
357
|
+
return `unexpected '${char}'`;
|
|
358
|
+
if (stack[stack.length - 1] !== want)
|
|
359
|
+
return `'${stack[stack.length - 1]}' closed by '${char}'`;
|
|
360
|
+
stack.pop();
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
if (inString)
|
|
364
|
+
return 'unterminated string';
|
|
365
|
+
if (stack.length === 0)
|
|
366
|
+
return '';
|
|
367
|
+
const closers = new Map();
|
|
368
|
+
for (const open of stack) {
|
|
369
|
+
const close = open === '{' ? '}' : ']';
|
|
370
|
+
closers.set(close, (closers.get(close) ?? 0) + 1);
|
|
371
|
+
}
|
|
372
|
+
return `unclosed ${[...closers.entries()].map(([close, count]) => `'${close}'×${String(count)}`).join('+')}`;
|
|
373
|
+
}
|
|
374
|
+
/**
|
|
375
|
+
* Parse and validate a historian response. Throws {@link HistorianOutputError}
|
|
376
|
+
* on any contract violation — the run then fails closed (no compartment row,
|
|
377
|
+
* no mutation) rather than storing a half-digest.
|
|
378
|
+
*/
|
|
379
|
+
export function parseHistorianOutput(text) {
|
|
380
|
+
const open = text.indexOf('{');
|
|
381
|
+
const close = text.lastIndexOf('}');
|
|
382
|
+
if (open === -1 || close <= open) {
|
|
383
|
+
throw new HistorianOutputError('no JSON object found in the response');
|
|
384
|
+
}
|
|
385
|
+
let parsed;
|
|
386
|
+
try {
|
|
387
|
+
parsed = JSON.parse(text.slice(open, close + 1));
|
|
388
|
+
}
|
|
389
|
+
catch (error) {
|
|
390
|
+
const imbalance = describeJsonImbalance(text.slice(open, close + 1));
|
|
391
|
+
throw new HistorianOutputError(`invalid JSON (${errText(error)})${imbalance === '' ? '' : ` [${imbalance}]`}`);
|
|
392
|
+
}
|
|
393
|
+
if (parsed === null || typeof parsed !== 'object') {
|
|
394
|
+
throw new HistorianOutputError('response is not a JSON object');
|
|
395
|
+
}
|
|
396
|
+
const record = parsed;
|
|
397
|
+
const summary = record.summary;
|
|
398
|
+
if (summary === null || typeof summary !== 'object') {
|
|
399
|
+
throw new HistorianOutputError('summary must be an object with p1..p4');
|
|
400
|
+
}
|
|
401
|
+
const tiers = summary;
|
|
402
|
+
const importanceRaw = record.importance;
|
|
403
|
+
const importance = typeof importanceRaw === 'number' && Number.isFinite(importanceRaw)
|
|
404
|
+
? Math.min(100, Math.max(0, Math.round(importanceRaw)))
|
|
405
|
+
: importanceRaw === undefined
|
|
406
|
+
? 50
|
|
407
|
+
: undefined;
|
|
408
|
+
if (importance === undefined)
|
|
409
|
+
throw new HistorianOutputError('importance must be a number when present');
|
|
410
|
+
return {
|
|
411
|
+
tiers: {
|
|
412
|
+
p1: asTier(tiers.p1, 'p1'),
|
|
413
|
+
p2: asTier(tiers.p2, 'p2'),
|
|
414
|
+
p3: asTier(tiers.p3, 'p3'),
|
|
415
|
+
p4: asTier(tiers.p4, 'p4'),
|
|
416
|
+
},
|
|
417
|
+
importance,
|
|
418
|
+
facts: asFactArray(record.facts),
|
|
419
|
+
events: asStringArray(record.events),
|
|
420
|
+
// WP5: absent on every digest written before the field existed, and on any
|
|
421
|
+
// model that follows the instruction by omitting an empty array.
|
|
422
|
+
userObservations: asStringArray(record.userObservations),
|
|
423
|
+
};
|
|
424
|
+
}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Own-corpus bootstrap — the plugin's memory database, owned end to end.
|
|
3
|
+
*
|
|
4
|
+
* Cutover (user ruling 2026-10-13): before small-scale distribution the
|
|
5
|
+
* adapter stops sharing upstream Magic Context's `context.db`. Dual-writer
|
|
6
|
+
* conflicts were structural, not incidental — upstream releases near-daily
|
|
7
|
+
* (0.47/0.48/0.49 within four days), the fence had zero version margin, and
|
|
8
|
+
* MC 0.48's indexCoordination treats every non-MC connection as a foreign
|
|
9
|
+
* body. The corpus now lives in the plugin's OWN data dir; the legacy file is
|
|
10
|
+
* read exactly once, by the importer (2026-10-13, retired after the run; the
|
|
11
|
+
* recipe lives in git history at 5c45d04e6), and
|
|
12
|
+
* never touched again. Upstream is followed for FEATURES only, never for
|
|
13
|
+
* storage compatibility.
|
|
14
|
+
*
|
|
15
|
+
* Schema lineage: the shape is inherited verbatim from the upstream v95
|
|
16
|
+
* store the corpus was imported from (validated against the live DDL
|
|
17
|
+
* 2026-10-13), so `MEMORY_SCHEMA_MAX_VERSION` keeps guarding the SAME
|
|
18
|
+
* numbering — but from this module on, every new `schema_migrations` row is
|
|
19
|
+
* OURS: a version above the ceiling can only mean this build is older than
|
|
20
|
+
* the corpus, which must fence exactly as before.
|
|
21
|
+
*
|
|
22
|
+
* Table set: exactly what the openers touch (memory-bridge / memory-write /
|
|
23
|
+
* profile-write / memory-search / vector-lane), nothing more. Upstream
|
|
24
|
+
* subsystems the adapter never reads (workspaces, authority guards, tags,
|
|
25
|
+
* session_meta, primers, git_commits_fts …) are deliberately absent:
|
|
26
|
+
* - `workspaces` absence is by design — domain resolution reads the
|
|
27
|
+
* adapter's own domains tables (design v2, no workspace takeover);
|
|
28
|
+
* - authority-guard triggers protected upstream's module-managed projects;
|
|
29
|
+
* no such modules exist here, so the writer's hasAuthority=false path is
|
|
30
|
+
* the honest one.
|
|
31
|
+
*/
|
|
32
|
+
import { existsSync, mkdirSync, statSync } from 'node:fs';
|
|
33
|
+
import { dirname } from 'node:path';
|
|
34
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
35
|
+
/** The version this build creates and understands. Own lineage, inherited shape. */
|
|
36
|
+
export const CORPUS_SCHEMA_VERSION = 95;
|
|
37
|
+
/** Marker row for a corpus imported from the legacy shared store. The main
|
|
38
|
+
* fence lane reads `MAX(version) WHERE version < 10000`, so sidecar markers
|
|
39
|
+
* at ≥ 10000 never influence it. */
|
|
40
|
+
export const CORPUS_IMPORT_MARKER_VERSION = 10_001;
|
|
41
|
+
const SCHEMA_DDL = `
|
|
42
|
+
CREATE TABLE schema_migrations (
|
|
43
|
+
version INTEGER PRIMARY KEY,
|
|
44
|
+
name TEXT,
|
|
45
|
+
applied_at INTEGER NOT NULL DEFAULT 0
|
|
46
|
+
);
|
|
47
|
+
|
|
48
|
+
CREATE TABLE memories (
|
|
49
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
50
|
+
project_path TEXT NOT NULL,
|
|
51
|
+
category TEXT NOT NULL,
|
|
52
|
+
content TEXT NOT NULL,
|
|
53
|
+
normalized_hash TEXT NOT NULL,
|
|
54
|
+
importance INTEGER,
|
|
55
|
+
scope TEXT NOT NULL DEFAULT 'project',
|
|
56
|
+
shareable INTEGER NOT NULL DEFAULT 0,
|
|
57
|
+
source_session_id TEXT,
|
|
58
|
+
source_type TEXT DEFAULT 'historian',
|
|
59
|
+
seen_count INTEGER DEFAULT 1,
|
|
60
|
+
retrieval_count INTEGER DEFAULT 0,
|
|
61
|
+
first_seen_at INTEGER NOT NULL,
|
|
62
|
+
created_at INTEGER NOT NULL,
|
|
63
|
+
updated_at INTEGER NOT NULL,
|
|
64
|
+
last_seen_at INTEGER NOT NULL,
|
|
65
|
+
last_retrieved_at INTEGER,
|
|
66
|
+
status TEXT DEFAULT 'active',
|
|
67
|
+
expires_at INTEGER,
|
|
68
|
+
verification_status TEXT DEFAULT 'unverified',
|
|
69
|
+
verified_at INTEGER,
|
|
70
|
+
classified_at INTEGER,
|
|
71
|
+
superseded_by_memory_id INTEGER,
|
|
72
|
+
merged_from TEXT,
|
|
73
|
+
metadata_json TEXT,
|
|
74
|
+
mural_cue TEXT,
|
|
75
|
+
mural_cue_hash TEXT,
|
|
76
|
+
mural_cue_at INTEGER,
|
|
77
|
+
mural_cue_rejection_count INTEGER NOT NULL DEFAULT 0,
|
|
78
|
+
UNIQUE(project_path, category, normalized_hash)
|
|
79
|
+
);
|
|
80
|
+
CREATE INDEX idx_memories_project_status_category ON memories(project_path, status, category);
|
|
81
|
+
CREATE INDEX idx_memories_project_status_expires ON memories(project_path, status, expires_at);
|
|
82
|
+
CREATE INDEX idx_memories_project_category_hash ON memories(project_path, category, normalized_hash);
|
|
83
|
+
|
|
84
|
+
CREATE VIRTUAL TABLE memories_fts USING fts5(
|
|
85
|
+
content, category, content='memories', content_rowid='id', tokenize='porter unicode61');
|
|
86
|
+
|
|
87
|
+
CREATE TRIGGER memories_ai AFTER INSERT ON memories BEGIN
|
|
88
|
+
INSERT INTO memories_fts(rowid, content, category) VALUES (new.id, new.content, new.category);
|
|
89
|
+
END;
|
|
90
|
+
CREATE TRIGGER memories_ad AFTER DELETE ON memories BEGIN
|
|
91
|
+
INSERT INTO memories_fts(memories_fts, rowid, content, category) VALUES ('delete', old.id, old.content, old.category);
|
|
92
|
+
END;
|
|
93
|
+
CREATE TRIGGER memories_au AFTER UPDATE ON memories BEGIN
|
|
94
|
+
INSERT INTO memories_fts(memories_fts, rowid, content, category) VALUES ('delete', old.id, old.content, old.category);
|
|
95
|
+
INSERT INTO memories_fts(rowid, content, category) VALUES (new.id, new.content, new.category);
|
|
96
|
+
END;
|
|
97
|
+
|
|
98
|
+
CREATE TABLE memory_embeddings (
|
|
99
|
+
memory_id INTEGER NOT NULL REFERENCES memories(id) ON DELETE CASCADE,
|
|
100
|
+
embedding BLOB NOT NULL,
|
|
101
|
+
model_id TEXT NOT NULL,
|
|
102
|
+
PRIMARY KEY(memory_id, model_id)
|
|
103
|
+
);
|
|
104
|
+
|
|
105
|
+
CREATE TABLE memory_mutation_log (
|
|
106
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
107
|
+
project_path TEXT NOT NULL,
|
|
108
|
+
mutation_type TEXT NOT NULL CHECK (mutation_type IN ('archive', 'delete', 'update', 'superseded')),
|
|
109
|
+
target_memory_id INTEGER NOT NULL,
|
|
110
|
+
superseded_by_id INTEGER,
|
|
111
|
+
category TEXT,
|
|
112
|
+
new_content TEXT,
|
|
113
|
+
queued_at INTEGER NOT NULL
|
|
114
|
+
);
|
|
115
|
+
CREATE INDEX idx_memory_mutation_log_project ON memory_mutation_log(project_path, id);
|
|
116
|
+
CREATE INDEX idx_memory_mutation_log_visibility ON memory_mutation_log(project_path, category, id, target_memory_id);
|
|
117
|
+
CREATE INDEX idx_memory_mutation_log_target ON memory_mutation_log(project_path, target_memory_id, id);
|
|
118
|
+
|
|
119
|
+
CREATE TABLE project_state (
|
|
120
|
+
project_path TEXT PRIMARY KEY,
|
|
121
|
+
project_memory_epoch INTEGER NOT NULL DEFAULT 0,
|
|
122
|
+
project_user_profile_version INTEGER NOT NULL DEFAULT 0,
|
|
123
|
+
updated_at INTEGER NOT NULL DEFAULT 0
|
|
124
|
+
);
|
|
125
|
+
|
|
126
|
+
CREATE TABLE user_memories (
|
|
127
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
128
|
+
content TEXT NOT NULL,
|
|
129
|
+
status TEXT NOT NULL DEFAULT 'active',
|
|
130
|
+
promoted_at INTEGER NOT NULL,
|
|
131
|
+
source_candidate_ids TEXT DEFAULT '[]',
|
|
132
|
+
created_at INTEGER NOT NULL,
|
|
133
|
+
updated_at INTEGER NOT NULL,
|
|
134
|
+
source_candidate_provenance TEXT
|
|
135
|
+
);
|
|
136
|
+
|
|
137
|
+
CREATE TABLE user_memory_candidates (
|
|
138
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
139
|
+
content TEXT NOT NULL,
|
|
140
|
+
session_id TEXT NOT NULL,
|
|
141
|
+
source_compartment_start INTEGER,
|
|
142
|
+
source_compartment_end INTEGER,
|
|
143
|
+
created_at INTEGER NOT NULL
|
|
144
|
+
);
|
|
145
|
+
`;
|
|
146
|
+
/**
|
|
147
|
+
* Make sure an own-corpus store exists at `dbPath`. Idempotent, fail-open:
|
|
148
|
+
* every non-`exists` outcome still lets the caller open the store and hit
|
|
149
|
+
* the same refusals the openers would have raised anyway. An EXISTING file
|
|
150
|
+
* is never touched — an operator-pointed legacy store keeps its shape and
|
|
151
|
+
* the fence keeps guarding it.
|
|
152
|
+
*/
|
|
153
|
+
export function ensureOwnCorpusStore(dbPath) {
|
|
154
|
+
try {
|
|
155
|
+
const existed = existsSync(dbPath);
|
|
156
|
+
if (existed && statSync(dbPath).size > 0)
|
|
157
|
+
return { outcome: 'exists' };
|
|
158
|
+
mkdirSync(dirname(dbPath), { recursive: true });
|
|
159
|
+
const db = new DatabaseSync(dbPath);
|
|
160
|
+
try {
|
|
161
|
+
db.exec('PRAGMA journal_mode = WAL');
|
|
162
|
+
db.exec('BEGIN');
|
|
163
|
+
db.exec(SCHEMA_DDL);
|
|
164
|
+
db.prepare('INSERT INTO schema_migrations(version, name, applied_at) VALUES(?, ?, ?)')
|
|
165
|
+
.run(CORPUS_SCHEMA_VERSION, 'own-corpus init (inherited upstream v95 shape)', Date.now());
|
|
166
|
+
db.exec('COMMIT');
|
|
167
|
+
}
|
|
168
|
+
finally {
|
|
169
|
+
db.close();
|
|
170
|
+
}
|
|
171
|
+
return { outcome: existed ? 'repaired-empty' : 'created', detail: `v${String(CORPUS_SCHEMA_VERSION)}` };
|
|
172
|
+
}
|
|
173
|
+
catch (error) {
|
|
174
|
+
return { outcome: 'failed', detail: error instanceof Error ? error.message : String(error) };
|
|
175
|
+
}
|
|
176
|
+
}
|