@dreamforgestudiollc/emet 0.0.0-stage → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +136 -0
- package/CHANGELOG.md +296 -0
- package/CHARTER.md +125 -0
- package/LICENSE +22 -0
- package/README.md +783 -2
- package/docs/AUTHORSHIP.md +26 -0
- package/docs/BENCHMARKS.md +261 -0
- package/docs/BOOTSTRAP.md +127 -0
- package/docs/BOTS.md +79 -0
- package/docs/COVERAGE.md +64 -0
- package/docs/DEPLOY.md +246 -0
- package/docs/DISCIPLINES.md +29 -0
- package/docs/GUARDRAILS-AND-MONORAILS.md +107 -0
- package/docs/GUIDED-SETUP.md +296 -0
- package/docs/PERMISSIONS.md +118 -0
- package/docs/ROADMAP.md +59 -0
- package/docs/SETUP.md +259 -0
- package/guides/NEURODIVERGENT_SUPPORT.md +278 -0
- package/package.json +70 -4
- package/server.json +39 -0
- package/src/aleph.js +395 -0
- package/src/annotations.js +135 -0
- package/src/approve-throttle.js +54 -0
- package/src/atlas-role.js +58 -0
- package/src/conformance.js +688 -0
- package/src/content_analyzer.js +450 -0
- package/src/database.js +811 -0
- package/src/disciplines.js +26 -0
- package/src/dispatch.js +279 -0
- package/src/env.js +52 -0
- package/src/floor.js +187 -0
- package/src/gaps.js +476 -0
- package/src/guides.js +48 -0
- package/src/http-remote.js +269 -0
- package/src/index.js +390 -0
- package/src/invites.js +167 -0
- package/src/layers.js +449 -0
- package/src/lessons-to-checks.js +366 -0
- package/src/logger.js +299 -0
- package/src/metadata.js +61 -0
- package/src/module-path.js +32 -0
- package/src/mongo-connect.js +48 -0
- package/src/named-items.js +208 -0
- package/src/oauth-provider.js +838 -0
- package/src/persons.js +116 -0
- package/src/provenance.js +257 -0
- package/src/redact.js +231 -0
- package/src/request-log.js +124 -0
- package/src/scopes.js +97 -0
- package/src/security-headers.js +26 -0
- package/src/session-graph.js +945 -0
- package/src/session.js +1097 -0
- package/src/setup.js +1313 -0
- package/src/startup-pages.js +354 -0
- package/src/startup-payload.js +126 -0
- package/src/tools.js +4271 -0
- package/src/transcript-session.js +106 -0
- package/src/validation.js +1163 -0
- package/src/version.js +2 -0
- package/templates/BOARD.md +72 -0
- package/templates/BOTHANDOFF.md +57 -0
- package/templates/CHARACTER.md +30 -0
- package/templates/HANDOFF.md +63 -0
- package/templates/IDENTITY.md +29 -0
- package/templates/INDEX.md +82 -0
- package/templates/INSTALL.md +29 -0
- package/templates/PROCEDURE.md +39 -0
- package/templates/ROSTER.md +43 -0
- package/templates/SEED.md +65 -0
- package/templates/STAGING.md +37 -0
- package/templates/STANDARD.md +99 -0
- package/templates/THREAD.md +43 -0
- package/templates/USER.md +29 -0
package/src/tools.js
ADDED
|
@@ -0,0 +1,4271 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* EMET - Enduring Memory & Epistemic Tiers
|
|
3
|
+
* Copyright (c) 2026 Dreamforge Studio LLC
|
|
4
|
+
*
|
|
5
|
+
* Derived from CASCADE Memory System
|
|
6
|
+
* Copyright (c) 2025-2026 CIPS Corp (C.I.P.S. LLC)
|
|
7
|
+
* Licensed under the MIT License - see LICENSE
|
|
8
|
+
*
|
|
9
|
+
* https://github.com/For-Sunny/cascade-memory-enterprise
|
|
10
|
+
*
|
|
11
|
+
* Tools Module - MCP tool definitions and handlers
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { connectMongo } from './mongo-connect.js';
|
|
15
|
+
import { resolveModulePath } from './module-path.js';
|
|
16
|
+
import { replacedByOf } from './setup.js';
|
|
17
|
+
import { currentActor, writerStamp, boundSourceTag, LOCAL_SUBJECT } from './persons.js';
|
|
18
|
+
import {
|
|
19
|
+
ValidationError,
|
|
20
|
+
VALID_LAYERS,
|
|
21
|
+
CONTENT_LIMITS,
|
|
22
|
+
NUMERIC_LIMITS,
|
|
23
|
+
validateLayer,
|
|
24
|
+
validateContent,
|
|
25
|
+
validateMetadata,
|
|
26
|
+
relocatedFields,
|
|
27
|
+
isPlainObject,
|
|
28
|
+
validateLimit,
|
|
29
|
+
validateQueryOptions,
|
|
30
|
+
RECOGNISED_FILTER_KEYS,
|
|
31
|
+
MARKER_FIELDS
|
|
32
|
+
} from './validation.js';
|
|
33
|
+
// The layer charter is the vocabulary home (2026-09-07): which layers may be
|
|
34
|
+
// revised, the importance band, and whether a provenance pointer is required
|
|
35
|
+
// are all read from it, not from constants here.
|
|
36
|
+
import { policyFor, bandFor, resolveImportance, FALLBACK_LAYER } from './layers.js';
|
|
37
|
+
// The full routing decision (confidence, signals, emotional intensity) is kept
|
|
38
|
+
// on every write so the receipt can say how the entry was placed (B1/B2).
|
|
39
|
+
import { ContentAnalyzer } from './content_analyzer.js';
|
|
40
|
+
const ROUTER = new ContentAnalyzer();
|
|
41
|
+
// Template validation (STANDARD 9). Runs inside writeDoc() on every document
|
|
42
|
+
// write, with no argument that can switch it off (9a L2). It marks; it never
|
|
43
|
+
// rejects, alters, or blocks a write.
|
|
44
|
+
import {
|
|
45
|
+
validateDocument, templateFor, isGovernedByShippedCopy, isDropDoc, buildAcceptance,
|
|
46
|
+
loadShipped, parseTemplate, checkRecord, VERDICT as CONFORMANCE_VERDICT
|
|
47
|
+
} from './conformance.js';
|
|
48
|
+
// Shipped guides (2026-09-28): read on demand; the release copy governs.
|
|
49
|
+
import { readShippedGuide } from './guides.js';
|
|
50
|
+
|
|
51
|
+
import {
|
|
52
|
+
EmetError,
|
|
53
|
+
DatabaseError,
|
|
54
|
+
ErrorCodes,
|
|
55
|
+
StatusCodes,
|
|
56
|
+
MEMORY_LAYERS,
|
|
57
|
+
MONGODB_DB,
|
|
58
|
+
COLLECTION_PREFIX,
|
|
59
|
+
determineLayer,
|
|
60
|
+
containsRegex,
|
|
61
|
+
sortFor,
|
|
62
|
+
buildRankPipeline,
|
|
63
|
+
buildMemoryFilter,
|
|
64
|
+
sanitizeErrorMessage,
|
|
65
|
+
sanitizeDetails
|
|
66
|
+
} from './database.js';
|
|
67
|
+
|
|
68
|
+
// MongoDB for Tier 2 transcript storage
|
|
69
|
+
import { setting } from './env.js';
|
|
70
|
+
// The session graph's governed-tool list (2026-09-29): one list, used for the schemas below.
|
|
71
|
+
import { GRAPH_WRITE_TOOLS, isBotTag, mainRecordIds, botRecordDir } from './session-graph.js';
|
|
72
|
+
// Startup page completion (2026-10-03): which startup pages a host never fetched, read at session open.
|
|
73
|
+
import { pageCompletion, noteOpenMissing, cachedLoad } from './startup-pages.js';
|
|
74
|
+
// Secret redaction (2026-09-07). Every write path below passes its content
|
|
75
|
+
// through this before the digest is computed, so the stored sha256 is the
|
|
76
|
+
// digest of what was stored. The result of each write reports `redacted` -
|
|
77
|
+
// the count of masked spans - because a control that acts silently is a
|
|
78
|
+
// summary, and the record must show that something was removed.
|
|
79
|
+
import { redactWithReport, redactDeep } from './redact.js';
|
|
80
|
+
import { EMET_VERSION } from './version.js';
|
|
81
|
+
|
|
82
|
+
// MongoDB connection URI — must be set via MONGODB_URI environment variable in .mcp.json
|
|
83
|
+
const MONGODB_URI = process.env.MONGODB_URI;
|
|
84
|
+
if (!MONGODB_URI) {
|
|
85
|
+
console.error("[EMET] WARNING: MONGODB_URI not set — MongoDB tools will fail");
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// ============================================
|
|
89
|
+
// RATE LIMITING
|
|
90
|
+
// ============================================
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Rate Limiter Configuration
|
|
94
|
+
*/
|
|
95
|
+
export const RATE_LIMIT_CONFIG = {
|
|
96
|
+
GLOBAL_WINDOW_MS: 60000,
|
|
97
|
+
GLOBAL_MAX_REQUESTS: 300,
|
|
98
|
+
TOOL_WINDOW_MS: 60000,
|
|
99
|
+
TOOL_MAX_REQUESTS: {
|
|
100
|
+
recall: 120,
|
|
101
|
+
semantic_recall: 120,
|
|
102
|
+
query_layer: 100,
|
|
103
|
+
emet_status: 30,
|
|
104
|
+
save_to_layer: 60
|
|
105
|
+
},
|
|
106
|
+
DEFAULT_TOOL_MAX: 60,
|
|
107
|
+
CLEANUP_INTERVAL_MS: 300000
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Rate Limit Error class
|
|
112
|
+
*/
|
|
113
|
+
export class RateLimitError extends Error {
|
|
114
|
+
constructor(message, retryAfterMs = 60000) {
|
|
115
|
+
super(message);
|
|
116
|
+
this.name = 'RateLimitError';
|
|
117
|
+
this.retryAfterMs = retryAfterMs;
|
|
118
|
+
this.statusCode = 429;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* In-Memory Rate Limiter
|
|
124
|
+
*/
|
|
125
|
+
export class RateLimiter {
|
|
126
|
+
constructor(logger = null) {
|
|
127
|
+
this.buckets = new Map();
|
|
128
|
+
this.logger = logger;
|
|
129
|
+
|
|
130
|
+
this.cleanupInterval = setInterval(() => this.cleanup(), RATE_LIMIT_CONFIG.CLEANUP_INTERVAL_MS);
|
|
131
|
+
|
|
132
|
+
if (this.cleanupInterval.unref) {
|
|
133
|
+
this.cleanupInterval.unref();
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
this.log('debug', 'Rate limiter initialized', {
|
|
137
|
+
globalLimit: RATE_LIMIT_CONFIG.GLOBAL_MAX_REQUESTS,
|
|
138
|
+
cleanupIntervalMs: RATE_LIMIT_CONFIG.CLEANUP_INTERVAL_MS
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
log(level, message, context = {}) {
|
|
143
|
+
if (this.logger) {
|
|
144
|
+
this.logger[level](message, context);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
_bucket(key) {
|
|
149
|
+
const k = key || 'stdio';
|
|
150
|
+
let bucket = this.buckets.get(k);
|
|
151
|
+
if (!bucket) {
|
|
152
|
+
bucket = { global: [], tools: new Map() };
|
|
153
|
+
this.buckets.set(k, bucket);
|
|
154
|
+
}
|
|
155
|
+
return bucket;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
checkLimit(toolName, bucketKey = 'stdio') {
|
|
159
|
+
const now = Date.now();
|
|
160
|
+
this.pruneExpired(now);
|
|
161
|
+
const bucket = this._bucket(bucketKey);
|
|
162
|
+
|
|
163
|
+
const globalWindowStart = now - RATE_LIMIT_CONFIG.GLOBAL_WINDOW_MS;
|
|
164
|
+
const globalCount = bucket.global.filter(ts => ts > globalWindowStart).length;
|
|
165
|
+
|
|
166
|
+
if (globalCount >= RATE_LIMIT_CONFIG.GLOBAL_MAX_REQUESTS) {
|
|
167
|
+
const oldestInWindow = Math.min(...bucket.global.filter(ts => ts > globalWindowStart));
|
|
168
|
+
const retryAfterMs = (oldestInWindow + RATE_LIMIT_CONFIG.GLOBAL_WINDOW_MS) - now;
|
|
169
|
+
|
|
170
|
+
return {
|
|
171
|
+
allowed: false,
|
|
172
|
+
reason: `Global rate limit exceeded: ${globalCount}/${RATE_LIMIT_CONFIG.GLOBAL_MAX_REQUESTS} requests per minute`,
|
|
173
|
+
retryAfterMs: Math.max(retryAfterMs, 1000)
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
const toolWindowStart = now - RATE_LIMIT_CONFIG.TOOL_WINDOW_MS;
|
|
178
|
+
const toolTimestamps = bucket.tools.get(toolName) || [];
|
|
179
|
+
const toolCount = toolTimestamps.filter(ts => ts > toolWindowStart).length;
|
|
180
|
+
const toolMax = RATE_LIMIT_CONFIG.TOOL_MAX_REQUESTS[toolName] || RATE_LIMIT_CONFIG.DEFAULT_TOOL_MAX;
|
|
181
|
+
|
|
182
|
+
if (toolCount >= toolMax) {
|
|
183
|
+
const oldestToolInWindow = Math.min(...toolTimestamps.filter(ts => ts > toolWindowStart));
|
|
184
|
+
const retryAfterMs = (oldestToolInWindow + RATE_LIMIT_CONFIG.TOOL_WINDOW_MS) - now;
|
|
185
|
+
|
|
186
|
+
return {
|
|
187
|
+
allowed: false,
|
|
188
|
+
reason: `Tool '${toolName}' rate limit exceeded: ${toolCount}/${toolMax} requests per minute`,
|
|
189
|
+
retryAfterMs: Math.max(retryAfterMs, 1000)
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
return { allowed: true };
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
recordRequest(toolName, bucketKey = 'stdio') {
|
|
197
|
+
const now = Date.now();
|
|
198
|
+
const bucket = this._bucket(bucketKey);
|
|
199
|
+
bucket.global.push(now);
|
|
200
|
+
|
|
201
|
+
if (!bucket.tools.has(toolName)) {
|
|
202
|
+
bucket.tools.set(toolName, []);
|
|
203
|
+
}
|
|
204
|
+
bucket.tools.get(toolName).push(now);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
pruneExpired(now = Date.now()) {
|
|
208
|
+
const globalCutoff = now - RATE_LIMIT_CONFIG.GLOBAL_WINDOW_MS;
|
|
209
|
+
const toolCutoff = now - RATE_LIMIT_CONFIG.TOOL_WINDOW_MS;
|
|
210
|
+
|
|
211
|
+
for (const [key, bucket] of this.buckets) {
|
|
212
|
+
bucket.global = bucket.global.filter(ts => ts > globalCutoff);
|
|
213
|
+
for (const [toolName, timestamps] of bucket.tools.entries()) {
|
|
214
|
+
const pruned = timestamps.filter(ts => ts > toolCutoff);
|
|
215
|
+
if (pruned.length === 0) bucket.tools.delete(toolName);
|
|
216
|
+
else bucket.tools.set(toolName, pruned);
|
|
217
|
+
}
|
|
218
|
+
if (bucket.global.length === 0 && bucket.tools.size === 0) this.buckets.delete(key);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
cleanup() {
|
|
223
|
+
const count = (map) => {
|
|
224
|
+
let n = 0;
|
|
225
|
+
for (const bucket of map.values()) {
|
|
226
|
+
n += bucket.global.length + Array.from(bucket.tools.values()).reduce((sum, arr) => sum + arr.length, 0);
|
|
227
|
+
}
|
|
228
|
+
return n;
|
|
229
|
+
};
|
|
230
|
+
const before = count(this.buckets);
|
|
231
|
+
this.pruneExpired();
|
|
232
|
+
const after = count(this.buckets);
|
|
233
|
+
|
|
234
|
+
if (before !== after) {
|
|
235
|
+
this.log('debug', 'Rate limiter cleanup completed', {
|
|
236
|
+
entriesBefore: before,
|
|
237
|
+
entriesAfter: after,
|
|
238
|
+
entriesRemoved: before - after
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
getStatus(bucketKey = 'stdio') {
|
|
244
|
+
const now = Date.now();
|
|
245
|
+
const globalWindowStart = now - RATE_LIMIT_CONFIG.GLOBAL_WINDOW_MS;
|
|
246
|
+
const bucket = this.buckets.get(bucketKey || 'stdio') || { global: [], tools: new Map() };
|
|
247
|
+
|
|
248
|
+
const status = {
|
|
249
|
+
bucket: bucketKey || 'stdio',
|
|
250
|
+
global: {
|
|
251
|
+
current: bucket.global.filter(ts => ts > globalWindowStart).length,
|
|
252
|
+
limit: RATE_LIMIT_CONFIG.GLOBAL_MAX_REQUESTS,
|
|
253
|
+
windowMs: RATE_LIMIT_CONFIG.GLOBAL_WINDOW_MS
|
|
254
|
+
},
|
|
255
|
+
tools: {}
|
|
256
|
+
};
|
|
257
|
+
|
|
258
|
+
const toolWindowStart = now - RATE_LIMIT_CONFIG.TOOL_WINDOW_MS;
|
|
259
|
+
for (const [toolName, timestamps] of bucket.tools.entries()) {
|
|
260
|
+
const count = timestamps.filter(ts => ts > toolWindowStart).length;
|
|
261
|
+
const limit = RATE_LIMIT_CONFIG.TOOL_MAX_REQUESTS[toolName] || RATE_LIMIT_CONFIG.DEFAULT_TOOL_MAX;
|
|
262
|
+
status.tools[toolName] = { current: count, limit };
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
return status;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
stop() {
|
|
269
|
+
if (this.cleanupInterval) {
|
|
270
|
+
clearInterval(this.cleanupInterval);
|
|
271
|
+
this.cleanupInterval = null;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/** Stdio shares one bucket per process. HTTP is per person and client id. */
|
|
277
|
+
export function rateBucket(actor) {
|
|
278
|
+
if (!actor || actor.transport !== 'http') return 'stdio';
|
|
279
|
+
return `${actor.personId || 'anonymous'}|${actor.clientId || 'anonymous'}`;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// ============================================
|
|
283
|
+
// ERROR HANDLING
|
|
284
|
+
// ============================================
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* Wrap ValidationError to match EmetError interface
|
|
288
|
+
*/
|
|
289
|
+
function wrapValidationError(error) {
|
|
290
|
+
return {
|
|
291
|
+
success: false,
|
|
292
|
+
error: {
|
|
293
|
+
code: ErrorCodes.VALIDATION_ERROR,
|
|
294
|
+
message: sanitizeErrorMessage(error.message),
|
|
295
|
+
statusCode: StatusCodes.BAD_REQUEST,
|
|
296
|
+
timestamp: Date.now(),
|
|
297
|
+
field: error.field || 'unknown',
|
|
298
|
+
validationMessage: sanitizeErrorMessage(error.validationMessage || error.message)
|
|
299
|
+
}
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/**
|
|
304
|
+
* Central error handler for all tool operations
|
|
305
|
+
*/
|
|
306
|
+
export function handleError(error, toolName, logger = null) {
|
|
307
|
+
let errorResponse;
|
|
308
|
+
|
|
309
|
+
if (error instanceof EmetError) {
|
|
310
|
+
errorResponse = error.toSafeJSON();
|
|
311
|
+
} else if (error instanceof ValidationError) {
|
|
312
|
+
errorResponse = wrapValidationError(error);
|
|
313
|
+
} else if (error instanceof RateLimitError) {
|
|
314
|
+
errorResponse = {
|
|
315
|
+
success: false,
|
|
316
|
+
error: {
|
|
317
|
+
code: ErrorCodes.RATE_LIMIT_EXCEEDED,
|
|
318
|
+
message: sanitizeErrorMessage(error.message),
|
|
319
|
+
statusCode: StatusCodes.RATE_LIMITED,
|
|
320
|
+
timestamp: Date.now(),
|
|
321
|
+
retryAfterMs: error.retryAfterMs,
|
|
322
|
+
retryAfterSeconds: Math.ceil(error.retryAfterMs / 1000)
|
|
323
|
+
}
|
|
324
|
+
};
|
|
325
|
+
} else if (error && error.code && error.code.startsWith('SQLITE')) {
|
|
326
|
+
errorResponse = {
|
|
327
|
+
success: false,
|
|
328
|
+
error: {
|
|
329
|
+
code: ErrorCodes.DATABASE_ERROR,
|
|
330
|
+
message: 'Database operation failed',
|
|
331
|
+
statusCode: StatusCodes.INTERNAL_ERROR,
|
|
332
|
+
timestamp: Date.now(),
|
|
333
|
+
details: {
|
|
334
|
+
sqliteCode: error.code
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
};
|
|
338
|
+
} else {
|
|
339
|
+
errorResponse = {
|
|
340
|
+
success: false,
|
|
341
|
+
error: {
|
|
342
|
+
code: ErrorCodes.INTERNAL_ERROR,
|
|
343
|
+
message: sanitizeErrorMessage(error?.message || 'Unknown error'),
|
|
344
|
+
statusCode: StatusCodes.INTERNAL_ERROR,
|
|
345
|
+
timestamp: Date.now()
|
|
346
|
+
}
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
errorResponse.error.tool = toolName;
|
|
351
|
+
|
|
352
|
+
// Read at call time. Never attach a stack on the HTTP server.
|
|
353
|
+
if (process.env.DEBUG === 'true' && error?.stack && currentActor().transport !== 'http') {
|
|
354
|
+
errorResponse.error._debug = {
|
|
355
|
+
originalMessage: error.message,
|
|
356
|
+
stack: error.stack.split('\n').slice(0, 5).join('\n')
|
|
357
|
+
};
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
if (logger) {
|
|
361
|
+
logger.error('Tool error occurred', {
|
|
362
|
+
tool: toolName,
|
|
363
|
+
errorName: error?.name || 'Error',
|
|
364
|
+
errorCode: error?.code || 'UNKNOWN',
|
|
365
|
+
error
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
return {
|
|
370
|
+
content: [{
|
|
371
|
+
type: "text",
|
|
372
|
+
text: JSON.stringify(errorResponse)
|
|
373
|
+
}],
|
|
374
|
+
isError: true
|
|
375
|
+
};
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
/**
|
|
379
|
+
* Create success response with consistent format
|
|
380
|
+
*/
|
|
381
|
+
export function createSuccessResponse(data, toolName) {
|
|
382
|
+
// A LIST payload can carry an advisory the caller needs to SEE - an ignored
|
|
383
|
+
// filter key, say - and an array has nowhere to put one: JSON.stringify drops
|
|
384
|
+
// non-index properties, and wrapping the array in an object would break every
|
|
385
|
+
// caller that reads `data` as a list. So a tool attaches advisories with
|
|
386
|
+
// attachWarnings() and they are hoisted here, beside `data`.
|
|
387
|
+
//
|
|
388
|
+
// Restricted to arrays deliberately. Object payloads such as the write
|
|
389
|
+
// receipt already carry their own `warnings` field INSIDE `data`; hoisting
|
|
390
|
+
// that one would duplicate it and change a shape callers already read.
|
|
391
|
+
const warnings = Array.isArray(data) && Array.isArray(data.warnings) && data.warnings.length > 0
|
|
392
|
+
? data.warnings
|
|
393
|
+
: null;
|
|
394
|
+
|
|
395
|
+
return {
|
|
396
|
+
content: [{
|
|
397
|
+
type: "text",
|
|
398
|
+
text: JSON.stringify({
|
|
399
|
+
success: true,
|
|
400
|
+
tool: toolName,
|
|
401
|
+
timestamp: Date.now(),
|
|
402
|
+
...(warnings ? { warnings } : {}),
|
|
403
|
+
data
|
|
404
|
+
})
|
|
405
|
+
// Compact on purpose (code audit 2026-09-27, 2.6): the two-space indent was
|
|
406
|
+
// ~12% of every response - 9,400 characters of the initialize payload -
|
|
407
|
+
// and hosts parse JSON; nobody reads the whitespace.
|
|
408
|
+
}]
|
|
409
|
+
};
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
/**
|
|
413
|
+
* Attach caller-facing advisories to an array payload without changing what the
|
|
414
|
+
* array itself serialises to. Non-enumerable, so JSON.stringify(list) is
|
|
415
|
+
* unaffected and any code iterating the list sees exactly what it saw before;
|
|
416
|
+
* createSuccessResponse is what lifts them into the response envelope.
|
|
417
|
+
* No-op unless both arguments are arrays and there is something to say.
|
|
418
|
+
*/
|
|
419
|
+
export function attachWarnings(list, warnings) {
|
|
420
|
+
if (!Array.isArray(list) || !Array.isArray(warnings) || warnings.length === 0) {
|
|
421
|
+
return list;
|
|
422
|
+
}
|
|
423
|
+
Object.defineProperty(list, 'warnings', {
|
|
424
|
+
value: warnings,
|
|
425
|
+
enumerable: false,
|
|
426
|
+
configurable: true
|
|
427
|
+
});
|
|
428
|
+
return list;
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
// ============================================
|
|
432
|
+
// MEMORY OPERATIONS
|
|
433
|
+
// ============================================
|
|
434
|
+
|
|
435
|
+
/**
|
|
436
|
+
* Save a memory record.
|
|
437
|
+
*/
|
|
438
|
+
export async function saveMemory(dbManager, content, layer = null, metadata = {}, logger = null, auditOperation = null, options = {}) {
|
|
439
|
+
const startTime = Date.now();
|
|
440
|
+
const requestId = logger?._generateRequestId?.() || `req-${Date.now()}`;
|
|
441
|
+
|
|
442
|
+
try {
|
|
443
|
+
// Redact BEFORE validation and BEFORE the digest: what is stored, hashed
|
|
444
|
+
// and read back is the scrubbed text. The count travels to the result.
|
|
445
|
+
const redaction = redactWithReport(content);
|
|
446
|
+
const validatedContent = validateContent(redaction.text);
|
|
447
|
+
const validatedLayer = layer ? validateLayer(layer) : null;
|
|
448
|
+
|
|
449
|
+
// The tier-1 -> tier-2 pointer is pulled out BEFORE metadata validation and
|
|
450
|
+
// stored as top-level columns, not inside metadata. When this was written
|
|
451
|
+
// metadata was a JSON string the store could not filter into; it is a
|
|
452
|
+
// subdocument now, but the pointer stays top-level because it is
|
|
453
|
+
// indexed there and the reverse lookup depends on that index.
|
|
454
|
+
const { derived_from: rawProvenance, ...restMetadata } = metadata || {};
|
|
455
|
+
let provenance = validateProvenance(rawProvenance);
|
|
456
|
+
const metaRedaction = redactDeep(restMetadata);
|
|
457
|
+
redaction.hits += metaRedaction.hits;
|
|
458
|
+
const incoming = { ...(metaRedaction.value || {}) };
|
|
459
|
+
// The accountability markers are the door's verdict, not the caller's
|
|
460
|
+
// claim: whatever the caller sent under these names is dropped and the
|
|
461
|
+
// door writes what it actually found.
|
|
462
|
+
for (const marker of MARKER_FIELDS) delete incoming[marker];
|
|
463
|
+
// A null importance / emotional_intensity is "not supplied", not a value.
|
|
464
|
+
// Left in place, the validator would turn a null importance into its own
|
|
465
|
+
// fixed 0.7 and the charter band below would never be consulted.
|
|
466
|
+
if (incoming.importance === null || incoming.importance === undefined) delete incoming.importance;
|
|
467
|
+
if (incoming.emotional_intensity === null || incoming.emotional_intensity === undefined) delete incoming.emotional_intensity;
|
|
468
|
+
// Bucket words are resolved after the layer is known. Pull them off so
|
|
469
|
+
// validateMetadata does not reject a string on the numeric field.
|
|
470
|
+
const rawImportance = incoming.importance;
|
|
471
|
+
const rawBucket = incoming.importance_bucket;
|
|
472
|
+
const rawDegree = incoming.importance_degree;
|
|
473
|
+
delete incoming.importance;
|
|
474
|
+
delete incoming.importance_bucket;
|
|
475
|
+
delete incoming.importance_degree;
|
|
476
|
+
// E3: absent (undefined) is the only condition that earns the marker. An
|
|
477
|
+
// explicit null is the caller saying "unknown", which is an answer.
|
|
478
|
+
const assertionOriginAbsent = incoming.assertion_origin === undefined;
|
|
479
|
+
const validatedMetadata = validateMetadata(incoming);
|
|
480
|
+
// A6: what the whitelist moved under metadata.custom is reported in the
|
|
481
|
+
// receipt, so a field that landed somewhere other than where the caller put
|
|
482
|
+
// it is visible at write time instead of discovered at read time.
|
|
483
|
+
const relocatedToCustom = relocatedFields(validatedMetadata);
|
|
484
|
+
const warnings = [];
|
|
485
|
+
|
|
486
|
+
// B1: the full routing decision is computed on every write, whether or not
|
|
487
|
+
// the caller named a layer, so the receipt can say how the entry was placed.
|
|
488
|
+
// B5 (decided 2026-09-07): an unlabelled write is NOT placed by either
|
|
489
|
+
// router. Both were measured against the record (keyword 41.4%, analyzer
|
|
490
|
+
// 36.1%) and neither is fit to put material in a durable layer by guess.
|
|
491
|
+
// It lands in the charter's fallback layer, flagged routing.method
|
|
492
|
+
// 'fallback', and both routers' opinions are recorded beside it so the
|
|
493
|
+
// decision is reviewable. Naming a layer is the norm; the receipt says so.
|
|
494
|
+
const decision = ROUTER.analyze(validatedContent, {});
|
|
495
|
+
const keywordLayer = determineLayer(validatedContent, validatedMetadata);
|
|
496
|
+
const targetLayer = validatedLayer || FALLBACK_LAYER;
|
|
497
|
+
// Lesson fields are refused at the door when the reader would never count
|
|
498
|
+
// them (2026-09-29, a decision record): a malformed key, a key off the meta
|
|
499
|
+
// layer, or a disposition that settles nothing. Checked on the caller's
|
|
500
|
+
// metadata as sent, before anything is written.
|
|
501
|
+
{
|
|
502
|
+
const { checkLessonFields } = await import('./lessons-to-checks.js');
|
|
503
|
+
const lessonProblem = checkLessonFields(targetLayer, incoming);
|
|
504
|
+
if (lessonProblem) throw new ValidationError('metadata.lesson', `${lessonProblem} Nothing was written.`);
|
|
505
|
+
}
|
|
506
|
+
if (!validatedLayer) {
|
|
507
|
+
warnings.push(
|
|
508
|
+
`no layer given: landed in ${FALLBACK_LAYER} (fallback). Name the layer - ` +
|
|
509
|
+
`the routers are recorded in routing but do not place entries.`
|
|
510
|
+
);
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
// A3: the source tag is decided HERE and nowhere else.
|
|
514
|
+
// Stdio: the caller's tag wins; the configured default is used and marked;
|
|
515
|
+
// nothing at all is marked too.
|
|
516
|
+
// HTTP: the tag bound at approval wins. A token issued before tags were
|
|
517
|
+
// stored (no bound tag) may use the caller's tag only when that tag is an
|
|
518
|
+
// exact member of EMET_SOURCE_TAGS. The shared default is not applied there.
|
|
519
|
+
const sourceDecision = resolvedSourceTag(validatedMetadata.source);
|
|
520
|
+
if (sourceDecision.from === 'caller') {
|
|
521
|
+
const defaultTag = setting('SOURCE_TAG');
|
|
522
|
+
if (sourceDecision.tag) {
|
|
523
|
+
validatedMetadata.source = sourceDecision.tag;
|
|
524
|
+
} else if (typeof defaultTag === 'string' && defaultTag.trim() !== '') {
|
|
525
|
+
validatedMetadata.source = defaultTag;
|
|
526
|
+
validatedMetadata.source_defaulted = true;
|
|
527
|
+
} else {
|
|
528
|
+
validatedMetadata.source = null;
|
|
529
|
+
validatedMetadata.source_missing = true;
|
|
530
|
+
}
|
|
531
|
+
} else {
|
|
532
|
+
validatedMetadata.source = sourceDecision.tag;
|
|
533
|
+
delete validatedMetadata.source_defaulted;
|
|
534
|
+
if (sourceDecision.tag) delete validatedMetadata.source_missing;
|
|
535
|
+
else validatedMetadata.source_missing = true;
|
|
536
|
+
}
|
|
537
|
+
// Registry check: only when a registry is configured (a fresh install has
|
|
538
|
+
// none yet), and only for a tag that exists - a missing tag is already marked.
|
|
539
|
+
const registry = String(setting('SOURCE_TAGS') ?? '')
|
|
540
|
+
.split(',').map((s) => s.trim()).filter((s) => s !== '');
|
|
541
|
+
if (registry.length > 0 && typeof validatedMetadata.source === 'string' &&
|
|
542
|
+
!registry.includes(validatedMetadata.source)) {
|
|
543
|
+
validatedMetadata.source_unregistered = true;
|
|
544
|
+
}
|
|
545
|
+
// Person comes from the login, never from the caller's claim. Stdio is
|
|
546
|
+
// one fixed local subject. A caller client_id never survives in custom.
|
|
547
|
+
{
|
|
548
|
+
const stamp = writerStamp();
|
|
549
|
+
validatedMetadata.person_id = stamp.person_id;
|
|
550
|
+
delete validatedMetadata.client_id;
|
|
551
|
+
if (validatedMetadata.custom && typeof validatedMetadata.custom === 'object') {
|
|
552
|
+
delete validatedMetadata.custom.client_id;
|
|
553
|
+
delete validatedMetadata.custom.person_id;
|
|
554
|
+
if (Object.keys(validatedMetadata.custom).length === 0) delete validatedMetadata.custom;
|
|
555
|
+
}
|
|
556
|
+
validatedMetadata.client_id = stamp.client_id;
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
// A4: importance is a named bucket. The float is the bucket midpoint
|
|
560
|
+
// unless the caller named a degree (low/mid/high) or still sent a number.
|
|
561
|
+
const band = bandFor(targetLayer);
|
|
562
|
+
const resolved = resolveImportance(targetLayer, rawImportance, {
|
|
563
|
+
importance_bucket: rawBucket,
|
|
564
|
+
importance_degree: rawDegree
|
|
565
|
+
});
|
|
566
|
+
validatedMetadata.importance = resolved.importance;
|
|
567
|
+
if (resolved.bucket) validatedMetadata.importance_bucket = resolved.bucket;
|
|
568
|
+
if (resolved.degree) validatedMetadata.importance_degree = resolved.degree;
|
|
569
|
+
if (resolved.defaulted) validatedMetadata.importance_defaulted = true;
|
|
570
|
+
if (resolved.importance < band.min || resolved.importance > band.max) {
|
|
571
|
+
warnings.push(`importance ${resolved.importance} is outside the ${targetLayer} band ${band.min}–${band.max}`);
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
// A5: the charter says whether this layer wants a provenance pointer.
|
|
575
|
+
// 2026-09-29 (a decision record): the pointer is STAMPED when the caller names
|
|
576
|
+
// its transcript session, and a layer whose charter REQUIRES provenance
|
|
577
|
+
// (semantic, procedural) REFUSES a write that has no source at all. The
|
|
578
|
+
// transport is stateless, so the server never guesses which host's session
|
|
579
|
+
// a write belongs to - the caller names it (save_to_layer `session_id`),
|
|
580
|
+
// and the server reads that session's own counter. The stamped exchange is
|
|
581
|
+
// the one in progress: the highest stored index plus one, which the append
|
|
582
|
+
// after the reply stores. Until it lands the pointer is pending, and
|
|
583
|
+
// emet_gaps reports one that never does.
|
|
584
|
+
let provenanceStamped = false;
|
|
585
|
+
if (!provenance && options && options.session_id !== undefined && options.session_id !== null) {
|
|
586
|
+
const sid = typeof options.session_id === 'string' ? options.session_id.trim() : '';
|
|
587
|
+
if (!sid || sid.length > 200) throw new ValidationError('session_id', 'must be the non-empty transcript session id from emet_session_open. Nothing was written.');
|
|
588
|
+
if (typeof dbManager.transcriptPosition !== 'function') throw new ValidationError('session_id', 'this store cannot read transcript positions, so the pointer cannot be stamped - pass derived_from instead. Nothing was written.');
|
|
589
|
+
const pos = await dbManager.transcriptPosition(sid);
|
|
590
|
+
const ex = pos.highest + 1;
|
|
591
|
+
provenance = validateProvenance({ session_id: pos.base, exchange_start: ex, exchange_end: ex });
|
|
592
|
+
provenanceStamped = true;
|
|
593
|
+
}
|
|
594
|
+
const provenancePolicy = policyFor(targetLayer).provenance;
|
|
595
|
+
if (!provenance && provenancePolicy === 'required') {
|
|
596
|
+
throw new ValidationError('derived_from',
|
|
597
|
+
`${targetLayer} entries must carry their source. Pass session_id (your transcript session from ` +
|
|
598
|
+
`emet_session_open - the server stamps the exchange), or derived_from {session_id, exchange_start, ` +
|
|
599
|
+
`exchange_end}, or derived_from {entries: [ids]} for an entry consolidated from other entries. Nothing was written.`);
|
|
600
|
+
}
|
|
601
|
+
if (!provenance && provenancePolicy === 'encouraged') {
|
|
602
|
+
validatedMetadata.provenance_missing = true;
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
// E3: the door never infers the origin; it only records that none was given.
|
|
606
|
+
if (assertionOriginAbsent) {
|
|
607
|
+
validatedMetadata.assertion_origin = null;
|
|
608
|
+
validatedMetadata.assertion_origin_missing = true;
|
|
609
|
+
}
|
|
610
|
+
{
|
|
611
|
+
const { CONTENT_ORIGINS } = await import('./validation.js');
|
|
612
|
+
const given = validatedMetadata.content_origin;
|
|
613
|
+
if (typeof given === 'string' && CONTENT_ORIGINS.includes(given)) {
|
|
614
|
+
validatedMetadata.content_origin = given;
|
|
615
|
+
} else if (validatedMetadata.assertion_origin === 'imported') {
|
|
616
|
+
validatedMetadata.content_origin = 'external';
|
|
617
|
+
} else if (validatedMetadata.assertion_origin === 'assistant') {
|
|
618
|
+
validatedMetadata.content_origin = 'assistant';
|
|
619
|
+
} else if (validatedMetadata.assertion_origin === 'user' || validatedMetadata.assertion_origin === 'consolidated') {
|
|
620
|
+
validatedMetadata.content_origin = 'self';
|
|
621
|
+
} else {
|
|
622
|
+
validatedMetadata.content_origin = null;
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
// E4: the redaction count lives in the stored metadata as well as the
|
|
627
|
+
// receipt, so "which host keeps pasting secrets" is a query, not a memory.
|
|
628
|
+
if (redaction.hits > 0) validatedMetadata.redacted_spans = redaction.hits;
|
|
629
|
+
|
|
630
|
+
// B2: emotional intensity - caller, then analyzer, then the fixed baseline.
|
|
631
|
+
let emotionalIntensity;
|
|
632
|
+
let emotionalIntensitySource;
|
|
633
|
+
if (validatedMetadata.emotional_intensity !== undefined) {
|
|
634
|
+
emotionalIntensity = validatedMetadata.emotional_intensity;
|
|
635
|
+
emotionalIntensitySource = 'caller';
|
|
636
|
+
} else if (Number.isFinite(decision.emotional_intensity)) {
|
|
637
|
+
emotionalIntensity = decision.emotional_intensity;
|
|
638
|
+
emotionalIntensitySource = 'analyzer';
|
|
639
|
+
} else {
|
|
640
|
+
emotionalIntensity = 0.5;
|
|
641
|
+
emotionalIntensitySource = 'default';
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
// B1: the routing record. A caller-supplied routing object is kept as-is.
|
|
645
|
+
if (!isPlainObject(validatedMetadata.routing)) {
|
|
646
|
+
validatedMetadata.routing = validatedLayer
|
|
647
|
+
? {
|
|
648
|
+
method: 'explicit',
|
|
649
|
+
layer: targetLayer,
|
|
650
|
+
analyzer_layer: decision.layer,
|
|
651
|
+
confidence: decision.confidence,
|
|
652
|
+
emotional_intensity_source: emotionalIntensitySource
|
|
653
|
+
}
|
|
654
|
+
: {
|
|
655
|
+
method: 'fallback',
|
|
656
|
+
layer: targetLayer,
|
|
657
|
+
keyword_layer: keywordLayer,
|
|
658
|
+
analyzer_layer: decision.layer,
|
|
659
|
+
confidence: decision.confidence,
|
|
660
|
+
signals: Object.keys(decision.signals || {}).slice(0, 8),
|
|
661
|
+
technical_density: decision.technical_density,
|
|
662
|
+
emotional_intensity_source: emotionalIntensitySource
|
|
663
|
+
};
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
logger?.debug('Saving memory', {
|
|
667
|
+
requestId,
|
|
668
|
+
layer: targetLayer,
|
|
669
|
+
contentLength: validatedContent.length,
|
|
670
|
+
hasMetadata: Object.keys(validatedMetadata).length > 0
|
|
671
|
+
});
|
|
672
|
+
|
|
673
|
+
await dbManager.getConnection(targetLayer);
|
|
674
|
+
|
|
675
|
+
const timestamp = Date.now() / 1000;
|
|
676
|
+
// Decided above (A4): the caller's value or the charter midpoint, marked.
|
|
677
|
+
const importance = validatedMetadata.importance;
|
|
678
|
+
|
|
679
|
+
const columns = provenanceColumns(provenance);
|
|
680
|
+
// 2026-09-05: the digest is computed BEFORE the write and stored
|
|
681
|
+
// as a column, not only compared in the read-back receipt. Without it the
|
|
682
|
+
// aleph claim held for exactly one instant - verified at write, never again.
|
|
683
|
+
const contentSha256 = digest(validatedContent);
|
|
684
|
+
let prevSha256 = null;
|
|
685
|
+
try {
|
|
686
|
+
const head = await dbManager.collectionFor(targetLayer)
|
|
687
|
+
.find({ sha256: { $exists: true, $nin: [null, ''] } })
|
|
688
|
+
.sort({ id: -1 }).limit(1).toArray();
|
|
689
|
+
prevSha256 = (head[0] && head[0].sha256) || null;
|
|
690
|
+
} catch { prevSha256 = null; }
|
|
691
|
+
const { chainDigest } = await import('./aleph.js');
|
|
692
|
+
const chainSha256 = chainDigest(prevSha256, contentSha256);
|
|
693
|
+
// C1 (2026-09-07): the metadata is attested too, in its own column, over
|
|
694
|
+
// its canonical JSON - computed AFTER every marker above has been added,
|
|
695
|
+
// so the digest covers the metadata as stored, not as received.
|
|
696
|
+
const metadataSha256 = digestMetadata(validatedMetadata);
|
|
697
|
+
const signingKey = setting('SIGNING_KEY') || null;
|
|
698
|
+
const signature = signDigests(contentSha256, metadataSha256, signingKey);
|
|
699
|
+
// The row as a document (stage 1 of the write-path rewrite, 2026-09-27).
|
|
700
|
+
// `event` (a mirror of content) was retired with the SQL read path,
|
|
701
|
+
// 2026-09-28; older rows keep theirs, untouched.
|
|
702
|
+
const row = {
|
|
703
|
+
timestamp,
|
|
704
|
+
content: validatedContent,
|
|
705
|
+
context: validatedMetadata.context || '',
|
|
706
|
+
emotional_intensity: emotionalIntensity,
|
|
707
|
+
importance,
|
|
708
|
+
// 2026-09-05: stored as a real subdocument, not a JSON string.
|
|
709
|
+
// The promoted columns below stay because they are indexed and the vector
|
|
710
|
+
// filter reads them.
|
|
711
|
+
metadata: validatedMetadata,
|
|
712
|
+
// Promoted top-level columns (2026-08-24): what the liveness predicate reads.
|
|
713
|
+
superseded_by: null,
|
|
714
|
+
valid_until: _extractValidUntil(validatedMetadata),
|
|
715
|
+
// is_live stays true when a row is superseded. The end of an entry is
|
|
716
|
+
// superseded_at plus superseded_by (or valid_until when it expires).
|
|
717
|
+
// A false value is a historical mark from before this rule; reads still
|
|
718
|
+
// return that row and say it has ended. Nothing is hidden for age.
|
|
719
|
+
is_live: true,
|
|
720
|
+
// Provenance. Explicit nulls rather than absent fields: "no pointer" and
|
|
721
|
+
// "this field was never part of the schema" must not look the same.
|
|
722
|
+
source_session_id: columns.source_session_id,
|
|
723
|
+
source_exchange_start: columns.source_exchange_start,
|
|
724
|
+
source_exchange_end: columns.source_exchange_end,
|
|
725
|
+
derived_from_ids: columns.derived_from_ids,
|
|
726
|
+
sha256: contentSha256,
|
|
727
|
+
metadata_sha256: metadataSha256,
|
|
728
|
+
signature,
|
|
729
|
+
prev_sha256: prevSha256,
|
|
730
|
+
chain_sha256: chainSha256
|
|
731
|
+
};
|
|
732
|
+
|
|
733
|
+
// Option A correction back-pointer (the owner, 2026-10-02, the owner's decision record item 4): a new entry may name the
|
|
734
|
+
// entry it partly corrects. The parent must exist in the same layer, and is checked BEFORE the insert so a
|
|
735
|
+
// refused correction writes nothing.
|
|
736
|
+
if (validatedMetadata.corrects !== undefined && validatedMetadata.corrects !== null) {
|
|
737
|
+
const parent = await dbManager.readMemory(targetLayer, validatedMetadata.corrects);
|
|
738
|
+
if (!parent) {
|
|
739
|
+
throw new ValidationError('corrects', `No entry with id ${validatedMetadata.corrects} in layer '${targetLayer}' - a correction names an entry in its own layer. Nothing was written.`);
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
const memoryId = await dbManager.insertMemory(targetLayer, row);
|
|
744
|
+
|
|
745
|
+
// aleph: read the entry back and compare before reporting it saved.
|
|
746
|
+
// Until this existed, save_to_layer returned an id the caller had no reason
|
|
747
|
+
// to doubt and no way to check, while the README promised every write was
|
|
748
|
+
// verified. The id is checked by CONTENT rather than existence because the
|
|
749
|
+
// dangerous failure is an id collision - a counter read from the wrong
|
|
750
|
+
// collection restarts at 1 and silently overwrites real entries.
|
|
751
|
+
const stored = await dbManager.readMemory(targetLayer, memoryId);
|
|
752
|
+
const receipt = verifyLayerEntry(stored, { id: memoryId, content: validatedContent, layer: targetLayer, metadata: validatedMetadata });
|
|
753
|
+
const correction = (validatedMetadata.corrects !== undefined && validatedMetadata.corrects !== null)
|
|
754
|
+
? await markCorrectedBy(dbManager, targetLayer, validatedMetadata.corrects, memoryId)
|
|
755
|
+
: null;
|
|
756
|
+
// the second reader N4: a correction whose parent could not be marked is still saved,
|
|
757
|
+
// but the caller is told in words, not only by a boolean.
|
|
758
|
+
if (correction && correction.marked !== true) {
|
|
759
|
+
warnings.push(`corrects: ${correction.parent_id} - the parent entry was not marked corrected_by (missing in ${targetLayer}, or the mark did not hold); the correction is saved, the link is one-sided until the parent is marked`);
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
const durationMs = Date.now() - startTime;
|
|
763
|
+
|
|
764
|
+
if (logger && auditOperation) {
|
|
765
|
+
logger.audit(auditOperation.MEMORY_SAVE, {
|
|
766
|
+
requestId,
|
|
767
|
+
layer: targetLayer,
|
|
768
|
+
memoryId,
|
|
769
|
+
contentLength: validatedContent.length,
|
|
770
|
+
importance,
|
|
771
|
+
emotionalIntensity,
|
|
772
|
+
durationMs,
|
|
773
|
+
success: true
|
|
774
|
+
});
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
logger?.info('Memory saved successfully', {
|
|
778
|
+
requestId,
|
|
779
|
+
layer: targetLayer,
|
|
780
|
+
memoryId,
|
|
781
|
+
durationMs
|
|
782
|
+
});
|
|
783
|
+
|
|
784
|
+
// C3: the receipt is the fixture. One shape on both transports, keys in a
|
|
785
|
+
// fixed order, and it says what happened to the write: every marker the
|
|
786
|
+
// door set is echoed here, and `warnings` is always an array so a clean
|
|
787
|
+
// write and a flagged write have the same shape.
|
|
788
|
+
const md = validatedMetadata;
|
|
789
|
+
return {
|
|
790
|
+
layer: targetLayer,
|
|
791
|
+
id: memoryId,
|
|
792
|
+
timestamp,
|
|
793
|
+
verified: true,
|
|
794
|
+
sha256: receipt.sha256,
|
|
795
|
+
// digest_stored: the read-back saw a sha256 COLUMN on the row and it agreed
|
|
796
|
+
// with the content. false means the row has no column (pre-#12 build) - the
|
|
797
|
+
// receipt above was computed, not read. The receipt is its own witness.
|
|
798
|
+
digest_stored: receipt.digest_stored === true,
|
|
799
|
+
// C1: the metadata digest, attested in its own column and read back.
|
|
800
|
+
metadata_sha256: receipt.metadata_sha256,
|
|
801
|
+
metadata_digest_stored: receipt.metadata_digest_stored === true,
|
|
802
|
+
signed: signatureIntegrityOf(stored, signingKey),
|
|
803
|
+
signature_stored: typeof stored?.signature === 'string' && stored.signature.length > 0,
|
|
804
|
+
// Number of secret spans masked before storage. Zero is the normal case;
|
|
805
|
+
// anything else is worth the caller's attention.
|
|
806
|
+
redacted: redaction.hits,
|
|
807
|
+
provenance: provenance || null,
|
|
808
|
+
...(provenanceStamped ? { provenance_stamped: 'from the named transcript session; the exchange is the one in progress, stored by the append after this reply' } : {}),
|
|
809
|
+
...(md.provenance_missing ? { provenance_missing: true } : {}),
|
|
810
|
+
source: md.source,
|
|
811
|
+
...(md.source_defaulted ? { source_defaulted: true } : {}),
|
|
812
|
+
...(md.source_unregistered ? { source_unregistered: true } : {}),
|
|
813
|
+
...(md.source_missing ? { source_missing: true } : {}),
|
|
814
|
+
assertion_origin: md.assertion_origin,
|
|
815
|
+
...(md.assertion_origin_missing ? { assertion_origin_missing: true } : {}),
|
|
816
|
+
importance,
|
|
817
|
+
...(md.importance_defaulted ? { importance_defaulted: true } : {}),
|
|
818
|
+
routing: md.routing,
|
|
819
|
+
...(correction ? { corrects: correction.parent_id, parent_marked: correction.marked } : {}),
|
|
820
|
+
// Present only when the whitelist moved something: an absent key means
|
|
821
|
+
// every field landed where the caller put it.
|
|
822
|
+
...(relocatedToCustom.length > 0 ? { relocated_to_custom: relocatedToCustom } : {}),
|
|
823
|
+
warnings
|
|
824
|
+
};
|
|
825
|
+
} catch (error) {
|
|
826
|
+
const durationMs = Date.now() - startTime;
|
|
827
|
+
|
|
828
|
+
if (logger && auditOperation) {
|
|
829
|
+
logger.audit(auditOperation.MEMORY_SAVE, {
|
|
830
|
+
requestId,
|
|
831
|
+
layer: layer || 'auto',
|
|
832
|
+
contentLength: content ? content.length : 0,
|
|
833
|
+
durationMs,
|
|
834
|
+
success: false,
|
|
835
|
+
errorCode: error.code || 'UNKNOWN',
|
|
836
|
+
errorMessage: error.message
|
|
837
|
+
});
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
logger?.error('Failed to save memory', {
|
|
841
|
+
requestId,
|
|
842
|
+
error,
|
|
843
|
+
layer: layer || 'auto',
|
|
844
|
+
durationMs
|
|
845
|
+
});
|
|
846
|
+
|
|
847
|
+
// AlephError is an EmetError, so it already propagates unchanged here - which
|
|
848
|
+
// is deliberate. "The store does not contain what it should" must never be
|
|
849
|
+
// flattened into "the save failed", because the two lead to opposite actions.
|
|
850
|
+
if (error instanceof EmetError || error instanceof ValidationError) {
|
|
851
|
+
throw error;
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
throw new DatabaseError(
|
|
855
|
+
'Failed to save memory',
|
|
856
|
+
'save_memory',
|
|
857
|
+
{ layer: layer || 'auto' }
|
|
858
|
+
);
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
/**
|
|
863
|
+
* Recall memories by keyword.
|
|
864
|
+
*/
|
|
865
|
+
// valid_until is an end date, not a hide. recall, query_layer and semantic_recall
|
|
866
|
+
// return an expired row marked ended, with valid_until as that date. The current
|
|
867
|
+
// entry is listed first. Nothing is dropped for age.
|
|
868
|
+
/** Pull valid_until out of metadata for promotion to a top-level column. */
|
|
869
|
+
function _extractValidUntil(metadata) {
|
|
870
|
+
if (!metadata || typeof metadata !== 'object') return null;
|
|
871
|
+
const vu = (metadata.valid_until !== undefined && metadata.valid_until !== null)
|
|
872
|
+
? metadata.valid_until
|
|
873
|
+
: (metadata.custom && metadata.custom.valid_until !== undefined ? metadata.custom.valid_until : null);
|
|
874
|
+
const n = Number(vu);
|
|
875
|
+
return (vu === null || vu === undefined || Number.isNaN(n)) ? null : n;
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
function isMemoryExpired(metadata, nowSec = Date.now() / 1000) {
|
|
879
|
+
const md = parseMetadata(metadata);
|
|
880
|
+
const vu = (md.valid_until !== undefined && md.valid_until !== null) ? md.valid_until
|
|
881
|
+
: (md.custom && md.custom.valid_until !== undefined ? md.custom.valid_until : null);
|
|
882
|
+
if (vu === null || vu === undefined) return false;
|
|
883
|
+
const n = Number(vu);
|
|
884
|
+
if (!Number.isFinite(n) || n <= 0) return false;
|
|
885
|
+
const expirySec = n > 1e12 ? n / 1000 : n; // tolerate milliseconds
|
|
886
|
+
return expirySec < nowSec;
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
// ============================================
|
|
890
|
+
// SUPERSESSION
|
|
891
|
+
// A revision is a NEW entry that declares what it replaces (metadata.supersedes = <id>);
|
|
892
|
+
// the parent entry is marked metadata.superseded_by = <new id>. Nothing is edited or
|
|
893
|
+
// deleted - retention plus obsolescence marking (ISO 7.5.3 pattern; write_doc
|
|
894
|
+
// versioning is the template).
|
|
895
|
+
//
|
|
896
|
+
// READ PATHS RETURN SUPERSEDED ENTRIES, MARKED. User's decision 2026-09-13:
|
|
897
|
+
// "a parent should still surface to give context to the revision". Retention plus
|
|
898
|
+
// obsolescence marking means the superseded record is IDENTIFIED, not hidden -
|
|
899
|
+
// hiding it by default meant a revision arrived with no sight of what it revised,
|
|
900
|
+
// and an entry that recorded a belief at a time stopped standing as the record of
|
|
901
|
+
// it. include_superseded: false does not hide the parent. The current entry is
|
|
902
|
+
// listed first; the parent stays, marked with its end date and what replaced it.
|
|
903
|
+
// Episodic revision is PROHIBITED - events are immutable.
|
|
904
|
+
// ============================================
|
|
905
|
+
|
|
906
|
+
/** The id of the entry that replaced this one, or null. */
|
|
907
|
+
function supersededByOf(memoryOrMetadata) {
|
|
908
|
+
const row = memoryOrMetadata && memoryOrMetadata.metadata !== undefined ? memoryOrMetadata : null;
|
|
909
|
+
const md = parseMetadata(row ? row.metadata : memoryOrMetadata);
|
|
910
|
+
let sb = (md.superseded_by !== undefined && md.superseded_by !== null) ? md.superseded_by
|
|
911
|
+
: (md.custom && md.custom.superseded_by !== undefined && md.custom.superseded_by !== null ? md.custom.superseded_by : null);
|
|
912
|
+
// The promoted top-level column is authoritative where it is present: it is what
|
|
913
|
+
// the store filters on, and a row can carry it without the metadata copy.
|
|
914
|
+
if ((sb === null || sb === undefined) && row && row.superseded_by !== undefined && row.superseded_by !== null) {
|
|
915
|
+
sb = row.superseded_by;
|
|
916
|
+
}
|
|
917
|
+
if (sb === null || sb === undefined) return null;
|
|
918
|
+
const n = Number(sb);
|
|
919
|
+
return Number.isFinite(n) && n > 0 ? n : null;
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
function isMemorySuperseded(metadata) {
|
|
923
|
+
return supersededByOf(metadata) !== null;
|
|
924
|
+
}
|
|
925
|
+
|
|
926
|
+
/**
|
|
927
|
+
* Walk superseded_by from an entry to the live end of its chain.
|
|
928
|
+
* Returns { live, ids } - ids is every id walked, starting with the entry
|
|
929
|
+
* given, ending with the live one. Bounded so a cycle (a broken_supersession
|
|
930
|
+
* gap) terminates instead of spinning; the walk stops at the first id that
|
|
931
|
+
* cannot be read and reports what it reached.
|
|
932
|
+
*/
|
|
933
|
+
export async function resolveSupersessionChain(read, entry, maxHops = 50) {
|
|
934
|
+
const ids = [entry.id];
|
|
935
|
+
let current = entry;
|
|
936
|
+
for (let hop = 0; hop < maxHops; hop++) {
|
|
937
|
+
const next = supersededByOf(current);
|
|
938
|
+
if (next === null || next === undefined) return { live: current, ids };
|
|
939
|
+
if (ids.includes(Number(next))) return { live: null, ids, cycle: true };
|
|
940
|
+
const row = await read(Number(next));
|
|
941
|
+
if (!row) return { live: null, ids, missing: Number(next) };
|
|
942
|
+
ids.push(row.id);
|
|
943
|
+
current = row;
|
|
944
|
+
}
|
|
945
|
+
return { live: null, ids, truncated: true };
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
/**
|
|
949
|
+
* Option A correction back-pointer (the owner, 2026-10-02, a decision record item 4; the reviewer's sketch in his
|
|
950
|
+
* second read of an episodic record). Episodic entries are immutable events, so a PARTIAL correction cannot
|
|
951
|
+
* supersede its parent: supersession would put an end date on the whole parent and present every item
|
|
952
|
+
* in it as replaced. Instead the new entry carries metadata.corrects = <parent id>, and the parent is MARKED:
|
|
953
|
+
* its corrected_by array gains the new id. The parent's content, digest and is_live are untouched; its
|
|
954
|
+
* metadata is re-attested in the same write (as reviseMemory does for superseded_by), then read back.
|
|
955
|
+
* Sanctioned in tests/write-path.test.js on the same grounds as reviseMemory and retireDoc: a mark on an
|
|
956
|
+
* existing row, never a content write or a removal.
|
|
957
|
+
*/
|
|
958
|
+
/**
|
|
959
|
+
* The re-attestation $set for a sanctioned metadata mark (supersession, a
|
|
960
|
+
* corrected_by mark): the new metadata digest and, when a signing key is set
|
|
961
|
+
* and the row's existing signature verifies against its metadata before the
|
|
962
|
+
* marks (aleph.js signatureCoversPriorMetadata), a fresh signature over the
|
|
963
|
+
* same content digest and the new metadata digest (2026-10-02, the reviewer's
|
|
964
|
+
* finding: the marks left signed:false behind). With no key, or a signature
|
|
965
|
+
* that does not verify, the signature is left exactly as it is: never re-signed
|
|
966
|
+
* blindly, so a row changed outside the sanctioned path keeps reading
|
|
967
|
+
* signed:false.
|
|
968
|
+
*/
|
|
969
|
+
export function resignedAttestation(row, newMetadata) {
|
|
970
|
+
const metadataSha256 = digestMetadata(newMetadata);
|
|
971
|
+
const set = { metadata_sha256: metadataSha256 };
|
|
972
|
+
const key = setting('SIGNING_KEY') || null;
|
|
973
|
+
if (key && signatureCoversPriorMetadata(row, key)) set.signature = signDigests(row.sha256, metadataSha256, key);
|
|
974
|
+
return set;
|
|
975
|
+
}
|
|
976
|
+
|
|
977
|
+
export async function markCorrectedBy(dbManager, layer, parentId, childId) {
|
|
978
|
+
await dbManager.getConnection(layer);
|
|
979
|
+
const col = dbManager.collectionFor(layer);
|
|
980
|
+
const parent = await col.findOne({ id: parentId });
|
|
981
|
+
if (!parent) return { parent_id: parentId, marked: false };
|
|
982
|
+
const cid = Number(childId);
|
|
983
|
+
// Race-safe (the second reader N3 / the reviewer, 2026-10-02): the id is ADDED atomically
|
|
984
|
+
// ($addToSet on the column and on the metadata copy), never written back
|
|
985
|
+
// from a stale read, so two simultaneous corrections of one parent both
|
|
986
|
+
// land. A legacy string-metadata row falls back to read-modify-write.
|
|
987
|
+
if (parent.metadata && typeof parent.metadata === 'object') {
|
|
988
|
+
await col.updateOne({ id: parentId }, { $addToSet: { corrected_by: cid, 'metadata.corrected_by': cid } });
|
|
989
|
+
} else {
|
|
990
|
+
const md = parseMetadata(parent.metadata);
|
|
991
|
+
const list = Array.isArray(md.corrected_by) ? md.corrected_by.map(Number) : [];
|
|
992
|
+
if (!list.includes(cid)) list.push(cid);
|
|
993
|
+
md.corrected_by = list;
|
|
994
|
+
await col.updateOne({ id: parentId }, { $set: { metadata: md, corrected_by: list } });
|
|
995
|
+
}
|
|
996
|
+
// Re-attest from what is actually stored. The digest is set only if the
|
|
997
|
+
// stored metadata is still exactly the metadata it was computed from: the
|
|
998
|
+
// compare-and-set filter is the WHOLE metadata value, not only corrected_by
|
|
999
|
+
// (the reviewer, 2026-10-02), so a concurrent change to any metadata field in
|
|
1000
|
+
// between (another mark, a superseded_by, anything) makes the filter miss,
|
|
1001
|
+
// and the loop re-reads and re-digests. The last re-attest therefore always
|
|
1002
|
+
// covers what is stored. The value compared is the one just read back, so
|
|
1003
|
+
// MongoDB's order-sensitive embedded-document equality matches it as stored.
|
|
1004
|
+
let back = null;
|
|
1005
|
+
for (let attempt = 0; attempt < 5; attempt++) {
|
|
1006
|
+
back = await col.findOne({ id: parentId });
|
|
1007
|
+
if (!back) break;
|
|
1008
|
+
const md = parseMetadata(back.metadata);
|
|
1009
|
+
const filter = { id: parentId, metadata: back.metadata };
|
|
1010
|
+
const res = await col.updateOne(filter, { $set: resignedAttestation(back, md) });
|
|
1011
|
+
if (res && res.matchedCount === 0) continue;
|
|
1012
|
+
back = await col.findOne({ id: parentId });
|
|
1013
|
+
break;
|
|
1014
|
+
}
|
|
1015
|
+
const backMd = back ? parseMetadata(back.metadata) : {};
|
|
1016
|
+
const marked = !!(back && Array.isArray(backMd.corrected_by) && backMd.corrected_by.map(Number).includes(cid)
|
|
1017
|
+
&& metadataIntegrityOf(back) === true && back.content === parent.content);
|
|
1018
|
+
return { parent_id: parentId, marked };
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
const correctedByOf = (row) => {
|
|
1022
|
+
const md = parseMetadata(row && row.metadata) || {};
|
|
1023
|
+
const v = Array.isArray(row && row.corrected_by) ? row.corrected_by : md.corrected_by;
|
|
1024
|
+
return Array.isArray(v) && v.length ? v.map(Number) : [];
|
|
1025
|
+
};
|
|
1026
|
+
|
|
1027
|
+
export async function reviseMemory(dbManager, layer, target_id, content, metadata = {}, logger = null) {
|
|
1028
|
+
try {
|
|
1029
|
+
const validatedLayer = validateLayer(layer, true);
|
|
1030
|
+
// Which layers may be revised is the charter's call (layers.js), not a
|
|
1031
|
+
// string comparison here. Today only episodic says false.
|
|
1032
|
+
if (policyFor(validatedLayer).revisable === false) {
|
|
1033
|
+
throw new ValidationError('layer', 'Episodic revision is prohibited - events are immutable history. Save a new episodic entry instead.');
|
|
1034
|
+
}
|
|
1035
|
+
const targetId = Number(target_id);
|
|
1036
|
+
if (!Number.isFinite(targetId) || !Number.isInteger(targetId) || targetId <= 0) {
|
|
1037
|
+
throw new ValidationError('target_id', 'target_id must be a positive integer');
|
|
1038
|
+
}
|
|
1039
|
+
const validatedContent = validateContent(content);
|
|
1040
|
+
// A7: metadata is validated exactly ONCE, inside saveMemory, with
|
|
1041
|
+
// `supersedes` already present as a first-class field. Validating here and
|
|
1042
|
+
// again there is what used to relocate it under custom on the second pass.
|
|
1043
|
+
if (metadata !== null && metadata !== undefined && !isPlainObject(metadata)) {
|
|
1044
|
+
throw new ValidationError('metadata', 'Metadata must be an object', typeof metadata);
|
|
1045
|
+
}
|
|
1046
|
+
// 2026-09-29 (a decision record): a revision with no source of its own cites
|
|
1047
|
+
// the entry it revises, so revising a semantic or procedural entry is never
|
|
1048
|
+
// refused for want of a pointer. A caller-supplied derived_from wins.
|
|
1049
|
+
const revisionMetadata = {
|
|
1050
|
+
...(metadata || {}),
|
|
1051
|
+
supersedes: targetId,
|
|
1052
|
+
...((metadata && metadata.derived_from) ? {} : { derived_from: { entries: [targetId] } })
|
|
1053
|
+
};
|
|
1054
|
+
|
|
1055
|
+
await dbManager.getConnection(validatedLayer);
|
|
1056
|
+
const col = dbManager.collectionFor(validatedLayer);
|
|
1057
|
+
|
|
1058
|
+
const target = await col.findOne({ id: targetId });
|
|
1059
|
+
if (!target) {
|
|
1060
|
+
throw new ValidationError('target_id', `No entry with id ${targetId} in layer '${validatedLayer}'`);
|
|
1061
|
+
}
|
|
1062
|
+
const targetMd = parseMetadata(target.metadata);
|
|
1063
|
+
if (isMemorySuperseded(targetMd)) {
|
|
1064
|
+
throw new ValidationError('target_id', `Entry ${targetId} is already superseded by ${targetMd.superseded_by} - revise the CURRENT entry, not the archived one`);
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
const saved = await saveMemory(dbManager, validatedContent, validatedLayer, revisionMetadata, logger, null);
|
|
1068
|
+
|
|
1069
|
+
targetMd.superseded_by = saved.id;
|
|
1070
|
+
targetMd.superseded_at = saved.timestamp;
|
|
1071
|
+
// Write BOTH the metadata copy and the promoted top-level column. The
|
|
1072
|
+
// top-level one is what a reader follows (indexed); the metadata copy
|
|
1073
|
+
// stays for audit. Always written as a subdocument - a parent still
|
|
1074
|
+
// carrying the legacy string shape is converted by this touch.
|
|
1075
|
+
// is_live is left as it was. Supersession is the end date and the link
|
|
1076
|
+
// to the new entry, not a flag that hides the old one.
|
|
1077
|
+
await col.updateOne({ id: targetId }, { $set: {
|
|
1078
|
+
metadata: targetMd,
|
|
1079
|
+
superseded_by: saved.id,
|
|
1080
|
+
superseded_at: saved.timestamp,
|
|
1081
|
+
// C1: marking a parent is the ONE sanctioned mutation of stored metadata,
|
|
1082
|
+
// so the parent is re-attested in the same write. Without this every
|
|
1083
|
+
// superseded row would read as a metadata mismatch from then on. With a
|
|
1084
|
+
// signing key, a parent whose signature verified before the mark is
|
|
1085
|
+
// re-signed in the same write (resignedAttestation), so it does not read
|
|
1086
|
+
// signed:false; the check runs on the row as read, before targetMd's marks.
|
|
1087
|
+
...resignedAttestation(target, targetMd)
|
|
1088
|
+
} });
|
|
1089
|
+
|
|
1090
|
+
// Read-back verification, both sides (verification over performance)
|
|
1091
|
+
const newDoc = await col.findOne({ id: saved.id });
|
|
1092
|
+
const parentDoc = await col.findOne({ id: targetId });
|
|
1093
|
+
const parentMd = parentDoc ? parseMetadata(parentDoc.metadata) : null;
|
|
1094
|
+
const topLevelOk = !!(parentDoc
|
|
1095
|
+
&& Number(parentDoc.superseded_by) === Number(saved.id)
|
|
1096
|
+
&& parentDoc.superseded_at === saved.timestamp
|
|
1097
|
+
&& parentDoc.is_live !== false);
|
|
1098
|
+
// C1: the parent's re-attestation must read back true, or the revision is
|
|
1099
|
+
// reported unverified - a parent that fails its own metadata digest after
|
|
1100
|
+
// being marked is exactly the inconsistency the column exists to expose.
|
|
1101
|
+
const parentMetaOk = metadataIntegrityOf(parentDoc) === true;
|
|
1102
|
+
const verified = !!(newDoc && parentMd && Number(parentMd.superseded_by) === Number(saved.id) && topLevelOk && parentMetaOk);
|
|
1103
|
+
|
|
1104
|
+
logger?.info('Memory revised via supersession', { layer: validatedLayer, supersedes: targetId, newId: saved.id, verified });
|
|
1105
|
+
// C3: the same receipt saveMemory produced, plus what supersession added.
|
|
1106
|
+
// `verified` here covers both sides (child stored, parent marked).
|
|
1107
|
+
return {
|
|
1108
|
+
...saved,
|
|
1109
|
+
verified,
|
|
1110
|
+
supersedes: targetId,
|
|
1111
|
+
parent_marked_superseded: verified
|
|
1112
|
+
};
|
|
1113
|
+
} catch (error) {
|
|
1114
|
+
if (error instanceof ValidationError || error instanceof EmetError) throw error;
|
|
1115
|
+
throw new DatabaseError(`Failed to revise memory: ${error.message}`, 'revise_memory');
|
|
1116
|
+
}
|
|
1117
|
+
}
|
|
1118
|
+
|
|
1119
|
+
/**
|
|
1120
|
+
* How many rows a recall or layer query keeps after the store has ranked them.
|
|
1121
|
+
* The store sorts by the same keys as the page (current before ended, then
|
|
1122
|
+
* keyword hits and importance, or time when the query asks for the latest),
|
|
1123
|
+
* and only then applies this bound. Same floor semantic recall uses for its
|
|
1124
|
+
* candidate set: 20 times the page, and at least 200.
|
|
1125
|
+
*/
|
|
1126
|
+
export const RECALL_CANDIDATE_FLOOR = 200;
|
|
1127
|
+
export const RECALL_CANDIDATE_FACTOR = 20;
|
|
1128
|
+
export function candidateBound(limit) {
|
|
1129
|
+
const page = Number(limit);
|
|
1130
|
+
const n = Number.isFinite(page) && page > 0 ? page : 10;
|
|
1131
|
+
return Math.max(n * RECALL_CANDIDATE_FACTOR, RECALL_CANDIDATE_FLOOR);
|
|
1132
|
+
}
|
|
1133
|
+
|
|
1134
|
+
function queryTokens(query) {
|
|
1135
|
+
return String(query ?? '').trim().split(/\s+/).filter((k) => k.length > 0).slice(0, 20);
|
|
1136
|
+
}
|
|
1137
|
+
|
|
1138
|
+
/** Whole-token latest / newest / most recent. latest-budget is one token, so it is not this. */
|
|
1139
|
+
function latestPhraseSpan(parts) {
|
|
1140
|
+
for (let i = 0; i < parts.length; i++) {
|
|
1141
|
+
if (/^(latest|newest)$/i.test(parts[i])) return [i, i + 1];
|
|
1142
|
+
if (/^most$/i.test(parts[i]) && /^recent$/i.test(parts[i + 1] || '')) return [i, i + 2];
|
|
1143
|
+
}
|
|
1144
|
+
return null;
|
|
1145
|
+
}
|
|
1146
|
+
|
|
1147
|
+
/** "not latest" and "not the latest" (and the newest / most recent forms). */
|
|
1148
|
+
function phraseIsNegated(parts, start) {
|
|
1149
|
+
const prev = parts[start - 1];
|
|
1150
|
+
const before = parts[start - 2];
|
|
1151
|
+
if (prev && /^not$/i.test(prev)) return true;
|
|
1152
|
+
if (prev && /^the$/i.test(prev) && before && /^not$/i.test(before)) return true;
|
|
1153
|
+
return false;
|
|
1154
|
+
}
|
|
1155
|
+
|
|
1156
|
+
/**
|
|
1157
|
+
* Natural-language time order. A standalone latest, newest, or most recent
|
|
1158
|
+
* reorders rows that already match the other words. It does not match inside
|
|
1159
|
+
* a hyphenated token, and "not the latest" does not ask for time order.
|
|
1160
|
+
* With no other words left, the word stays an ordinary keyword and this is false.
|
|
1161
|
+
* Pass order: 'latest' when time order should not depend on the wording.
|
|
1162
|
+
*/
|
|
1163
|
+
export function queryAsksForLatest(query) {
|
|
1164
|
+
const parts = queryTokens(query);
|
|
1165
|
+
const span = latestPhraseSpan(parts);
|
|
1166
|
+
if (!span || phraseIsNegated(parts, span[0])) return false;
|
|
1167
|
+
const rest = parts.filter((_, i) => i < span[0] || i >= span[1]);
|
|
1168
|
+
return rest.length > 0;
|
|
1169
|
+
}
|
|
1170
|
+
|
|
1171
|
+
/** Words the match set is built from. A latest phrase is removed only when other words remain. */
|
|
1172
|
+
export function recallKeywords(query) {
|
|
1173
|
+
const parts = queryTokens(query);
|
|
1174
|
+
const span = latestPhraseSpan(parts);
|
|
1175
|
+
if (!span || phraseIsNegated(parts, span[0])) return parts;
|
|
1176
|
+
const rest = parts.filter((_, i) => i < span[0] || i >= span[1]);
|
|
1177
|
+
return rest.length > 0 ? rest : parts;
|
|
1178
|
+
}
|
|
1179
|
+
|
|
1180
|
+
function wantsTimeOrder(query, options) {
|
|
1181
|
+
return !!(options && options.order === 'latest') || queryAsksForLatest(query);
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
/** How many of the keywords appear in content or context. One count per keyword.
|
|
1185
|
+
* Used only when the store did not already compute `_hits`. The store path
|
|
1186
|
+
* keeps that count: JavaScript `RegExp` `i` does not fold ẞ and ß. */
|
|
1187
|
+
export function keywordHitCount(memory, keywords) {
|
|
1188
|
+
const content = memory && memory.content ? String(memory.content) : '';
|
|
1189
|
+
const context = memory && memory.context ? String(memory.context) : '';
|
|
1190
|
+
let n = 0;
|
|
1191
|
+
for (const k of keywords) {
|
|
1192
|
+
const spec = containsRegex(k);
|
|
1193
|
+
const re = new RegExp(spec.$regex, spec.$options);
|
|
1194
|
+
if (re.test(content) || re.test(context)) n += 1;
|
|
1195
|
+
}
|
|
1196
|
+
return n;
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1199
|
+
function finiteNumber(value) {
|
|
1200
|
+
const n = Number(value);
|
|
1201
|
+
return Number.isFinite(n) ? n : null;
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1204
|
+
/**
|
|
1205
|
+
* End date and replacement, for a row that is still returned.
|
|
1206
|
+
* ended is true when a later entry replaced it, valid_until has passed,
|
|
1207
|
+
* or an older write set is_live false. The row is not dropped.
|
|
1208
|
+
*/
|
|
1209
|
+
function columnExpiry(value) {
|
|
1210
|
+
const n = Number(value);
|
|
1211
|
+
if (value == null || !Number.isFinite(n) || n <= 0) return { validUntil: null, expired: false };
|
|
1212
|
+
const expirySec = n > 1e12 ? n / 1000 : n;
|
|
1213
|
+
return { validUntil: n, expired: expirySec < Date.now() / 1000 };
|
|
1214
|
+
}
|
|
1215
|
+
|
|
1216
|
+
export function endMarks(memory) {
|
|
1217
|
+
const md = parseMetadata(memory && memory.metadata) || {};
|
|
1218
|
+
const supersededBy = supersededByOf(memory);
|
|
1219
|
+
const supersededAt = finiteNumber(memory && memory.superseded_at != null ? memory.superseded_at : md.superseded_at);
|
|
1220
|
+
const fromMetadata = _extractValidUntil(md);
|
|
1221
|
+
const stamped = fromMetadata != null
|
|
1222
|
+
? { validUntil: fromMetadata, expired: isMemoryExpired(md) }
|
|
1223
|
+
: columnExpiry(memory && memory.valid_until);
|
|
1224
|
+
const validUntil = stamped.validUntil;
|
|
1225
|
+
const expired = stamped.expired;
|
|
1226
|
+
const historicalHide = !!(memory && memory.is_live === false);
|
|
1227
|
+
const ended = supersededBy !== null || expired || historicalHide;
|
|
1228
|
+
let end_date = null;
|
|
1229
|
+
if (supersededBy !== null) end_date = supersededAt;
|
|
1230
|
+
else if (expired) end_date = validUntil;
|
|
1231
|
+
return {
|
|
1232
|
+
superseded_by: supersededBy,
|
|
1233
|
+
superseded_at: supersededBy !== null ? supersededAt : null,
|
|
1234
|
+
is_superseded: supersededBy !== null,
|
|
1235
|
+
valid_until: validUntil,
|
|
1236
|
+
is_expired: expired,
|
|
1237
|
+
ended,
|
|
1238
|
+
end_date,
|
|
1239
|
+
replaced_by: supersededBy
|
|
1240
|
+
};
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
function importanceRank(row) {
|
|
1244
|
+
const n = Number(row && row.importance);
|
|
1245
|
+
return Number.isFinite(n) ? n : 0;
|
|
1246
|
+
}
|
|
1247
|
+
|
|
1248
|
+
/** Store `_ended` when the row came from the rank pipeline. Otherwise the marked flag. */
|
|
1249
|
+
function endedRank(row) {
|
|
1250
|
+
if (row && row._endedRank != null) return row._endedRank;
|
|
1251
|
+
return row && row.ended ? 1 : 0;
|
|
1252
|
+
}
|
|
1253
|
+
|
|
1254
|
+
/** Current entries first, then more keyword hits, then higher importance. Id breaks ties. Not time. */
|
|
1255
|
+
export function compareByRelevance(a, b) {
|
|
1256
|
+
const ae = endedRank(a);
|
|
1257
|
+
const be = endedRank(b);
|
|
1258
|
+
if (ae !== be) return ae - be;
|
|
1259
|
+
const ah = a && a.hits ? a.hits : 0;
|
|
1260
|
+
const bh = b && b.hits ? b.hits : 0;
|
|
1261
|
+
if (ah !== bh) return bh - ah;
|
|
1262
|
+
const ai = importanceRank(a);
|
|
1263
|
+
const bi = importanceRank(b);
|
|
1264
|
+
if (ai !== bi) return bi - ai;
|
|
1265
|
+
return (Number(a && a.id) || 0) - (Number(b && b.id) || 0);
|
|
1266
|
+
}
|
|
1267
|
+
|
|
1268
|
+
/** Current entries first, then newest timestamp. Used only for an explicit latest request. */
|
|
1269
|
+
export function compareByRecency(a, b) {
|
|
1270
|
+
const ae = endedRank(a);
|
|
1271
|
+
const be = endedRank(b);
|
|
1272
|
+
if (ae !== be) return ae - be;
|
|
1273
|
+
const at = Number(a && a.timestamp) || 0;
|
|
1274
|
+
const bt = Number(b && b.timestamp) || 0;
|
|
1275
|
+
if (at !== bt) return bt - at;
|
|
1276
|
+
return (Number(a && a.id) || 0) - (Number(b && b.id) || 0);
|
|
1277
|
+
}
|
|
1278
|
+
|
|
1279
|
+
/**
|
|
1280
|
+
* Default: the ranked list, cut at `limit` (current entries already sit first).
|
|
1281
|
+
* history: true returns ended rows only, up to the same limit, so a full page
|
|
1282
|
+
* of current matches cannot make the ended ones unreachable.
|
|
1283
|
+
*/
|
|
1284
|
+
export function applyReadLimit(rows, limit, history) {
|
|
1285
|
+
const cap = Number(limit);
|
|
1286
|
+
const n = Number.isFinite(cap) && cap > 0 ? cap : rows.length;
|
|
1287
|
+
const pool = history === true ? rows.filter((r) => endedRank(r) === 1) : rows;
|
|
1288
|
+
return pool.slice(0, n);
|
|
1289
|
+
}
|
|
1290
|
+
|
|
1291
|
+
function recalledRow(memory, layer, extra = {}) {
|
|
1292
|
+
return {
|
|
1293
|
+
...(layer ? { layer } : {}),
|
|
1294
|
+
id: memory.id,
|
|
1295
|
+
timestamp: memory.timestamp,
|
|
1296
|
+
content: memory.content,
|
|
1297
|
+
context: memory.context,
|
|
1298
|
+
importance: memory.importance,
|
|
1299
|
+
emotional_intensity: memory.emotional_intensity,
|
|
1300
|
+
metadata: parseMetadata(memory.metadata),
|
|
1301
|
+
...endMarks(memory),
|
|
1302
|
+
corrected_by: correctedByOf(memory),
|
|
1303
|
+
integrity: integrityOf(memory),
|
|
1304
|
+
metadata_integrity: metadataIntegrityOf(memory),
|
|
1305
|
+
signed: signatureIntegrityOf(memory, setting('SIGNING_KEY') || null),
|
|
1306
|
+
...extra
|
|
1307
|
+
};
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1310
|
+
/** Layer query order. Current entries come first in every order, including timestamp. */
|
|
1311
|
+
export function compareQueryRows(a, b, orderBy) {
|
|
1312
|
+
const spec = sortFor(orderBy);
|
|
1313
|
+
const column = Object.keys(spec)[0];
|
|
1314
|
+
const dir = spec[column];
|
|
1315
|
+
const ae = endedRank(a);
|
|
1316
|
+
const be = endedRank(b);
|
|
1317
|
+
if (ae !== be) return ae - be;
|
|
1318
|
+
const av = a ? a[column] : undefined;
|
|
1319
|
+
const bv = b ? b[column] : undefined;
|
|
1320
|
+
if (av !== bv) {
|
|
1321
|
+
const aNull = av == null;
|
|
1322
|
+
const bNull = bv == null;
|
|
1323
|
+
if (aNull || bNull) {
|
|
1324
|
+
if (!(aNull && bNull)) {
|
|
1325
|
+
const nullsLow = aNull ? -1 : 1;
|
|
1326
|
+
return dir === 1 ? nullsLow : -nullsLow;
|
|
1327
|
+
}
|
|
1328
|
+
} else if (av > bv) return dir === 1 ? 1 : -1;
|
|
1329
|
+
else return dir === 1 ? -1 : 1;
|
|
1330
|
+
}
|
|
1331
|
+
return (Number(a && a.id) || 0) - (Number(b && b.id) || 0);
|
|
1332
|
+
}
|
|
1333
|
+
|
|
1334
|
+
export function collectSemanticHits(docs, layer) {
|
|
1335
|
+
const out = [];
|
|
1336
|
+
for (const d of docs || []) out.push(recalledRow(d, layer, { score: d.score }));
|
|
1337
|
+
return out;
|
|
1338
|
+
}
|
|
1339
|
+
|
|
1340
|
+
/** Vector score is the relevance. Importance breaks a score tie. Time only if the query asks. */
|
|
1341
|
+
export function rankSemanticHits(rows, query) {
|
|
1342
|
+
if (queryAsksForLatest(query)) return [...rows].sort(compareByRecency);
|
|
1343
|
+
return [...rows].sort((a, b) => {
|
|
1344
|
+
const ae = a && a.ended ? 1 : 0;
|
|
1345
|
+
const be = b && b.ended ? 1 : 0;
|
|
1346
|
+
if (ae !== be) return ae - be;
|
|
1347
|
+
const as = Number(a && a.score) || 0;
|
|
1348
|
+
const bs = Number(b && b.score) || 0;
|
|
1349
|
+
if (as !== bs) return bs - as;
|
|
1350
|
+
const ai = importanceRank(a);
|
|
1351
|
+
const bi = importanceRank(b);
|
|
1352
|
+
if (ai !== bi) return bi - ai;
|
|
1353
|
+
return (Number(a && a.id) || 0) - (Number(b && b.id) || 0);
|
|
1354
|
+
});
|
|
1355
|
+
}
|
|
1356
|
+
|
|
1357
|
+
export async function recallMemories(dbManager, query, layer = null, limit = 10, logger = null, auditOperation = null, options = null) {
|
|
1358
|
+
const startTime = Date.now();
|
|
1359
|
+
const requestId = logger?._generateRequestId?.() || `req-${Date.now()}`;
|
|
1360
|
+
|
|
1361
|
+
try {
|
|
1362
|
+
if (query === null || query === undefined || (typeof query === 'string' && query.trim() === '')) {
|
|
1363
|
+
throw new ValidationError('query', 'Query is required');
|
|
1364
|
+
}
|
|
1365
|
+
if (typeof query !== 'string') {
|
|
1366
|
+
throw new ValidationError('query', 'Query must be a string');
|
|
1367
|
+
}
|
|
1368
|
+
if (query.length > CONTENT_LIMITS.MAX_QUERY_LENGTH) {
|
|
1369
|
+
throw new ValidationError('query', `Query exceeds maximum length of ${CONTENT_LIMITS.MAX_QUERY_LENGTH} characters`);
|
|
1370
|
+
}
|
|
1371
|
+
const validatedQuery = query.trim();
|
|
1372
|
+
const validatedLayer = layer ? validateLayer(layer) : null;
|
|
1373
|
+
const validatedLimit = validateLimit(limit);
|
|
1374
|
+
if (options && options.order != null && options.order !== 'latest') {
|
|
1375
|
+
throw new ValidationError('order', "order must be 'latest' when it is set");
|
|
1376
|
+
}
|
|
1377
|
+
|
|
1378
|
+
logger?.debug('Recalling memories', {
|
|
1379
|
+
requestId,
|
|
1380
|
+
queryLength: validatedQuery.length,
|
|
1381
|
+
layer: validatedLayer || 'all',
|
|
1382
|
+
limit: validatedLimit
|
|
1383
|
+
});
|
|
1384
|
+
|
|
1385
|
+
const layers = validatedLayer ? [validatedLayer] : Object.keys(MEMORY_LAYERS);
|
|
1386
|
+
const results = [];
|
|
1387
|
+
|
|
1388
|
+
// order: 'latest' is the explicit time order. A standalone latest phrase
|
|
1389
|
+
// only reorders rows that match the other words. It never widens the match
|
|
1390
|
+
// set, and a bare "latest" stays a keyword.
|
|
1391
|
+
const asksForLatest = wantsTimeOrder(validatedQuery, options);
|
|
1392
|
+
const keywords = recallKeywords(validatedQuery);
|
|
1393
|
+
const keywordFilter = keywords.length === 0
|
|
1394
|
+
? null
|
|
1395
|
+
: { $or: keywords.flatMap((k) => [{ content: containsRegex(k) }, { context: containsRegex(k) }]) };
|
|
1396
|
+
// Rank in the store, then limit. A column sort before the limit would
|
|
1397
|
+
// drop a better keyword match, or a current row behind newer ended ones.
|
|
1398
|
+
const bound = candidateBound(validatedLimit);
|
|
1399
|
+
const history = !!(options && options.history === true);
|
|
1400
|
+
|
|
1401
|
+
for (const currentLayer of layers) {
|
|
1402
|
+
await dbManager.getConnection(currentLayer);
|
|
1403
|
+
|
|
1404
|
+
const memories = keywordFilter === null ? [] : await dbManager.aggregateMemories(
|
|
1405
|
+
currentLayer,
|
|
1406
|
+
buildRankPipeline({
|
|
1407
|
+
filter: keywordFilter,
|
|
1408
|
+
keywords,
|
|
1409
|
+
latest: asksForLatest,
|
|
1410
|
+
bound,
|
|
1411
|
+
history,
|
|
1412
|
+
kind: 'recall'
|
|
1413
|
+
})
|
|
1414
|
+
);
|
|
1415
|
+
|
|
1416
|
+
for (const memory of memories) {
|
|
1417
|
+
// The pipeline's _hits is the count the window was ranked with.
|
|
1418
|
+
// A JavaScript recount folds a different set of letters and can drop
|
|
1419
|
+
// every row the store kept. keywordHitCount is only for a row that
|
|
1420
|
+
// did not come from that pipeline.
|
|
1421
|
+
const fromStore = memory._hits != null;
|
|
1422
|
+
const hits = fromStore
|
|
1423
|
+
? (Number(memory._hits) || 0)
|
|
1424
|
+
: (keywords.length === 0 ? 0 : keywordHitCount(memory, keywords));
|
|
1425
|
+
if (!fromStore && keywords.length > 0 && hits === 0) continue;
|
|
1426
|
+
const row = recalledRow(memory, currentLayer);
|
|
1427
|
+
row.hits = hits;
|
|
1428
|
+
if (memory._ended != null) row._endedRank = Number(memory._ended) === 1 ? 1 : 0;
|
|
1429
|
+
results.push(row);
|
|
1430
|
+
}
|
|
1431
|
+
}
|
|
1432
|
+
|
|
1433
|
+
results.sort(asksForLatest ? compareByRecency : compareByRelevance);
|
|
1434
|
+
const finalResults = applyReadLimit(results, validatedLimit, options && options.history === true);
|
|
1435
|
+
for (const row of finalResults) {
|
|
1436
|
+
delete row.hits;
|
|
1437
|
+
delete row._endedRank;
|
|
1438
|
+
}
|
|
1439
|
+
const durationMs = Date.now() - startTime;
|
|
1440
|
+
|
|
1441
|
+
if (logger && auditOperation) {
|
|
1442
|
+
logger.audit(auditOperation.MEMORY_RECALL, {
|
|
1443
|
+
requestId,
|
|
1444
|
+
queryLength: validatedQuery.length,
|
|
1445
|
+
layer: validatedLayer || 'all',
|
|
1446
|
+
layersSearched: layers.length,
|
|
1447
|
+
limit: validatedLimit,
|
|
1448
|
+
resultsFound: finalResults.length,
|
|
1449
|
+
durationMs,
|
|
1450
|
+
success: true
|
|
1451
|
+
});
|
|
1452
|
+
}
|
|
1453
|
+
|
|
1454
|
+
logger?.info('Memories recalled successfully', {
|
|
1455
|
+
requestId,
|
|
1456
|
+
resultsFound: finalResults.length,
|
|
1457
|
+
durationMs
|
|
1458
|
+
});
|
|
1459
|
+
|
|
1460
|
+
return finalResults;
|
|
1461
|
+
} catch (error) {
|
|
1462
|
+
const durationMs = Date.now() - startTime;
|
|
1463
|
+
|
|
1464
|
+
if (logger && auditOperation) {
|
|
1465
|
+
logger.audit(auditOperation.MEMORY_RECALL, {
|
|
1466
|
+
requestId,
|
|
1467
|
+
queryLength: query ? query.length : 0,
|
|
1468
|
+
layer: layer || 'all',
|
|
1469
|
+
durationMs,
|
|
1470
|
+
success: false,
|
|
1471
|
+
errorCode: error.code || 'UNKNOWN',
|
|
1472
|
+
errorMessage: error.message
|
|
1473
|
+
});
|
|
1474
|
+
}
|
|
1475
|
+
|
|
1476
|
+
logger?.error('Failed to recall memories', {
|
|
1477
|
+
requestId,
|
|
1478
|
+
error,
|
|
1479
|
+
layer: layer || 'all',
|
|
1480
|
+
durationMs
|
|
1481
|
+
});
|
|
1482
|
+
|
|
1483
|
+
if (error instanceof EmetError || error instanceof ValidationError) {
|
|
1484
|
+
throw error;
|
|
1485
|
+
}
|
|
1486
|
+
|
|
1487
|
+
throw new DatabaseError(
|
|
1488
|
+
'Failed to recall memories',
|
|
1489
|
+
'recall_memories',
|
|
1490
|
+
{ layer: layer || 'all' }
|
|
1491
|
+
);
|
|
1492
|
+
}
|
|
1493
|
+
}
|
|
1494
|
+
|
|
1495
|
+
/**
|
|
1496
|
+
* Query specific layer
|
|
1497
|
+
*/
|
|
1498
|
+
export async function queryLayer(dbManager, layer, options = {}, logger = null, auditOperation = null) {
|
|
1499
|
+
const startTime = Date.now();
|
|
1500
|
+
const requestId = logger?._generateRequestId?.() || `req-${Date.now()}`;
|
|
1501
|
+
|
|
1502
|
+
try {
|
|
1503
|
+
const validatedLayer = validateLayer(layer, true);
|
|
1504
|
+
const validatedOptions = validateQueryOptions(options);
|
|
1505
|
+
// Superseded and expired rows are returned, marked. include_superseded: false
|
|
1506
|
+
// does not hide them. Current entries are ordered first in every order,
|
|
1507
|
+
// including timestamp. history: true returns the ended rows on their own.
|
|
1508
|
+
|
|
1509
|
+
logger?.debug('Querying layer', {
|
|
1510
|
+
requestId,
|
|
1511
|
+
layer: validatedLayer,
|
|
1512
|
+
hasFilters: !!validatedOptions.filters,
|
|
1513
|
+
limit: validatedOptions.limit
|
|
1514
|
+
});
|
|
1515
|
+
|
|
1516
|
+
await dbManager.getConnection(validatedLayer);
|
|
1517
|
+
const limit = validatedOptions.limit;
|
|
1518
|
+
let filter = {};
|
|
1519
|
+
|
|
1520
|
+
if (validatedOptions._deprecated_where_used) {
|
|
1521
|
+
logger?.warn('Deprecated WHERE clause usage', {
|
|
1522
|
+
requestId,
|
|
1523
|
+
message: 'Arbitrary WHERE clauses are deprecated for security. Use structured filters instead.'
|
|
1524
|
+
});
|
|
1525
|
+
}
|
|
1526
|
+
|
|
1527
|
+
// An unrecognised filter key is dropped by the whitelist in validateFilters,
|
|
1528
|
+
// leaving the query unfiltered - which reads to the caller as a broken
|
|
1529
|
+
// filter rather than a rejected argument. Say so instead of swallowing it.
|
|
1530
|
+
if (validatedOptions._unknown_filter_keys) {
|
|
1531
|
+
logger?.warn('Unrecognised filter keys ignored', {
|
|
1532
|
+
requestId,
|
|
1533
|
+
layer: validatedLayer,
|
|
1534
|
+
ignored: validatedOptions._unknown_filter_keys,
|
|
1535
|
+
recognised: RECOGNISED_FILTER_KEYS,
|
|
1536
|
+
message: 'These filter keys are not recognised and were ignored; the query did not filter on them.'
|
|
1537
|
+
});
|
|
1538
|
+
}
|
|
1539
|
+
|
|
1540
|
+
filter = buildMemoryFilter(validatedOptions.filters);
|
|
1541
|
+
|
|
1542
|
+
if (!validatedOptions.filters && validatedOptions.params && Array.isArray(validatedOptions.params) && validatedOptions.params.length > 0) {
|
|
1543
|
+
const searchTerm = validatedOptions.params[0];
|
|
1544
|
+
if (typeof searchTerm === 'string') {
|
|
1545
|
+
if (searchTerm.length > CONTENT_LIMITS.MAX_QUERY_LENGTH) {
|
|
1546
|
+
throw new ValidationError('params[0]', `Search term exceeds maximum length of ${CONTENT_LIMITS.MAX_QUERY_LENGTH}`);
|
|
1547
|
+
}
|
|
1548
|
+
filter = { $or: [{ content: containsRegex(searchTerm) }, { context: containsRegex(searchTerm) }] };
|
|
1549
|
+
}
|
|
1550
|
+
}
|
|
1551
|
+
|
|
1552
|
+
|
|
1553
|
+
// Rank in the store with the same keys as the page, then limit. Ended rows
|
|
1554
|
+
// stay in the window; they are not dropped for being expired.
|
|
1555
|
+
const memories = await dbManager.aggregateMemories(validatedLayer, buildRankPipeline({
|
|
1556
|
+
filter,
|
|
1557
|
+
orderBy: validatedOptions.order_by,
|
|
1558
|
+
bound: candidateBound(limit),
|
|
1559
|
+
history: !!(options && options.history === true),
|
|
1560
|
+
kind: 'query'
|
|
1561
|
+
}));
|
|
1562
|
+
|
|
1563
|
+
// CHAIN RESOLUTION (2026-09-15, the consolidated revision). A lookup BY ID
|
|
1564
|
+
// that lands on a superseded entry also returns the live end of its chain,
|
|
1565
|
+
// marked, so a cross-reference written against any id in the chain keeps
|
|
1566
|
+
// resolving after every later revision. Before this, every revise_memory
|
|
1567
|
+
// minted a new id and a hand-kept name->id index had to be revised in the
|
|
1568
|
+
// same pass; the index was the reminder standing in for the control.
|
|
1569
|
+
const requestedId = validatedOptions.filters && validatedOptions.filters.id !== undefined ? Number(validatedOptions.filters.id) : null;
|
|
1570
|
+
if (requestedId !== null && memories.length === 1 && supersededByOf(memories[0]) !== null) {
|
|
1571
|
+
const chain = await resolveSupersessionChain((id) => dbManager.readMemory(validatedLayer, id), memories[0]);
|
|
1572
|
+
if (chain.live && !memories.some(m => m.id === chain.live.id)) {
|
|
1573
|
+
chain.live._resolved_via = { requested_id: requestedId, chain: chain.ids };
|
|
1574
|
+
memories.unshift(chain.live);
|
|
1575
|
+
}
|
|
1576
|
+
}
|
|
1577
|
+
|
|
1578
|
+
const results = memories.map(m => {
|
|
1579
|
+
const row = recalledRow(m, null, {
|
|
1580
|
+
sha256: m.sha256 || null,
|
|
1581
|
+
prev_sha256: m.prev_sha256 || null,
|
|
1582
|
+
chain_sha256: m.chain_sha256 || null,
|
|
1583
|
+
chain: chainIntegrityOf(m),
|
|
1584
|
+
// Present only on the live entry a chain lookup resolved to: which id was
|
|
1585
|
+
// asked for and every id walked to get here, oldest first.
|
|
1586
|
+
...(m._resolved_via ? { resolved_via: m._resolved_via } : {})
|
|
1587
|
+
});
|
|
1588
|
+
if (m._ended != null) row._endedRank = Number(m._ended) === 1 ? 1 : 0;
|
|
1589
|
+
return row;
|
|
1590
|
+
});
|
|
1591
|
+
results.sort((a, b) => compareQueryRows(a, b, validatedOptions.order_by));
|
|
1592
|
+
const via = results.findIndex((row) => row.resolved_via);
|
|
1593
|
+
if (via > 0) {
|
|
1594
|
+
const [resolved] = results.splice(via, 1);
|
|
1595
|
+
results.unshift(resolved);
|
|
1596
|
+
}
|
|
1597
|
+
const ranked = applyReadLimit(results, limit, options && options.history === true);
|
|
1598
|
+
for (const row of ranked) delete row._endedRank;
|
|
1599
|
+
// Keep the name `results` for the audit log and the warning attach below.
|
|
1600
|
+
results.length = 0;
|
|
1601
|
+
results.push(...ranked);
|
|
1602
|
+
|
|
1603
|
+
const durationMs = Date.now() - startTime;
|
|
1604
|
+
|
|
1605
|
+
if (logger && auditOperation) {
|
|
1606
|
+
logger.audit(auditOperation.MEMORY_QUERY, {
|
|
1607
|
+
requestId,
|
|
1608
|
+
layer: validatedLayer,
|
|
1609
|
+
filterCount: validatedOptions.filters ? Object.keys(validatedOptions.filters).length : 0,
|
|
1610
|
+
limit,
|
|
1611
|
+
orderBy: validatedOptions.order_by || 'importance DESC',
|
|
1612
|
+
resultsFound: results.length,
|
|
1613
|
+
durationMs,
|
|
1614
|
+
success: true
|
|
1615
|
+
});
|
|
1616
|
+
}
|
|
1617
|
+
|
|
1618
|
+
logger?.info('Layer query completed', {
|
|
1619
|
+
requestId,
|
|
1620
|
+
layer: validatedLayer,
|
|
1621
|
+
resultsFound: results.length,
|
|
1622
|
+
durationMs
|
|
1623
|
+
});
|
|
1624
|
+
|
|
1625
|
+
// The caller cannot read a server-side log line, and on the HTTP transport
|
|
1626
|
+
// there is not even one to read - that transport's logger discards warn().
|
|
1627
|
+
// So the ignored keys ride back in the response envelope, where the person
|
|
1628
|
+
// who wrote the bad filter will actually see them. This is the surface that
|
|
1629
|
+
// would have caught the 2026-09-08 phantom defect as it happened.
|
|
1630
|
+
if (validatedOptions._unknown_filter_keys) {
|
|
1631
|
+
attachWarnings(results, [
|
|
1632
|
+
`Ignored unrecognised filter key(s): ${validatedOptions._unknown_filter_keys.join(', ')}. `
|
|
1633
|
+
+ `The query did NOT filter on them. `
|
|
1634
|
+
+ `Recognised filter keys: ${RECOGNISED_FILTER_KEYS.join(', ')}.`
|
|
1635
|
+
]);
|
|
1636
|
+
}
|
|
1637
|
+
|
|
1638
|
+
return results;
|
|
1639
|
+
} catch (error) {
|
|
1640
|
+
const durationMs = Date.now() - startTime;
|
|
1641
|
+
|
|
1642
|
+
if (logger && auditOperation) {
|
|
1643
|
+
logger.audit(auditOperation.MEMORY_QUERY, {
|
|
1644
|
+
requestId,
|
|
1645
|
+
layer: layer || 'unknown',
|
|
1646
|
+
durationMs,
|
|
1647
|
+
success: false,
|
|
1648
|
+
errorCode: error.code || 'UNKNOWN',
|
|
1649
|
+
errorMessage: error.message
|
|
1650
|
+
});
|
|
1651
|
+
}
|
|
1652
|
+
|
|
1653
|
+
logger?.error('Failed to query layer', {
|
|
1654
|
+
requestId,
|
|
1655
|
+
error,
|
|
1656
|
+
layer: layer || 'unknown',
|
|
1657
|
+
durationMs
|
|
1658
|
+
});
|
|
1659
|
+
|
|
1660
|
+
if (error instanceof EmetError || error instanceof ValidationError) {
|
|
1661
|
+
throw error;
|
|
1662
|
+
}
|
|
1663
|
+
|
|
1664
|
+
throw new DatabaseError(
|
|
1665
|
+
'Failed to query layer',
|
|
1666
|
+
'query_layer',
|
|
1667
|
+
{ layer: layer || 'unknown' }
|
|
1668
|
+
);
|
|
1669
|
+
}
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
/**
|
|
1673
|
+
* Get server status.
|
|
1674
|
+
*/
|
|
1675
|
+
export async function getStatus(dbManager, logger = null) {
|
|
1676
|
+
try {
|
|
1677
|
+
const status = {
|
|
1678
|
+
database: `mongodb:${MONGODB_DB}`,
|
|
1679
|
+
storage_backend: 'mongodb',
|
|
1680
|
+
system: 'EMET',
|
|
1681
|
+
version: EMET_VERSION,
|
|
1682
|
+
layers: {},
|
|
1683
|
+
total_memories: 0,
|
|
1684
|
+
health: 'healthy'
|
|
1685
|
+
};
|
|
1686
|
+
|
|
1687
|
+
for (const layer of Object.keys(MEMORY_LAYERS)) {
|
|
1688
|
+
try {
|
|
1689
|
+
await dbManager.getConnection(layer);
|
|
1690
|
+
const count = (await dbManager.countMemories(layer)) || 0;
|
|
1691
|
+
|
|
1692
|
+
status.layers[layer] = {
|
|
1693
|
+
status: 'connected',
|
|
1694
|
+
count,
|
|
1695
|
+
path: `mongodb:${MONGODB_DB}.${COLLECTION_PREFIX}${layer}`
|
|
1696
|
+
};
|
|
1697
|
+
status.total_memories += count;
|
|
1698
|
+
} catch (error) {
|
|
1699
|
+
status.layers[layer] = { status: 'error', error: error.message };
|
|
1700
|
+
status.health = 'degraded';
|
|
1701
|
+
}
|
|
1702
|
+
}
|
|
1703
|
+
|
|
1704
|
+
logger?.info(`EMET status: ${status.total_memories} total memories, health: ${status.health}`);
|
|
1705
|
+
|
|
1706
|
+
return status;
|
|
1707
|
+
} catch (error) {
|
|
1708
|
+
logger?.error('Error getting system status:', { error });
|
|
1709
|
+
|
|
1710
|
+
if (error instanceof EmetError || error instanceof ValidationError) {
|
|
1711
|
+
throw error;
|
|
1712
|
+
}
|
|
1713
|
+
|
|
1714
|
+
throw new DatabaseError(
|
|
1715
|
+
'Failed to get system status',
|
|
1716
|
+
'emet_status'
|
|
1717
|
+
);
|
|
1718
|
+
}
|
|
1719
|
+
}
|
|
1720
|
+
|
|
1721
|
+
/**
|
|
1722
|
+
* Query session transcripts - the live tier-2 store AND the legacy archive.
|
|
1723
|
+
*
|
|
1724
|
+
* Two collections hold word-for-word dialog. `transcripts` is the live store
|
|
1725
|
+
* written by save_transcript (one document per session). `exchanges` is an
|
|
1726
|
+
* archive that stopped receiving writes around June 2026 (one document per
|
|
1727
|
+
* exchange, imported from earlier hosts). Until 2026-09 the archive had its own
|
|
1728
|
+
* reader, query_exchanges; folding it in here means one reader for tier 2 and
|
|
1729
|
+
* lets the five overlapping tools go (the user's decision, 2026-09-14).
|
|
1730
|
+
*
|
|
1731
|
+
* Nothing is migrated. Archive rows are grouped by session_id into the same
|
|
1732
|
+
* session shape the live store returns, every result carries `source` naming
|
|
1733
|
+
* the collection it came from, and the two sets are merged in session_start
|
|
1734
|
+
* order. A session_id filter applies to both.
|
|
1735
|
+
*/
|
|
1736
|
+
export async function queryTranscripts(timestamp_start, timestamp_end, session_id = null, limit = 50, logger = null) {
|
|
1737
|
+
const startTime = Date.now();
|
|
1738
|
+
try {
|
|
1739
|
+
if (typeof timestamp_start !== 'number' || typeof timestamp_end !== 'number')
|
|
1740
|
+
throw new ValidationError('timestamp_start and timestamp_end must be numbers (Unix timestamps)');
|
|
1741
|
+
if (timestamp_start >= timestamp_end)
|
|
1742
|
+
throw new ValidationError('timestamp_start must be less than timestamp_end');
|
|
1743
|
+
// Security audit 2026-09-29: session_id reached the query unchecked, so an
|
|
1744
|
+
// object such as {"$ne": null} was an operator, not an id. Every other
|
|
1745
|
+
// doc_id / session_id door already refused a non-string.
|
|
1746
|
+
if (session_id !== null && session_id !== undefined && typeof session_id !== 'string')
|
|
1747
|
+
throw new ValidationError('session_id', 'must be a string');
|
|
1748
|
+
const effectiveLimit = Math.min(Math.max(1, limit || 50), 500);
|
|
1749
|
+
const client = await connectMongo(MONGODB_URI, { serverSelectionTimeoutMS: 5000, maxPoolSize: 10 });
|
|
1750
|
+
const db = client.db(MONGODB_DB);
|
|
1751
|
+
const windowStart = new Date(timestamp_start * 1000);
|
|
1752
|
+
const windowEnd = new Date(timestamp_end * 1000);
|
|
1753
|
+
|
|
1754
|
+
// Live store: one document per session, filtered by session_start.
|
|
1755
|
+
const liveQuery = { _type: { $ne: 'architecture_note' }, session_start: { $gte: windowStart, $lte: windowEnd } };
|
|
1756
|
+
if (session_id) liveQuery.session_id = session_id;
|
|
1757
|
+
const live = await db.collection('transcripts').find(liveQuery).sort({ session_start: 1 }).limit(effectiveLimit).toArray();
|
|
1758
|
+
for (const row of live) row.source = 'transcripts';
|
|
1759
|
+
|
|
1760
|
+
// Archive: one document per exchange. A session is in the window if any of
|
|
1761
|
+
// its exchanges is (rows carry session_start, but the CLI-era rows have a
|
|
1762
|
+
// session_start a month before their timestamps, so the row time is the
|
|
1763
|
+
// honest key). Every row of a matched session is returned, so a session is
|
|
1764
|
+
// never returned with a hole in it.
|
|
1765
|
+
const archived = await archivedSessions(db, windowStart, windowEnd, session_id, effectiveLimit);
|
|
1766
|
+
|
|
1767
|
+
await client.close();
|
|
1768
|
+
const merged = [...live, ...archived]
|
|
1769
|
+
.sort((a, b) => new Date(a.session_start) - new Date(b.session_start))
|
|
1770
|
+
.slice(0, effectiveLimit);
|
|
1771
|
+
const duration = Date.now() - startTime;
|
|
1772
|
+
return {
|
|
1773
|
+
count: merged.length,
|
|
1774
|
+
sessions: merged,
|
|
1775
|
+
sources: { transcripts: live.length, exchanges: archived.length },
|
|
1776
|
+
query: { timestamp_start, timestamp_end, session_id: session_id || 'all', limit: effectiveLimit },
|
|
1777
|
+
performance: { duration_ms: duration }
|
|
1778
|
+
};
|
|
1779
|
+
} catch (error) {
|
|
1780
|
+
if (error instanceof ValidationError || error instanceof EmetError) throw error;
|
|
1781
|
+
throw new DatabaseError(`Failed to query transcripts: ${error.message}`, 'query_transcripts');
|
|
1782
|
+
}
|
|
1783
|
+
}
|
|
1784
|
+
|
|
1785
|
+
/**
|
|
1786
|
+
* Group archive exchanges into sessions. Exported for its test; not a tool.
|
|
1787
|
+
* Pure over its inputs apart from the two reads, so the grouping can be tested
|
|
1788
|
+
* against rows without a database - see groupArchivedExchanges.
|
|
1789
|
+
*/
|
|
1790
|
+
async function archivedSessions(db, windowStart, windowEnd, session_id, limit) {
|
|
1791
|
+
const col = db.collection('exchanges');
|
|
1792
|
+
const rowFilter = { timestamp: { $gte: windowStart, $lte: windowEnd } };
|
|
1793
|
+
if (session_id) rowFilter.session_id = session_id;
|
|
1794
|
+
const ids = await col.distinct('session_id', rowFilter);
|
|
1795
|
+
if (ids.length === 0) return [];
|
|
1796
|
+
const rows = await col.find({ session_id: { $in: ids } }).sort({ session_id: 1, exchange_index: 1, timestamp: 1 }).toArray();
|
|
1797
|
+
return groupArchivedExchanges(rows).slice(0, limit);
|
|
1798
|
+
}
|
|
1799
|
+
|
|
1800
|
+
/**
|
|
1801
|
+
* The archive's rows, grouped by session_id into the live store's session
|
|
1802
|
+
* shape. Exchange indices are kept as stored (the archive counts from 0; the
|
|
1803
|
+
* live store from 1) - renumbering would be a rewrite of the record.
|
|
1804
|
+
*/
|
|
1805
|
+
export function groupArchivedExchanges(rows) {
|
|
1806
|
+
const bySession = new Map();
|
|
1807
|
+
for (const r of rows) {
|
|
1808
|
+
const key = r.session_id || `unkeyed_${r.conversation_uuid || r.channel || 'archive'}`;
|
|
1809
|
+
let s = bySession.get(key);
|
|
1810
|
+
if (!s) {
|
|
1811
|
+
s = {
|
|
1812
|
+
session_id: key,
|
|
1813
|
+
source: 'exchanges',
|
|
1814
|
+
channel: r.channel || null,
|
|
1815
|
+
conversation_name: r.conversation_name || null,
|
|
1816
|
+
conversation_uuid: r.conversation_uuid || null,
|
|
1817
|
+
session_start: r.session_start || r.timestamp,
|
|
1818
|
+
session_end: r.timestamp,
|
|
1819
|
+
exchange_count: 0,
|
|
1820
|
+
exchanges: []
|
|
1821
|
+
};
|
|
1822
|
+
bySession.set(key, s);
|
|
1823
|
+
}
|
|
1824
|
+
if (r.session_start && new Date(r.session_start) < new Date(s.session_start)) s.session_start = r.session_start;
|
|
1825
|
+
if (r.timestamp && new Date(r.timestamp) > new Date(s.session_end)) s.session_end = r.timestamp;
|
|
1826
|
+
s.exchanges.push({
|
|
1827
|
+
exchange_index: r.exchange_index,
|
|
1828
|
+
user_message: r.user_message,
|
|
1829
|
+
assistant_message: r.assistant_message,
|
|
1830
|
+
timestamp: r.timestamp
|
|
1831
|
+
});
|
|
1832
|
+
s.exchange_count = s.exchanges.length;
|
|
1833
|
+
}
|
|
1834
|
+
const out = [...bySession.values()];
|
|
1835
|
+
for (const s of out) s.exchanges.sort((a, b) => (a.exchange_index ?? 0) - (b.exchange_index ?? 0));
|
|
1836
|
+
return out.sort((a, b) => new Date(a.session_start) - new Date(b.session_start));
|
|
1837
|
+
}
|
|
1838
|
+
|
|
1839
|
+
/** One message field, and the whole transcript request. */
|
|
1840
|
+
export const EXCHANGE_FIELD_MAX = 32000;
|
|
1841
|
+
export const EXCHANGE_REQUEST_MAX = 256000;
|
|
1842
|
+
|
|
1843
|
+
export function assertExchangeBudget(parts) {
|
|
1844
|
+
let total = 0;
|
|
1845
|
+
for (const part of parts) {
|
|
1846
|
+
const text = String(part ?? '');
|
|
1847
|
+
if (text.length > EXCHANGE_FIELD_MAX) {
|
|
1848
|
+
throw new ValidationError(`an exchange field exceeds ${EXCHANGE_FIELD_MAX} characters. Nothing was written.`);
|
|
1849
|
+
}
|
|
1850
|
+
total += text.length;
|
|
1851
|
+
}
|
|
1852
|
+
if (total > EXCHANGE_REQUEST_MAX) {
|
|
1853
|
+
throw new ValidationError(`this transcript request exceeds ${EXCHANGE_REQUEST_MAX} characters. Nothing was written.`);
|
|
1854
|
+
}
|
|
1855
|
+
}
|
|
1856
|
+
|
|
1857
|
+
/**
|
|
1858
|
+
* Save a full session transcript to the transcripts collection.
|
|
1859
|
+
* Hardcoded rule: full word-for-word dialog only - no summaries.
|
|
1860
|
+
* The caller passes exchanges. The server does not read a file path.
|
|
1861
|
+
* Every host writes here, so continuity is shared.
|
|
1862
|
+
*/
|
|
1863
|
+
export async function saveTranscript(session_id, session_start, session_end, exchanges = null, channel = null, logger = null) {
|
|
1864
|
+
const startTime = Date.now();
|
|
1865
|
+
const requestId = logger?._generateRequestId?.() || `req-${Date.now()}`;
|
|
1866
|
+
|
|
1867
|
+
try {
|
|
1868
|
+
if (!session_id || typeof session_id !== 'string') {
|
|
1869
|
+
throw new ValidationError('session_id must be a non-empty string');
|
|
1870
|
+
}
|
|
1871
|
+
if (typeof session_start !== 'number') {
|
|
1872
|
+
throw new ValidationError('session_start must be a Unix timestamp (number)');
|
|
1873
|
+
}
|
|
1874
|
+
|
|
1875
|
+
const resolvedChannel = channel || 'claude_project';
|
|
1876
|
+
if (!Array.isArray(exchanges) || exchanges.length === 0) {
|
|
1877
|
+
throw new ValidationError('exchanges must be a non-empty array');
|
|
1878
|
+
}
|
|
1879
|
+
assertExchangeBudget(exchanges.flatMap((ex) => [ex && ex.user_message, ex && ex.assistant_message]));
|
|
1880
|
+
const resolvedExchanges = exchanges;
|
|
1881
|
+
|
|
1882
|
+
const client = await connectMongo(MONGODB_URI, { serverSelectionTimeoutMS: 5000, maxPoolSize: 10 });
|
|
1883
|
+
const db = client.db(MONGODB_DB);
|
|
1884
|
+
const col = db.collection('transcripts');
|
|
1885
|
+
try { await col.createIndex({ 'exchanges.client_id': 1 }, { sparse: true }); } catch { /* an index miss never blocks the write */ }
|
|
1886
|
+
|
|
1887
|
+
const existingCount = await col.countDocuments();
|
|
1888
|
+
if (existingCount === 0) {
|
|
1889
|
+
await col.insertOne({
|
|
1890
|
+
_type: 'architecture_note',
|
|
1891
|
+
created_at: new Date(),
|
|
1892
|
+
note: 'Full session dialog, stored verbatim. Search by timestamp window.'
|
|
1893
|
+
});
|
|
1894
|
+
}
|
|
1895
|
+
|
|
1896
|
+
// Verbatim means every word the user said, not every secret they
|
|
1897
|
+
// pasted. A masked span is visible in the record as '*****'; the rest of
|
|
1898
|
+
// the exchange is untouched. This is the one edit the verbatim rule allows.
|
|
1899
|
+
let redactedSpans = 0;
|
|
1900
|
+
const scrub = (s) => { const r = redactWithReport(s || ''); redactedSpans += r.hits; return r.text; };
|
|
1901
|
+
|
|
1902
|
+
const sessionDoc = {
|
|
1903
|
+
session_id,
|
|
1904
|
+
channel: resolvedChannel,
|
|
1905
|
+
session_start: new Date(session_start * 1000),
|
|
1906
|
+
session_end: session_end ? new Date(session_end * 1000) : new Date(),
|
|
1907
|
+
exchange_count: resolvedExchanges.length,
|
|
1908
|
+
exchanges: resolvedExchanges.map((ex, i) => stampExchange(storedExchange(ex, i, scrub), resolvedChannel)),
|
|
1909
|
+
created_at: new Date()
|
|
1910
|
+
};
|
|
1911
|
+
if (redactedSpans > 0) sessionDoc.redacted_spans = redactedSpans;
|
|
1912
|
+
const timestampsDefaulted = sessionDoc.exchanges.filter(e => e.timestamp_defaulted).length;
|
|
1913
|
+
|
|
1914
|
+
const result = await col.insertOne(sessionDoc);
|
|
1915
|
+
await client.close();
|
|
1916
|
+
|
|
1917
|
+
const duration = Date.now() - startTime;
|
|
1918
|
+
return {
|
|
1919
|
+
success: true,
|
|
1920
|
+
session_id,
|
|
1921
|
+
channel: resolvedChannel,
|
|
1922
|
+
inserted_id: result.insertedId.toString(),
|
|
1923
|
+
exchange_count: resolvedExchanges.length,
|
|
1924
|
+
redacted: redactedSpans,
|
|
1925
|
+
...(timestampsDefaulted > 0 ? { timestamps_defaulted: timestampsDefaulted } : {}),
|
|
1926
|
+
performance: { duration_ms: duration }
|
|
1927
|
+
};
|
|
1928
|
+
|
|
1929
|
+
} catch (error) {
|
|
1930
|
+
if (error instanceof ValidationError || error instanceof EmetError) throw error;
|
|
1931
|
+
throw new DatabaseError(`Failed to save session transcript: ${error.message}`, 'save_transcript');
|
|
1932
|
+
}
|
|
1933
|
+
}
|
|
1934
|
+
|
|
1935
|
+
/**
|
|
1936
|
+
* One exchange as stored, for both transcript doors (save and revise).
|
|
1937
|
+
*
|
|
1938
|
+
* A missing timestamp is still filled with the save time - the record needs a
|
|
1939
|
+
* Date - but it is now MARKED `timestamp_defaulted: true`, so the fill is
|
|
1940
|
+
* visible and queryable instead of indistinguishable from an observed time.
|
|
1941
|
+
* Found 2026-09-10: two exchanges saved without timestamps read back as if both
|
|
1942
|
+
* had happened at the save instant, with nothing to say otherwise. Same
|
|
1943
|
+
* principle as the layer door - a missing field is written as a marker, never
|
|
1944
|
+
* silently filled - applied to the one path that still filled silently.
|
|
1945
|
+
* Semantics otherwise unchanged: a falsy timestamp is treated as absent.
|
|
1946
|
+
*/
|
|
1947
|
+
export function storedExchange(ex, i, scrub, now = new Date()) {
|
|
1948
|
+
const observed = Boolean(ex.timestamp);
|
|
1949
|
+
return {
|
|
1950
|
+
exchange_index: ex.exchange_index !== undefined ? ex.exchange_index : i,
|
|
1951
|
+
user_message: scrub(ex.user_message),
|
|
1952
|
+
assistant_message: scrub(ex.assistant_message),
|
|
1953
|
+
timestamp: observed ? new Date(ex.timestamp * 1000) : now,
|
|
1954
|
+
...(observed ? {} : { timestamp_defaulted: true })
|
|
1955
|
+
};
|
|
1956
|
+
}
|
|
1957
|
+
|
|
1958
|
+
function stampExchange(row, source) {
|
|
1959
|
+
const stamp = writerStamp();
|
|
1960
|
+
return {
|
|
1961
|
+
...row,
|
|
1962
|
+
source: source || setting('SOURCE_TAG') || null,
|
|
1963
|
+
person_id: stamp.person_id,
|
|
1964
|
+
client_id: stamp.client_id
|
|
1965
|
+
};
|
|
1966
|
+
}
|
|
1967
|
+
|
|
1968
|
+
/** Store wait for the boot drop list. Tests set EMET_DROP_LIST_TIMEOUT_MS short so a missing store does not stall the suite. */
|
|
1969
|
+
function dropListTimeoutMs() {
|
|
1970
|
+
const n = Number(process.env.EMET_DROP_LIST_TIMEOUT_MS);
|
|
1971
|
+
return Number.isFinite(n) && n > 0 ? n : 1500;
|
|
1972
|
+
}
|
|
1973
|
+
|
|
1974
|
+
/** Unhandled drops from the store. Used when the startup cache has no list. */
|
|
1975
|
+
async function unhandledDropIds() {
|
|
1976
|
+
const client = await Promise.race([
|
|
1977
|
+
connectMongo(MONGODB_URI, { serverSelectionTimeoutMS: dropListTimeoutMs(), maxPoolSize: 10 }),
|
|
1978
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error('unhandled drop list timed out')), dropListTimeoutMs()))
|
|
1979
|
+
]);
|
|
1980
|
+
try {
|
|
1981
|
+
const open = await client.db(MONGODB_DB).collection('documents')
|
|
1982
|
+
.find({ doc_id: { $regex: '^drops/' }, retired: { $ne: true } }, { projection: { doc_id: 1 } })
|
|
1983
|
+
.sort({ updated_at: -1 })
|
|
1984
|
+
.toArray();
|
|
1985
|
+
return open.map((d) => d.doc_id).filter((id) => typeof id === 'string');
|
|
1986
|
+
} finally { await client.close(); }
|
|
1987
|
+
}
|
|
1988
|
+
|
|
1989
|
+
/** New York calendar day, the drop-cap boundary. */
|
|
1990
|
+
function nyDay(date) {
|
|
1991
|
+
return new Intl.DateTimeFormat('en-CA', { timeZone: 'America/New_York', year: 'numeric', month: '2-digit', day: '2-digit' }).format(date);
|
|
1992
|
+
}
|
|
1993
|
+
/**
|
|
1994
|
+
* 20/day bucket. Stdio is one bucket for the process. HTTP with no OAuth
|
|
1995
|
+
* client stays http:visitor, and a caller-supplied source never moves it.
|
|
1996
|
+
* An approval-bound source is 20 per source per New York day. Any other
|
|
1997
|
+
* OAuth client is its own bucket.
|
|
1998
|
+
*/
|
|
1999
|
+
function dropBucket(clientId, _source) {
|
|
2000
|
+
const actor = currentActor();
|
|
2001
|
+
if (actor.transport === 'http' || clientId === 'http:visitor' || clientId === null) {
|
|
2002
|
+
if (!clientId || clientId === 'http:visitor') return 'http:visitor';
|
|
2003
|
+
const bound = boundSourceTag();
|
|
2004
|
+
if (bound) return `source:${bound}`;
|
|
2005
|
+
return `client:${clientId}`;
|
|
2006
|
+
}
|
|
2007
|
+
if (!clientId || clientId === 'stdio') return 'stdio';
|
|
2008
|
+
const bound = boundSourceTag();
|
|
2009
|
+
if (bound) return `source:${bound}`;
|
|
2010
|
+
return `client:${clientId}`;
|
|
2011
|
+
}
|
|
2012
|
+
|
|
2013
|
+
/** Drops this open must stamp. Always from the store, so an expired or evicted startup cache cannot stamp an empty list. Never caller input. */
|
|
2014
|
+
export async function dropsSeenForOpen(load_id) {
|
|
2015
|
+
try {
|
|
2016
|
+
return await unhandledDropIds();
|
|
2017
|
+
} catch (e) {
|
|
2018
|
+
const boot = cachedLoad(load_id);
|
|
2019
|
+
const cached = boot && Array.isArray(boot.unhandled_drops) ? boot.unhandled_drops.filter((id) => typeof id === 'string') : [];
|
|
2020
|
+
if (cached.length) return cached;
|
|
2021
|
+
throw new DatabaseError(
|
|
2022
|
+
`emet_session_open could not load unhandled drops from the store, so no session was opened: ${e && e.message}. Nothing was stamped.`,
|
|
2023
|
+
'emet_session_open'
|
|
2024
|
+
);
|
|
2025
|
+
}
|
|
2026
|
+
}
|
|
2027
|
+
|
|
2028
|
+
export async function sessionOpen({ slug, load_id } = {}, logger = null, dbManager = null, now = new Date()) {
|
|
2029
|
+
const { sessionBaseId, sessionTimeZone, partId, firstUnclosedBase } = await import('./transcript-session.js');
|
|
2030
|
+
const { sessionStatus } = await import('./session-graph.js');
|
|
2031
|
+
const tz = sessionTimeZone();
|
|
2032
|
+
const requested = sessionBaseId(now, slug || 'session', tz.zone);
|
|
2033
|
+
// A closed session is never reopened (2026-10-01): a closed id is replaced by
|
|
2034
|
+
// the next free <id>-2, -3, ... and the response says so. If the session
|
|
2035
|
+
// state cannot be read, nothing is opened - an unchecked id could be a
|
|
2036
|
+
// closed one.
|
|
2037
|
+
let resolved;
|
|
2038
|
+
try {
|
|
2039
|
+
resolved = await firstUnclosedBase(requested, (b) => sessionStatus(b, dbManager));
|
|
2040
|
+
} catch (e) {
|
|
2041
|
+
if (e && e.allClosed) throw new ValidationError('slug', `${e.message} Nothing was opened.`);
|
|
2042
|
+
throw new DatabaseError(`emet_session_open could not check whether ${requested} is already closed, so no session was opened: ${e && e.message}. ` +
|
|
2043
|
+
'Call emet_session_open again with the same slug and load_id.', 'emet_session_open');
|
|
2044
|
+
}
|
|
2045
|
+
const base = resolved.base;
|
|
2046
|
+
const session_id = partId(base, 1);
|
|
2047
|
+
const suffixed = base !== requested;
|
|
2048
|
+
const phrase = (x) => (x.reason === 'closed' ? `${x.id} is already closed` : `${x.id} is ${x.reason}`);
|
|
2049
|
+
const notice = suffixed
|
|
2050
|
+
? `${resolved.skipped.map(phrase).join('; ')}, so this session was opened as ${base} instead. ` +
|
|
2051
|
+
(resolved.closed.length
|
|
2052
|
+
? 'A closed session is never reopened: nothing was written to it and nothing was merged into it.'
|
|
2053
|
+
: 'Nothing was merged into another session.')
|
|
2054
|
+
: null;
|
|
2055
|
+
logger?.info?.('transcript session open', { session_id, ...(suffixed ? { requested } : {}) });
|
|
2056
|
+
// Startup page completion (2026-10-03). Through dispatch, an open with no
|
|
2057
|
+
// load_id or with pages missing is refused before this runs (the
|
|
2058
|
+
// startup_pages gate); a direct call is told what was checked.
|
|
2059
|
+
const pages = startupPagesCheck(load_id, logger, session_id);
|
|
2060
|
+
// Opened with pages missing: a page fetched from now on is recorded as late,
|
|
2061
|
+
// under the label the open's lab row carries (the id asked for).
|
|
2062
|
+
if (pages && pages.checked && pages.complete === false) {
|
|
2063
|
+
try { noteOpenMissing(pages.load_id, requested, pages.missing); } catch { /* never fails the open */ }
|
|
2064
|
+
}
|
|
2065
|
+
// Server-side, from the stored drop list the board session must see.
|
|
2066
|
+
// Never args.drops_seen. A cache miss loads the store. A failed load does
|
|
2067
|
+
// not stamp an empty seen-list (that would fail open): the open stands,
|
|
2068
|
+
// and a board edit is refused until the list is known.
|
|
2069
|
+
let drops_seen = [];
|
|
2070
|
+
let drops_unverified = false;
|
|
2071
|
+
try {
|
|
2072
|
+
drops_seen = await dropsSeenForOpen(load_id);
|
|
2073
|
+
} catch (e) {
|
|
2074
|
+
drops_seen = null;
|
|
2075
|
+
drops_unverified = true;
|
|
2076
|
+
}
|
|
2077
|
+
return {
|
|
2078
|
+
session_id,
|
|
2079
|
+
base,
|
|
2080
|
+
drops_seen,
|
|
2081
|
+
...(drops_unverified ? { drops_unverified: true } : {}),
|
|
2082
|
+
next_exchange_index: 1,
|
|
2083
|
+
time_zone: tz.zone,
|
|
2084
|
+
...(tz.note ? { time_zone_note: tz.note } : {}),
|
|
2085
|
+
...(suffixed ? {
|
|
2086
|
+
suffix_applied: { requested, closed: resolved.closed, skipped: resolved.skipped, opened_as: base, reason: 'closed sessions are never reopened, and a suffixed id is never shared' },
|
|
2087
|
+
notice
|
|
2088
|
+
} : {}),
|
|
2089
|
+
...(pages ? { startup_pages: pages } : {}),
|
|
2090
|
+
instructions:
|
|
2091
|
+
(suffixed ? `Tell the user, in one line: ${notice} ` : '') +
|
|
2092
|
+
(pages && pages.notice ? `${pages.notice} ` : '') +
|
|
2093
|
+
'After EVERY reply, call emet_transcript_append with this session_id, the user message, and your reply. One exchange per call. Do not wait until the session ends. Do not reconstruct earlier turns.'
|
|
2094
|
+
};
|
|
2095
|
+
}
|
|
2096
|
+
|
|
2097
|
+
/** Which pages of a startup load were never served (the session graph's startup_pages gate reads it here). */
|
|
2098
|
+
export function startupPageCompletion(loadId = null) {
|
|
2099
|
+
return pageCompletion(loadId);
|
|
2100
|
+
}
|
|
2101
|
+
|
|
2102
|
+
const LOAD_ID_ECHO_MAX = 64;
|
|
2103
|
+
export const NO_LOAD_ID_NOTICE =
|
|
2104
|
+
'Startup pages not checked: no load_id. Pass load_id from your startup pages (startup_page.load_id on every ' +
|
|
2105
|
+
'emet_initialize page) to emet_session_open, so the check runs on your own startup.';
|
|
2106
|
+
/** What emet_session_open says about the startup pages: null when there is nothing to say. Never throws. */
|
|
2107
|
+
export function startupPagesCheck(loadId, logger = null, sessionId = null, now = Date.now()) {
|
|
2108
|
+
// No load_id (2026-10-03): the open is not tied to a startup, so nothing was
|
|
2109
|
+
// checked - say so, never stay silent, and never check the latest startup
|
|
2110
|
+
// this server built in its place (it may be another host's).
|
|
2111
|
+
if (!loadId || (typeof loadId === 'string' && !loadId.trim())) {
|
|
2112
|
+
return { load_id: null, checked: false, reason: 'no load_id', notice: NO_LOAD_ID_NOTICE };
|
|
2113
|
+
}
|
|
2114
|
+
let c;
|
|
2115
|
+
try { c = pageCompletion(loadId, now); } catch { return null; }
|
|
2116
|
+
if (!c) {
|
|
2117
|
+
// A host-supplied id is echoed back capped, so a bogus one cannot bloat the reply.
|
|
2118
|
+
let echoed;
|
|
2119
|
+
try { echoed = String(loadId); } catch { return null; }
|
|
2120
|
+
const cps = Array.from(echoed); // whole characters: never a lone surrogate in the echo
|
|
2121
|
+
if (cps.length > LOAD_ID_ECHO_MAX) echoed = `${cps.slice(0, LOAD_ID_ECHO_MAX).join('')}…`;
|
|
2122
|
+
return { load_id: echoed, checked: false, reason: 'no record of that startup load on this server in the last 60 minutes (it expired, or the server restarted)' };
|
|
2123
|
+
}
|
|
2124
|
+
if (c.complete) return { load_id: c.load_id, checked: true, complete: true };
|
|
2125
|
+
const list = c.missing.join(', ');
|
|
2126
|
+
const notice = `Startup pages ${list} of ${c.of} (load ${c.load_id}) were never fetched, so you are working from part of your memory. ` +
|
|
2127
|
+
`Fetch each now with emet_initialize {"page": N, "load_id": "${c.load_id}"} before you rely on memory; if the server says the startup changed, ` +
|
|
2128
|
+
'call emet_initialize {} and read every page. This session was opened; nothing was blocked.';
|
|
2129
|
+
try { logger?.warn?.('session opened with startup pages missing', { session_id: sessionId, load_id: c.load_id, missing: c.missing, of: c.of }); } catch { /* a failing logger never fails the open */ }
|
|
2130
|
+
return { load_id: c.load_id, checked: true, complete: false, of: c.of, served: c.served, missing: c.missing, notice };
|
|
2131
|
+
}
|
|
2132
|
+
|
|
2133
|
+
export async function transcriptAppend(args = {}, logger = null) {
|
|
2134
|
+
const session_id = args.session_id;
|
|
2135
|
+
const user_message = args.user_message;
|
|
2136
|
+
const assistant_message = args.assistant_message;
|
|
2137
|
+
const exchange_index = args.exchange_index;
|
|
2138
|
+
const session_start = args.session_start;
|
|
2139
|
+
const channel = args.channel;
|
|
2140
|
+
redactWithReport(String(user_message || '') + String(assistant_message || ''));
|
|
2141
|
+
if (!session_id || typeof session_id !== 'string') {
|
|
2142
|
+
throw new ValidationError('session_id is required');
|
|
2143
|
+
}
|
|
2144
|
+
if (typeof user_message !== 'string' || typeof assistant_message !== 'string') {
|
|
2145
|
+
throw new ValidationError('user_message and assistant_message must be strings');
|
|
2146
|
+
}
|
|
2147
|
+
assertExchangeBudget([user_message, assistant_message]);
|
|
2148
|
+
const {
|
|
2149
|
+
partId, partNumber, baseFromPart, nextExchangeIndex, shouldStartNewPart, EXCHANGES_PER_PART
|
|
2150
|
+
} = await import('./transcript-session.js');
|
|
2151
|
+
|
|
2152
|
+
const client = await connectMongo(MONGODB_URI, { serverSelectionTimeoutMS: 5000, maxPoolSize: 10 });
|
|
2153
|
+
try {
|
|
2154
|
+
const col = client.db(MONGODB_DB).collection('transcripts');
|
|
2155
|
+
let part = session_id;
|
|
2156
|
+
let live = await col.findOne({ session_id: part, superseded_by: { $in: [null, undefined] } })
|
|
2157
|
+
|| await col.findOne({ session_id: part });
|
|
2158
|
+
|
|
2159
|
+
if (live && shouldStartNewPart(live.exchange_count || (live.exchanges || []).length)) {
|
|
2160
|
+
const n = partNumber(part) + 1;
|
|
2161
|
+
part = partId(baseFromPart(part), n);
|
|
2162
|
+
live = await col.findOne({ session_id: part, superseded_by: { $in: [null, undefined] } });
|
|
2163
|
+
}
|
|
2164
|
+
|
|
2165
|
+
const exchanges = (live && live.exchanges) || [];
|
|
2166
|
+
const idx = Number.isFinite(exchange_index) ? exchange_index : nextExchangeIndex(exchanges);
|
|
2167
|
+
if (exchanges.some((e) => Number(e.exchange_index) === idx)) {
|
|
2168
|
+
return { session_id: part, exchange_index: idx, already_present: true, exchange_count: exchanges.length };
|
|
2169
|
+
}
|
|
2170
|
+
|
|
2171
|
+
const scrub = (s) => redactWithReport(s || '').text;
|
|
2172
|
+
const stamp = writerStamp();
|
|
2173
|
+
const row = {
|
|
2174
|
+
...storedExchange({
|
|
2175
|
+
exchange_index: idx,
|
|
2176
|
+
user_message,
|
|
2177
|
+
assistant_message,
|
|
2178
|
+
timestamp: Date.now() / 1000
|
|
2179
|
+
}, idx - 1, scrub),
|
|
2180
|
+
source: channel || setting('SOURCE_TAG') || null,
|
|
2181
|
+
person_id: stamp.person_id,
|
|
2182
|
+
client_id: stamp.client_id
|
|
2183
|
+
};
|
|
2184
|
+
|
|
2185
|
+
// Build 1 (2026-09-27): the first append of a session must carry the
|
|
2186
|
+
// measured machine line, or say there is no local shell. The rule lived in
|
|
2187
|
+
// host instructions and was skipped twice on 2026-09-26; the server now
|
|
2188
|
+
// refuses to start a transcript without it. Known names come from the
|
|
2189
|
+
// user's install notes; with none, the check is reported as not enforced.
|
|
2190
|
+
let machineCheck = null;
|
|
2191
|
+
if (!live && partNumber(part) === 1 && idx === 1) {
|
|
2192
|
+
const { knownMachinesFromInstallNotes, machineLineCheck } = await import('./lessons-to-checks.js');
|
|
2193
|
+
const { installDoc } = await import('./setup.js');
|
|
2194
|
+
const notes = await client.db(MONGODB_DB).collection('documents')
|
|
2195
|
+
.findOne({ doc_id: installDoc(), retired: { $ne: true } }, { projection: { content: 1 } });
|
|
2196
|
+
machineCheck = machineLineCheck(assistant_message, knownMachinesFromInstallNotes(notes && notes.content));
|
|
2197
|
+
if (!machineCheck.ok) throw new ValidationError(machineCheck.reason);
|
|
2198
|
+
}
|
|
2199
|
+
|
|
2200
|
+
if (!live) {
|
|
2201
|
+
const start = typeof session_start === 'number' ? session_start : Date.now() / 1000;
|
|
2202
|
+
await col.insertOne({
|
|
2203
|
+
session_id: part,
|
|
2204
|
+
channel: channel || setting('SOURCE_TAG') || 'mcp',
|
|
2205
|
+
session_start: new Date(start * 1000),
|
|
2206
|
+
session_end: new Date(),
|
|
2207
|
+
exchange_count: 1,
|
|
2208
|
+
exchanges: [row],
|
|
2209
|
+
created_at: new Date()
|
|
2210
|
+
});
|
|
2211
|
+
return {
|
|
2212
|
+
session_id: part, exchange_index: idx, exchange_count: 1, rolled: part !== session_id,
|
|
2213
|
+
...(machineCheck ? { machine_line: machineCheck.enforced ? `checked: ${machineCheck.matched}` : machineCheck.reason } : {})
|
|
2214
|
+
};
|
|
2215
|
+
}
|
|
2216
|
+
|
|
2217
|
+
await col.updateOne(
|
|
2218
|
+
{ _id: live._id },
|
|
2219
|
+
{ $push: { exchanges: row }, $inc: { exchange_count: 1 }, $set: { session_end: new Date() } }
|
|
2220
|
+
);
|
|
2221
|
+
logger?.info?.('transcript append', { session_id: part, exchange_index: idx });
|
|
2222
|
+
return {
|
|
2223
|
+
session_id: part,
|
|
2224
|
+
exchange_index: idx,
|
|
2225
|
+
exchange_count: (live.exchange_count || exchanges.length) + 1,
|
|
2226
|
+
rolled: part !== session_id,
|
|
2227
|
+
next_part_after: EXCHANGES_PER_PART
|
|
2228
|
+
};
|
|
2229
|
+
} finally {
|
|
2230
|
+
await client.close();
|
|
2231
|
+
}
|
|
2232
|
+
}
|
|
2233
|
+
|
|
2234
|
+
/**
|
|
2235
|
+
* Supersede a stored transcript part with a corrected one.
|
|
2236
|
+
*
|
|
2237
|
+
* Everything else in this store is non-destructive: layer entries supersede,
|
|
2238
|
+
* documents version with the prior copy archived. Transcripts had no such path,
|
|
2239
|
+
* so a part written with a mispaired or missing exchange could only be annotated
|
|
2240
|
+
* from the outside and left standing - which is how a record ends up complete in
|
|
2241
|
+
* shape and wrong in content.
|
|
2242
|
+
*
|
|
2243
|
+
* This does NOT edit or delete. It writes a NEW document under the same
|
|
2244
|
+
* session_id carrying an incremented revision and a pointer to its parent, and
|
|
2245
|
+
* marks the parent superseded. Both remain readable; readers take the live one.
|
|
2246
|
+
* The reason is REQUIRED - a correction with no stated cause is a silent rewrite
|
|
2247
|
+
* wearing a version number, and the point of keeping the parent is that someone
|
|
2248
|
+
* can see what changed and why.
|
|
2249
|
+
*/
|
|
2250
|
+
export async function reviseTranscript(session_id, exchanges, reason, session_start = null, session_end = null, channel = null, logger = null) {
|
|
2251
|
+
const startTime = Date.now();
|
|
2252
|
+
try {
|
|
2253
|
+
if (!session_id || typeof session_id !== 'string')
|
|
2254
|
+
throw new ValidationError('session_id must be a non-empty string');
|
|
2255
|
+
if (!Array.isArray(exchanges) || exchanges.length === 0)
|
|
2256
|
+
throw new ValidationError('exchanges must be a non-empty array - a revision replaces the whole part, not a field within it');
|
|
2257
|
+
if (!reason || typeof reason !== 'string' || reason.trim().length < 10)
|
|
2258
|
+
throw new ValidationError('reason is required and must describe what was wrong - at least 10 characters. An unexplained correction is a silent rewrite.');
|
|
2259
|
+
|
|
2260
|
+
const client = await connectMongo(MONGODB_URI, { serverSelectionTimeoutMS: 5000, maxPoolSize: 10 });
|
|
2261
|
+
const db = client.db(MONGODB_DB);
|
|
2262
|
+
const col = db.collection('transcripts');
|
|
2263
|
+
|
|
2264
|
+
// The live document is the one nothing supersedes.
|
|
2265
|
+
const parent = await col.findOne({ session_id, superseded_by: { $in: [null, undefined] } })
|
|
2266
|
+
|| await col.findOne({ session_id });
|
|
2267
|
+
if (!parent) {
|
|
2268
|
+
await client.close();
|
|
2269
|
+
throw new ValidationError(`No transcript found under session_id "${session_id}". Use save_transcript to write a new part; revise_transcript only corrects one that exists.`);
|
|
2270
|
+
}
|
|
2271
|
+
|
|
2272
|
+
const revision = (typeof parent.revision === 'number' ? parent.revision : 1) + 1;
|
|
2273
|
+
let redactedSpans = 0;
|
|
2274
|
+
const scrub = (s) => { const r = redactWithReport(s || ''); redactedSpans += r.hits; return r.text; };
|
|
2275
|
+
const revised = {
|
|
2276
|
+
session_id,
|
|
2277
|
+
channel: channel || parent.channel || null,
|
|
2278
|
+
session_start: session_start ? new Date(session_start * 1000) : parent.session_start,
|
|
2279
|
+
session_end: session_end ? new Date(session_end * 1000) : parent.session_end,
|
|
2280
|
+
exchange_count: exchanges.length,
|
|
2281
|
+
exchanges: exchanges.map((ex, i) => stampExchange(storedExchange(ex, i, scrub), channel || parent.channel)),
|
|
2282
|
+
...(redactedSpans > 0 ? { redacted_spans: redactedSpans } : {}),
|
|
2283
|
+
revision,
|
|
2284
|
+
supersedes: parent._id,
|
|
2285
|
+
superseded_by: null,
|
|
2286
|
+
revision_reason: reason.trim(),
|
|
2287
|
+
revised_at: new Date(),
|
|
2288
|
+
created_at: new Date()
|
|
2289
|
+
};
|
|
2290
|
+
|
|
2291
|
+
const result = await col.insertOne(revised);
|
|
2292
|
+
await col.updateOne({ _id: parent._id }, { $set: { superseded_by: result.insertedId } });
|
|
2293
|
+
|
|
2294
|
+
// Read back rather than trust the write, and confirm the parent survived -
|
|
2295
|
+
// a supersession that lost the original would be a deletion with extra steps.
|
|
2296
|
+
const stored = await col.findOne({ _id: result.insertedId });
|
|
2297
|
+
const parentAfter = await col.findOne({ _id: parent._id });
|
|
2298
|
+
await client.close();
|
|
2299
|
+
|
|
2300
|
+
const verified = Boolean(stored)
|
|
2301
|
+
&& stored.exchange_count === exchanges.length
|
|
2302
|
+
&& Array.isArray(stored.exchanges)
|
|
2303
|
+
&& stored.exchanges.length === exchanges.length
|
|
2304
|
+
&& Boolean(parentAfter)
|
|
2305
|
+
&& String(parentAfter.superseded_by) === String(result.insertedId);
|
|
2306
|
+
|
|
2307
|
+
return {
|
|
2308
|
+
success: true,
|
|
2309
|
+
verified,
|
|
2310
|
+
session_id,
|
|
2311
|
+
revision,
|
|
2312
|
+
inserted_id: result.insertedId.toString(),
|
|
2313
|
+
supersedes_id: parent._id.toString(),
|
|
2314
|
+
parent_retained: Boolean(parentAfter),
|
|
2315
|
+
parent_marked_superseded: Boolean(parentAfter && parentAfter.superseded_by),
|
|
2316
|
+
exchange_count: exchanges.length,
|
|
2317
|
+
...(revised.exchanges.some(e => e.timestamp_defaulted)
|
|
2318
|
+
? { timestamps_defaulted: revised.exchanges.filter(e => e.timestamp_defaulted).length } : {}),
|
|
2319
|
+
previous_exchange_count: parent.exchange_count ?? null,
|
|
2320
|
+
redacted: redactedSpans,
|
|
2321
|
+
revision_reason: reason.trim(),
|
|
2322
|
+
note: verified
|
|
2323
|
+
? 'The prior version is retained and marked superseded. Readers take the revision; the original stays auditable.'
|
|
2324
|
+
: 'Write completed but read-back did not confirm. Do not report this as corrected - check the collection.',
|
|
2325
|
+
performance: { duration_ms: Date.now() - startTime }
|
|
2326
|
+
};
|
|
2327
|
+
} catch (error) {
|
|
2328
|
+
if (error instanceof ValidationError || error instanceof EmetError) throw error;
|
|
2329
|
+
throw new DatabaseError(`Failed to revise transcript: ${error.message}`, 'revise_transcript');
|
|
2330
|
+
}
|
|
2331
|
+
}
|
|
2332
|
+
|
|
2333
|
+
// ============================================
|
|
2334
|
+
// DOCUMENT STORE - versioned continuity documents
|
|
2335
|
+
// Added 2026-07-18: HANDOFF/STATE/threads/bootstrap served interface-independently
|
|
2336
|
+
// ============================================
|
|
2337
|
+
|
|
2338
|
+
async function _docsClient() {
|
|
2339
|
+
const client = await connectMongo(MONGODB_URI, { serverSelectionTimeoutMS: 5000, maxPoolSize: 10 });
|
|
2340
|
+
const db = client.db(MONGODB_DB);
|
|
2341
|
+
const col = db.collection('documents');
|
|
2342
|
+
const hist = db.collection('documents_history');
|
|
2343
|
+
if (!docIndexesReady) {
|
|
2344
|
+
try {
|
|
2345
|
+
await col.createIndex({ person_id: 1 }, { sparse: true });
|
|
2346
|
+
await col.createIndex({ client_id: 1 }, { sparse: true });
|
|
2347
|
+
await hist.createIndex({ person_id: 1 }, { sparse: true });
|
|
2348
|
+
await hist.createIndex({ client_id: 1 }, { sparse: true });
|
|
2349
|
+
docIndexesReady = true;
|
|
2350
|
+
} catch { /* an index miss never blocks the write */ }
|
|
2351
|
+
}
|
|
2352
|
+
return { client, col, hist };
|
|
2353
|
+
}
|
|
2354
|
+
let docIndexesReady = false;
|
|
2355
|
+
|
|
2356
|
+
/**
|
|
2357
|
+
* The default page for a read_doc TOOL call, in bytes (2026-09-26; bytes and
|
|
2358
|
+
* 18,000 since 2026-10-03). A host that caps tool output returned a
|
|
2359
|
+
* 69,000-character board as an overflow file rather than as the board, and a
|
|
2360
|
+
* host with no file fallback would simply lose it. Hosts cap the result they
|
|
2361
|
+
* RECEIVE (commonly 20,000 bytes), and that is the page as JSON text - every
|
|
2362
|
+
* newline, quote and backslash costs a byte more there - plus the envelope
|
|
2363
|
+
* (sha256, version, conformance, the page note). So a page is cut where its
|
|
2364
|
+
* JSON-escaped bytes plus READ_DOC_ENVELOPE_SPARE would pass the limit. The
|
|
2365
|
+
* tool pages at this size unless the caller asks otherwise; `limit: 0` returns
|
|
2366
|
+
* the whole document. Internal callers pass no page and always get the whole
|
|
2367
|
+
* document - patch_doc and friends must never operate on a slice.
|
|
2368
|
+
*/
|
|
2369
|
+
export const READ_DOC_PAGE_DEFAULT = Number(setting('READ_DOC_PAGE', 18000)) || 18000;
|
|
2370
|
+
|
|
2371
|
+
/** Bytes of a page's limit kept for the read_doc result's envelope (2026-10-03). */
|
|
2372
|
+
export const READ_DOC_ENVELOPE_SPARE = 1500;
|
|
2373
|
+
|
|
2374
|
+
/** The bytes a string takes as JSON text, quotes included: what a host receives for it. */
|
|
2375
|
+
export function jsonEscapedBytes(s) {
|
|
2376
|
+
return Buffer.byteLength(JSON.stringify(s), 'utf8');
|
|
2377
|
+
}
|
|
2378
|
+
|
|
2379
|
+
/** UTF-8 length of one code point (a lone surrogate encodes as U+FFFD: 3 bytes). */
|
|
2380
|
+
function utf8Length(cp) {
|
|
2381
|
+
return cp < 0x80 ? 1 : cp < 0x800 ? 2 : cp < 0x10000 ? 3 : 4;
|
|
2382
|
+
}
|
|
2383
|
+
|
|
2384
|
+
/** JSON-escaped length of one code point, as JSON.stringify writes it (a lone surrogate is \uXXXX: 6 bytes). */
|
|
2385
|
+
function jsonLength(cp) {
|
|
2386
|
+
if (cp === 0x22 || cp === 0x5c || cp === 0x08 || cp === 0x09 || cp === 0x0a || cp === 0x0c || cp === 0x0d) return 2;
|
|
2387
|
+
if (cp < 0x20) return 6;
|
|
2388
|
+
if (cp >= 0xd800 && cp <= 0xdfff) return 6;
|
|
2389
|
+
return utf8Length(cp);
|
|
2390
|
+
}
|
|
2391
|
+
|
|
2392
|
+
/**
|
|
2393
|
+
* Cut one page from a document's content. Returns the content and, when the
|
|
2394
|
+
* page is not the whole document, a `page` block saying where it sits.
|
|
2395
|
+
*
|
|
2396
|
+
* `offset`, `next_offset`, `returned` and `size` are UTF-8 BYTES of the whole
|
|
2397
|
+
* document. `limit` bounds what the host receives: a page ends before its
|
|
2398
|
+
* JSON-escaped bytes (jsonEscapedBytes, quotes included) plus
|
|
2399
|
+
* READ_DOC_ENVELOPE_SPARE would pass `limit`; `returned_escaped` is that
|
|
2400
|
+
* escaped size. A page never splits a character (a code point), and
|
|
2401
|
+
* `next_offset` is always the byte where the next character starts. An `offset` that falls inside a character is moved back to
|
|
2402
|
+
* that character's first byte, so nothing is skipped. A `limit` smaller than
|
|
2403
|
+
* the character at `offset` still returns that one character, so paging always
|
|
2404
|
+
* moves forward. Joining the pages in order gives back the document exactly.
|
|
2405
|
+
*/
|
|
2406
|
+
export function pageContent(full, offset = 0, limit = null) {
|
|
2407
|
+
const text = String(full ?? '');
|
|
2408
|
+
const n = text.length;
|
|
2409
|
+
const unitsAt = (i) => {
|
|
2410
|
+
const cp = text.codePointAt(i);
|
|
2411
|
+
return { bytes: utf8Length(cp), json: jsonLength(cp), units: cp > 0xffff ? 2 : 1 };
|
|
2412
|
+
};
|
|
2413
|
+
let size = 0;
|
|
2414
|
+
for (let i = 0; i < n;) { const c = unitsAt(i); size += c.bytes; i += c.units; }
|
|
2415
|
+
const want = Math.max(0, Math.min(Number(offset) || 0, size));
|
|
2416
|
+
const lim = limit === null || limit === undefined ? 0 : Math.max(0, Number(limit) || 0);
|
|
2417
|
+
// Start: the first byte of the character holding byte `want`.
|
|
2418
|
+
let i = 0; let start = 0;
|
|
2419
|
+
while (i < n) { const c = unitsAt(i); if (start + c.bytes > want) break; start += c.bytes; i += c.units; }
|
|
2420
|
+
// End: the last whole character whose JSON-escaped page, plus the envelope
|
|
2421
|
+
// spare, stays within `lim` (at least one character, so paging moves forward).
|
|
2422
|
+
let j = i; let end = start;
|
|
2423
|
+
if (lim === 0) { j = n; end = size; } else {
|
|
2424
|
+
let escaped = 2; // the quotes
|
|
2425
|
+
while (j < n) {
|
|
2426
|
+
const c = unitsAt(j);
|
|
2427
|
+
if (escaped + c.json + READ_DOC_ENVELOPE_SPARE > lim && j > i) break;
|
|
2428
|
+
escaped += c.json; end += c.bytes; j += c.units;
|
|
2429
|
+
}
|
|
2430
|
+
}
|
|
2431
|
+
const content = text.slice(i, j);
|
|
2432
|
+
if (start === 0 && end === size) return { content, page: null };
|
|
2433
|
+
return {
|
|
2434
|
+
content,
|
|
2435
|
+
page: {
|
|
2436
|
+
unit: 'bytes',
|
|
2437
|
+
offset: start,
|
|
2438
|
+
returned: end - start,
|
|
2439
|
+
returned_escaped: jsonEscapedBytes(content),
|
|
2440
|
+
envelope_spare: READ_DOC_ENVELOPE_SPARE,
|
|
2441
|
+
size,
|
|
2442
|
+
size_bytes: size,
|
|
2443
|
+
next_offset: end < size ? end : null,
|
|
2444
|
+
note: end < size
|
|
2445
|
+
? 'Partial: this is one page of the document. Call read_doc again with offset = next_offset for the rest, and pass expected_version (this result\'s version) so a change between pages is refused rather than mixed. sha256 is the WHOLE document\'s digest.'
|
|
2446
|
+
: 'Final page. sha256 is the WHOLE document\'s digest.'
|
|
2447
|
+
}
|
|
2448
|
+
};
|
|
2449
|
+
}
|
|
2450
|
+
|
|
2451
|
+
/**
|
|
2452
|
+
* The version pin for a paged read (2026-10-03). A caller reading page 2 onward
|
|
2453
|
+
* passes the `version` (or `sha256`) its first page returned; if the document
|
|
2454
|
+
* was rewritten in between, the read is refused, so pages of two versions are
|
|
2455
|
+
* never joined. A shipped guide has no version: pin it by sha256. Throws a
|
|
2456
|
+
* ValidationError naming what changed; returns nothing when the pin holds.
|
|
2457
|
+
*/
|
|
2458
|
+
export function checkReadPin(doc_id, current, opts = {}) {
|
|
2459
|
+
const wantV = opts.expected_version;
|
|
2460
|
+
const wantS = opts.expected_sha256;
|
|
2461
|
+
const hasV = wantV !== undefined && wantV !== null && wantV !== '';
|
|
2462
|
+
const hasS = typeof wantS === 'string' && wantS.trim() !== '';
|
|
2463
|
+
if (!hasV && !hasS) return;
|
|
2464
|
+
if (hasV && !(Number.isInteger(Number(wantV)) && Number(wantV) >= 1)) {
|
|
2465
|
+
throw new ValidationError('expected_version', 'expected_version is the whole-number `version` an earlier page of this document returned.');
|
|
2466
|
+
}
|
|
2467
|
+
const vChanged = hasV && current.version !== undefined && current.version !== null && Number(wantV) !== Number(current.version);
|
|
2468
|
+
const sChanged = hasS && String(current.sha256 || '').toLowerCase() !== wantS.trim().toLowerCase();
|
|
2469
|
+
if (!vChanged && !sChanged) return;
|
|
2470
|
+
const was = [hasV ? `version ${Number(wantV)}` : null, hasS ? `sha256 ${wantS.trim().slice(0, 12)}...` : null].filter(Boolean).join(', ');
|
|
2471
|
+
const now = [current.version !== undefined && current.version !== null ? `version ${current.version}` : null,
|
|
2472
|
+
current.sha256 ? `sha256 ${String(current.sha256).slice(0, 12)}...` : null].filter(Boolean).join(', ');
|
|
2473
|
+
throw new ValidationError('expected_version',
|
|
2474
|
+
`${doc_id}: document changed - restart from offset 0. You read earlier pages of ${was}; it is now ${now}. ` +
|
|
2475
|
+
'Pages of two versions are never joined, so nothing was returned. Call read_doc again with offset 0 and no expected_version, ' +
|
|
2476
|
+
'discard the pages you already have, and read every page of the new version.');
|
|
2477
|
+
}
|
|
2478
|
+
|
|
2479
|
+
export async function readDoc(doc_id, logger = null, opts = {}) {
|
|
2480
|
+
try {
|
|
2481
|
+
if (!doc_id || typeof doc_id !== 'string') throw new ValidationError('doc_id must be a non-empty string');
|
|
2482
|
+
const guide = readShippedGuide(doc_id);
|
|
2483
|
+
if (guide) {
|
|
2484
|
+
checkReadPin(doc_id, guide, opts);
|
|
2485
|
+
const { content, page } = pageContent(guide.content, opts.offset ?? 0, opts.limit ?? null);
|
|
2486
|
+
return { ...guide, content, size_unit: 'characters', size_bytes: Buffer.byteLength(guide.content, 'utf8'), ...(page ? { page } : {}) };
|
|
2487
|
+
}
|
|
2488
|
+
const { client, col } = await _docsClient();
|
|
2489
|
+
try {
|
|
2490
|
+
const doc = await col.findOne({ doc_id });
|
|
2491
|
+
if (!doc) throw new ValidationError(`Document not found: ${doc_id}`);
|
|
2492
|
+
checkReadPin(doc_id, doc, opts);
|
|
2493
|
+
const { content, page } = pageContent(doc.content, opts.offset ?? 0, opts.limit ?? null);
|
|
2494
|
+
// A retired document is still returned in full - retain and mark, never
|
|
2495
|
+
// withhold. The caller is TOLD it is obsolete so it is not used as
|
|
2496
|
+
// current, which is the whole of "prevent unintended use".
|
|
2497
|
+
return { doc_id: doc.doc_id, content, sha256: doc.sha256, version: doc.version,
|
|
2498
|
+
updated_at: doc.updated_at, updated_by: doc.updated_by || null,
|
|
2499
|
+
client_id: doc.client_id || null, session_id: doc.session_id || null,
|
|
2500
|
+
path_hint: doc.path_hint || null, size: doc.content.length,
|
|
2501
|
+
size_unit: 'characters', size_bytes: Buffer.byteLength(doc.content, 'utf8'),
|
|
2502
|
+
...(page ? { page } : {}),
|
|
2503
|
+
// The conformance verdict travels with the document the same way
|
|
2504
|
+
// `integrity` travels with a layer row: a reader is told what the
|
|
2505
|
+
// record knows about itself rather than having to ask separately.
|
|
2506
|
+
// null on documents written before validation existed - a dated
|
|
2507
|
+
// boundary, not a defect.
|
|
2508
|
+
conformance: doc.conformance || null,
|
|
2509
|
+
signed: signatureIntegrityOf(doc, setting('SIGNING_KEY') || null),
|
|
2510
|
+
...(doc.retired === true ? {
|
|
2511
|
+
retired: true, retired_at: doc.retired_at || null,
|
|
2512
|
+
retired_by: doc.retired_by || null,
|
|
2513
|
+
retired_reason: doc.retired_reason || null,
|
|
2514
|
+
note: 'This document is marked OBSOLETE. It is retained in full and readable, but it is not current - do not act on it as live state.'
|
|
2515
|
+
} : {}) };
|
|
2516
|
+
} finally { await client.close(); }
|
|
2517
|
+
} catch (error) {
|
|
2518
|
+
if (error instanceof ValidationError || error instanceof EmetError) throw error;
|
|
2519
|
+
throw new DatabaseError(`Failed to read document: ${error.message}`, 'read_doc');
|
|
2520
|
+
}
|
|
2521
|
+
}
|
|
2522
|
+
|
|
2523
|
+
/**
|
|
2524
|
+
* Read a retained revision of a document, or list the revisions retained.
|
|
2525
|
+
*
|
|
2526
|
+
* Every write archives the prior copy to documents_history (commitment 2:
|
|
2527
|
+
* nothing is deleted). Until 2026-09-15 nothing could read one back - the
|
|
2528
|
+
* retention held but an audit asking "what did this say at version N" had no
|
|
2529
|
+
* tool to answer with. Read-only; the write-path invariant test confirms it.
|
|
2530
|
+
*/
|
|
2531
|
+
export async function readDocHistory(doc_id, version = null, logger = null) {
|
|
2532
|
+
try {
|
|
2533
|
+
if (!doc_id || typeof doc_id !== 'string') throw new ValidationError('doc_id must be a non-empty string');
|
|
2534
|
+
if (version !== null && version !== undefined && (!Number.isInteger(version) || version < 1)) throw new ValidationError('version must be a positive integer');
|
|
2535
|
+
const { client, col, hist } = await _docsClient();
|
|
2536
|
+
try {
|
|
2537
|
+
const current = await col.findOne({ doc_id });
|
|
2538
|
+
if (version === null || version === undefined) {
|
|
2539
|
+
const retained = await hist.find({ doc_id }, { projection: { content: 0 } }).sort({ version: 1 }).toArray();
|
|
2540
|
+
const rows = retained.map(r => ({ version: r.version, sha256: r.sha256 || null, size: r.size ?? null,
|
|
2541
|
+
updated_at: r.updated_at || null, updated_by: r.updated_by || null, client_id: r.client_id || null, session_id: r.session_id || null, archived_at: r.archived_at || null,
|
|
2542
|
+
conformance: r.conformance ? r.conformance.verdict : null }));
|
|
2543
|
+
return { doc_id, current_version: current ? current.version : null, retained_count: rows.length, retained: rows,
|
|
2544
|
+
note: current ? null : 'No current document by this id; only retained revisions, if any.' };
|
|
2545
|
+
}
|
|
2546
|
+
if (current && current.version === version) {
|
|
2547
|
+
return { doc_id, version, is_current: true, content: current.content, sha256: current.sha256, size: current.content.length,
|
|
2548
|
+
updated_at: current.updated_at, updated_by: current.updated_by || null,
|
|
2549
|
+
client_id: current.client_id || null, session_id: current.session_id || null,
|
|
2550
|
+
conformance: current.conformance || null };
|
|
2551
|
+
}
|
|
2552
|
+
// More than one archived row can carry a version when a write was retried;
|
|
2553
|
+
// the most recently archived is the one the store last held at that number.
|
|
2554
|
+
const row = await hist.find({ doc_id, version }).sort({ archived_at: -1 }).limit(1).next();
|
|
2555
|
+
if (!row) throw new ValidationError(`No retained revision ${version} of ${doc_id}`);
|
|
2556
|
+
const integrity = row.sha256 ? (digest(row.content) === row.sha256) : 'unattested';
|
|
2557
|
+
return { doc_id, version, is_current: false, content: row.content, sha256: row.sha256 || null, size: row.content.length,
|
|
2558
|
+
updated_at: row.updated_at || null, updated_by: row.updated_by || null, archived_at: row.archived_at || null,
|
|
2559
|
+
client_id: row.client_id || null, session_id: row.session_id || null,
|
|
2560
|
+
conformance: row.conformance || null,
|
|
2561
|
+
// Re-hashed on read, as a layer row is: a retained revision that
|
|
2562
|
+
// does not match its own digest is returned and flagged, never hidden.
|
|
2563
|
+
integrity };
|
|
2564
|
+
} finally { await client.close(); }
|
|
2565
|
+
} catch (error) {
|
|
2566
|
+
if (error instanceof ValidationError || error instanceof EmetError) throw error;
|
|
2567
|
+
throw new DatabaseError(`Failed to read document history: ${error.message}`, 'read_doc_history');
|
|
2568
|
+
}
|
|
2569
|
+
}
|
|
2570
|
+
|
|
2571
|
+
/**
|
|
2572
|
+
* Records are superseded, never amended (2026-09-26, the owner: "enforce
|
|
2573
|
+
* supersession ISO practices"). ISO 9001 7.5.3 / ISO 13485 4.2.5 separate
|
|
2574
|
+
* DOCUMENTS, which are revised under control, from RECORDS, which are evidence
|
|
2575
|
+
* of what happened and are not altered. The handoff is a record: it attests to
|
|
2576
|
+
* one session's close. So:
|
|
2577
|
+
* - patch_doc is refused on a record - no in-place amendment;
|
|
2578
|
+
* - a new revision goes through write_doc WHOLE and must name the revision it
|
|
2579
|
+
* supersedes (`supersedes`), which must be the one currently in force;
|
|
2580
|
+
* - the prior revision is retained in documents_history and marked obsolete
|
|
2581
|
+
* there (superseded_by_version), as it is for every document.
|
|
2582
|
+
* The record set is the handoff plus any ids in EMET_RECORD_DOCS (comma-list).
|
|
2583
|
+
*/
|
|
2584
|
+
export function recordDocIds() {
|
|
2585
|
+
const extra = String(setting('RECORD_DOCS', '') || '').split(',').map((s) => s.trim()).filter(Boolean);
|
|
2586
|
+
return [setting('HANDOFF_DOC', 'HANDOFF.md'), ...extra];
|
|
2587
|
+
}
|
|
2588
|
+
export function isRecordDoc(doc_id) {
|
|
2589
|
+
return recordDocIds().includes(doc_id);
|
|
2590
|
+
}
|
|
2591
|
+
/** Every document that is not a record, a drop, or a shipped copy. */
|
|
2592
|
+
export function isWorkingDoc(doc_id) {
|
|
2593
|
+
return !isRecordDoc(doc_id) && !isDropDoc(doc_id) && !isGovernedByShippedCopy(doc_id);
|
|
2594
|
+
}
|
|
2595
|
+
function stateDoc() {
|
|
2596
|
+
return setting('STATE_DOC', 'STATE.md');
|
|
2597
|
+
}
|
|
2598
|
+
function _refuseRecordAmendment(doc_id, tool) {
|
|
2599
|
+
if (isRecordDoc(doc_id)) {
|
|
2600
|
+
throw new ValidationError(
|
|
2601
|
+
`${tool}: ${doc_id} is a record. Records are superseded, not amended (ISO 9001 7.5.3 / 13485 4.2.5). ` +
|
|
2602
|
+
`Issue a new revision with write_doc - the whole record, with \`supersedes\` set to the version now in force. ` +
|
|
2603
|
+
`The prior revision is retained and marked obsolete. Nothing was changed.`
|
|
2604
|
+
);
|
|
2605
|
+
}
|
|
2606
|
+
}
|
|
2607
|
+
|
|
2608
|
+
/**
|
|
2609
|
+
* The main records are the hub's (C1, 2026-10-01). write_doc and patch_doc refuse the main HANDOFF.md or STATE.md when the
|
|
2610
|
+
* write's source is a bot tag (bots/<tag>/HANDOFF.md listed in
|
|
2611
|
+
* EMET_RECORD_DOCS), and nothing is written. So do the other doors that can
|
|
2612
|
+
* change a main record (the reviewer's F1 in review): retire_doc, rename_doc (from
|
|
2613
|
+
* or onto one) and accept_nonconformance - retire and rename set retired_by,
|
|
2614
|
+
* not updated_by, so the close alone could not see them. The source is the one the door
|
|
2615
|
+
* stamps as updated_by: the caller's, else EMET_SOURCE_TAG. A records rule like
|
|
2616
|
+
* the record-amendment refusal above, so it applies in every graph mode.
|
|
2617
|
+
*
|
|
2618
|
+
* Two further guards on the same doors (2026-10-01, the owner's approval to
|
|
2619
|
+
* make the bot tags fully functional; the reviewer's review of the bot tags):
|
|
2620
|
+
* - REGISTERED HOSTS ONLY on the main records. With a source registry
|
|
2621
|
+
* configured (EMET_SOURCE_TAGS, parsed as save_to_layer parses it), the
|
|
2622
|
+
* main handoff and board take writes only from a registered tag. An
|
|
2623
|
+
* unlisted or misspelled bot tag, a write naming no source (which falls
|
|
2624
|
+
* back to EMET_SOURCE_TAG), and cloud-scheduled-* runs are refused - the
|
|
2625
|
+
* transcript registry's cloud-scheduled exception is deliberately not
|
|
2626
|
+
* copied: unattended runs do not write the hub's records. With no registry
|
|
2627
|
+
* (a fresh install) nothing changes.
|
|
2628
|
+
* - EACH BOT IN ITS OWN DIRECTORY. A bot tag writes under bots/ only inside
|
|
2629
|
+
* bots/<its own tag>/: not another bot's handoff, not bots/ROSTER.md.
|
|
2630
|
+
* Hosts (the hub) keep writing anywhere under bots/.
|
|
2631
|
+
*/
|
|
2632
|
+
export function sourceRegistry() {
|
|
2633
|
+
return String(setting('SOURCE_TAGS') ?? '').split(',').map((s) => s.trim()).filter((s) => s !== '');
|
|
2634
|
+
}
|
|
2635
|
+
|
|
2636
|
+
/**
|
|
2637
|
+
* The tag a write is stamped with.
|
|
2638
|
+
* `from: 'bound'` — HTTP, the tag stored at approval. The caller's tag is ignored.
|
|
2639
|
+
* `from: 'legacy'` — HTTP, the token has no bound tag. The caller's tag is kept
|
|
2640
|
+
* only when it is an exact member of EMET_SOURCE_TAGS. An empty registry has no
|
|
2641
|
+
* members, so the tag stays unset.
|
|
2642
|
+
* `from: 'caller'` — stdio. The caller tag is returned as given; saveMemory may
|
|
2643
|
+
* still apply EMET_SOURCE_TAG when it is empty.
|
|
2644
|
+
*/
|
|
2645
|
+
export function resolvedSourceTag(caller) {
|
|
2646
|
+
const bound = boundSourceTag();
|
|
2647
|
+
const tag = typeof caller === 'string' ? caller.trim() : '';
|
|
2648
|
+
if (typeof bound === 'string' && bound) return { tag: bound, from: 'bound' };
|
|
2649
|
+
if (bound === undefined) return { tag: tag || null, from: 'caller' };
|
|
2650
|
+
if (tag && sourceRegistry().includes(tag)) return { tag, from: 'legacy' };
|
|
2651
|
+
return { tag: null, from: 'legacy' };
|
|
2652
|
+
}
|
|
2653
|
+
|
|
2654
|
+
// Look-alike doc ids (the second reader's recheck d1-d5, 2026-10-02): a `.` or `..` segment,
|
|
2655
|
+
// a leading `/`, or a `bots/` prefix in the wrong case would let a write pass
|
|
2656
|
+
// the prefix checks below while naming a record they are meant to fence (d1:
|
|
2657
|
+
// bots/tester-bot/../intake-bot/HANDOFF.md passed the own-dir check). Doc
|
|
2658
|
+
// ids are flat strings, never resolved as paths, so these shapes are refused
|
|
2659
|
+
// outright, before any guard, on every door that calls this.
|
|
2660
|
+
//
|
|
2661
|
+
// Hardening (the reviewer's scope on the doc-id fix, 2026-10-02): also refused are
|
|
2662
|
+
// control and invisible-format characters (\p{Cc}, \p{Cf}) anywhere; leading or
|
|
2663
|
+
// trailing whitespace on the id or on any segment; a backslash; the `/`, `.`/`..`
|
|
2664
|
+
// and `bots/` checks run on the NFKC form too (so full-width dots and slashes,
|
|
2665
|
+
// the two-dot leader and full-width letters fold first); a first segment that
|
|
2666
|
+
// folds to `bots` must be exactly the ASCII `bots`; and a case or Unicode variant
|
|
2667
|
+
// of a main-record id (HANDOFF.md -> handoff.md). No confusables table: a
|
|
2668
|
+
// Cyrillic look-alike is a different id and falls to the ordinary guards. `%2e`
|
|
2669
|
+
// is not decoded: ids are never URL-decoded, so it is just three characters.
|
|
2670
|
+
const FOLD = (s) => s.normalize('NFKC').toLowerCase();
|
|
2671
|
+
export function lookalikeDocIdReason(doc_id) {
|
|
2672
|
+
if (typeof doc_id !== 'string') return null;
|
|
2673
|
+
if (/[\p{Cc}\p{Cf}]/u.test(doc_id)) return 'has a control or invisible formatting character';
|
|
2674
|
+
if (doc_id.includes('\\')) return 'has a backslash';
|
|
2675
|
+
if (doc_id !== doc_id.trim() || doc_id.split('/').some((seg) => seg !== seg.trim())) {
|
|
2676
|
+
return 'has leading or trailing whitespace on the id or on a segment';
|
|
2677
|
+
}
|
|
2678
|
+
const nfkc = doc_id.normalize('NFKC');
|
|
2679
|
+
if (doc_id.startsWith('/')) return 'starts with "/"';
|
|
2680
|
+
if (nfkc.startsWith('/')) return 'starts with a character that normalises to "/"';
|
|
2681
|
+
if (doc_id.split('/').some((seg) => seg === '.' || seg === '..')) return 'has a "." or ".." segment';
|
|
2682
|
+
if (nfkc.split('/').some((seg) => seg === '.' || seg === '..')) return 'has a segment that normalises to "." or ".."';
|
|
2683
|
+
const first = doc_id.split('/')[0];
|
|
2684
|
+
const firstFolded = FOLD(nfkc.split('/')[0]);
|
|
2685
|
+
if (doc_id.slice(0, 5).toLowerCase() === 'bots/' && !doc_id.startsWith('bots/')) {
|
|
2686
|
+
return `starts with "${doc_id.slice(0, 5)}" - the bot records folder is "bots/", lower case`;
|
|
2687
|
+
}
|
|
2688
|
+
if (firstFolded === 'bots' && first !== 'bots') {
|
|
2689
|
+
return `starts with "${first}", which folds to "bots" - the bot records folder is the ASCII "bots", lower case`;
|
|
2690
|
+
}
|
|
2691
|
+
const mains = mainRecordIds();
|
|
2692
|
+
if (!mains.includes(doc_id) && mains.some((m) => typeof m === 'string' && FOLD(m) === FOLD(doc_id))) {
|
|
2693
|
+
return 'is a case or Unicode variant of a main record id';
|
|
2694
|
+
}
|
|
2695
|
+
return null;
|
|
2696
|
+
}
|
|
2697
|
+
|
|
2698
|
+
export function refuseLookalikeDocId(doc_id, tool) {
|
|
2699
|
+
const why = lookalikeDocIdReason(doc_id);
|
|
2700
|
+
if (why) {
|
|
2701
|
+
throw new ValidationError(
|
|
2702
|
+
`${tool}: doc_id ${JSON.stringify(doc_id)} ${why}. Doc ids are not paths and are not resolved; write the id ` +
|
|
2703
|
+
`exactly as it is meant to be stored. Nothing was written.`
|
|
2704
|
+
);
|
|
2705
|
+
}
|
|
2706
|
+
}
|
|
2707
|
+
|
|
2708
|
+
// Cleanup path (the reviewer's scope): a look-alike id may already exist in a store
|
|
2709
|
+
// written before these checks. retire_doc, and rename_doc FROM such an id, are let
|
|
2710
|
+
// through for a registered host tag only: EMET_SOURCE_TAGS set, the tag listed,
|
|
2711
|
+
// and not a bot tag. Never for a write, a patch, a rename onto it, or a bot.
|
|
2712
|
+
function mayCleanUpLookalike(tag) {
|
|
2713
|
+
const registry = sourceRegistry();
|
|
2714
|
+
return !!tag && !isBotTag(tag) && registry.length > 0 && registry.includes(tag);
|
|
2715
|
+
}
|
|
2716
|
+
|
|
2717
|
+
export function refuseBotMainRecord(doc_id, updated_by, tool, options = {}) {
|
|
2718
|
+
const tag = updated_by || setting('SOURCE_TAG') || null;
|
|
2719
|
+
if (!(options.cleanup === true && mayCleanUpLookalike(tag))) refuseLookalikeDocId(doc_id, tool);
|
|
2720
|
+
const bot = !!tag && isBotTag(tag);
|
|
2721
|
+
if (bot && typeof doc_id === 'string' && doc_id.startsWith('bots/') && !doc_id.startsWith(botRecordDir(tag))) {
|
|
2722
|
+
throw new ValidationError(
|
|
2723
|
+
`${tool}: ${doc_id} is outside ${botRecordDir(tag)} and ${tag} is a bot tag - a bot writes only its own ` +
|
|
2724
|
+
`${botRecordDir(tag)} records; other bots' records and the roster are not its to change. ` +
|
|
2725
|
+
`Hand it to the hub instead. Nothing was written.`
|
|
2726
|
+
);
|
|
2727
|
+
}
|
|
2728
|
+
// Registry guard on bots/ too (the reviewer's review of the bot guards; the second reader's fix (a), 2026-10-02): with EMET_SOURCE_TAGS
|
|
2729
|
+
// set, a non-bot write under bots/ is refused unless its tag is registered; a missing source counts as
|
|
2730
|
+
// unregistered. Without this, a misspelled or unregistered tag is not a bot tag, so the cross-bot guard
|
|
2731
|
+
// above never fires and it can write any bot's records or the roster. A listed bot is already fenced to
|
|
2732
|
+
// its own bots/<tag>/ above; registered hosts (the hub) still write anywhere under bots/.
|
|
2733
|
+
const registry = sourceRegistry();
|
|
2734
|
+
if (!bot && registry.length && typeof doc_id === 'string' && doc_id.startsWith('bots/') && !(tag && registry.includes(tag))) {
|
|
2735
|
+
throw new ValidationError(
|
|
2736
|
+
`${tool}: ${doc_id} is a bot record and ${tag ? `"${tag}" is not a registered source tag` : 'this write names no source tag'} - ` +
|
|
2737
|
+
`only a registered host or the bot itself writes under bots/. Nothing was written.`
|
|
2738
|
+
);
|
|
2739
|
+
}
|
|
2740
|
+
if (!mainRecordIds().includes(doc_id)) return;
|
|
2741
|
+
if (bot) {
|
|
2742
|
+
throw new ValidationError(
|
|
2743
|
+
`${tool}: ${doc_id} is a main record and ${tag} is a bot tag - a bot session closes against ` +
|
|
2744
|
+
`${botRecordDir(tag)} records; the main handoff and board are the hub's. Write the bot's own ` +
|
|
2745
|
+
`${botRecordDir(tag)}HANDOFF.md instead. Nothing was written.`
|
|
2746
|
+
);
|
|
2747
|
+
}
|
|
2748
|
+
if (options && options.serverProvision === true) return;
|
|
2749
|
+
if (registry.length && !(tag && registry.includes(tag))) {
|
|
2750
|
+
throw new ValidationError(
|
|
2751
|
+
`${tool}: ${doc_id} is a main record and ` +
|
|
2752
|
+
`${tag ? `"${tag}" is not a registered source tag` : 'this write names no source tag'} - only a registered ` +
|
|
2753
|
+
`host writes the main handoff and board. Pass the host's registered \`source\` (EMET_SOURCE_TAGS). Nothing was written.`
|
|
2754
|
+
);
|
|
2755
|
+
}
|
|
2756
|
+
}
|
|
2757
|
+
|
|
2758
|
+
/**
|
|
2759
|
+
* retire_doc and rename_doc are not bot tools at all (the owner, 2026-10-01,
|
|
2760
|
+
* decision 1, as relayed: "retire_doc isn't needed for bots: keep it
|
|
2761
|
+
* the owner/hub only and out of every bot profile"; the owner's decision record; stage 1 of the
|
|
2762
|
+
* host-aware startup amendment, admitted by the reviewer 2026-10-02). A profile is not a
|
|
2763
|
+
* security boundary, so the door refuses them under ANY bot tag, for every
|
|
2764
|
+
* doc_id - not only the main records refuseBotMainRecord covers. The owner's decision record (item 1)
|
|
2765
|
+
* covers retire_doc only; rename_doc stays owner-only under a later decision record
|
|
2766
|
+
* (owner-confirmed 2026-10-02: every other owner-only tool is
|
|
2767
|
+
* unchanged), and it retires its old id internally, so it is refused too. The tag is the
|
|
2768
|
+
* one the door would stamp: the caller's, else EMET_SOURCE_TAG. Hosts and the
|
|
2769
|
+
* hub have no bots/<tag>/HANDOFF.md record and are unaffected. Nothing is
|
|
2770
|
+
* written when refused.
|
|
2771
|
+
*/
|
|
2772
|
+
export function refuseBotRetireRename(tag, tool) {
|
|
2773
|
+
const t = tag || setting('SOURCE_TAG') || null;
|
|
2774
|
+
if (!t || !isBotTag(t)) return;
|
|
2775
|
+
throw new ValidationError(
|
|
2776
|
+
'source',
|
|
2777
|
+
`${tool} is not a bot tool: ${t} is a bot tag, and ` +
|
|
2778
|
+
(tool === 'retire_doc'
|
|
2779
|
+
? `retiring documents is the owner's and the hub's (decision 1, the owner's decision record item 1)`
|
|
2780
|
+
: `renaming documents stays owner-only (the owner's decision record)`) +
|
|
2781
|
+
`. Ask the hub. Nothing was changed.`
|
|
2782
|
+
);
|
|
2783
|
+
}
|
|
2784
|
+
|
|
2785
|
+
export async function writeDoc(doc_id, rawContent, updated_by = null, path_hint = null, logger = null, options = {}) {
|
|
2786
|
+
try {
|
|
2787
|
+
if (!doc_id || typeof doc_id !== 'string') throw new ValidationError('doc_id must be a non-empty string');
|
|
2788
|
+
// write_doc, patch_doc (which lands here) and rename_doc's copy (reissued_from).
|
|
2789
|
+
refuseBotMainRecord(doc_id, updated_by, options && options.reissued_from ? 'rename_doc' : 'write_doc', options && options.serverProvision === true ? { serverProvision: true } : {});
|
|
2790
|
+
if (typeof rawContent !== 'string' || rawContent.length === 0) throw new ValidationError('content must be a non-empty string');
|
|
2791
|
+
if (rawContent.length > 2000000) throw new ValidationError('content exceeds 2MB limit');
|
|
2792
|
+
// patch_doc lands here too, so one
|
|
2793
|
+
// scrub covers every document write. The digest is of the stored text.
|
|
2794
|
+
const redaction = redactWithReport(rawContent);
|
|
2795
|
+
const content = redaction.text;
|
|
2796
|
+
const sha256 = digest(content);
|
|
2797
|
+
const signingKey = setting('SIGNING_KEY') || null;
|
|
2798
|
+
const signature = signDigests(sha256, '', signingKey);
|
|
2799
|
+
const { client, col, hist } = await _docsClient();
|
|
2800
|
+
try {
|
|
2801
|
+
const prev = await col.findOne({ doc_id });
|
|
2802
|
+
const version = (prev && prev.version ? prev.version : 0) + 1;
|
|
2803
|
+
// A record in force is superseded only by a revision that names it.
|
|
2804
|
+
// Checked before anything is archived, so a refused write leaves no trace.
|
|
2805
|
+
if (prev && isRecordDoc(doc_id) && options && options.restore_of && (options.supersedes === undefined || options.supersedes === null)) {
|
|
2806
|
+
options = { ...options, supersedes: prev.version };
|
|
2807
|
+
}
|
|
2808
|
+
if (prev && isRecordDoc(doc_id)) {
|
|
2809
|
+
const named = options && options.supersedes;
|
|
2810
|
+
if (named === undefined || named === null) {
|
|
2811
|
+
throw new ValidationError(
|
|
2812
|
+
`write_doc: ${doc_id} is a record and v${prev.version} is in force. A new revision must name what it ` +
|
|
2813
|
+
`supersedes - pass supersedes: ${prev.version}. Nothing was changed.`
|
|
2814
|
+
);
|
|
2815
|
+
}
|
|
2816
|
+
if (Number(named) !== prev.version) {
|
|
2817
|
+
throw new ValidationError(
|
|
2818
|
+
`write_doc: ${doc_id} - supersedes names v${named}, but v${prev.version} is in force. Re-read the record ` +
|
|
2819
|
+
`and supersede the current revision. Nothing was changed.`
|
|
2820
|
+
);
|
|
2821
|
+
}
|
|
2822
|
+
const reason = options && options.reason;
|
|
2823
|
+
if (typeof reason !== 'string' || reason.trim().length < 10) {
|
|
2824
|
+
throw new ValidationError(
|
|
2825
|
+
`write_doc: ${doc_id} is a record. A new revision needs reason (at least 10 characters) as well as supersedes. Nothing was changed.`
|
|
2826
|
+
);
|
|
2827
|
+
}
|
|
2828
|
+
}
|
|
2829
|
+
const registry = sourceRegistry();
|
|
2830
|
+
const tag = updated_by || '';
|
|
2831
|
+
const registered = registry.length === 0 || registry.includes(tag);
|
|
2832
|
+
const ownBotFolder = isBotTag(updated_by) && doc_id.startsWith(`bots/${updated_by}/`);
|
|
2833
|
+
const serverProvision = options && options.serverProvision === true;
|
|
2834
|
+
if (!registered && !ownBotFolder && !serverProvision && !(isDropDoc(doc_id) && !prev)) {
|
|
2835
|
+
throw new ValidationError(`write_doc: ${updated_by || 'no source'} is not a registered source. This session may read, and may write a new drop only. Nothing was written.`);
|
|
2836
|
+
}
|
|
2837
|
+
if (isDropDoc(doc_id) && !prev) {
|
|
2838
|
+
const { writerStamp } = await import('./persons.js');
|
|
2839
|
+
const bucket = dropBucket(writerStamp().client_id, updated_by);
|
|
2840
|
+
const day = nyDay(new Date());
|
|
2841
|
+
const same = (await col.find({ doc_id: { $regex: '^drops/' } }).toArray()).filter((d) => (d.drop_bucket || dropBucket(d.client_id, d.updated_by)) === bucket && nyDay(d.updated_at instanceof Date ? d.updated_at : new Date(d.updated_at || 0)) === day);
|
|
2842
|
+
if (same.length >= 20) throw new ValidationError(`write_doc: this connected app already wrote 20 drops today. Nothing was written.`);
|
|
2843
|
+
options = { ...options, drop_bucket: bucket };
|
|
2844
|
+
}
|
|
2845
|
+
const floorExempt = options && options.provision;
|
|
2846
|
+
const internal = options && (options.provision || options.edited || options.setupReissue || options.restore_of);
|
|
2847
|
+
if (options && options.provision && prev && doc_id === stateDoc()) {
|
|
2848
|
+
throw new ValidationError('The board already exists. Provision does not replace it. Nothing was written.');
|
|
2849
|
+
}
|
|
2850
|
+
if (doc_id === stateDoc() && !(options && (options.provision || options.edited || options.restore_of))) {
|
|
2851
|
+
throw new ValidationError(prev
|
|
2852
|
+
? 'The board is never written. Edit it with patch_doc, with `reason` (and `origin` for an added row). Nothing was written.'
|
|
2853
|
+
: 'The board is provisioned by the server at setup (emet_setup_complete). Nothing was written.');
|
|
2854
|
+
}
|
|
2855
|
+
if (prev && isWorkingDoc(doc_id) && !internal) {
|
|
2856
|
+
throw new ValidationError(
|
|
2857
|
+
`write_doc: ${doc_id} exists, so it is edited, not rewritten. Use patch_doc with \`reason\`. Nothing was written.`
|
|
2858
|
+
);
|
|
2859
|
+
}
|
|
2860
|
+
if (isDropDoc(doc_id) && (prev || (options && options.restore_of))) {
|
|
2861
|
+
throw new ValidationError(
|
|
2862
|
+
`write_doc: ${doc_id} is a drop and already exists, retired drops included. A drop is never patched or restored. Use a new id. Nothing was written.`
|
|
2863
|
+
);
|
|
2864
|
+
}
|
|
2865
|
+
if (isGovernedByShippedCopy(doc_id) && !(options && options.provision)) {
|
|
2866
|
+
const { readFileSync } = await import('node:fs');
|
|
2867
|
+
const shippedPath = resolveModulePath(import.meta.url, ['..', doc_id]);
|
|
2868
|
+
let shipped = '';
|
|
2869
|
+
try { shipped = readFileSync(shippedPath, 'utf8'); } catch { shipped = ''; }
|
|
2870
|
+
if (content !== shipped) {
|
|
2871
|
+
throw new ValidationError(`write_doc: ${doc_id} is governed by the copy shipped with this release. Only that text may be stored. Nothing was written.`);
|
|
2872
|
+
}
|
|
2873
|
+
}
|
|
2874
|
+
const sid = (options && options.session_id) || null;
|
|
2875
|
+
const base = String(sid || '').replace(/_verbatim_part\d+$/, '').trim() || null;
|
|
2876
|
+
const baselines = prev && prev.size_baselines && typeof prev.size_baselines === 'object' ? { ...prev.size_baselines } : {};
|
|
2877
|
+
let baseline;
|
|
2878
|
+
if (base && Number.isInteger(baselines[base])) baseline = baselines[base];
|
|
2879
|
+
else {
|
|
2880
|
+
baseline = prev ? String(prev.content || '').length : content.length;
|
|
2881
|
+
if (base) baselines[base] = baseline;
|
|
2882
|
+
}
|
|
2883
|
+
if (prev && !floorExempt && baseline > 0 && content.length * 2 < baseline) {
|
|
2884
|
+
const pct = Math.floor((content.length / baseline) * 100);
|
|
2885
|
+
throw new ValidationError(
|
|
2886
|
+
`This change would leave ${doc_id} at ${pct}% of its size when this session first touched it. Condensing below half is not done on the EMET server. Nothing was written.`
|
|
2887
|
+
);
|
|
2888
|
+
}
|
|
2889
|
+
// Cross-session floor. The per-session floor is measured from this
|
|
2890
|
+
// session's first touch, so a new session can shrink a document once.
|
|
2891
|
+
// It cannot shrink it again below half of the largest size stored in
|
|
2892
|
+
// the last 24 hours. The window is a security floor, not memory decay.
|
|
2893
|
+
const CROSS_SESSION_FLOOR_MS = 24 * 60 * 60 * 1000;
|
|
2894
|
+
const nowMs = Date.now();
|
|
2895
|
+
const recentSamples = (prev && Array.isArray(prev.size_samples) ? prev.size_samples : [])
|
|
2896
|
+
.filter((s) => s && typeof s.size === 'number' && s.at && (nowMs - new Date(s.at).getTime()) <= CROSS_SESSION_FLOOR_MS);
|
|
2897
|
+
let peak = 0;
|
|
2898
|
+
for (const s of recentSamples) peak = Math.max(peak, s.size);
|
|
2899
|
+
if (prev && prev.updated_at && (nowMs - new Date(prev.updated_at).getTime()) <= CROSS_SESSION_FLOOR_MS) {
|
|
2900
|
+
peak = Math.max(peak, String(prev.content || '').length);
|
|
2901
|
+
}
|
|
2902
|
+
if (prev && !floorExempt && peak > 0 && content.length * 2 < peak) {
|
|
2903
|
+
const pct = Math.floor((content.length / peak) * 100);
|
|
2904
|
+
throw new ValidationError(
|
|
2905
|
+
`This change would leave ${doc_id} at ${pct}% of its largest size in the last 24 hours. A write may not fall below half of that peak. Nothing was written.`
|
|
2906
|
+
);
|
|
2907
|
+
}
|
|
2908
|
+
const sizeSamples = recentSamples.slice();
|
|
2909
|
+
if (prev) sizeSamples.push({ size: String(prev.content || '').length, at: prev.updated_at || new Date() });
|
|
2910
|
+
if (prev) {
|
|
2911
|
+
const { _id, size_baselines: _liveBaselines, ...rest } = prev;
|
|
2912
|
+
// Retained and marked obsolete (ISO 7.5.3 / 4.2.5): the archived copy
|
|
2913
|
+
// says which revision replaced it. Baselines stay on the live row only.
|
|
2914
|
+
await hist.insertOne({ ...rest, archived_at: new Date(), superseded_by_version: version });
|
|
2915
|
+
}
|
|
2916
|
+
const now = new Date();
|
|
2917
|
+
// Validation runs HERE: after redaction, on the exact text that will be
|
|
2918
|
+
// stored, with the previous revision already in hand for the
|
|
2919
|
+
// carry-forward comparison. It marks and never rejects, so its
|
|
2920
|
+
// result is a field on the row rather than a branch in this function.
|
|
2921
|
+
// Writing the field whole also enforces L6 - the fresh object carries no
|
|
2922
|
+
// acceptance, so an acceptance covers the one revision it was given to.
|
|
2923
|
+
const conformance = validateDocument(doc_id, content, prev ? prev.content : null);
|
|
2924
|
+
// L5, set only when renameDoc reaches this door. The destination passes
|
|
2925
|
+
// validation as "untemplated" when no template maps to it - which is the
|
|
2926
|
+
// evasion rather than the answer if the id it came FROM was governed.
|
|
2927
|
+
// Marked here, inside the one door, rather than by a second write after
|
|
2928
|
+
// the fact: a marker applied by a separate updateOne would be exactly the
|
|
2929
|
+
// kind of side entrance the write-path invariant exists to prevent.
|
|
2930
|
+
const from = options && options.reissued_from;
|
|
2931
|
+
if (from && templateFor(from) && !templateFor(doc_id)) {
|
|
2932
|
+
conformance.verdict = CONFORMANCE_VERDICT.NONCONFORMING;
|
|
2933
|
+
conformance.failures = [...(conformance.failures || []),
|
|
2934
|
+
`reissued from ${from}, which ${templateFor(from)} governs, to ${doc_id}, which no template governs (L5)`];
|
|
2935
|
+
}
|
|
2936
|
+
const writtenRow = {
|
|
2937
|
+
doc_id, content, sha256, signature, version, size: content.length, updated_at: now,
|
|
2938
|
+
conformance,
|
|
2939
|
+
supersedes_version: prev ? prev.version : null,
|
|
2940
|
+
updated_by: updated_by || (isDropDoc(doc_id) ? 'visitor' : (setting('SOURCE_TAG') || null)),
|
|
2941
|
+
edit_reason: (options && options.reason) || null,
|
|
2942
|
+
edit_origin: (options && options.origin) || null,
|
|
2943
|
+
person_id: (await import('./persons.js').then((m) => m.writerStamp())).person_id,
|
|
2944
|
+
client_id: (await import('./persons.js').then((m) => m.writerStamp())).client_id,
|
|
2945
|
+
session_id: sid,
|
|
2946
|
+
restored_from_version: (options && options.restore_of) || null,
|
|
2947
|
+
size_at_session_start: baseline,
|
|
2948
|
+
size_session_id: base,
|
|
2949
|
+
size_baselines: baselines,
|
|
2950
|
+
size_samples: typeof sizeSamples !== 'undefined' ? sizeSamples : (prev && prev.size_samples) || [],
|
|
2951
|
+
drop_bucket: (options && options.drop_bucket) || (prev && prev.drop_bucket) || null,
|
|
2952
|
+
path_hint: path_hint || (prev && prev.path_hint) || null,
|
|
2953
|
+
// Writing a retired document brings it back into the working view.
|
|
2954
|
+
// Without this a retired id would take writes and stay invisible, which
|
|
2955
|
+
// is a trap; it is also the un-retire path, so retirement is not a
|
|
2956
|
+
// one-way door.
|
|
2957
|
+
retired: false,
|
|
2958
|
+
size_note: 'baseline is the size when this session first touched the document'
|
|
2959
|
+
};
|
|
2960
|
+
if (options && options.provision && doc_id === stateDoc()) {
|
|
2961
|
+
const ins = await col.updateOne({ doc_id }, { $setOnInsert: writtenRow }, { upsert: true });
|
|
2962
|
+
if (ins && ins.upsertedCount === 0) {
|
|
2963
|
+
throw new ValidationError('The board already exists. Provision does not replace it. Nothing was written.');
|
|
2964
|
+
}
|
|
2965
|
+
} else {
|
|
2966
|
+
await col.updateOne({ doc_id }, { $set: writtenRow }, { upsert: true });
|
|
2967
|
+
}
|
|
2968
|
+
// aleph: the write is not done because the call returned.
|
|
2969
|
+
const check = await col.findOne({ doc_id });
|
|
2970
|
+
verifyDocument(check, { doc_id, sha256, version });
|
|
2971
|
+
return { doc_id, sha256, version, size: content.length, updated_at: now, verified: true,
|
|
2972
|
+
signed: signatureIntegrityOf(check, signingKey),
|
|
2973
|
+
signature_stored: typeof check?.signature === 'string' && check.signature.length > 0,
|
|
2974
|
+
redacted: redaction.hits, conformance };
|
|
2975
|
+
} finally { await client.close(); }
|
|
2976
|
+
} catch (error) {
|
|
2977
|
+
if (error instanceof ValidationError || error instanceof EmetError) throw error;
|
|
2978
|
+
throw new DatabaseError(`Failed to write document: ${error.message}`, 'write_doc');
|
|
2979
|
+
}
|
|
2980
|
+
}
|
|
2981
|
+
|
|
2982
|
+
export async function listDocs(prefix = null, logger = null, include_retired = false) {
|
|
2983
|
+
try {
|
|
2984
|
+
const { client, col } = await _docsClient();
|
|
2985
|
+
try {
|
|
2986
|
+
const q = prefix ? { doc_id: { $regex: '^' + prefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') } } : {};
|
|
2987
|
+
// Retired documents stay in the list. Current documents come first.
|
|
2988
|
+
// include_retired is accepted for older callers and does not hide anything.
|
|
2989
|
+
logger?.debug?.('Listing documents', { include_retired: include_retired === true });
|
|
2990
|
+
const docs = await col.find(q, { projection: { content: 0 } }).toArray();
|
|
2991
|
+
docs.sort((a, b) => {
|
|
2992
|
+
const ar = a.retired === true ? 1 : 0;
|
|
2993
|
+
const br = b.retired === true ? 1 : 0;
|
|
2994
|
+
if (ar !== br) return ar - br;
|
|
2995
|
+
return String(a.doc_id).localeCompare(String(b.doc_id));
|
|
2996
|
+
});
|
|
2997
|
+
const retiredCount = docs.filter(d => d.retired === true).length;
|
|
2998
|
+
return { count: docs.length, include_retired: true,
|
|
2999
|
+
retired_shown: retiredCount,
|
|
3000
|
+
documents: docs.map(d => {
|
|
3001
|
+
const replaced_by = d.retired === true ? replacedByOf(d) : null;
|
|
3002
|
+
return {
|
|
3003
|
+
doc_id: d.doc_id, sha256: d.sha256, version: d.version, size: d.size !== undefined ? d.size : null,
|
|
3004
|
+
updated_at: d.updated_at, updated_by: d.updated_by || null, path_hint: d.path_hint || null,
|
|
3005
|
+
...(d.conformance ? { conformance: d.conformance.verdict,
|
|
3006
|
+
...(d.conformance.failures && d.conformance.failures.length ? { nonconformances: d.conformance.failures.length } : {}) } : {}),
|
|
3007
|
+
...(d.retired === true ? { retired: true, retired_at: d.retired_at || null, retired_reason: d.retired_reason || null, replaced_by } : {}) };
|
|
3008
|
+
}) };
|
|
3009
|
+
} finally { await client.close(); }
|
|
3010
|
+
} catch (error) {
|
|
3011
|
+
if (error instanceof ValidationError || error instanceof EmetError) throw error;
|
|
3012
|
+
throw new DatabaseError(`Failed to list documents: ${error.message}`, 'list_docs');
|
|
3013
|
+
}
|
|
3014
|
+
}
|
|
3015
|
+
|
|
3016
|
+
// ============================================
|
|
3017
|
+
// DOC-STORE LIFECYCLE (added 2026-09-08)
|
|
3018
|
+
//
|
|
3019
|
+
// ⭐ THERE IS NO DELETE, AND THERE IS NOT GOING TO BE ONE. the user's decision,
|
|
3020
|
+
// 2026-09-08, on ISO grounds: document control is retain-and-mark-obsolete, not
|
|
3021
|
+
// remove. ISO 7.5.3's requirement is to issue the revision, mark the prior copy
|
|
3022
|
+
// obsolete, RETAIN it, and prevent its unintended use. Removal fails the third
|
|
3023
|
+
// clause outright, and "recoverable from the history collection" does not fix
|
|
3024
|
+
// that - a record you must reconstruct from an archive is not a retained record.
|
|
3025
|
+
//
|
|
3026
|
+
// The layers have always worked this way: entries are superseded, never edited
|
|
3027
|
+
// and never deleted, and episodic events are immutable. The document store was
|
|
3028
|
+
// the one place that lacked the mechanism, and the honest fix is to give it the
|
|
3029
|
+
// SAME one rather than to give it a delete verb.
|
|
3030
|
+
//
|
|
3031
|
+
// So: retire_doc marks a document obsolete. The content stays exactly where it
|
|
3032
|
+
// is, at its own id, readable forever. list_docs and the startup scan keep it,
|
|
3033
|
+
// after the current documents, marked with retired_at and what replaced it.
|
|
3034
|
+
// That is the mark the seed needed, and it never needed a removal to solve.
|
|
3035
|
+
//
|
|
3036
|
+
// An earlier draft of this section deleted rows and archived them first. It was
|
|
3037
|
+
// written from seeds/cascade-tools-js-improvements.md §C, which proposed
|
|
3038
|
+
// delete_doc / rename_doc - a proposal authored by an instance, never decided by
|
|
3039
|
+
// the user. It is exactly the audit candidate the meta-layer index has open: content
|
|
3040
|
+
// in the record that reads as the user's and is not.
|
|
3041
|
+
// ============================================
|
|
3042
|
+
|
|
3043
|
+
/**
|
|
3044
|
+
* Mark a document obsolete. Nothing is removed and nothing is archived, because
|
|
3045
|
+
* nothing is lost: the row keeps its id, its content, its version and its
|
|
3046
|
+
* digest. list_docs and startup keep it, marked, after the current documents.
|
|
3047
|
+
*
|
|
3048
|
+
* This is the document-store analogue of revise_memory marking a parent
|
|
3049
|
+
* superseded, and it is a sanctioned write for the same reason: it is a
|
|
3050
|
+
* supersession marker, not a content write, and it has exactly one door.
|
|
3051
|
+
* Reversible by writing the document again - write_doc clears the flag.
|
|
3052
|
+
*/
|
|
3053
|
+
export async function retireDoc(doc_id, reason = null, retired_by = null, logger = null, options = {}) {
|
|
3054
|
+
try {
|
|
3055
|
+
if (!doc_id || typeof doc_id !== 'string') throw new ValidationError('doc_id must be a non-empty string');
|
|
3056
|
+
refuseBotMainRecord(doc_id, retired_by, 'retire_doc', { cleanup: true });
|
|
3057
|
+
refuseBotRetireRename(retired_by, 'retire_doc');
|
|
3058
|
+
if (doc_id === stateDoc()) throw new ValidationError('retire_doc: the board is never retired. Edit it with patch_doc and a reason. Nothing was changed.');
|
|
3059
|
+
if (!isDropDoc(doc_id) && !(options && options.fromRename)) throw new ValidationError(`retire_doc: ${doc_id} is not a drop. Working documents are edited, not retired. Nothing was changed.`);
|
|
3060
|
+
if (typeof reason !== 'string' || reason.trim().length < 10) throw new ValidationError('retire_doc needs reason (at least 10 characters), naming where the drop went. Nothing was changed.');
|
|
3061
|
+
|
|
3062
|
+
const { client, col } = await _docsClient();
|
|
3063
|
+
try {
|
|
3064
|
+
const doc = await col.findOne({ doc_id });
|
|
3065
|
+
if (!doc) throw new ValidationError(`Document not found: ${doc_id}`);
|
|
3066
|
+
if (doc.retired) {
|
|
3067
|
+
return { doc_id, retired: true, already_retired: true, verified: true,
|
|
3068
|
+
retired_at: doc.retired_at, retired_reason: doc.retired_reason || null,
|
|
3069
|
+
version: doc.version, sha256: doc.sha256 };
|
|
3070
|
+
}
|
|
3071
|
+
|
|
3072
|
+
const now = new Date();
|
|
3073
|
+
await col.updateOne({ doc_id }, { $set: {
|
|
3074
|
+
retired: true,
|
|
3075
|
+
retired_at: now,
|
|
3076
|
+
retired_by: retired_by || setting('SOURCE_TAG') || null,
|
|
3077
|
+
retired_reason: reason || null,
|
|
3078
|
+
retired_person_id: (await import('./persons.js').then((m) => m.currentPersonId())) || null,
|
|
3079
|
+
retired_client_id: (await import('./persons.js').then((m) => m.currentClientId())) || 'stdio',
|
|
3080
|
+
retired_session_id: (options && options.session_id) || null
|
|
3081
|
+
} });
|
|
3082
|
+
|
|
3083
|
+
// aleph: the mark is not set because the call returned. Content and
|
|
3084
|
+
// digest are re-checked too - retiring must not touch either.
|
|
3085
|
+
const check = await col.findOne({ doc_id });
|
|
3086
|
+
if (!check || check.retired !== true) {
|
|
3087
|
+
throw new DatabaseError(`Retirement mark did not persist: ${doc_id}`, 'retire_doc');
|
|
3088
|
+
}
|
|
3089
|
+
if (check.sha256 !== doc.sha256 || check.version !== doc.version) {
|
|
3090
|
+
throw new DatabaseError(`Retiring altered the document: ${doc_id}`, 'retire_doc');
|
|
3091
|
+
}
|
|
3092
|
+
|
|
3093
|
+
// L5 - retirement is not an escape hatch. retireDoc is the one document
|
|
3094
|
+
// write that deliberately does NOT pass through writeDoc(): it changes
|
|
3095
|
+
// visibility, never content, and re-running the whole content pipeline
|
|
3096
|
+
// for a flag would be a second door for no gain. That is exactly why the
|
|
3097
|
+
// marker has to be set HERE rather than assumed to have been set there.
|
|
3098
|
+
// Nothing is blocked - the retirement stands and carries its mark.
|
|
3099
|
+
let conformance = null;
|
|
3100
|
+
if (isGovernedByShippedCopy(doc_id) || templateFor(doc_id)) {
|
|
3101
|
+
conformance = {
|
|
3102
|
+
template: templateFor(doc_id), template_status: null,
|
|
3103
|
+
verdict: CONFORMANCE_VERDICT.NONCONFORMING,
|
|
3104
|
+
failures: [`${doc_id} is a controlled document and has been retired; the checks that governed it stop here (L5)`],
|
|
3105
|
+
warnings: [], checked_at: new Date().toISOString()
|
|
3106
|
+
};
|
|
3107
|
+
await col.updateOne({ doc_id }, { $set: { conformance } });
|
|
3108
|
+
}
|
|
3109
|
+
|
|
3110
|
+
return { doc_id, retired: true, verified: true, retired_at: now, conformance,
|
|
3111
|
+
retired_reason: reason || null, version: check.version,
|
|
3112
|
+
sha256: check.sha256, content_unchanged: true,
|
|
3113
|
+
note: 'Content retained at this id. list_docs keeps it after the current documents, marked with retired_at and, when a rename replaced it, what replaced it. read_doc still returns it. Nothing was deleted.' };
|
|
3114
|
+
} finally { await client.close(); }
|
|
3115
|
+
} catch (error) {
|
|
3116
|
+
if (error instanceof ValidationError || error instanceof EmetError) throw error;
|
|
3117
|
+
throw new DatabaseError(`Failed to retire document: ${error.message}`, 'retire_doc');
|
|
3118
|
+
}
|
|
3119
|
+
}
|
|
3120
|
+
|
|
3121
|
+
export async function renameDoc(old_id, new_id, updated_by = null, logger = null, reason = null, options = {}) {
|
|
3122
|
+
try {
|
|
3123
|
+
if (!old_id || typeof old_id !== 'string') throw new ValidationError('old_id must be a non-empty string');
|
|
3124
|
+
if (!new_id || typeof new_id !== 'string') throw new ValidationError('new_id must be a non-empty string');
|
|
3125
|
+
if (old_id === new_id) throw new ValidationError('old_id and new_id name the same document');
|
|
3126
|
+
refuseBotMainRecord(old_id, updated_by, 'rename_doc', { cleanup: true });
|
|
3127
|
+
refuseBotMainRecord(new_id, updated_by, 'rename_doc');
|
|
3128
|
+
refuseBotRetireRename(updated_by, 'rename_doc');
|
|
3129
|
+
if (typeof reason !== 'string' || reason.trim().length < 10) throw new ValidationError('rename_doc needs reason (at least 10 characters). Nothing was changed.');
|
|
3130
|
+
if (old_id === stateDoc() || new_id === stateDoc()) throw new ValidationError('rename_doc: the board is never renamed. Nothing was changed.');
|
|
3131
|
+
if (isDropDoc(old_id) || isDropDoc(new_id)) throw new ValidationError('rename_doc: a drop is write-once. Use a new id. Nothing was changed.');
|
|
3132
|
+
|
|
3133
|
+
let content, path_hint;
|
|
3134
|
+
const { client, col } = await _docsClient();
|
|
3135
|
+
try {
|
|
3136
|
+
const source = await col.findOne({ doc_id: old_id });
|
|
3137
|
+
if (!source) throw new ValidationError(`Document not found: ${old_id}`);
|
|
3138
|
+
// Never clobber. A rename onto an occupied id is a MERGE decision, and a
|
|
3139
|
+
// merge is not something to guess at on the caller's behalf.
|
|
3140
|
+
const occupied = await col.findOne({ doc_id: new_id });
|
|
3141
|
+
if (occupied) {
|
|
3142
|
+
throw new ValidationError(`Destination already exists: ${new_id} (version ${occupied.version}). Rename refused; nothing was changed.`);
|
|
3143
|
+
}
|
|
3144
|
+
content = source.content;
|
|
3145
|
+
path_hint = source.path_hint || null;
|
|
3146
|
+
} finally { await client.close(); }
|
|
3147
|
+
|
|
3148
|
+
// One write path: the new id is created through writeDoc, so it inherits
|
|
3149
|
+
// the same redaction, digest, versioning and sha256 read-back as any other
|
|
3150
|
+
// write rather than getting a second, parallel implementation.
|
|
3151
|
+
// `reissued_from` carries the L5 question to the door, so the mark is set
|
|
3152
|
+
// by the same write that creates the document rather than by a second one.
|
|
3153
|
+
const written = await writeDoc(new_id, content, updated_by, path_hint, logger, { reissued_from: old_id, reason, session_id: options && options.session_id });
|
|
3154
|
+
|
|
3155
|
+
// Ordering is deliberate: the copy exists and is verified BEFORE the old id
|
|
3156
|
+
// is marked obsolete. Nothing is removed at any point - a "rename" here is a
|
|
3157
|
+
// reissue plus an obsolescence mark, which is what document control actually
|
|
3158
|
+
// asks for. The old id keeps its content and stays readable at its own id.
|
|
3159
|
+
const retired = await retireDoc(old_id, `Reissued as ${new_id}`, updated_by, logger, { fromRename: true });
|
|
3160
|
+
|
|
3161
|
+
return {
|
|
3162
|
+
from: old_id, to: new_id, verified: written.verified === true && retired.verified === true,
|
|
3163
|
+
conformance: written.conformance || null,
|
|
3164
|
+
sha256: written.sha256, version: written.version, size: written.size,
|
|
3165
|
+
redacted: written.redacted,
|
|
3166
|
+
old_id_retired: true, old_id_retired_at: retired.retired_at,
|
|
3167
|
+
old_content_retained: true,
|
|
3168
|
+
note: `${old_id} is retained and readable at its own id, marked obsolete, and list_docs keeps it after the current documents with retired_at and replaced_by ${new_id}. Nothing was deleted.`
|
|
3169
|
+
};
|
|
3170
|
+
} catch (error) {
|
|
3171
|
+
if (error instanceof ValidationError || error instanceof EmetError) throw error;
|
|
3172
|
+
throw new DatabaseError(`Failed to rename document: ${error.message}`, 'rename_doc');
|
|
3173
|
+
}
|
|
3174
|
+
}
|
|
3175
|
+
|
|
3176
|
+
async function unretiredBootDrops(session_id) {
|
|
3177
|
+
if (!session_id || typeof session_id !== 'string' || !String(session_id).trim()) {
|
|
3178
|
+
throw new ValidationError('a board edit or restore needs session_id from emet_session_open, with a verified drop list. Nothing was written.');
|
|
3179
|
+
}
|
|
3180
|
+
const base = String(session_id).replace(/_verbatim_part\d+$/, '').trim();
|
|
3181
|
+
if (!base) {
|
|
3182
|
+
throw new ValidationError('a board edit or restore needs session_id from emet_session_open, with a verified drop list. Nothing was written.');
|
|
3183
|
+
}
|
|
3184
|
+
const { COLLECTION_PREFIX } = await import('./database.js');
|
|
3185
|
+
const client = await connectMongo(MONGODB_URI, { serverSelectionTimeoutMS: dropListTimeoutMs(), maxPoolSize: 10 });
|
|
3186
|
+
try {
|
|
3187
|
+
const db = client.db(MONGODB_DB);
|
|
3188
|
+
const row = await db.collection(`${COLLECTION_PREFIX}sessions`).findOne({ _id: base });
|
|
3189
|
+
if (!row || row.drops_unverified === true || !Array.isArray(row.drops_seen)) {
|
|
3190
|
+
throw new ValidationError('unhandled drops could not be loaded at this session boot, so the board cannot be edited or restored until the list is known. Nothing was written.');
|
|
3191
|
+
}
|
|
3192
|
+
const listed = row.drops_seen;
|
|
3193
|
+
if (!listed.length) return [];
|
|
3194
|
+
const open = await db.collection('documents').find({ doc_id: { $in: listed }, retired: { $ne: true } }).toArray();
|
|
3195
|
+
return open.map((d) => d.doc_id);
|
|
3196
|
+
} finally { await client.close(); }
|
|
3197
|
+
}
|
|
3198
|
+
|
|
3199
|
+
export async function restoreDoc(doc_id, version, updated_by = null, logger = null, reason = null, options = {}) {
|
|
3200
|
+
refuseBotMainRecord(doc_id, updated_by, 'restore_doc');
|
|
3201
|
+
refuseBotRetireRename(updated_by, 'restore_doc');
|
|
3202
|
+
if (typeof reason !== 'string' || reason.trim().length < 10) throw new ValidationError('restore_doc needs reason (at least 10 characters). Nothing was written.');
|
|
3203
|
+
if (!updated_by) throw new ValidationError('restore_doc needs a source. Nothing was written.');
|
|
3204
|
+
if (isDropDoc(doc_id)) throw new ValidationError('restore_doc: a drop is never restored. Nothing was written.');
|
|
3205
|
+
if (isRecordDoc(doc_id)) throw new ValidationError('restore_doc: a record is not restored. Records are superseded via revise_memory, not restored. Nothing was written.');
|
|
3206
|
+
if (isGovernedByShippedCopy(doc_id)) throw new ValidationError('restore_doc: a shipped copy is not restored. Nothing was written.');
|
|
3207
|
+
const allowed = (setting('RESTORE_TAGS') || setting('SOURCE_TAG') || '').split(',').map((s) => s.trim()).filter(Boolean);
|
|
3208
|
+
if (!allowed.includes(updated_by)) throw new ValidationError(`restore_doc: ${updated_by} is not in EMET_RESTORE_TAGS. Nothing was written.`);
|
|
3209
|
+
const prior = await readDocHistory(doc_id, version, logger);
|
|
3210
|
+
if (!prior || typeof prior.content !== 'string') throw new ValidationError(`No retained revision ${version} of ${doc_id}`);
|
|
3211
|
+
if (prior.integrity !== true) throw new ValidationError(`restore_doc: retained revision ${version} of ${doc_id} does not match its digest. Nothing was written.`);
|
|
3212
|
+
const verdict = prior.conformance && prior.conformance.verdict;
|
|
3213
|
+
const accepted = prior.conformance && prior.conformance.accepted;
|
|
3214
|
+
if (!verdict) {
|
|
3215
|
+
if (templateFor(doc_id)) throw new ValidationError(`restore_doc: ${doc_id} is templated and the retained revision has no conformance verdict. Only an untemplated document can be restored without a verdict. Nothing was written.`);
|
|
3216
|
+
} else if (verdict !== 'conforms' && verdict !== 'untemplated' && !accepted) {
|
|
3217
|
+
throw new ValidationError(`restore_doc: retained revision ${version} of ${doc_id} was not conforming. Nothing was written.`);
|
|
3218
|
+
}
|
|
3219
|
+
const current = await readDoc(doc_id, logger).catch(() => null);
|
|
3220
|
+
const checked = validateDocument(doc_id, prior.content, current && current.content);
|
|
3221
|
+
const lost = (checked.failures || []).filter((f) => /no disposition/.test(f));
|
|
3222
|
+
if (doc_id === stateDoc() && (await unretiredBootDrops(options && options.session_id)).length) {
|
|
3223
|
+
throw new ValidationError('restore_doc: unhandled drops were listed at this session boot and must be retired with retire_doc before the board is restored. Nothing was written.');
|
|
3224
|
+
}
|
|
3225
|
+
const written = await writeDoc(doc_id, prior.content, updated_by, null, logger, { restore_of: version, reason, session_id: options && options.session_id, supersedes: current && current.version });
|
|
3226
|
+
return { ...written, rows_lost: lost, restored_from_version: version };
|
|
3227
|
+
}
|
|
3228
|
+
|
|
3229
|
+
// ============================================
|
|
3230
|
+
// TEMPLATE VALIDATION - freestanding (STANDARD 9)
|
|
3231
|
+
// ============================================
|
|
3232
|
+
//
|
|
3233
|
+
// The same check the door runs, callable without writing anything. Three uses,
|
|
3234
|
+
// all of them the reason it exists separately: auditing records written before
|
|
3235
|
+
// validation existed, counting what adoption will mark BEFORE it marks it, and
|
|
3236
|
+
// checking a new local template against the standard before anyone fills it.
|
|
3237
|
+
|
|
3238
|
+
export async function validateDoc(options = {}, logger = null) {
|
|
3239
|
+
const { doc_id = null, content = null, template_id = null } = options || {};
|
|
3240
|
+
try {
|
|
3241
|
+
if (!doc_id && !content) {
|
|
3242
|
+
throw new ValidationError(
|
|
3243
|
+
'Give a doc_id to check a stored document, or content (with template_id) to check text that is not stored yet.'
|
|
3244
|
+
);
|
|
3245
|
+
}
|
|
3246
|
+
|
|
3247
|
+
if (content) {
|
|
3248
|
+
const tpl_id = template_id || (doc_id ? templateFor(doc_id) : null);
|
|
3249
|
+
if (!tpl_id) {
|
|
3250
|
+
return { checked: doc_id || '(content)', template: null, verdict: CONFORMANCE_VERDICT.UNTEMPLATED,
|
|
3251
|
+
failures: [], warnings: ['No template_id was given and no map entry matches, so nothing governs this content.'] };
|
|
3252
|
+
}
|
|
3253
|
+
const template = loadShipped(tpl_id);
|
|
3254
|
+
if (!template) throw new ValidationError(`No template ${tpl_id} ships with this release.`);
|
|
3255
|
+
const r = checkRecord(content, template, { doc_id });
|
|
3256
|
+
return { checked: doc_id || '(content)', template: tpl_id, template_status: template.status,
|
|
3257
|
+
verdict: r.failures.length ? CONFORMANCE_VERDICT.NONCONFORMING : CONFORMANCE_VERDICT.CONFORMS,
|
|
3258
|
+
failures: r.failures, warnings: r.warnings, stored: false,
|
|
3259
|
+
note: 'Checked against the copy shipped with this release (L1). Nothing was written.' };
|
|
3260
|
+
}
|
|
3261
|
+
|
|
3262
|
+
const { client, col, hist } = await _docsClient();
|
|
3263
|
+
try {
|
|
3264
|
+
const doc = await col.findOne({ doc_id });
|
|
3265
|
+
if (!doc) throw new ValidationError(`Document not found: ${doc_id}`);
|
|
3266
|
+
// The carry-forward check needs the revision before this one, which is
|
|
3267
|
+
// exactly what the history collection archived when this one was written.
|
|
3268
|
+
const prior = await hist.find({ doc_id }).sort({ version: -1 }).limit(1).toArray();
|
|
3269
|
+
const result = validateDocument(doc_id, doc.content, prior.length ? prior[0].content : null);
|
|
3270
|
+
return {
|
|
3271
|
+
checked: doc_id, version: doc.version, stored: false,
|
|
3272
|
+
compared_against_revision: prior.length ? prior[0].version : null,
|
|
3273
|
+
...result,
|
|
3274
|
+
stored_verdict: doc.conformance ? doc.conformance.verdict : null,
|
|
3275
|
+
note: 'Nothing was written. To record a disposition for a standing marker, use accept_nonconformance; to clear one, revise the document so it passes.'
|
|
3276
|
+
};
|
|
3277
|
+
} finally { await client.close(); }
|
|
3278
|
+
} catch (error) {
|
|
3279
|
+
if (error instanceof ValidationError || error instanceof EmetError) throw error;
|
|
3280
|
+
throw new DatabaseError(`Failed to validate document: ${error.message}`, 'validate_doc');
|
|
3281
|
+
}
|
|
3282
|
+
}
|
|
3283
|
+
|
|
3284
|
+
/**
|
|
3285
|
+
* Record the written disposition STANDARD 9 requires for a nonconformance that
|
|
3286
|
+
* will not be corrected. The marker is KEPT, with the acceptance beside it -
|
|
3287
|
+
* never deleted, and never silently cleared. It covers the one revision it
|
|
3288
|
+
* names (L6); the next revision is validated fresh.
|
|
3289
|
+
*/
|
|
3290
|
+
export async function acceptNonconformance(doc_id, reason, accepted_by = null, logger = null, options = {}) {
|
|
3291
|
+
try {
|
|
3292
|
+
refuseBotMainRecord(doc_id, accepted_by, 'accept_nonconformance');
|
|
3293
|
+
if (!doc_id || typeof doc_id !== 'string') throw new ValidationError('doc_id must be a non-empty string');
|
|
3294
|
+
if (typeof reason !== 'string' || reason.trim().length < 10) {
|
|
3295
|
+
throw new ValidationError(
|
|
3296
|
+
'An acceptance needs a written reason of at least 10 characters. It is the disposition of ' +
|
|
3297
|
+
'record for a nonconformance and is what an audit reads later - the same rule revise_transcript uses.'
|
|
3298
|
+
);
|
|
3299
|
+
}
|
|
3300
|
+
const { client, col } = await _docsClient();
|
|
3301
|
+
try {
|
|
3302
|
+
const doc = await col.findOne({ doc_id });
|
|
3303
|
+
if (!doc) throw new ValidationError(`Document not found: ${doc_id}`);
|
|
3304
|
+
const current = doc.conformance;
|
|
3305
|
+
if (!current || current.verdict !== CONFORMANCE_VERDICT.NONCONFORMING) {
|
|
3306
|
+
throw new ValidationError(
|
|
3307
|
+
`${doc_id} version ${doc.version} carries no nonconformance marker, so there is nothing to accept. ` +
|
|
3308
|
+
`Its verdict is "${current ? current.verdict : 'not yet checked'}".`
|
|
3309
|
+
);
|
|
3310
|
+
}
|
|
3311
|
+
const { currentPersonId, currentClientId } = await import('./persons.js');
|
|
3312
|
+
const accepted = buildAcceptance({
|
|
3313
|
+
reason, by: accepted_by || setting('SOURCE_TAG') || null, version: doc.version,
|
|
3314
|
+
person_id: currentPersonId() || null,
|
|
3315
|
+
client_id: currentClientId() || 'stdio',
|
|
3316
|
+
session_id: (options && options.session_id) || null
|
|
3317
|
+
});
|
|
3318
|
+
await col.updateOne({ doc_id }, { $set: { 'conformance.accepted': accepted } });
|
|
3319
|
+
// aleph: the disposition is not recorded because the call returned.
|
|
3320
|
+
const check = await col.findOne({ doc_id });
|
|
3321
|
+
const stored = check && check.conformance && check.conformance.accepted;
|
|
3322
|
+
if (!stored || stored.reason !== accepted.reason || Number(stored.covers_version) !== Number(doc.version)) {
|
|
3323
|
+
throw new DatabaseError(`The acceptance did not persist on ${doc_id}.`, 'accept_nonconformance');
|
|
3324
|
+
}
|
|
3325
|
+
return {
|
|
3326
|
+
doc_id, verified: true, version: doc.version, covers_version: doc.version,
|
|
3327
|
+
accepted, failures_accepted: current.failures || [],
|
|
3328
|
+
note: 'The marker is retained with this acceptance beside it. It covers this revision only - the next write is validated fresh, and accepted markers stay counted at session start, separately from open ones.'
|
|
3329
|
+
};
|
|
3330
|
+
} finally { await client.close(); }
|
|
3331
|
+
} catch (error) {
|
|
3332
|
+
if (error instanceof ValidationError || error instanceof EmetError) throw error;
|
|
3333
|
+
throw new DatabaseError(`Failed to accept nonconformance: ${error.message}`, 'accept_nonconformance');
|
|
3334
|
+
}
|
|
3335
|
+
}
|
|
3336
|
+
|
|
3337
|
+
// ============================================
|
|
3338
|
+
// DOC-STORE PARTIAL UPDATES (added 2026-08-24)
|
|
3339
|
+
// Server-side read-modify-write. The whole point: existing content never
|
|
3340
|
+
// crosses the wire, so document size stops being bounded by an MCP client's
|
|
3341
|
+
// context/output budget. All three delegate the actual write to writeDoc(),
|
|
3342
|
+
// which owns archiving to documents_history, version increment, and the
|
|
3343
|
+
// sha256 read-back -- one write path, not four.
|
|
3344
|
+
// ============================================
|
|
3345
|
+
|
|
3346
|
+
function _assertStr(v, name) {
|
|
3347
|
+
if (typeof v !== 'string' || v.length === 0) {
|
|
3348
|
+
throw new ValidationError(`${name} must be a non-empty string`);
|
|
3349
|
+
}
|
|
3350
|
+
}
|
|
3351
|
+
|
|
3352
|
+
function _assertVersion(cur, expected_version, tool) {
|
|
3353
|
+
if (expected_version === undefined || expected_version === null) return;
|
|
3354
|
+
if (cur.version !== expected_version) {
|
|
3355
|
+
throw new ValidationError(
|
|
3356
|
+
`${tool}: version conflict - document is at v${cur.version}, caller expected v${expected_version}. Re-read and retry.`
|
|
3357
|
+
);
|
|
3358
|
+
}
|
|
3359
|
+
}
|
|
3360
|
+
|
|
3361
|
+
/** Find `needle` in `hay`; require exactly one occurrence. Returns [before, after]. */
|
|
3362
|
+
function _splitUnique(hay, needle, tool, what, doc_id) {
|
|
3363
|
+
const parts = hay.split(needle);
|
|
3364
|
+
const n = parts.length - 1;
|
|
3365
|
+
if (n === 0) throw new ValidationError(`${tool}: ${what} not found in ${doc_id}`);
|
|
3366
|
+
if (n > 1) {
|
|
3367
|
+
throw new ValidationError(
|
|
3368
|
+
`${tool}: ${what} matches ${n} times in ${doc_id} - must match exactly once. Add surrounding context to disambiguate.`
|
|
3369
|
+
);
|
|
3370
|
+
}
|
|
3371
|
+
return [parts[0], parts[1]];
|
|
3372
|
+
}
|
|
3373
|
+
|
|
3374
|
+
export async function patchDoc(doc_id, old_str, new_str, updated_by = null, expected_version = null, logger = null, options = {}) {
|
|
3375
|
+
try {
|
|
3376
|
+
_assertStr(doc_id, 'doc_id');
|
|
3377
|
+
refuseBotMainRecord(doc_id, updated_by, 'patch_doc');
|
|
3378
|
+
_refuseRecordAmendment(doc_id, 'patch_doc');
|
|
3379
|
+
if (!(Array.isArray(options.edits) && options.edits.length)) _assertStr(old_str, 'old_str');
|
|
3380
|
+
const reason = options && options.reason;
|
|
3381
|
+
if (typeof reason !== 'string' || reason.trim().length < 10) {
|
|
3382
|
+
throw new ValidationError('patch_doc needs reason (at least 10 characters). Nothing was written.');
|
|
3383
|
+
}
|
|
3384
|
+
const cur = await readDoc(doc_id, logger);
|
|
3385
|
+
_assertVersion(cur, expected_version, 'patch_doc');
|
|
3386
|
+
if (isDropDoc(doc_id)) throw new ValidationError('patch_doc: a drop is write-once. Use a new id. Nothing was written.');
|
|
3387
|
+
const edits = Array.isArray(options.edits) && options.edits.length
|
|
3388
|
+
? options.edits
|
|
3389
|
+
: [{ old_str, new_str }];
|
|
3390
|
+
for (const edit of edits) {
|
|
3391
|
+
if (typeof edit.old_str !== 'string' || edit.old_str.length === 0) throw new ValidationError('old_str must be a non-empty string');
|
|
3392
|
+
if (typeof edit.new_str !== 'string') throw new ValidationError('new_str must be a string (empty string allowed = delete)');
|
|
3393
|
+
_splitUnique(cur.content, edit.old_str, 'patch_doc', 'old_str', doc_id);
|
|
3394
|
+
}
|
|
3395
|
+
const spans = edits.map((edit) => {
|
|
3396
|
+
const at = cur.content.indexOf(edit.old_str);
|
|
3397
|
+
return { at, len: edit.old_str.length, new_str: edit.new_str };
|
|
3398
|
+
});
|
|
3399
|
+
const ordered = [...spans].sort((a, b) => a.at - b.at);
|
|
3400
|
+
for (let i = 1; i < ordered.length; i++) {
|
|
3401
|
+
if (ordered[i].at < ordered[i - 1].at + ordered[i - 1].len) throw new ValidationError('patch_doc: edits overlap. Nothing was written.');
|
|
3402
|
+
}
|
|
3403
|
+
let next = cur.content;
|
|
3404
|
+
for (const span of [...spans].sort((a, b) => b.at - a.at)) next = next.slice(0, span.at) + span.new_str + next.slice(span.at + span.len);
|
|
3405
|
+
if (doc_id === stateDoc()) {
|
|
3406
|
+
if ((await unretiredBootDrops(options && options.session_id)).length) throw new ValidationError('patch_doc: unhandled drops were listed at this session boot and must be retired with retire_doc before the board is edited. Nothing was written.');
|
|
3407
|
+
const ids = (text) => new Set([...String(text || '').matchAll(/\bB-\d+\b/g)].map((m) => m[0]));
|
|
3408
|
+
const added = [...ids(next)].filter((id) => !ids(cur.content).has(id));
|
|
3409
|
+
if (added.length && !(options && options.origin)) throw new ValidationError('patch_doc: an added board row needs origin. Nothing was written.');
|
|
3410
|
+
const checked = validateDocument(doc_id, next, cur.content);
|
|
3411
|
+
if (checked && checked.verdict === 'nonconforming') {
|
|
3412
|
+
throw new ValidationError(`patch_doc: the board edit is nonconforming. ${(checked.failures || []).join('; ')} Nothing was written.`);
|
|
3413
|
+
}
|
|
3414
|
+
}
|
|
3415
|
+
const res = await writeDoc(doc_id, next, updated_by, null, logger, { edited: true, reason, origin: options.origin || null, session_id: options.session_id || null });
|
|
3416
|
+
return { ...res, previous_version: cur.version, replaced: edits.length,
|
|
3417
|
+
size_delta: next.length - cur.content.length };
|
|
3418
|
+
} catch (error) {
|
|
3419
|
+
if (error instanceof ValidationError || error instanceof EmetError) throw error;
|
|
3420
|
+
throw new DatabaseError(`Failed to patch document: ${error.message}`, 'patch_doc');
|
|
3421
|
+
}
|
|
3422
|
+
}
|
|
3423
|
+
|
|
3424
|
+
// Restored 2026-09-19: both constants were dropped in an earlier commit while the function
|
|
3425
|
+
// kept referencing them, which made every semantic_recall call throw
|
|
3426
|
+
// "SEMANTIC_RECALL_LAYERS is not defined".
|
|
3427
|
+
const SEMANTIC_RECALL_LAYERS = ['episodic', 'semantic', 'procedural', 'meta', 'identity', 'working'];
|
|
3428
|
+
const SEMANTIC_RECALL_INDEX = process.env.EMET_VECTOR_INDEX || 'emet_content_auto';
|
|
3429
|
+
|
|
3430
|
+
export async function semanticRecall(query, layer = null, limit = 10, logger = null, options = null, dbManager = null) {
|
|
3431
|
+
try {
|
|
3432
|
+
if (!query || typeof query !== 'string' || query.trim() === '') {
|
|
3433
|
+
throw new ValidationError('query', 'query must be a non-empty string');
|
|
3434
|
+
}
|
|
3435
|
+
if (query.length > CONTENT_LIMITS.MAX_QUERY_LENGTH) {
|
|
3436
|
+
throw new ValidationError('query', `query exceeds maximum length of ${CONTENT_LIMITS.MAX_QUERY_LENGTH}`);
|
|
3437
|
+
}
|
|
3438
|
+
const validatedLimit = validateLimit(limit);
|
|
3439
|
+
const layers = layer ? [validateLayer(layer)] : SEMANTIC_RECALL_LAYERS;
|
|
3440
|
+
// vectorRecall is the in-memory stand-in. Production has no such method
|
|
3441
|
+
// and uses Atlas. The stand-in lets a test see history and rank without a
|
|
3442
|
+
// vector index. It does not change what Atlas returns.
|
|
3443
|
+
const useStandIn = !!(dbManager && typeof dbManager.vectorRecall === 'function');
|
|
3444
|
+
|
|
3445
|
+
const client = useStandIn ? null : await connectMongo(MONGODB_URI, { serverSelectionTimeoutMS: 5000, maxPoolSize: 10 });
|
|
3446
|
+
try {
|
|
3447
|
+
const db = client ? client.db(MONGODB_DB) : null;
|
|
3448
|
+
const results = [];
|
|
3449
|
+
for (const l of layers) {
|
|
3450
|
+
if (useStandIn) {
|
|
3451
|
+
const docs = await dbManager.vectorRecall(l, query.trim(), validatedLimit);
|
|
3452
|
+
results.push(...collectSemanticHits(docs, l));
|
|
3453
|
+
continue;
|
|
3454
|
+
}
|
|
3455
|
+
const col = db.collection(`${COLLECTION_PREFIX}${l}`);
|
|
3456
|
+
const docs = await col.aggregate([
|
|
3457
|
+
{ $vectorSearch: {
|
|
3458
|
+
index: SEMANTIC_RECALL_INDEX,
|
|
3459
|
+
path: 'content',
|
|
3460
|
+
query: query.trim(),
|
|
3461
|
+
numCandidates: candidateBound(validatedLimit),
|
|
3462
|
+
limit: validatedLimit,
|
|
3463
|
+
// THE INDEX-LEVEL PRE-FILTER ON is_live WAS REMOVED 2026-09-13.
|
|
3464
|
+
// It excluded superseded rows from Atlas before ranking, which is
|
|
3465
|
+
// exactly what the user's decision reverses: a parent surfaces so
|
|
3466
|
+
// a revision is read in the context of what it revised.
|
|
3467
|
+
//
|
|
3468
|
+
// THE COST, STATED RATHER THAN DISCOVERED LATER: this is a ranked
|
|
3469
|
+
// top-N path, so both halves of a supersession chain can now occupy
|
|
3470
|
+
// slots that previously held two distinct entries. That is the price
|
|
3471
|
+
// of the context, and it is a real one on this path in a way it is
|
|
3472
|
+
// not on the unranked paths. If it bites, the fix is to rank the
|
|
3473
|
+
// chain as one and return the live entry with its parent attached -
|
|
3474
|
+
// NOT to quietly re-hide the parent.
|
|
3475
|
+
//
|
|
3476
|
+
// is_live remains a `filter` field on the configured vector index on
|
|
3477
|
+
// all six layers; nothing was removed from the index. Retained note:
|
|
3478
|
+
// {superseded_by: {$eq: null}} returns ZERO rows here because vector
|
|
3479
|
+
// filters do not match null, which is why the boolean exists at all.
|
|
3480
|
+
} },
|
|
3481
|
+
{ $project: { _id: 0, id: 1, timestamp: 1, content: 1, context: 1,
|
|
3482
|
+
importance: 1, emotional_intensity: 1, metadata: 1, sha256: 1, metadata_sha256: 1,
|
|
3483
|
+
superseded_by: 1, superseded_at: 1, valid_until: 1, is_live: 1,
|
|
3484
|
+
score: { $meta: 'vectorSearchScore' } } }
|
|
3485
|
+
]).toArray();
|
|
3486
|
+
// Expired and superseded hits stay. collectSemanticHits marks them;
|
|
3487
|
+
// it does not drop them. Current entries rank ahead of ended ones.
|
|
3488
|
+
results.push(...collectSemanticHits(docs, l));
|
|
3489
|
+
}
|
|
3490
|
+
const ranked = rankSemanticHits(results, query);
|
|
3491
|
+
const top = applyReadLimit(ranked, validatedLimit, options && options.history === true);
|
|
3492
|
+
logger?.info('Semantic recall completed', { layers: layers.length, resultsFound: top.length });
|
|
3493
|
+
return { count: top.length, query: query.trim(), layers_searched: layers, results: top };
|
|
3494
|
+
} finally { if (client) await client.close(); }
|
|
3495
|
+
} catch (error) {
|
|
3496
|
+
if (error instanceof ValidationError || error instanceof EmetError) throw error;
|
|
3497
|
+
throw new DatabaseError(`Failed semantic recall: ${error.message}`, 'semantic_recall');
|
|
3498
|
+
}
|
|
3499
|
+
}
|
|
3500
|
+
|
|
3501
|
+
|
|
3502
|
+
|
|
3503
|
+
// ============================================
|
|
3504
|
+
// TOOL DEFINITIONS
|
|
3505
|
+
// ============================================
|
|
3506
|
+
|
|
3507
|
+
/**
|
|
3508
|
+
* MCP tool definitions
|
|
3509
|
+
*/
|
|
3510
|
+
|
|
3511
|
+
// ============================================
|
|
3512
|
+
// CORPUS RECALL (Stage B, 2026-07-27)
|
|
3513
|
+
// Semantic search over an optional corpus collection you populate yourself.
|
|
3514
|
+
// ============================================
|
|
3515
|
+
|
|
3516
|
+
const CORPUS_RECALL_INDEX = process.env.EMET_CORPUS_INDEX || 'corpus_content_auto';
|
|
3517
|
+
// Corpus names are whatever you populate. Empty by default - this tool returns
|
|
3518
|
+
// nothing until you load documents into the corpus collection yourself.
|
|
3519
|
+
const CORPUS_VALUES = (process.env.EMET_CORPORA || '').split(',').map(s => s.trim()).filter(Boolean);
|
|
3520
|
+
|
|
3521
|
+
export async function corpusRecall(query, corpus = null, limit = 5, logger = null) {
|
|
3522
|
+
try {
|
|
3523
|
+
if (!query || typeof query !== 'string' || query.trim() === '') {
|
|
3524
|
+
throw new ValidationError('query', 'query must be a non-empty string');
|
|
3525
|
+
}
|
|
3526
|
+
if (query.length > CONTENT_LIMITS.MAX_QUERY_LENGTH) {
|
|
3527
|
+
throw new ValidationError('query', `query exceeds maximum length of ${CONTENT_LIMITS.MAX_QUERY_LENGTH}`);
|
|
3528
|
+
}
|
|
3529
|
+
const validatedLimit = Math.min(Math.max(parseInt(limit) || 5, 1), 25);
|
|
3530
|
+
// An empty EMET_CORPORA means "unrestricted", not "nothing allowed".
|
|
3531
|
+
if (corpus && CORPUS_VALUES.length && !CORPUS_VALUES.includes(corpus)) {
|
|
3532
|
+
throw new ValidationError('corpus', `corpus must be one of: ${CORPUS_VALUES.join(', ')}`);
|
|
3533
|
+
}
|
|
3534
|
+
const client = await connectMongo(MONGODB_URI, { serverSelectionTimeoutMS: 5000, maxPoolSize: 10 });
|
|
3535
|
+
try {
|
|
3536
|
+
const db = client.db(MONGODB_DB);
|
|
3537
|
+
const col = db.collection(process.env.EMET_CORPUS_COLLECTION || 'corpus_chunks');
|
|
3538
|
+
// Native corpus filter on the index (filter field added 2026-07-27); dedupe post-search.
|
|
3539
|
+
const fetchN = validatedLimit * 3;
|
|
3540
|
+
const vs = {
|
|
3541
|
+
index: CORPUS_RECALL_INDEX,
|
|
3542
|
+
path: 'content',
|
|
3543
|
+
query: query.trim(),
|
|
3544
|
+
numCandidates: Math.max(fetchN * 20, 300),
|
|
3545
|
+
limit: fetchN
|
|
3546
|
+
};
|
|
3547
|
+
if (corpus) vs.filter = { corpus: { $eq: corpus } };
|
|
3548
|
+
const docs = await col.aggregate([
|
|
3549
|
+
{ $vectorSearch: vs },
|
|
3550
|
+
{ $project: { _id: 1, corpus: 1, video_id: 1, video_title: 1, url: 1, chunk_seq: 1,
|
|
3551
|
+
start_sec: 1, content: 1, epistemic: 1, work: 1, story_num: 1, topic: 1,
|
|
3552
|
+
score: { $meta: 'vectorSearchScore' } } }
|
|
3553
|
+
]).toArray();
|
|
3554
|
+
const filtered = corpus ? docs.filter(d => d.corpus === corpus) : docs;
|
|
3555
|
+
const perKey = {}; const out = [];
|
|
3556
|
+
for (const d of filtered) {
|
|
3557
|
+
const key = `${d.corpus}:${d.video_id || d.work || d.video_title}`;
|
|
3558
|
+
perKey[key] = (perKey[key] || 0) + 1;
|
|
3559
|
+
if (perKey[key] <= 2) out.push(d);
|
|
3560
|
+
if (out.length >= validatedLimit) break;
|
|
3561
|
+
}
|
|
3562
|
+
const results = out.map(d => ({
|
|
3563
|
+
corpus: d.corpus,
|
|
3564
|
+
title: d.video_title,
|
|
3565
|
+
url: d.url ? (d.start_sec != null ? `${d.url}&t=${Math.floor(d.start_sec)}s` : d.url) : null,
|
|
3566
|
+
start_sec: d.start_sec != null ? d.start_sec : null,
|
|
3567
|
+
epistemic: d.epistemic || null,
|
|
3568
|
+
score: d.score,
|
|
3569
|
+
content: d.content,
|
|
3570
|
+
chunk_id: d._id
|
|
3571
|
+
}));
|
|
3572
|
+
logger?.info('Corpus recall completed', { corpus: corpus || 'all', resultsFound: results.length });
|
|
3573
|
+
return { count: results.length, query: query.trim(), corpus: corpus || 'all', results };
|
|
3574
|
+
} finally { await client.close(); }
|
|
3575
|
+
} catch (error) {
|
|
3576
|
+
if (error instanceof ValidationError || error instanceof EmetError) throw error;
|
|
3577
|
+
throw new DatabaseError(`Failed corpus recall: ${error.message}`, 'corpus_recall');
|
|
3578
|
+
}
|
|
3579
|
+
}
|
|
3580
|
+
|
|
3581
|
+
import { digest, verifyDocument, verifyLayerEntry, integrityOf, digestMetadata, metadataIntegrityOf, signDigests, signatureIntegrityOf, signatureCoversPriorMetadata, chainIntegrityOf, AlephError } from './aleph.js';
|
|
3582
|
+
import { parseMetadata } from './metadata.js';
|
|
3583
|
+
export { parseMetadata, isLegacyMetadataString, isMetadataPath } from './metadata.js';
|
|
3584
|
+
import { validateProvenance, provenanceColumns } from './provenance.js';
|
|
3585
|
+
export { validateProvenance, provenanceColumns, resolveProvenance, crossCheckProvenance } from './provenance.js';
|
|
3586
|
+
export { digest, verifyDocument, verifyLayerEntry, integrityOf, canonicalJson, digestMetadata, metadataIntegrityOf, signDigests, signatureIntegrityOf, signatureCoversPriorMetadata, AlephError } from './aleph.js';
|
|
3587
|
+
import { annotate } from './annotations.js';
|
|
3588
|
+
export { sessionClose, CLOSE_PROTOCOL } from './session.js';
|
|
3589
|
+
export { findGaps, gapCounts, GAP_CLASSES } from './gaps.js';
|
|
3590
|
+
export { TOOL_ANNOTATIONS } from './annotations.js';
|
|
3591
|
+
|
|
3592
|
+
export { DISCIPLINE_SUMMARY } from './disciplines.js';
|
|
3593
|
+
export { FLOOR, FLOOR_REASSERTION, FLOOR_SECTIONS, PRECEDENCE, readFloor } from './floor.js';
|
|
3594
|
+
|
|
3595
|
+
export {
|
|
3596
|
+
initialize,
|
|
3597
|
+
setupStatus,
|
|
3598
|
+
setupComplete,
|
|
3599
|
+
verifyConnection,
|
|
3600
|
+
SETUP_INTERVIEW
|
|
3601
|
+
} from './setup.js';
|
|
3602
|
+
|
|
3603
|
+
/**
|
|
3604
|
+
* Who may mint a guest or member code. The local stdio actor has no person id
|
|
3605
|
+
* and is the owner. HTTP still requires a loaded person whose role is owner.
|
|
3606
|
+
* An in-process actor that names a person is checked the same way.
|
|
3607
|
+
*/
|
|
3608
|
+
async function codeCreator(dbManager, refusal) {
|
|
3609
|
+
const { currentActor, getPerson } = await import('./persons.js');
|
|
3610
|
+
const actor = currentActor();
|
|
3611
|
+
if (actor.transport === 'stdio' || (actor.transport !== 'http' && !actor.personId)) {
|
|
3612
|
+
return actor.personId || LOCAL_SUBJECT;
|
|
3613
|
+
}
|
|
3614
|
+
if (!actor.personId) throw new ValidationError(refusal);
|
|
3615
|
+
const person = await getPerson(dbManager.db, actor.personId);
|
|
3616
|
+
if (!person || person.role !== 'owner') throw new ValidationError(refusal);
|
|
3617
|
+
return actor.personId;
|
|
3618
|
+
}
|
|
3619
|
+
|
|
3620
|
+
export async function inviteCreate(dbManager, { display_name, scopes, days, source } = {}, logger = null) {
|
|
3621
|
+
if (!display_name || !String(display_name).trim()) {
|
|
3622
|
+
throw new ValidationError('display_name is required — the name the guest will be known by');
|
|
3623
|
+
}
|
|
3624
|
+
await dbManager.getConnection('working');
|
|
3625
|
+
const { createInvite, INVITES_FROZEN, INVITES_FROZEN_REASON } = await import('./invites.js');
|
|
3626
|
+
if (INVITES_FROZEN) throw new ValidationError(INVITES_FROZEN_REASON);
|
|
3627
|
+
const { registeredSourceTags } = await import('./setup.js');
|
|
3628
|
+
const actor = await codeCreator(dbManager, 'Only the owner can create invites');
|
|
3629
|
+
const requested = Array.isArray(scopes) ? scopes : ['emet.read'];
|
|
3630
|
+
if (requested.length !== 1 || requested[0] !== 'emet.read') {
|
|
3631
|
+
throw new ValidationError('A guest invite accepts only emet.read');
|
|
3632
|
+
}
|
|
3633
|
+
const registry = registeredSourceTags();
|
|
3634
|
+
const tag = typeof source === 'string' ? source.trim() : '';
|
|
3635
|
+
if (registry.length && tag && !registry.includes(tag)) {
|
|
3636
|
+
throw new ValidationError('source', 'A guest invite source must be a registered tag. Nothing was created.');
|
|
3637
|
+
}
|
|
3638
|
+
const out = await createInvite(dbManager.db, {
|
|
3639
|
+
created_by: actor,
|
|
3640
|
+
display_name,
|
|
3641
|
+
scopes: ['emet.read'],
|
|
3642
|
+
days: days || 14,
|
|
3643
|
+
source_tag: tag || null
|
|
3644
|
+
});
|
|
3645
|
+
logger?.info?.('Invite created', { display_name: out.display_name });
|
|
3646
|
+
return {
|
|
3647
|
+
...out,
|
|
3648
|
+
note: 'Show the code once. It is not stored in the clear. The guest pastes it on the connect screen instead of the passphrase.'
|
|
3649
|
+
};
|
|
3650
|
+
}
|
|
3651
|
+
|
|
3652
|
+
export async function memberAccessCreate(dbManager, { display_name, days, source } = {}, logger = null) {
|
|
3653
|
+
if (!display_name || !String(display_name).trim()) {
|
|
3654
|
+
throw new ValidationError('display_name is required \u2014 the name the member will be known by');
|
|
3655
|
+
}
|
|
3656
|
+
await dbManager.getConnection('working');
|
|
3657
|
+
const { createMemberAccess, MEMBER_ACCESS_FROZEN, MEMBER_ACCESS_FROZEN_REASON } = await import('./invites.js');
|
|
3658
|
+
if (MEMBER_ACCESS_FROZEN) throw new ValidationError(MEMBER_ACCESS_FROZEN_REASON);
|
|
3659
|
+
const { registeredSourceTags } = await import('./setup.js');
|
|
3660
|
+
const actor = await codeCreator(dbManager, 'Only the owner can create EMET Member Access');
|
|
3661
|
+
const registry = registeredSourceTags();
|
|
3662
|
+
const tag = typeof source === 'string' ? source.trim() : '';
|
|
3663
|
+
if (registry.length && tag && !registry.includes(tag)) {
|
|
3664
|
+
throw new ValidationError('source', 'A member code source must be a registered tag. Nothing was created.');
|
|
3665
|
+
}
|
|
3666
|
+
const out = await createMemberAccess(dbManager.db, { created_by: actor, display_name, days: days || 14, source_tag: tag || null });
|
|
3667
|
+
logger?.info?.('EMET Member Access created', { display_name: out.display_name });
|
|
3668
|
+
return {
|
|
3669
|
+
...out,
|
|
3670
|
+
label: 'EMET Member Access',
|
|
3671
|
+
note: 'Show the code once. It is not stored in the clear. This is not EMET Guest Access. The person pastes it on the connect screen. They get a new person id with read and write access, never the owner identity.'
|
|
3672
|
+
};
|
|
3673
|
+
}
|
|
3674
|
+
|
|
3675
|
+
// Raw tool definitions. Exported as TOOLS below, with annotations attached.
|
|
3676
|
+
const TOOL_DEFINITIONS = [
|
|
3677
|
+
{
|
|
3678
|
+
name: "emet_initialize",
|
|
3679
|
+
description: "START HERE. Call this FIRST in every session, before answering anything that might have prior context, and always when the user opens with 'initialize', 'start', 'begin', 'hello' or similar. It is the only setup step a user should ever have to know about. A large startup comes in pages: read `page_notice` first and call again with the page it names until the last page, before replying. If this memory store is new it returns a short interview to run conversationally; if it is established it returns who you are, how you speak (`character`, when the store keeps one - authoritative on voice for the whole session), who you are serving, and where things were left. The user never needs to know which case applied.",
|
|
3680
|
+
inputSchema: {
|
|
3681
|
+
type: "object",
|
|
3682
|
+
properties: {
|
|
3683
|
+
recent_limit: { type: "number", description: "How many recent context entries to return (default 5)." },
|
|
3684
|
+
page: { description: "The startup is served in pages when it is larger than one result (each page fits under 20,000 bytes). Omit for page 1; every page but the last says which page to ask for next - call again with that page, before replying, until a page says it is the last. \"all\" returns the whole startup in one result, for a host with no size limit." },
|
|
3685
|
+
load_id: { type: "string", description: "The load_id your startup returned in `startup_page.load_id`: every emet_initialize result carries one (each page of a paged startup, the whole startup with page \"all\" or when it fits one result, and a store that is not set up yet). Pass it with every later page so all pages come from the same startup, and pass the same load_id to emet_session_open so its startup page check runs on this startup." }
|
|
3686
|
+
}
|
|
3687
|
+
}
|
|
3688
|
+
},
|
|
3689
|
+
{
|
|
3690
|
+
name: "emet_status",
|
|
3691
|
+
description: "Diagnostic in one call (replaces emet_setup_status, get_status and emet_verify_connection, 2026-09-27). Reports environment, database, collections, vector index, whether identity and user documents exist, and per-layer counts and health. Pass probe: true to also prove the write path - connect, write a probe document, read the same value back, remove it; a read alone cannot reveal a broken write path. Prefer emet_initialize for a normal session start.",
|
|
3692
|
+
inputSchema: {
|
|
3693
|
+
type: "object",
|
|
3694
|
+
properties: {
|
|
3695
|
+
probe: { type: "boolean", description: "Also write and read back a probe document (default false). Use after a deploy or a configuration change." }
|
|
3696
|
+
}
|
|
3697
|
+
}
|
|
3698
|
+
},
|
|
3699
|
+
{
|
|
3700
|
+
name: "emet_setup_complete",
|
|
3701
|
+
description: "First-time setup, the owner's only: record the user's interview answers as bootstrap/IDENTITY.md, bootstrap/CHARACTER.md and bootstrap/USER.md and as an identity-layer entry, and create the owner person. Every write is read-back verified. Refused under a bot tag. On a store that is already set up it is refused unless reissue is true (the owner's explicit word): a re-run writes new versions of the user's identity documents; prior versions are archived, never destroyed.",
|
|
3702
|
+
inputSchema: {
|
|
3703
|
+
type: "object",
|
|
3704
|
+
properties: {
|
|
3705
|
+
answers: {
|
|
3706
|
+
type: "object",
|
|
3707
|
+
description: "Answers keyed exactly as the interview questions returned by emet_initialize.",
|
|
3708
|
+
properties: {
|
|
3709
|
+
assistant_name: { type: "string" },
|
|
3710
|
+
assistant_character: { type: "string" },
|
|
3711
|
+
user_name: { type: "string" },
|
|
3712
|
+
user_context: { type: "string" },
|
|
3713
|
+
user_preferences: { type: "string" },
|
|
3714
|
+
source_tag: { type: "string" }
|
|
3715
|
+
},
|
|
3716
|
+
required: ["assistant_name", "assistant_character", "user_name", "user_context"]
|
|
3717
|
+
},
|
|
3718
|
+
reissue: { type: "boolean", description: "Only on the owner's explicit word: re-run setup on a store that is already set up. Without it, a re-run is refused and nothing is written." },
|
|
3719
|
+
source: { type: "string", description: "Your source tag. Setup is refused under a bot tag." }
|
|
3720
|
+
},
|
|
3721
|
+
required: ["answers"]
|
|
3722
|
+
}
|
|
3723
|
+
},
|
|
3724
|
+
{
|
|
3725
|
+
name: "corpus_recall",
|
|
3726
|
+
description: "Semantic search over an optional supplementary knowledge corpus that the user has loaded separately from memory. Returns nothing unless a corpus has been populated; this is not where session memory lives.",
|
|
3727
|
+
inputSchema: {
|
|
3728
|
+
type: "object",
|
|
3729
|
+
properties: {
|
|
3730
|
+
query: {
|
|
3731
|
+
type: "string",
|
|
3732
|
+
description: "Natural-language query (raw text; concept match, phrased however the experience is phrased)"
|
|
3733
|
+
},
|
|
3734
|
+
corpus: {
|
|
3735
|
+
type: "string",
|
|
3736
|
+
description: "Optional: restrict to one corpus name"
|
|
3737
|
+
},
|
|
3738
|
+
limit: {
|
|
3739
|
+
type: "number",
|
|
3740
|
+
description: "Max results (default 5, max 25)"
|
|
3741
|
+
}
|
|
3742
|
+
},
|
|
3743
|
+
required: ["query"]
|
|
3744
|
+
}
|
|
3745
|
+
},
|
|
3746
|
+
|
|
3747
|
+
{
|
|
3748
|
+
name: "recall",
|
|
3749
|
+
description: "Search memories by keyword. Ranked by how many of your words match, then by importance. Current entries come first in every order. Pass order: 'latest' for time order. A standalone latest, newest, or most recent, together with other words, also asks for time order: it reorders those matches and does not change which rows match. A hyphenated word such as latest-budget, a negated 'not the latest', and a bare 'latest' are ordinary keywords. Expired and superseded entries stay in the result, marked with their end date and what replaced them. Pass history: true to read ended entries when current ones fill the limit.",
|
|
3750
|
+
inputSchema: {
|
|
3751
|
+
type: "object",
|
|
3752
|
+
properties: {
|
|
3753
|
+
query: {
|
|
3754
|
+
type: "string",
|
|
3755
|
+
description: "Search query to match against memory content. order: 'latest' is how you ask for time order. A standalone latest, newest, or most recent reorders the other words' matches."
|
|
3756
|
+
},
|
|
3757
|
+
layer: {
|
|
3758
|
+
type: "string",
|
|
3759
|
+
enum: ["episodic", "semantic", "procedural", "meta", "identity", "working"],
|
|
3760
|
+
description: "Optional: Search only in specific layer"
|
|
3761
|
+
},
|
|
3762
|
+
limit: {
|
|
3763
|
+
type: "number",
|
|
3764
|
+
description: "Maximum number of results to return (default: 10)"
|
|
3765
|
+
},
|
|
3766
|
+
history: {
|
|
3767
|
+
type: "boolean",
|
|
3768
|
+
description: "Return ended entries (superseded or expired), marked, up to limit. Default false: ended entries are included after current ones until limit."
|
|
3769
|
+
},
|
|
3770
|
+
order: {
|
|
3771
|
+
type: "string",
|
|
3772
|
+
enum: ["latest"],
|
|
3773
|
+
description: "Pass 'latest' for time order: current matches first, then newer ones. This does not change which rows match the query."
|
|
3774
|
+
}
|
|
3775
|
+
},
|
|
3776
|
+
required: ["query"]
|
|
3777
|
+
}
|
|
3778
|
+
},
|
|
3779
|
+
{
|
|
3780
|
+
name: "query_layer",
|
|
3781
|
+
description: "Query a specific memory layer with structured filters",
|
|
3782
|
+
inputSchema: {
|
|
3783
|
+
type: "object",
|
|
3784
|
+
properties: {
|
|
3785
|
+
layer: {
|
|
3786
|
+
type: "string",
|
|
3787
|
+
enum: ["episodic", "semantic", "procedural", "meta", "identity", "working"],
|
|
3788
|
+
description: "Memory layer to query"
|
|
3789
|
+
},
|
|
3790
|
+
options: {
|
|
3791
|
+
type: "object",
|
|
3792
|
+
description: "Query options with structured filters",
|
|
3793
|
+
properties: {
|
|
3794
|
+
filters: {
|
|
3795
|
+
type: "object",
|
|
3796
|
+
description: "Structured filter conditions (safe parameterized queries)",
|
|
3797
|
+
properties: {
|
|
3798
|
+
importance_min: { type: "number", description: "Minimum importance (0-1)" },
|
|
3799
|
+
importance_max: { type: "number", description: "Maximum importance (0-1)" },
|
|
3800
|
+
importance_bucket: { type: "string", description: "Named bucket: constitutive, durable, binding, event, or scratch. Maps to that band's numeric range so older rows still match." },
|
|
3801
|
+
emotional_intensity_min: { type: "number", description: "Minimum emotional intensity (0-1)" },
|
|
3802
|
+
emotional_intensity_max: { type: "number", description: "Maximum emotional intensity (0-1)" },
|
|
3803
|
+
timestamp_after: { type: "number", description: "Unix timestamp - memories after this time" },
|
|
3804
|
+
timestamp_before: { type: "number", description: "Unix timestamp - memories before this time" },
|
|
3805
|
+
content_contains: { type: "string", description: "Text to search in content/event" },
|
|
3806
|
+
context_contains: { type: "string", description: "Text to search in context" },
|
|
3807
|
+
id: { type: "number", description: "Exact memory ID" }
|
|
3808
|
+
}
|
|
3809
|
+
},
|
|
3810
|
+
limit: { type: "number", description: "Max results (1-1000, default: 10)" },
|
|
3811
|
+
order_by: { type: "string", description: "Sort field and direction, e.g. 'importance DESC' or 'timestamp DESC'. Fields: id, timestamp, importance, emotional_intensity, content, context. Default is importance descending. Current entries come first in every order, including timestamp. Anything else falls back to importance descending." },
|
|
3812
|
+
include_superseded: { type: "boolean", description: "Accepted, and it does not hide anything. Superseded and expired entries are always returned, marked with their end date and what replaced them. Current entries are listed first." },
|
|
3813
|
+
history: { type: "boolean", description: "Return ended entries only, up to limit, so a page full of current entries cannot leave them out. Default false: ended entries follow the current ones until limit." }
|
|
3814
|
+
}
|
|
3815
|
+
}
|
|
3816
|
+
},
|
|
3817
|
+
required: ["layer"]
|
|
3818
|
+
}
|
|
3819
|
+
},
|
|
3820
|
+
{
|
|
3821
|
+
name: "save_to_layer",
|
|
3822
|
+
description: "Save memory to a specific layer with full control over metadata. PASS session_id - your transcript session from emet_session_open - on every save made inside a session: the server stamps derived_from with the exchange in progress. Semantic and procedural entries are REFUSED without a source (session_id, or metadata.derived_from); nothing is written when refused.",
|
|
3823
|
+
inputSchema: {
|
|
3824
|
+
type: "object",
|
|
3825
|
+
properties: {
|
|
3826
|
+
session_id: {
|
|
3827
|
+
type: "string",
|
|
3828
|
+
description: "Your transcript session id (from emet_session_open). The server reads that session's counter and stamps derived_from {session_id, exchange_start, exchange_end} with the exchange in progress. Ignored when metadata.derived_from is given. Required in practice for semantic and procedural saves made in a session."
|
|
3829
|
+
},
|
|
3830
|
+
layer: {
|
|
3831
|
+
type: "string",
|
|
3832
|
+
enum: ["episodic", "semantic", "procedural", "meta", "identity", "working"],
|
|
3833
|
+
description: "Target memory layer"
|
|
3834
|
+
},
|
|
3835
|
+
content: {
|
|
3836
|
+
type: "string",
|
|
3837
|
+
description: "Memory content to save"
|
|
3838
|
+
},
|
|
3839
|
+
metadata: {
|
|
3840
|
+
type: "object",
|
|
3841
|
+
description: "Full metadata control",
|
|
3842
|
+
properties: {
|
|
3843
|
+
importance: { description: "Bucket word (constitutive|durable|binding|event|scratch), optional :low|:mid|:high, or a number kept for compatibility." },
|
|
3844
|
+
importance_bucket: { type: "string", description: "constitutive|durable|binding|event|scratch" },
|
|
3845
|
+
importance_degree: { type: "string", description: "low|mid|high — only when degree inside the bucket matters" },
|
|
3846
|
+
emotional_intensity: { type: "number" },
|
|
3847
|
+
context: { type: "string" },
|
|
3848
|
+
source: { type: "string", description: "Which host produced this write. A shared default tag is ambiguity, not attribution." },
|
|
3849
|
+
tags: { type: "array", items: { type: "string" } },
|
|
3850
|
+
valid_until: { type: "number", description: "Unix seconds. After this the entry is marked ended and stays in recall and queries, with this date as its end date. It is not hidden and it is not deleted." },
|
|
3851
|
+
derived_from: {
|
|
3852
|
+
type: "object",
|
|
3853
|
+
description: "Where this entry came from, so the claim can be checked later. Either the transcript span it was drawn from - {session_id, exchange_start, exchange_end} - or, for an entry consolidated from other entries, {entries: [id, ...]}. One or the other, never both. REQUIRED on semantic and procedural entries unless session_id is passed (the server then stamps it): an assertion whose source cannot be reached is a claim the system merely happens to hold. Never guess - a nearest-timestamp guess recorded as a citation is worse than no citation."
|
|
3854
|
+
}
|
|
3855
|
+
}
|
|
3856
|
+
}
|
|
3857
|
+
},
|
|
3858
|
+
required: ["layer", "content"]
|
|
3859
|
+
}
|
|
3860
|
+
},
|
|
3861
|
+
{
|
|
3862
|
+
name: "revise_memory",
|
|
3863
|
+
description: "Supersede an existing entry with a revised one (supersession, not mutation). Creates a NEW entry carrying metadata.supersedes=<target_id> and marks the target with superseded_by and superseded_at. The old entry stays readable: it is not hidden and is_live is not cleared. Recall, query_layer and semantic_recall return it marked with that end date and the id of the new entry. The current entry is listed first. Episodic revision is prohibited - events are immutable.",
|
|
3864
|
+
inputSchema: {
|
|
3865
|
+
type: "object",
|
|
3866
|
+
properties: {
|
|
3867
|
+
layer: {
|
|
3868
|
+
type: "string",
|
|
3869
|
+
enum: ["semantic", "procedural", "meta", "identity", "working"],
|
|
3870
|
+
description: "Layer of the entry being revised (episodic is prohibited)"
|
|
3871
|
+
},
|
|
3872
|
+
target_id: {
|
|
3873
|
+
type: "number",
|
|
3874
|
+
description: "ID of the entry this revision supersedes"
|
|
3875
|
+
},
|
|
3876
|
+
content: {
|
|
3877
|
+
type: "string",
|
|
3878
|
+
description: "Full revised content - must stand alone without the superseded entry"
|
|
3879
|
+
},
|
|
3880
|
+
metadata: {
|
|
3881
|
+
type: "object",
|
|
3882
|
+
description: "Metadata for the new entry (supersedes is set automatically; include source tag)"
|
|
3883
|
+
}
|
|
3884
|
+
},
|
|
3885
|
+
required: ["layer", "target_id", "content"]
|
|
3886
|
+
}
|
|
3887
|
+
},
|
|
3888
|
+
{
|
|
3889
|
+
name: "revise_transcript",
|
|
3890
|
+
description: "Supersede a stored transcript part with a corrected one. Nothing is edited or deleted: a NEW document is written under the same session_id with an incremented revision and a pointer to its parent, and the parent is marked superseded. Use when a saved part has a mispaired, missing or wrong exchange. The reason parameter is required - an unexplained correction is a silent rewrite. To ADD a new part, use save_transcript instead. Allowed on a closed session (a correction deletes nothing), but there a revision may only correct the exchanges the part already holds - never add new ones - and must carry exactly the same exchange numbers (none dropped or repeated; numbers sent as text are stored as numbers).",
|
|
3891
|
+
inputSchema: {
|
|
3892
|
+
type: "object",
|
|
3893
|
+
properties: {
|
|
3894
|
+
session_id: {
|
|
3895
|
+
type: "string",
|
|
3896
|
+
description: "The session_id of the part to correct, e.g. '2026-09-04_x_verbatim_part14'. Must already exist."
|
|
3897
|
+
},
|
|
3898
|
+
exchanges: {
|
|
3899
|
+
type: "array",
|
|
3900
|
+
description: "The COMPLETE corrected exchange list for this part, verbatim. A revision replaces the whole part.",
|
|
3901
|
+
items: {
|
|
3902
|
+
type: "object",
|
|
3903
|
+
properties: {
|
|
3904
|
+
exchange_index: { type: "number" },
|
|
3905
|
+
user_message: { type: "string" },
|
|
3906
|
+
assistant_message: { type: "string" },
|
|
3907
|
+
timestamp: { type: "number" }
|
|
3908
|
+
}
|
|
3909
|
+
}
|
|
3910
|
+
},
|
|
3911
|
+
reason: {
|
|
3912
|
+
type: "string",
|
|
3913
|
+
description: "What was wrong and how this differs. Required, minimum 10 characters. This is the audit record of the correction."
|
|
3914
|
+
},
|
|
3915
|
+
session_start: { type: "number", description: "Optional. Defaults to the parent's." },
|
|
3916
|
+
session_end: { type: "number", description: "Optional. Defaults to the parent's." },
|
|
3917
|
+
source: { type: "string", description: "Source tag - which host produced this write. Defaults to the parent's." },
|
|
3918
|
+
channel: { type: "string", description: "Alias of `source` (the older name, kept one release). Prefer `source`." }
|
|
3919
|
+
},
|
|
3920
|
+
required: ["session_id", "exchanges", "reason"]
|
|
3921
|
+
}
|
|
3922
|
+
},
|
|
3923
|
+
{
|
|
3924
|
+
name: "query_transcripts",
|
|
3925
|
+
description: "Retrieve session transcripts - the second tier, word for word. Reads the live store written by save_transcript AND the legacy archive (exchanges written by earlier hosts, June 2025 to June 2026), grouped into the same session shape; every result carries `source` naming the collection it came from. Nothing is migrated between them. Use it when a layer entry is relevant but thin: the transcript around its timestamp holds the depth the layer compressed out.",
|
|
3926
|
+
inputSchema: {
|
|
3927
|
+
type: "object",
|
|
3928
|
+
properties: {
|
|
3929
|
+
timestamp_start: {
|
|
3930
|
+
type: "number",
|
|
3931
|
+
description: "Unix timestamp - start of time range (filters by session_start)"
|
|
3932
|
+
},
|
|
3933
|
+
timestamp_end: {
|
|
3934
|
+
type: "number",
|
|
3935
|
+
description: "Unix timestamp - end of time range (filters by session_start)"
|
|
3936
|
+
},
|
|
3937
|
+
session_id: {
|
|
3938
|
+
type: "string",
|
|
3939
|
+
description: "Optional: Filter by specific session ID (e.g. '2026-04-16_001')"
|
|
3940
|
+
},
|
|
3941
|
+
limit: {
|
|
3942
|
+
type: "number",
|
|
3943
|
+
description: "Maximum number of sessions to return (default: 50, max: 500)"
|
|
3944
|
+
}
|
|
3945
|
+
},
|
|
3946
|
+
required: ["timestamp_start", "timestamp_end"]
|
|
3947
|
+
}
|
|
3948
|
+
},
|
|
3949
|
+
{
|
|
3950
|
+
name: "save_transcript",
|
|
3951
|
+
description: "FALLBACK. The primary path is emet_session_open then emet_transcript_append after every reply; use this only where a host cannot append per reply. Saves the full word-for-word dialog of a session - full transcripts only, no summaries. Pass exchanges in the call. The server does not read a file. Refused for a session that was closed complete.",
|
|
3952
|
+
inputSchema: {
|
|
3953
|
+
type: "object",
|
|
3954
|
+
properties: {
|
|
3955
|
+
session_id: {
|
|
3956
|
+
type: "string",
|
|
3957
|
+
description: "Unique session identifier, e.g. 'claude_project_2026-05-23_001'"
|
|
3958
|
+
},
|
|
3959
|
+
session_start: {
|
|
3960
|
+
type: "number",
|
|
3961
|
+
description: "Unix timestamp — session start"
|
|
3962
|
+
},
|
|
3963
|
+
session_end: {
|
|
3964
|
+
type: "number",
|
|
3965
|
+
description: "Unix timestamp — session end (optional, defaults to now)"
|
|
3966
|
+
},
|
|
3967
|
+
exchanges: {
|
|
3968
|
+
type: "array",
|
|
3969
|
+
description: "Claude.ai Project mode: full array of exchange objects for this session",
|
|
3970
|
+
items: {
|
|
3971
|
+
type: "object",
|
|
3972
|
+
properties: {
|
|
3973
|
+
exchange_index: { type: "number" },
|
|
3974
|
+
user_message: { type: "string" },
|
|
3975
|
+
assistant_message: { type: "string" },
|
|
3976
|
+
timestamp: { type: "number", description: "Optional Unix timestamp for this exchange" }
|
|
3977
|
+
},
|
|
3978
|
+
required: ["user_message", "assistant_message"]
|
|
3979
|
+
}
|
|
3980
|
+
},
|
|
3981
|
+
source: { type: "string", description: "Source tag - which host produced this write. One name on every write since 2026-09-27." },
|
|
3982
|
+
channel: { type: "string", description: "Alias of `source` (the older name, kept one release). Prefer `source`." }
|
|
3983
|
+
},
|
|
3984
|
+
required: ["session_id", "session_start", "exchanges"]
|
|
3985
|
+
}
|
|
3986
|
+
},
|
|
3987
|
+
{
|
|
3988
|
+
name: "emet_session_open",
|
|
3989
|
+
description: "Open a transcript session. Returns session_id, dated in the configured local time zone (EMET_TIMEZONE, default America/New_York). A slug whose id is already CLOSED is never reopened: the session opens as <id>-2, -3, ... and the result's `notice` says so. Pass load_id from your startup pages (startup_page.load_id on every emet_initialize page) so the startup page check runs on your own startup; an open without load_id, or with a page of that startup never fetched, is refused (the refusal says how to pass). After every reply, call emet_transcript_append with that id and the one exchange that just happened. Do not dump the whole chat at close.",
|
|
3990
|
+
inputSchema: {
|
|
3991
|
+
type: "object",
|
|
3992
|
+
properties: {
|
|
3993
|
+
slug: { type: "string", description: "Short name for the session, e.g. grok-init" },
|
|
3994
|
+
load_id: { type: "string", description: "Required for the startup page check (by policy; the schema leaves it optional so older hosts can still open): the load_id from `startup_page` on your emet_initialize pages - page 1 returns it, every page carries it, and the last page names it. An open without load_id, or with a page of that load known not fetched, is refused before anything is written (gate startup_pages); the refusal names the exact calls that pass." }
|
|
3995
|
+
}
|
|
3996
|
+
}
|
|
3997
|
+
},
|
|
3998
|
+
{
|
|
3999
|
+
name: "emet_transcript_append",
|
|
4000
|
+
description: "Append ONE exchange to the session transcript. Call after every reply. Idempotent on exchange_index. Rolls to a new part after 8 exchanges. After a complete close the session takes exactly ONE more append - its closing exchange, numbered next, within EMET_CLOSE_GRACE_MINUTES (default 5) of the close; it is written into the session's last part whatever session_id it names, and the result's `closing_exchange` says it was accepted and that the session is now locked. A retry of that exchange number is answered as already saved and writes nothing. exchange_index, when given, must be a whole number of 1 or more (a number or its text). Any other append to a closed session is refused and nothing is written; that exchange belongs in the next session you open the usual way. THE FIRST APPEND OF A SESSION IS CHECKED: its assistant_message must carry the measured machine line - a machine name from the install notes plus the clock reading, as the shell returned them - or the words `no local shell MCP`. Without one of those the append is refused and the transcript does not start.",
|
|
4001
|
+
inputSchema: {
|
|
4002
|
+
type: "object",
|
|
4003
|
+
properties: {
|
|
4004
|
+
session_id: { type: "string" },
|
|
4005
|
+
user_message: { type: "string" },
|
|
4006
|
+
assistant_message: { type: "string" },
|
|
4007
|
+
exchange_index: { type: "number" },
|
|
4008
|
+
session_start: { type: "number" },
|
|
4009
|
+
source: { type: "string", description: "Source tag - which host produced this write. One name on every write since 2026-09-27." },
|
|
4010
|
+
channel: { type: "string", description: "Alias of `source` (the older name, kept one release). Prefer `source`." }
|
|
4011
|
+
},
|
|
4012
|
+
required: ["session_id", "user_message", "assistant_message"]
|
|
4013
|
+
}
|
|
4014
|
+
},
|
|
4015
|
+
{
|
|
4016
|
+
name: "read_doc",
|
|
4017
|
+
description: "Read a continuity document from the versioned document store. A document longer than one page (18,000 UTF-8 bytes by default) comes back in pages: the result then carries `page` with `next_offset` - call again with that offset (and `expected_version` set to the first page's `version`) until `next_offset` is null, and do not act on a document you have only part of. If the document changes between pages the read is refused: restart from offset 0. `sha256` and `size` (characters; `size_bytes` in UTF-8 bytes) always describe the WHOLE document.",
|
|
4018
|
+
inputSchema: { type: "object", properties: {
|
|
4019
|
+
doc_id: { type: "string", description: "Document id, e.g. 'STATE.md', 'HANDOFF.md', 'threads/pcb-laser-board.md', 'bootstrap/IDENTITY.md'" },
|
|
4020
|
+
offset: { type: "number", description: "Optional: UTF-8 byte offset to start from (default 0). Use the previous page's `page.next_offset`." },
|
|
4021
|
+
limit: { type: "number", description: "Optional: page size in bytes as the host receives it (default 18,000): a page ends before its JSON-escaped text plus 1,500 bytes kept for the result envelope would pass it, and never splits a character. 0 returns the whole document in one result." },
|
|
4022
|
+
expected_version: { type: "number", description: "Optional: the `version` your first page returned. Pass it with every later page; if the document changed in between, the read is refused (restart from offset 0) so pages of two versions are never joined." },
|
|
4023
|
+
expected_sha256: { type: "string", description: "Optional: the `sha256` your first page returned - the same pin as expected_version, and the only pin for a shipped guide (guides have no version)." }
|
|
4024
|
+
}, required: ["doc_id"] }
|
|
4025
|
+
},
|
|
4026
|
+
{
|
|
4027
|
+
name: "read_doc_history",
|
|
4028
|
+
description: "Read a RETAINED revision of a document, or list the revisions retained for it. Every write archives the prior copy (nothing is deleted); this is the reader for that archive. Without a version: the list of retained revisions with their digests and dates. With a version: that revision's full content, re-hashed against its stored digest. Read-only. Use it for an audit question - what did this document say at version N - or to compare a document with what it replaced.",
|
|
4029
|
+
inputSchema: { type: "object", properties: {
|
|
4030
|
+
doc_id: { type: "string", description: "Document id, e.g. 'CHARTER.md', 'STATE.md'" },
|
|
4031
|
+
version: { type: "number", description: "Optional: the revision to read. Omit to list what is retained." }
|
|
4032
|
+
}, required: ["doc_id"] }
|
|
4033
|
+
},
|
|
4034
|
+
{
|
|
4035
|
+
name: "write_doc",
|
|
4036
|
+
description: "Write or update a continuity document in the document store (versioned, prior copy archived to documents_history and marked obsolete, sha256 read-back verified). RECORDS - the handoff (HANDOFF.md) and any id in EMET_RECORD_DOCS - are superseded, never amended: patch_doc is refused on them, and a new revision is written here WHOLE with `supersedes` set to the version currently in force (ISO 9001 7.5.3 / 13485 4.2.5).",
|
|
4037
|
+
inputSchema: { type: "object", properties: {
|
|
4038
|
+
doc_id: { type: "string" },
|
|
4039
|
+
content: { type: "string" },
|
|
4040
|
+
supersedes: { type: "number", description: "Required when writing a new revision of a RECORD (the handoff): the version currently in force, which this revision supersedes. Ignored for ordinary documents." },
|
|
4041
|
+
reason: { type: "string", description: "Required on a record revision (at least 10 characters). Stored as edit_reason." },
|
|
4042
|
+
session_id: { type: "string", description: "Transcript session id. The floor is measured from the size when this session first touched the document." },
|
|
4043
|
+
source: { type: "string", description: "Source tag - which host produced this write. One name on every write since 2026-09-27." },
|
|
4044
|
+
updated_by: { type: "string", description: "Alias of `source` (the older name, kept one release). Prefer `source`." }
|
|
4045
|
+
}, required: ["doc_id", "content"] }
|
|
4046
|
+
},
|
|
4047
|
+
{
|
|
4048
|
+
name: "list_docs",
|
|
4049
|
+
description: "List continuity documents (metadata only; optional doc_id prefix filter, e.g. 'threads/'). Current documents come first. Retired documents stay in the list, marked with retired_at and, when a rename replaced them, what replaced them.",
|
|
4050
|
+
inputSchema: { type: "object", properties: {
|
|
4051
|
+
prefix: { type: "string" },
|
|
4052
|
+
include_retired: { type: "boolean", description: "Accepted. Retired documents are always listed, after the current ones. This flag does not hide them." }
|
|
4053
|
+
} }
|
|
4054
|
+
},
|
|
4055
|
+
{
|
|
4056
|
+
name: "retire_doc",
|
|
4057
|
+
description: "Mark a continuity document OBSOLETE. Nothing is deleted: the document keeps its id, its full content, its version and its digest, and read_doc still returns it, flagged retired, with the reason. list_docs and the startup scan keep it too, after the current documents, with retired_at and what replaced it. A retired drop is not an unhandled drop. This is document control as ISO 7.5.3 describes it - issue the revision, mark the prior copy obsolete, retain it - and it is the document-store counterpart of revise_memory superseding a layer entry. THE STORE HAS NO DELETE, BY DESIGN. Use for genuine litter: tombstones, merged staging documents, test scratch. Reversible: write_doc on the same id brings it back into the working view.",
|
|
4058
|
+
inputSchema: { type: "object", properties: {
|
|
4059
|
+
doc_id: { type: "string", description: "Document id to mark obsolete." },
|
|
4060
|
+
reason: { type: "string", description: "Why it is obsolete. Required, at least 10 characters. Drops only." },
|
|
4061
|
+
session_id: { type: "string", description: "Transcript session id." },
|
|
4062
|
+
source: { type: "string", description: "Source tag - which host produced this write. One name on every write since 2026-09-27." },
|
|
4063
|
+
retired_by: { type: "string", description: "Alias of `source` (the older name, kept one release). Prefer `source`." }
|
|
4064
|
+
}, required: ["doc_id", "reason"] }
|
|
4065
|
+
},
|
|
4066
|
+
{
|
|
4067
|
+
name: "rename_doc",
|
|
4068
|
+
description: "Reissue a continuity document under a new doc_id. The destination is written first through the normal write path (same redaction, digest, versioning and sha256 read-back), and only then is the old id marked obsolete with retire_doc. NOTHING IS DELETED: the old id keeps its content and stays readable at its own id, flagged retired with the reason 'Reissued as <new_id>'. list_docs keeps it after the current documents and names the new id in replaced_by. No tombstone document is needed, because the retired original IS the tombstone and it still carries the real content. Refuses if the destination already exists, because a rename onto an occupied id is a merge decision and merges are not guessed.",
|
|
4069
|
+
inputSchema: { type: "object", properties: {
|
|
4070
|
+
old_id: { type: "string", description: "Existing document id to reissue from. It is retired, not removed." },
|
|
4071
|
+
new_id: { type: "string", description: "Destination document id. Must not already exist." },
|
|
4072
|
+
reason: { type: "string", description: "Why the document is reissued. Required, at least 10 characters." },
|
|
4073
|
+
session_id: { type: "string", description: "Transcript session id." },
|
|
4074
|
+
source: { type: "string", description: "Source tag - which host produced this write. One name on every write since 2026-09-27." },
|
|
4075
|
+
updated_by: { type: "string", description: "Alias of `source` (the older name, kept one release). Prefer `source`." }
|
|
4076
|
+
}, required: ["old_id", "new_id", "reason"] }
|
|
4077
|
+
},
|
|
4078
|
+
{
|
|
4079
|
+
name: "restore_doc",
|
|
4080
|
+
description: "Write a retained revision back as the current document. The retained row is copied through the normal write path, with a reason. Nothing is deleted.",
|
|
4081
|
+
inputSchema: { type: "object", properties: {
|
|
4082
|
+
doc_id: { type: "string", description: "Document id to restore." },
|
|
4083
|
+
version: { type: "integer", description: "Retained revision number to write back." },
|
|
4084
|
+
reason: { type: "string", description: "Why this revision is restored. At least 10 characters." },
|
|
4085
|
+
session_id: { type: "string", description: "Transcript session id. Required on Path 2." },
|
|
4086
|
+
source: { type: "string", description: "Source tag. Must be in EMET_RESTORE_TAGS. A missing source is refused." }
|
|
4087
|
+
}, required: ["doc_id", "version", "reason", "source"] }
|
|
4088
|
+
},
|
|
4089
|
+
{
|
|
4090
|
+
name: "validate_doc",
|
|
4091
|
+
description: "Check a controlled document against the template that governs it, WITHOUT writing anything. The same check the write path runs on every document write - this is the freestanding form, for auditing records written before validation existed, for counting what adoption will mark before it marks anything, and for checking new content before it is stored. Pass doc_id to check a stored document (the previous revision is pulled from the archive so the carry-forward check runs). Pass content, with template_id or a doc_id the map covers, to check text that is not stored. Checks FORM and CONTINUITY only: sections present, in order and filled; no placeholder left unfilled; NOT COMPLETED carrying a reason; a board's counts matching its rows; every row on the previous revision still present or dispositioned. It cannot check whether an answer is true, or whether a 'none' states its scope honestly - those stay with the author and the reader.",
|
|
4092
|
+
inputSchema: { type: "object", properties: {
|
|
4093
|
+
doc_id: { type: "string", description: "Stored document to check. Alone, it is read from the store and compared against its own previous revision." },
|
|
4094
|
+
content: { type: "string", description: "Text to check instead of a stored document. Requires template_id, or a doc_id the map covers." },
|
|
4095
|
+
template_id: { type: "string", description: "Template to check against, e.g. 'templates/BOARD.md'. Read from the copy shipped with this release, never the store copy." }
|
|
4096
|
+
} }
|
|
4097
|
+
},
|
|
4098
|
+
{
|
|
4099
|
+
name: "accept_nonconformance",
|
|
4100
|
+
description: "Record the written disposition for a nonconformance that will not be corrected. A marker is never cleared silently: it leaves either because a later revision passes, or because someone accepts it in writing here. The marker is RETAINED with the acceptance beside it, and the acceptance covers only the revision it names - the next write is validated fresh, and accepted markers stay counted at session start, separately from open ones. The reason must be at least 10 characters: it is what an audit reads later.",
|
|
4101
|
+
inputSchema: { type: "object", properties: {
|
|
4102
|
+
doc_id: { type: "string", description: "Document carrying the open marker." },
|
|
4103
|
+
reason: { type: "string", description: "Why this nonconformance is accepted rather than corrected. At least 10 characters." },
|
|
4104
|
+
source: { type: "string", description: "Source tag - which host produced this write. One name on every write since 2026-09-27." },
|
|
4105
|
+
session_id: { type: "string", description: "Transcript session id." },
|
|
4106
|
+
accepted_by: { type: "string", description: "Alias of `source` (the older name, kept one release). Prefer `source`." }
|
|
4107
|
+
}, required: ["doc_id", "reason"] }
|
|
4108
|
+
},
|
|
4109
|
+
{
|
|
4110
|
+
name: "patch_doc",
|
|
4111
|
+
description: "Replace one exact substring in a document (str_replace semantics) without resending the whole file. THE PREFERRED WAY TO MAKE SMALL EDITS to large docs like STATE.md. old_str must match EXACTLY ONCE - zero or multiple matches are rejected rather than guessed. Pass new_str as an empty string to delete. Same versioning, archiving, and sha256 read-back as write_doc.",
|
|
4112
|
+
inputSchema: { type: "object", properties: {
|
|
4113
|
+
doc_id: { type: "string" },
|
|
4114
|
+
old_str: { type: "string", description: "Exact text to replace. Must occur exactly once in the current text. Omit when `edits` is set." },
|
|
4115
|
+
new_str: { type: "string", description: "Replacement text. Empty string deletes the matched text." },
|
|
4116
|
+
reason: { type: "string", description: "Required, at least 10 characters. A removed or closed board row says why." },
|
|
4117
|
+
origin: { type: "string", description: "Required when a board edit adds a row id. Names the session, chat or drop the row came from." },
|
|
4118
|
+
edits: { type: "array", description: "Several replacements in one revision. Each old_str must match the current text exactly once. Applied together, so a failed edit writes nothing.", items: { type: "object" } },
|
|
4119
|
+
session_id: { type: "string", description: "Transcript session id. The floor is measured from the size when this session first touched the document." },
|
|
4120
|
+
source: { type: "string", description: "Source tag - which host produced this write. One name on every write since 2026-09-27." },
|
|
4121
|
+
updated_by: { type: "string", description: "Alias of `source` (the older name, kept one release). Prefer `source`." },
|
|
4122
|
+
expected_version: { type: "number", description: "Optional optimistic-concurrency guard." }
|
|
4123
|
+
}, required: ["doc_id", "reason"] }
|
|
4124
|
+
},
|
|
4125
|
+
{
|
|
4126
|
+
name: "semantic_recall",
|
|
4127
|
+
description: "Semantic (meaning-based) memory search across all six memory layers via Atlas Vector Search autoEmbed. Finds conceptually related memories even without keyword overlap. Complements keyword-based recall/query_layer.",
|
|
4128
|
+
inputSchema: { type: "object", properties: {
|
|
4129
|
+
query: { type: "string", description: "Natural-language query - embedded automatically server-side (voyage-4)" },
|
|
4130
|
+
layer: { type: "string", enum: ["episodic", "semantic", "procedural", "meta", "identity", "working"], description: "Optional: restrict to one layer" },
|
|
4131
|
+
limit: { type: "number", description: "Max results (default 10)" },
|
|
4132
|
+
history: { type: "boolean", description: "Return ended entries only, up to limit. Default false: current matches come first and ended matches follow until limit, marked with their end date." }
|
|
4133
|
+
}, required: ["query"] }
|
|
4134
|
+
},
|
|
4135
|
+
{
|
|
4136
|
+
name: "emet_gaps",
|
|
4137
|
+
description: "The record's honesty about its own holes, as a query. Read-only; identifies and never fills. Counts, with the ids behind them, for each gap class: unsourced assertions, unattributed inferences, ambiguous attribution (by tag and by session), uncertain routing, unattestable rows (pre-2026-09-05 boundary), digest mismatches (the alarm - expect zero), pointer disagreements against the transcript store, broken supersession, expired-but-live rows, consolidation candidates (tag families; a list for a person), and redacted spans by source. Call it when asked how trustworthy the record is, before a consolidation pass, or after any migration. Closing a gap is a person's decision: revise_memory with a real derived_from where the source is knowable; otherwise the gap stands as a fact.",
|
|
4138
|
+
inputSchema: {
|
|
4139
|
+
type: "object",
|
|
4140
|
+
properties: {
|
|
4141
|
+
layer: { type: "string", enum: ["episodic", "semantic", "procedural", "meta", "identity", "working"], description: "Restrict to one layer. Default: all six." },
|
|
4142
|
+
since: { type: "number", description: "Unix seconds; only entries with timestamp >= since. Default: the whole store." },
|
|
4143
|
+
limit: { type: "number", description: "Max ids returned per class (1-500, default 50). Counts are always complete." }
|
|
4144
|
+
}
|
|
4145
|
+
}
|
|
4146
|
+
},
|
|
4147
|
+
{
|
|
4148
|
+
name: "emet_floor",
|
|
4149
|
+
description: "Read the floor: the standing description of how this assistant engages with the person using it. It ships in the server code, so setup cannot overwrite it and no install can lose it - the character document holds the user's modifiers on top of it. emet_initialize already returns the whole floor at session start; call this to read it again without re-running initialize. Use `compact: true` for the short form partway through a long session, when the start of the conversation is far behind. Read-only, and re-reading is not the same as being bound by it.",
|
|
4150
|
+
inputSchema: {
|
|
4151
|
+
type: "object",
|
|
4152
|
+
properties: {
|
|
4153
|
+
compact: { type: "boolean", description: "Return the short re-assertion form instead of the whole floor." },
|
|
4154
|
+
section: { type: "string", description: "Return one section only, e.g. 'neurodivergent_support'. Call with no arguments to see the section names." }
|
|
4155
|
+
}
|
|
4156
|
+
}
|
|
4157
|
+
},
|
|
4158
|
+
{
|
|
4159
|
+
name: "emet_invite_create",
|
|
4160
|
+
description: "EMET Guest Access: create a one-time code for a guest. They paste it on the connect screen instead of the passphrase. Default access is read-only. Only the owner can create a guest code. EMET Member Access is a different door.",
|
|
4161
|
+
inputSchema: {
|
|
4162
|
+
type: "object",
|
|
4163
|
+
properties: {
|
|
4164
|
+
display_name: { type: "string", description: "Name the guest will be known by in this store" },
|
|
4165
|
+
scopes: { type: "array", items: { type: "string", enum: ["emet.read"] }, description: "Guest invites accept only emet.read." },
|
|
4166
|
+
days: { type: "number", description: "Days until the code expires. Default 14." },
|
|
4167
|
+
source: { type: "string", description: "Source tag bound to this guest at approval. Exact membership of EMET_SOURCE_TAGS when a registry is set." }
|
|
4168
|
+
},
|
|
4169
|
+
required: ["display_name"]
|
|
4170
|
+
}
|
|
4171
|
+
},
|
|
4172
|
+
{
|
|
4173
|
+
name: "emet_member_access_create",
|
|
4174
|
+
description: "EMET Member Access: create a one-time code for a new person with read and write access. They paste it on the connect screen instead of the passphrase. This is not EMET Guest Access, and it never makes them the owner. Only the owner can create it. Off unless EMET_MEMBER_ACCESS=1.",
|
|
4175
|
+
inputSchema: {
|
|
4176
|
+
type: "object",
|
|
4177
|
+
properties: {
|
|
4178
|
+
display_name: { type: "string", description: "Name this person will be known by in this store. A name never selects an existing person." },
|
|
4179
|
+
days: { type: "number", description: "Days until the code expires. Default 14." },
|
|
4180
|
+
source: { type: "string", description: "Source tag bound to this member at approval. Exact membership of EMET_SOURCE_TAGS when a registry is set. Required for this member to write documents over HTTP while a registry is set." }
|
|
4181
|
+
},
|
|
4182
|
+
required: ["display_name"]
|
|
4183
|
+
}
|
|
4184
|
+
},
|
|
4185
|
+
{
|
|
4186
|
+
name: "emet_session_close",
|
|
4187
|
+
description: "Call this when the session is ending - the user says goodbye, asks you to wrap up, mentions restarting or opening a new session, or you are about to run out of room. An acknowledgement of the turn just completed - thanks, great, perfect - is not a session end; if it is ambiguous, ask rather than close. Checks whether this session's work has actually been written to the store and reports exactly what is missing. It does not write anything itself. RECEIPTS ARE REQUIRED: pass `receipts` naming what you wrote and read back for each close step - working documents, the episodic session record id, the board and its version, the transcript part ids and highest exchange index as query_transcripts returned them, and the handoff version (written last). Each receipt is compared with the store; omitted or mismatched receipts, a handoff that was not the last write, or a transcript no layer entry cites (derived_from) return state 'blocked'. Each missing line carries a `scope`: touch only what it names. THREE blocked closes on one session return state 'escalate': stop retrying, say the `speak` lines, and hand the user the list - the counter does not reset on a changed receipt, only with a new session. Say every line of the returned `speak` to the user, in order - a wrap that leaves it out is a failed close. Never tell the user their session is saved unless this returns state 'complete'.",
|
|
4188
|
+
inputSchema: {
|
|
4189
|
+
type: "object",
|
|
4190
|
+
properties: {
|
|
4191
|
+
receipts: {
|
|
4192
|
+
type: "object",
|
|
4193
|
+
description: "Required for state 'complete'. One entry per close step, taken from what the tools returned this session - not from recall.",
|
|
4194
|
+
properties: {
|
|
4195
|
+
working_docs: { type: "array", items: { type: "string" }, description: "Document ids updated this session before the handoff, e.g. ['threads/emet-publication.md']. [] if none." },
|
|
4196
|
+
episodic_id: { type: "number", description: "Id of the episodic session record, as save_to_layer returned it and query_layer read it back." },
|
|
4197
|
+
board_doc: { type: "string", description: "Board document id, usually 'STATE.md'." },
|
|
4198
|
+
board_version: { type: "number", description: "Board version after this session's update." },
|
|
4199
|
+
transcript_session_id: { type: "string", description: "Base transcript session id." },
|
|
4200
|
+
transcript_parts: { type: "array", items: { type: "string" }, description: "Part ids query_transcripts returned for this session." },
|
|
4201
|
+
transcript_exchanges: { type: "number", description: "Highest exchange index query_transcripts returned." },
|
|
4202
|
+
handoff_version: { type: "number", description: "Version of the handoff, written last." },
|
|
4203
|
+
handoff_doc: { type: "string", description: "Bot sessions only: the bot's own handoff record, bots/<source tag>/HANDOFF.md. A bot session needs no board receipt and never names the main HANDOFF.md or STATE.md." }
|
|
4204
|
+
}
|
|
4205
|
+
},
|
|
4206
|
+
session_id: { type: "string", description: "The session_id used (or to be used) for this session's transcript, e.g. '2026-09-03_short-description'. Without it the transcript cannot be checked; receipts.transcript_session_id is used when this is omitted." },
|
|
4207
|
+
session_start: { type: "number", description: "Unix seconds when the session began. Defaults to 12 hours ago, which makes the answer approximate." },
|
|
4208
|
+
thread_docs: { type: "array", items: { type: "string" }, description: "Additional document ids this session should have updated, e.g. ['threads/some-project.md']." },
|
|
4209
|
+
source: { type: "string", description: "Your host's source tag: the registered tag this session was written under. The close is never refused for it; an unregistered tag, one that did not write this session, or none (with no server default) gets a warning that the close is not attributed." }
|
|
4210
|
+
}
|
|
4211
|
+
}
|
|
4212
|
+
}
|
|
4213
|
+
];
|
|
4214
|
+
|
|
4215
|
+
/**
|
|
4216
|
+
* The tool list as hosts receive it, with MCP annotations attached.
|
|
4217
|
+
*
|
|
4218
|
+
* annotate() throws if any tool is missing an entry, so a new tool cannot reach
|
|
4219
|
+
* a host unannotated - which would make it look, to any host that asks before
|
|
4220
|
+
* acting, exactly as risky as an arbitrary shell command.
|
|
4221
|
+
*/
|
|
4222
|
+
/**
|
|
4223
|
+
* The session graph's two inputs on every write tool (2026-09-29):
|
|
4224
|
+
* `session_id` names the session the write belongs to, `exchange` the turn.
|
|
4225
|
+
* Added here, once, rather than hand-copied into each schema; a schema that
|
|
4226
|
+
* already declares one keeps its own description.
|
|
4227
|
+
*/
|
|
4228
|
+
const GRAPH_INPUTS = {
|
|
4229
|
+
session_id: { type: 'string', description: 'Your transcript session id from emet_session_open. The session graph reads this session\'s state before the write and returns `session_graph.next` - the next step. A governed write with no session is refused (open one with emet_session_open and load_id).' },
|
|
4230
|
+
exchange: { type: 'number', description: 'The exchange (turn) this write belongs to, counting from 1. A write for exchange n while exchange n-1 was never appended is refused by the session graph until n-1 is appended.' }
|
|
4231
|
+
};
|
|
4232
|
+
for (const t of TOOL_DEFINITIONS) {
|
|
4233
|
+
// One list of governed tools: session-graph.js. The append already carries
|
|
4234
|
+
// session_id and exchange_index, its own names for the same two inputs.
|
|
4235
|
+
if (!GRAPH_WRITE_TOOLS.has(t.name) || t.name === 'emet_transcript_append' || !t.inputSchema) continue;
|
|
4236
|
+
t.inputSchema.properties = t.inputSchema.properties || {};
|
|
4237
|
+
for (const [k, v] of Object.entries(GRAPH_INPUTS)) if (!t.inputSchema.properties[k]) t.inputSchema.properties[k] = v;
|
|
4238
|
+
}
|
|
4239
|
+
|
|
4240
|
+
export const TOOLS = annotate(TOOL_DEFINITIONS);
|
|
4241
|
+
|
|
4242
|
+
|
|
4243
|
+
// ============================================
|
|
4244
|
+
// EXPORTS
|
|
4245
|
+
// ============================================
|
|
4246
|
+
|
|
4247
|
+
export default {
|
|
4248
|
+
// Rate limiting
|
|
4249
|
+
RATE_LIMIT_CONFIG,
|
|
4250
|
+
RateLimitError,
|
|
4251
|
+
RateLimiter,
|
|
4252
|
+
|
|
4253
|
+
// Error handling
|
|
4254
|
+
handleError,
|
|
4255
|
+
createSuccessResponse,
|
|
4256
|
+
|
|
4257
|
+
// Memory operations
|
|
4258
|
+
saveMemory,
|
|
4259
|
+
recallMemories,
|
|
4260
|
+
semanticRecall,
|
|
4261
|
+
queryLayer,
|
|
4262
|
+
getStatus,
|
|
4263
|
+
queryTranscripts,
|
|
4264
|
+
reviseTranscript,
|
|
4265
|
+
saveTranscript,
|
|
4266
|
+
sessionOpen,
|
|
4267
|
+
transcriptAppend,
|
|
4268
|
+
|
|
4269
|
+
// Tool definitions
|
|
4270
|
+
TOOLS
|
|
4271
|
+
};
|