dsh-cortex-memory 0.1.0 → 0.2.0-rc.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +3 -3
- package/README.md +85 -146
- package/bench/ab-engine.mjs +60 -0
- package/bench/ann-bench.mjs +78 -0
- package/bench/hnsw-bench.mjs +79 -0
- package/bench/metrics.js +8 -0
- package/bench/migration-bench.mjs +69 -0
- package/bench/run.js +5 -2
- package/bench/synthetic.js +51 -7
- package/bench/vector-ab.mjs +76 -0
- package/bin/cortex.mjs +2 -3
- package/client.js +286 -4
- package/cordis.patch.yml +1 -1
- package/docs/install-windows.md +115 -27
- package/docs/models.md +165 -30
- package/docs/multimodal.md +112 -0
- package/docs/scopes.md +142 -0
- package/examples/README.md +72 -0
- package/examples/setup-cortex-bge.ps1 +66 -0
- package/examples/uninstall-cortex.ps1 +91 -0
- package/index.js +15 -1
- package/package.json +5 -12
- package/src/adapters/markdown-import.js +20 -1
- package/src/adapters/markdown.js +12 -2
- package/src/adapters/shiori.js +1 -1
- package/src/api/routes.js +148 -4
- package/src/api/validate.js +26 -0
- package/src/core/attachment-reader.js +31 -0
- package/src/core/attachments.js +39 -0
- package/src/core/config.js +36 -1
- package/src/core/model-config.js +3 -1
- package/src/core/profile.js +32 -0
- package/src/engines/develop.js +1 -1
- package/src/engines/encode.js +68 -30
- package/src/engines/extract.js +13 -8
- package/src/engines/growth.js +1 -1
- package/src/engines/reflect.js +1 -1
- package/src/engines/retrieve.js +44 -13
- package/src/host/runtime.js +1 -1
- package/src/memory/query.js +1 -1
- package/src/memory/tokenizer.js +19 -4
- package/src/security/scope-grants.js +73 -0
- package/src/self/appraisal.js +3 -3
- package/src/self/emotion.js +1 -1
- package/src/self/identity.js +1 -1
- package/src/self/relationship.js +2 -2
- package/src/service/cortex.js +191 -3
- package/src/store/database.js +33 -1
- package/src/store/repositories/edges.js +48 -4
- package/src/store/repositories/traces.js +23 -12
- package/src/store/schema.sql +1 -0
- package/src/tools/definitions.js +64 -5
- package/src/vector/external-ann.js +37 -0
- package/src/vector/hnsw.js +246 -0
- package/src/vector/index.js +85 -2
- package/docs/compatibility.md +0 -39
package/src/api/routes.js
CHANGED
|
@@ -10,6 +10,13 @@ function header(request, name) {
|
|
|
10
10
|
return Array.isArray(value) ? value[0] : value;
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
+
function agentOf(request) {
|
|
14
|
+
const value = header(request, 'x-cortex-agent');
|
|
15
|
+
if (typeof value !== 'string') return null;
|
|
16
|
+
const trimmed = value.trim();
|
|
17
|
+
return /^[A-Za-z0-9._:-]{1,120}$/.test(trimmed) ? trimmed : null;
|
|
18
|
+
}
|
|
19
|
+
|
|
13
20
|
function isLoopback(value) {
|
|
14
21
|
const address = String(value ?? '').toLowerCase().replace(/^\[|\]$/g, '');
|
|
15
22
|
if (address === 'localhost' || address === 'localhost.') return true;
|
|
@@ -69,6 +76,8 @@ function sendJson(response, status, value) {
|
|
|
69
76
|
}
|
|
70
77
|
|
|
71
78
|
function toInt(value, fallback, min, max) {
|
|
79
|
+
// Number(null) is 0, which would silently turn an omitted ?limit into min.
|
|
80
|
+
if (value == null || value === '') return fallback;
|
|
72
81
|
const n = Number(value);
|
|
73
82
|
return Number.isFinite(n) ? Math.max(min, Math.min(max, Math.trunc(n))) : fallback;
|
|
74
83
|
}
|
|
@@ -79,7 +88,7 @@ function scopeOf(request, url, body = null) {
|
|
|
79
88
|
return value && value.length <= 120 && /^[A-Za-z0-9._:-]+$/.test(value) ? value : 'global';
|
|
80
89
|
}
|
|
81
90
|
|
|
82
|
-
export function createApiHandler({ cortex, logger = null } = {}) {
|
|
91
|
+
export function createApiHandler({ cortex, logger = null, getExtractor = null } = {}) {
|
|
83
92
|
if (!cortex) throw new Error('api handler needs cortex');
|
|
84
93
|
|
|
85
94
|
async function handler(request, response) {
|
|
@@ -108,6 +117,7 @@ export function createApiHandler({ cortex, logger = null } = {}) {
|
|
|
108
117
|
if (route === '/skills') return sendJson(response, 200, { ok: true, data: cortex.engines.skills.list({ scopeId: scopeOf(request, url), limit: 100 }) });
|
|
109
118
|
if (route === '/models') return sendJson(response, 200, { ok: true, data: cortex.models.get() });
|
|
110
119
|
if (route === '/permissions') return sendJson(response, 200, { ok: true, data: cortex.permissions.status() });
|
|
120
|
+
if (route === '/security/grants') return sendJson(response, 200, { ok: true, data: cortex.security.grants() });
|
|
111
121
|
if (route === '/settings') return sendJson(response, 200, { ok: true, data: cortex.settings() });
|
|
112
122
|
if (route === '/report') return sendJson(response, 200, { ok: true, data: cortex.report(scopeOf(request, url)) });
|
|
113
123
|
if (route === '/metrics') return sendJson(response, 200, { ok: true, data: cortex.metrics() });
|
|
@@ -115,6 +125,7 @@ export function createApiHandler({ cortex, logger = null } = {}) {
|
|
|
115
125
|
if (route === '/digest') return sendJson(response, 200, { ok: true, data: cortex.digest.daily({ date: url.searchParams.get('date') ?? undefined }) }); if (route === '/audit') return sendJson(response, 200, { ok: true, data: cortex.repos.audit.list({ limit: toInt(url.searchParams.get('limit'), 50, 1, 200) }) });
|
|
116
126
|
if (route === '/graph') return sendJson(response, 200, { ok: true, data: cortex.graphSummary({ scopeId: scopeOf(request, url) }) });
|
|
117
127
|
if (route === '/graph/edges') return sendJson(response, 200, { ok: true, data: cortex.graphData({ scopeId: scopeOf(request, url) }) });
|
|
128
|
+
if (route === '/graph/candidates') return sendJson(response, 200, { ok: true, data: cortex.graphCandidates({ scopeId: scopeOf(request, url), limit: toInt(url.searchParams.get('limit'), 200, 1, 2000) }) });
|
|
118
129
|
if (route === '/search') {
|
|
119
130
|
const query = url.searchParams.get('q') ?? '';
|
|
120
131
|
const tiered = url.searchParams.get('tiered') === 'true';
|
|
@@ -206,6 +217,40 @@ export function createApiHandler({ cortex, logger = null } = {}) {
|
|
|
206
217
|
return;
|
|
207
218
|
}
|
|
208
219
|
try {
|
|
220
|
+
const scopedWriteRoutes = new Set([
|
|
221
|
+
'/write',
|
|
222
|
+
'/working/add',
|
|
223
|
+
'/working/purge',
|
|
224
|
+
'/goals/add',
|
|
225
|
+
'/graph/candidates/action',
|
|
226
|
+
'/retention/apply',
|
|
227
|
+
'/consolidate',
|
|
228
|
+
'/vector/rebuild',
|
|
229
|
+
'/emotion/record',
|
|
230
|
+
'/compress',
|
|
231
|
+
'/life/diary',
|
|
232
|
+
'/life/dream',
|
|
233
|
+
'/life/sleep',
|
|
234
|
+
]);
|
|
235
|
+
if (scopedWriteRoutes.has(route)) {
|
|
236
|
+
const writeScope = scopeOf(request, url, body);
|
|
237
|
+
if (!cortex.scopeWritable(writeScope, agentOf(request))) {
|
|
238
|
+
return sendJson(response, 403, { ok: false, error: 'scope-read-only', scopeId: writeScope });
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
const byIdWriteRoutes = {
|
|
242
|
+
'/working/update': 'working',
|
|
243
|
+
'/working/remove': 'working',
|
|
244
|
+
'/goals/update': 'goal',
|
|
245
|
+
'/goals/complete': 'goal',
|
|
246
|
+
};
|
|
247
|
+
if (byIdWriteRoutes[route]) {
|
|
248
|
+
const kind = byIdWriteRoutes[route];
|
|
249
|
+
const id = requireString(body.id, 'id', { max: 200 });
|
|
250
|
+
if (!cortex.scopeWritableById(kind, id, agentOf(request))) {
|
|
251
|
+
return sendJson(response, 403, { ok: false, error: 'scope-read-only' });
|
|
252
|
+
}
|
|
253
|
+
}
|
|
209
254
|
if (route === '/search') {
|
|
210
255
|
const input = validateSearch(body);
|
|
211
256
|
const useAsync = body.async === true || body.tiered === true && cortex.vector?.status?.().async === true || cortex.vector?.status?.().async === true;
|
|
@@ -218,7 +263,30 @@ export function createApiHandler({ cortex, logger = null } = {}) {
|
|
|
218
263
|
if (useAsync) return sendJson(response, 200, { ok: true, data: await cortex.searchAsync(input.query, { limit: input.limit ?? 10, budget: input.budget ?? 2200, scopeId: scopeOf(request, url, body) }) });
|
|
219
264
|
return sendJson(response, 200, { ok: true, data: cortex.search(input.query, { limit: input.limit ?? 10, budget: input.budget ?? 2200, scopeId: scopeOf(request, url, body) }) });
|
|
220
265
|
}
|
|
266
|
+
if (route === '/graph/candidates/action') {
|
|
267
|
+
const action = requireString(body.action, 'action', { max: 20 });
|
|
268
|
+
const result = cortex.graphCandidateAction({
|
|
269
|
+
action,
|
|
270
|
+
id: body.id ? requireString(body.id, 'id', { max: 200 }) : undefined,
|
|
271
|
+
scopeId: scopeOf(request, url, body),
|
|
272
|
+
fromId: body.fromId ? requireString(body.fromId, 'fromId', { max: 200 }) : undefined,
|
|
273
|
+
toId: body.toId ? requireString(body.toId, 'toId', { max: 200 }) : undefined,
|
|
274
|
+
rel: body.rel ? requireString(body.rel, 'rel', { max: 100 }) : undefined,
|
|
275
|
+
weight: body.weight,
|
|
276
|
+
confidence: body.confidence,
|
|
277
|
+
sourceEventId: body.sourceEventId ? requireString(body.sourceEventId, 'sourceEventId', { max: 200 }) : null,
|
|
278
|
+
sourceTraceId: body.sourceTraceId ? requireString(body.sourceTraceId, 'sourceTraceId', { max: 200 }) : null,
|
|
279
|
+
evidence: body.evidence ?? null,
|
|
280
|
+
metadata: body.metadata && typeof body.metadata === 'object' ? body.metadata : {},
|
|
281
|
+
});
|
|
282
|
+
if (!result) return sendJson(response, 404, { ok: false, error: 'candidate not found or not pending' });
|
|
283
|
+
return sendJson(response, 200, { ok: true, data: result });
|
|
284
|
+
}
|
|
221
285
|
if (route === '/write') {
|
|
286
|
+
const writeScope = scopeOf(request, url, body);
|
|
287
|
+
if (!cortex.scopeWritable(writeScope, agentOf(request))) {
|
|
288
|
+
return sendJson(response, 403, { ok: false, error: 'scope-read-only', scopeId: writeScope });
|
|
289
|
+
}
|
|
222
290
|
if (Array.isArray(body.items)) {
|
|
223
291
|
return sendJson(response, 200, { ok: true, data: cortex.writeMany(body.items.map((item) => ({ ...validateWrite(item), scopeId: scopeOf(request, url, item) }))) });
|
|
224
292
|
}
|
|
@@ -231,6 +299,12 @@ export function createApiHandler({ cortex, logger = null } = {}) {
|
|
|
231
299
|
}
|
|
232
300
|
if (route === '/forget') {
|
|
233
301
|
const input = validateForget(body);
|
|
302
|
+
if (!input.id || typeof input.id !== 'string' || !input.id.trim()) {
|
|
303
|
+
return sendJson(response, 400, { ok: false, error: 'traceId is required' });
|
|
304
|
+
}
|
|
305
|
+
if (!cortex.scopeWritableById('trace', input.id, agentOf(request))) {
|
|
306
|
+
return sendJson(response, 403, { ok: false, error: 'scope-read-only' });
|
|
307
|
+
}
|
|
234
308
|
return sendJson(response, 200, { ok: true, data: cortex.forget({ traceId: input.id, mode: input.mode }) });
|
|
235
309
|
}
|
|
236
310
|
if (route === '/reflect') {
|
|
@@ -256,7 +330,12 @@ export function createApiHandler({ cortex, logger = null } = {}) {
|
|
|
256
330
|
return sendJson(response, 200, { ok: true, data: cortex.exportMarkdown(requireString(body.dir, 'dir', { max: 4000 })) });
|
|
257
331
|
}
|
|
258
332
|
if (route === '/guard/sync') {
|
|
259
|
-
|
|
333
|
+
const dir = requireString(body.dir, 'dir', { max: 4000 });
|
|
334
|
+
const dryRun = body.dryRun === true;
|
|
335
|
+
if (!dryRun && !cortex.scopeWritable(scopeOf(request, url, body), agentOf(request))) {
|
|
336
|
+
return sendJson(response, 403, { ok: false, error: 'scope-read-only' });
|
|
337
|
+
}
|
|
338
|
+
return sendJson(response, 200, { ok: true, data: cortex.syncMarkdown(dir, { dryRun, scopeId: scopeOf(request, url, body) }) });
|
|
260
339
|
}
|
|
261
340
|
if (route === '/qa') {
|
|
262
341
|
const question = requireString(body.question, 'question', { max: 2000 });
|
|
@@ -264,6 +343,15 @@ export function createApiHandler({ cortex, logger = null } = {}) {
|
|
|
264
343
|
}
|
|
265
344
|
if (route === '/playbook') {
|
|
266
345
|
const action = String(body.action ?? 'add');
|
|
346
|
+
if (['vote', 'retire', 'approve'].includes(action)) {
|
|
347
|
+
const id = requireString(body.id, 'id', { max: 200 });
|
|
348
|
+
if (!cortex.scopeWritableById('playbook', id, agentOf(request))) {
|
|
349
|
+
return sendJson(response, 403, { ok: false, error: 'scope-read-only' });
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
if (['add', 'suggest'].includes(action) && !cortex.scopeWritable(scopeOf(request, url, body), agentOf(request))) {
|
|
353
|
+
return sendJson(response, 403, { ok: false, error: 'scope-read-only' });
|
|
354
|
+
}
|
|
267
355
|
if (action === 'add') {
|
|
268
356
|
return sendJson(response, 200, { ok: true, data: cortex.playbook.add({ title: requireString(body.title, 'title', { max: 200 }), body: requireString(body.body, 'body', { max: 4000 }), tags: Array.isArray(body.tags) ? body.tags.map(String).slice(0, 20) : [], validTo: body.validTo ? requireString(body.validTo, 'validTo', { max: 40 }) : null, scopeId: scopeOf(request, url, body) }) });
|
|
269
357
|
}
|
|
@@ -274,12 +362,68 @@ export function createApiHandler({ cortex, logger = null } = {}) {
|
|
|
274
362
|
if (action === 'render') return sendJson(response, 200, { ok: true, data: cortex.playbook.render({ scopeId: scopeOf(request, url, body) }) });
|
|
275
363
|
throw new Error('playbook action must be add, vote, retire, suggest, approve or render');
|
|
276
364
|
}
|
|
365
|
+
if (route === '/attachments/extract') {
|
|
366
|
+
const extractor = typeof getExtractor === 'function' ? getExtractor() : null;
|
|
367
|
+
if (!extractor || typeof extractor.extract !== 'function') {
|
|
368
|
+
return sendJson(response, 503, { ok: false, error: 'extractor-unavailable' });
|
|
369
|
+
}
|
|
370
|
+
try {
|
|
371
|
+
const extracted = await extractor.extract({
|
|
372
|
+
uri: requireString(body.uri, 'uri', { max: 2000 }),
|
|
373
|
+
mode: requireString(body.mode, 'mode', { max: 16 }),
|
|
374
|
+
language: body.language ? requireString(body.language, 'language', { max: 32 }) : undefined,
|
|
375
|
+
layout: body.layout ? requireString(body.layout, 'layout', { max: 16 }) : undefined,
|
|
376
|
+
});
|
|
377
|
+
const ingested = cortex.ingestExtractedText({
|
|
378
|
+
targetTraceId: requireString(body.traceId, 'traceId', { max: 200 }),
|
|
379
|
+
text: extracted.text,
|
|
380
|
+
uri: extracted.uri,
|
|
381
|
+
mode: extracted.mode,
|
|
382
|
+
engine: extracted.engine ?? null,
|
|
383
|
+
});
|
|
384
|
+
return sendJson(response, 200, { ok: true, data: { ...extracted, ...ingested } });
|
|
385
|
+
} catch (error) {
|
|
386
|
+
return sendJson(response, 400, { ok: false, error: error instanceof Error ? error.message : String(error) });
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
if (route === '/trace/attachments') {
|
|
390
|
+
try {
|
|
391
|
+
const result = cortex.attachments.attach({
|
|
392
|
+
traceId: requireString(body.id, 'id', { max: 200 }),
|
|
393
|
+
attachments: body.attachments,
|
|
394
|
+
});
|
|
395
|
+
return sendJson(response, 200, { ok: true, data: result });
|
|
396
|
+
} catch (error) {
|
|
397
|
+
return sendJson(response, 400, { ok: false, error: error instanceof Error ? error.message : String(error) });
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
if (route === '/attachments/inspect') {
|
|
401
|
+
if (cortex.settings().features?.attachmentReader !== true) {
|
|
402
|
+
return sendJson(response, 403, { ok: false, error: 'attachment-reader-disabled' });
|
|
403
|
+
}
|
|
404
|
+
try {
|
|
405
|
+
const result = cortex.attachments.inspect({ uri: requireString(body.uri, 'uri', { max: 2000 }) });
|
|
406
|
+
return sendJson(response, 200, { ok: true, data: result });
|
|
407
|
+
} catch (error) {
|
|
408
|
+
return sendJson(response, 400, { ok: false, error: error instanceof Error ? error.message : String(error) });
|
|
409
|
+
}
|
|
410
|
+
}
|
|
277
411
|
if (route === '/models') {
|
|
278
412
|
return sendJson(response, 200, { ok: true, data: cortex.models.save({ embedding: body.embedding ?? {}, rerank: body.rerank ?? {}, confirm: body.confirm === true }) });
|
|
279
413
|
}
|
|
280
414
|
if (route === '/permissions') {
|
|
281
415
|
return sendJson(response, 200, { ok: true, data: cortex.permissions.update({ key: requireString(body.key, 'key', { max: 80 }), value: body.value === true, confirm: body.confirm === true }) });
|
|
282
416
|
}
|
|
417
|
+
if (route === '/security/grants') {
|
|
418
|
+
const action = requireString(body.action, 'action', { max: 20 });
|
|
419
|
+
if (action === 'add') {
|
|
420
|
+
return sendJson(response, 200, { ok: true, data: cortex.security.addGrant({ grant: requireString(body.grant, 'grant', { max: 200 }), until: body.until ?? null }) });
|
|
421
|
+
}
|
|
422
|
+
if (action === 'revoke') {
|
|
423
|
+
return sendJson(response, 200, { ok: true, data: cortex.security.revokeGrant({ grant: requireString(body.grant, 'grant', { max: 200 }) }) });
|
|
424
|
+
}
|
|
425
|
+
return sendJson(response, 400, { ok: false, error: 'grant action must be add or revoke' });
|
|
426
|
+
}
|
|
283
427
|
if (route === '/permissions/rollback') {
|
|
284
428
|
return sendJson(response, 200, { ok: true, data: cortex.permissions.rollback() });
|
|
285
429
|
}
|
|
@@ -361,8 +505,8 @@ export function createApiHandler({ cortex, logger = null } = {}) {
|
|
|
361
505
|
return handler;
|
|
362
506
|
}
|
|
363
507
|
|
|
364
|
-
export function mountApi(ctx, { cortex, logger = null } = {}) {
|
|
365
|
-
const handler = createApiHandler({ cortex, logger });
|
|
508
|
+
export function mountApi(ctx, { cortex, logger = null, getExtractor = null } = {}) {
|
|
509
|
+
const handler = createApiHandler({ cortex, logger, getExtractor });
|
|
366
510
|
const mount = (server) => {
|
|
367
511
|
if (!server || typeof server.register !== 'function') return () => {};
|
|
368
512
|
try {
|
package/src/api/validate.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { CortexError, ERROR_CODES } from '../core/errors.js';
|
|
2
|
+
import { normalizeAttachments } from '../core/attachments.js';
|
|
2
3
|
|
|
3
4
|
function fail(message) {
|
|
4
5
|
throw new CortexError(ERROR_CODES.INVALID_INPUT, message);
|
|
@@ -32,6 +33,29 @@ export function optionalStringArray(value, name, { max = 50 } = {}) {
|
|
|
32
33
|
return value.map((item) => item.trim()).filter(Boolean);
|
|
33
34
|
}
|
|
34
35
|
|
|
36
|
+
function optionalFacts(value) {
|
|
37
|
+
if (value == null) return undefined;
|
|
38
|
+
if (!Array.isArray(value) || value.length > 50) fail('facts must be an array of at most 50 items');
|
|
39
|
+
return value.map((item, index) => {
|
|
40
|
+
if (!item || typeof item !== 'object' || Array.isArray(item)) fail(`facts[${index}] must be an object`);
|
|
41
|
+
return {
|
|
42
|
+
subject: requireString(item.subject, `facts[${index}].subject`, { max: 200 }),
|
|
43
|
+
predicate: requireString(item.predicate, `facts[${index}].predicate`, { max: 200 }),
|
|
44
|
+
object: requireString(item.object, `facts[${index}].object`, { optional: true, max: 200 }),
|
|
45
|
+
objectLiteral: requireString(item.objectLiteral, `facts[${index}].objectLiteral`, { optional: true, max: 2000 }),
|
|
46
|
+
confidence: optionalNumber(item.confidence, `facts[${index}].confidence`, { min: 0, max: 1 }),
|
|
47
|
+
};
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function optionalAttachments(value) {
|
|
52
|
+
try {
|
|
53
|
+
return normalizeAttachments(value);
|
|
54
|
+
} catch (error) {
|
|
55
|
+
return fail(error.message);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
35
59
|
export function validateWrite(body) {
|
|
36
60
|
return {
|
|
37
61
|
text: requireString(body.text, 'text'),
|
|
@@ -41,6 +65,8 @@ export function validateWrite(body) {
|
|
|
41
65
|
importance: optionalNumber(body.importance, 'importance', { min: 0, max: 1 }),
|
|
42
66
|
explicit: optionalBoolean(body.explicit, 'explicit'),
|
|
43
67
|
operationId: requireString(body.operationId, 'operationId', { optional: true, max: 200 }),
|
|
68
|
+
facts: optionalFacts(body.facts),
|
|
69
|
+
attachments: optionalAttachments(body.attachments),
|
|
44
70
|
};
|
|
45
71
|
}
|
|
46
72
|
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import crypto from 'node:crypto';
|
|
4
|
+
import { fileURLToPath } from 'node:url';
|
|
5
|
+
|
|
6
|
+
export const ATTACHMENT_READ_LIMIT = 50 * 1024 * 1024;
|
|
7
|
+
|
|
8
|
+
export function inspectLocalAttachment(uri, { maxBytes = ATTACHMENT_READ_LIMIT } = {}) {
|
|
9
|
+
const text = String(uri ?? '').trim();
|
|
10
|
+
if (!text) throw new Error('attachment uri is required');
|
|
11
|
+
let url;
|
|
12
|
+
try {
|
|
13
|
+
url = new URL(text);
|
|
14
|
+
} catch {
|
|
15
|
+
throw new Error('attachment uri must be a file:// URL');
|
|
16
|
+
}
|
|
17
|
+
if (url.protocol !== 'file:') throw new Error('only file:// attachments can be inspected');
|
|
18
|
+
const file = fileURLToPath(url);
|
|
19
|
+
const stat = fs.statSync(file);
|
|
20
|
+
if (!stat.isFile()) throw new Error('attachment is not a regular file');
|
|
21
|
+
if (stat.size > maxBytes) throw new Error(`attachment is larger than ${maxBytes} bytes`);
|
|
22
|
+
const sha256 = crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex');
|
|
23
|
+
return {
|
|
24
|
+
kind: 'file',
|
|
25
|
+
uri: text,
|
|
26
|
+
name: path.basename(file),
|
|
27
|
+
bytes: stat.size,
|
|
28
|
+
mtime: stat.mtime.toISOString(),
|
|
29
|
+
sha256,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
const KINDS = new Set(['image', 'audio', 'video', 'file']);
|
|
2
|
+
const HEX = /^[a-f0-9]{64}$/i;
|
|
3
|
+
|
|
4
|
+
function clean(value, max) {
|
|
5
|
+
return String(value ?? '').trim().slice(0, max);
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Reference-only multimodal metadata. No bytes are fetched, decoded or stored;
|
|
10
|
+
* callers remain responsible for the URI and for any model-side use.
|
|
11
|
+
*/
|
|
12
|
+
export function normalizeAttachments(value) {
|
|
13
|
+
if (value == null) return [];
|
|
14
|
+
if (!Array.isArray(value)) throw new Error('attachments must be an array');
|
|
15
|
+
if (value.length > 8) throw new Error('at most 8 attachments');
|
|
16
|
+
return value.map((item, index) => {
|
|
17
|
+
if (!item || typeof item !== 'object' || Array.isArray(item)) throw new Error(`attachments[${index}] must be an object`);
|
|
18
|
+
const kind = clean(item.kind ?? 'file', 20).toLowerCase();
|
|
19
|
+
if (!KINDS.has(kind)) throw new Error(`attachments[${index}].kind must be image/audio/video/file`);
|
|
20
|
+
const uri = clean(item.uri, 2000);
|
|
21
|
+
if (!uri) throw new Error(`attachments[${index}].uri is required`);
|
|
22
|
+
const out = { kind, uri };
|
|
23
|
+
const name = clean(item.name, 200);
|
|
24
|
+
const mime = clean(item.mime, 100);
|
|
25
|
+
const sha256 = clean(item.sha256, 64);
|
|
26
|
+
if (name) out.name = name;
|
|
27
|
+
if (mime) out.mime = mime;
|
|
28
|
+
if (item.bytes != null) {
|
|
29
|
+
const bytes = Number(item.bytes);
|
|
30
|
+
if (!Number.isInteger(bytes) || bytes < 0 || bytes > 1_000_000_000) throw new Error(`attachments[${index}].bytes must be 0..1e9`);
|
|
31
|
+
out.bytes = bytes;
|
|
32
|
+
}
|
|
33
|
+
if (sha256) {
|
|
34
|
+
if (!HEX.test(sha256)) throw new Error(`attachments[${index}].sha256 must be 64 hex chars`);
|
|
35
|
+
out.sha256 = sha256.toLowerCase();
|
|
36
|
+
}
|
|
37
|
+
return out;
|
|
38
|
+
});
|
|
39
|
+
}
|
package/src/core/config.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import os from 'node:os';
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { clamp } from './time.js';
|
|
4
|
+
import { profileDataRoot } from './profile.js';
|
|
4
5
|
|
|
5
6
|
const DSH_HOME = process.env.DSH_HOME || path.join(os.homedir(), '.dsh');
|
|
6
7
|
const DEFAULT_ROOT = process.env.CORTEX_HOME || path.join(DSH_HOME, 'cortex');
|
|
@@ -10,11 +11,17 @@ export const DEFAULT_CONFIG = Object.freeze({
|
|
|
10
11
|
dbFile: path.join(DEFAULT_ROOT, 'cortex.sqlite'),
|
|
11
12
|
scope: 'global',
|
|
12
13
|
contextBudget: 2200,
|
|
14
|
+
readOnlyScopes: Object.freeze([]),
|
|
15
|
+
scopeWriteGrants: Object.freeze([]),
|
|
13
16
|
identityBudget: 360,
|
|
14
17
|
workingBudget: 800,
|
|
15
18
|
recallTimeoutMs: 1200,
|
|
16
19
|
ftsLimit: 40,
|
|
17
20
|
vectorEnabled: false,
|
|
21
|
+
vectorAnnMin: 20000,
|
|
22
|
+
vectorAnnEngine: 'hnsw',
|
|
23
|
+
vectorAnnEndpoint: '',
|
|
24
|
+
vectorAnnTimeoutMs: 800,
|
|
18
25
|
llmEnabled: false,
|
|
19
26
|
asyncJobs: true,
|
|
20
27
|
dsr: Object.freeze({
|
|
@@ -37,6 +44,9 @@ export const DEFAULT_CONFIG = Object.freeze({
|
|
|
37
44
|
reflect: true,
|
|
38
45
|
growth: true,
|
|
39
46
|
vector: false,
|
|
47
|
+
vectorAnn: false,
|
|
48
|
+
vectorAnnExternal: false,
|
|
49
|
+
attachmentReader: false,
|
|
40
50
|
graph: true,
|
|
41
51
|
llm: false,
|
|
42
52
|
}),
|
|
@@ -56,7 +66,7 @@ function textPath(value, fallback, base = null) {
|
|
|
56
66
|
|
|
57
67
|
export function normalizeConfig(raw = {}) {
|
|
58
68
|
const input = raw && typeof raw === 'object' ? raw : {};
|
|
59
|
-
const dataRoot = textPath(input.dataRoot,
|
|
69
|
+
const dataRoot = textPath(input.dataRoot, process.env.CORTEX_HOME || profileDataRoot());
|
|
60
70
|
const dbFile = textPath(input.dbFile, path.join(dataRoot, 'cortex.sqlite'), dataRoot);
|
|
61
71
|
const dsrInput = input.dsr && typeof input.dsr === 'object' ? input.dsr : {};
|
|
62
72
|
const compressionInput = input.compression && typeof input.compression === 'object' ? input.compression : {};
|
|
@@ -69,6 +79,31 @@ export function normalizeConfig(raw = {}) {
|
|
|
69
79
|
dataRoot,
|
|
70
80
|
dbFile,
|
|
71
81
|
scope,
|
|
82
|
+
readOnlyScopes: Object.freeze(Array.isArray(input.readOnlyScopes)
|
|
83
|
+
? [...new Set(input.readOnlyScopes.filter((value) => typeof value === 'string' && value.trim()).map((value) => value.trim()).slice(0, 100))]
|
|
84
|
+
: []),
|
|
85
|
+
scopeWriteGrants: Object.freeze((() => {
|
|
86
|
+
if (!Array.isArray(input.scopeWriteGrants)) return [];
|
|
87
|
+
const out = [];
|
|
88
|
+
const seen = new Set();
|
|
89
|
+
for (const raw of input.scopeWriteGrants) {
|
|
90
|
+
if (typeof raw === 'string' && /^[^:\s]+:\S+$/.test(raw.trim())) {
|
|
91
|
+
const grant = raw.trim();
|
|
92
|
+
const key = `${grant}\u0000`;
|
|
93
|
+
if (!seen.has(key)) { seen.add(key); out.push(Object.freeze({ grant, until: null })); }
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
if (raw && typeof raw === 'object' && typeof raw.grant === 'string' && /^[^:\s]+:\S+$/.test(raw.grant.trim())) {
|
|
97
|
+
const grant = raw.grant.trim();
|
|
98
|
+
const until = raw.until == null ? null : String(raw.until).trim();
|
|
99
|
+
if (until && !Number.isFinite(Date.parse(until))) continue;
|
|
100
|
+
const key = `${grant}\u0000${until ?? ''}`;
|
|
101
|
+
if (!seen.has(key)) { seen.add(key); out.push(Object.freeze({ grant, until: until || null })); }
|
|
102
|
+
}
|
|
103
|
+
if (out.length >= 200) break;
|
|
104
|
+
}
|
|
105
|
+
return out;
|
|
106
|
+
})()),
|
|
72
107
|
contextBudget: numeric(input.contextBudget, DEFAULT_CONFIG.contextBudget, 256, 20000),
|
|
73
108
|
identityBudget: numeric(input.identityBudget, DEFAULT_CONFIG.identityBudget, 64, 4000),
|
|
74
109
|
workingBudget: numeric(input.workingBudget, DEFAULT_CONFIG.workingBudget, 128, 8000),
|
package/src/core/model-config.js
CHANGED
|
@@ -43,7 +43,9 @@ function clean(value) {
|
|
|
43
43
|
|
|
44
44
|
export function readModelConfig(dataRoot) {
|
|
45
45
|
try {
|
|
46
|
-
|
|
46
|
+
// Windows PowerShell's `Set-Content -Encoding utf8` writes a BOM.
|
|
47
|
+
// JSON.parse rejects it, which would silently downgrade to hash embeddings.
|
|
48
|
+
const parsed = JSON.parse(fs.readFileSync(modelConfigPath(dataRoot), 'utf8').replace(/^\uFEFF/, ''));
|
|
47
49
|
return normalizeModelConfig(parsed);
|
|
48
50
|
} catch {
|
|
49
51
|
return normalizeModelConfig({});
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import os from 'node:os';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
|
|
4
|
+
function homeFromEnv(env) {
|
|
5
|
+
return env.DSH_HOME || path.join(os.homedir(), '.dsh');
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export function detectProfileName({ argv = process.argv, env = process.env } = {}) {
|
|
9
|
+
// Explicit CLI/profile-path args outrank an inherited DSH_PROFILE from the
|
|
10
|
+
// desktop app; otherwise `dsh --profile clean` would still read desktop data.
|
|
11
|
+
const args = Array.isArray(argv) ? argv.map(String) : [];
|
|
12
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
13
|
+
const arg = args[index];
|
|
14
|
+
if (arg === '--profile' && args[index + 1]) return path.basename(args[index + 1].replace(/[\\/]+$/, ''));
|
|
15
|
+
if (arg.startsWith('--profile=')) return path.basename(arg.slice('--profile='.length).replace(/[\\/]+$/, ''));
|
|
16
|
+
}
|
|
17
|
+
for (const arg of args) {
|
|
18
|
+
const match = arg.match(/[\\/]profiles[\\/]([^\\/]+)[\\/]?$/);
|
|
19
|
+
if (match) return match[1];
|
|
20
|
+
}
|
|
21
|
+
const fromEnv = String(env.DSH_PROFILE ?? '').trim();
|
|
22
|
+
if (fromEnv) return fromEnv;
|
|
23
|
+
return null;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function profileDataRoot({ argv = process.argv, env = process.env } = {}) {
|
|
27
|
+
const profile = detectProfileName({ argv, env });
|
|
28
|
+
const home = homeFromEnv(env);
|
|
29
|
+
// desktop is the historical default; never silently point it at an empty DB.
|
|
30
|
+
if (!profile || profile === 'desktop') return path.join(home, 'cortex');
|
|
31
|
+
return path.join(home, 'cortex', 'profiles', profile);
|
|
32
|
+
}
|
package/src/engines/develop.js
CHANGED
package/src/engines/encode.js
CHANGED
|
@@ -59,6 +59,7 @@ export function createEncodeEngine({
|
|
|
59
59
|
traces,
|
|
60
60
|
entities = null,
|
|
61
61
|
facts = null,
|
|
62
|
+
edges = null,
|
|
62
63
|
audit = null,
|
|
63
64
|
vector = null,
|
|
64
65
|
config = {},
|
|
@@ -67,6 +68,60 @@ export function createEncodeEngine({
|
|
|
67
68
|
const minChars = config.minContentChars ?? 2;
|
|
68
69
|
const minCompression = config.minCompressionValue ?? 0.08;
|
|
69
70
|
|
|
71
|
+
function applyFacts(rawFacts, { scopeId, recordedAt, sourceEventId }) {
|
|
72
|
+
const resolved = [];
|
|
73
|
+
if (!Array.isArray(rawFacts) || !entities || !facts) return resolved;
|
|
74
|
+
for (const raw of rawFacts) {
|
|
75
|
+
if (!raw || !raw.subject || !raw.predicate) continue;
|
|
76
|
+
const subject = entities.upsert({ scopeId, name: raw.subject, type: raw.subjectType || 'concept' });
|
|
77
|
+
let objectId = null;
|
|
78
|
+
const objectLiteral = raw.objectLiteral ?? null;
|
|
79
|
+
if (raw.object && !raw.objectLiteral) {
|
|
80
|
+
const object = entities.upsert({ scopeId, name: raw.object, type: raw.objectType || 'concept' });
|
|
81
|
+
objectId = object.id;
|
|
82
|
+
}
|
|
83
|
+
const resolution = facts.resolve({
|
|
84
|
+
scopeId,
|
|
85
|
+
subjectId: subject.id,
|
|
86
|
+
predicate: raw.predicate,
|
|
87
|
+
objectId,
|
|
88
|
+
objectLiteral,
|
|
89
|
+
validFrom: raw.validFrom || recordedAt,
|
|
90
|
+
validTo: raw.validTo || null,
|
|
91
|
+
recordedAt,
|
|
92
|
+
confidence: raw.confidence ?? 0.7,
|
|
93
|
+
sourceEventId,
|
|
94
|
+
});
|
|
95
|
+
resolved.push(resolution);
|
|
96
|
+
}
|
|
97
|
+
return resolved;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function queueFactEdges(resolvedFacts, { scopeId, sourceEventId, sourceTraceId }) {
|
|
101
|
+
if (!edges || !Array.isArray(resolvedFacts) || resolvedFacts.length === 0) return [];
|
|
102
|
+
const queued = [];
|
|
103
|
+
for (const resolution of resolvedFacts) {
|
|
104
|
+
const fact = resolution?.fact;
|
|
105
|
+
if (!fact?.subjectId || !fact?.objectId || fact.status !== 'active') continue;
|
|
106
|
+
try {
|
|
107
|
+
queued.push(edges.addCandidate({
|
|
108
|
+
scopeId,
|
|
109
|
+
fromId: fact.subjectId,
|
|
110
|
+
toId: fact.objectId,
|
|
111
|
+
rel: fact.predicate,
|
|
112
|
+
weight: fact.confidence ?? 0.5,
|
|
113
|
+
confidence: fact.confidence ?? 0.5,
|
|
114
|
+
sourceEventId,
|
|
115
|
+
sourceTraceId,
|
|
116
|
+
evidence: { factId: fact.id ?? null, validFrom: fact.validFrom ?? null },
|
|
117
|
+
}));
|
|
118
|
+
} catch {
|
|
119
|
+
/* a bad candidate should not fail the write */
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
return queued;
|
|
123
|
+
}
|
|
124
|
+
|
|
70
125
|
function encode(input = {}) {
|
|
71
126
|
const original = String(input.text ?? input.content ?? '').trim();
|
|
72
127
|
const scanned = scanText(original);
|
|
@@ -101,18 +156,24 @@ export function createEncodeEngine({
|
|
|
101
156
|
});
|
|
102
157
|
|
|
103
158
|
const existing = traces.findByEvent(event.id);
|
|
104
|
-
if (existing)
|
|
159
|
+
if (existing) {
|
|
160
|
+
const factsApplied = applyFacts(input.facts, { scopeId, recordedAt, sourceEventId: event.id });
|
|
161
|
+
const edgeCandidates = queueFactEdges(factsApplied, { scopeId, sourceEventId: event.id, sourceTraceId: existing.id });
|
|
162
|
+
return { skipped: true, reason: 'idempotent', event, trace: existing, facts: factsApplied, edgeCandidates };
|
|
163
|
+
}
|
|
105
164
|
|
|
106
165
|
const duplicate = traces.findByHash(scopeId, hashText(text));
|
|
107
166
|
if (duplicate) {
|
|
167
|
+
const factsApplied = applyFacts(input.facts, { scopeId, recordedAt, sourceEventId: event.id });
|
|
168
|
+
const edgeCandidates = queueFactEdges(factsApplied, { scopeId, sourceEventId: event.id, sourceTraceId: duplicate.id });
|
|
108
169
|
audit?.record?.({
|
|
109
170
|
actor: input.actor || 'user',
|
|
110
171
|
action: 'trace.duplicate',
|
|
111
172
|
targetType: 'trace',
|
|
112
173
|
targetId: duplicate.id,
|
|
113
|
-
detail: { eventId: event.id, textHash: hashText(text) },
|
|
174
|
+
detail: { eventId: event.id, textHash: hashText(text), factsApplied: factsApplied.length, edgeCandidates: edgeCandidates.length },
|
|
114
175
|
});
|
|
115
|
-
return { skipped: true, reason: 'duplicate', event, trace: duplicate };
|
|
176
|
+
return { skipped: true, reason: 'duplicate', event, trace: duplicate, facts: factsApplied, edgeCandidates };
|
|
116
177
|
}
|
|
117
178
|
|
|
118
179
|
const trace = traces.insert({
|
|
@@ -123,6 +184,7 @@ export function createEncodeEngine({
|
|
|
123
184
|
summary: input.summary || null,
|
|
124
185
|
keywords: input.keywords || [],
|
|
125
186
|
tags: input.tags || [],
|
|
187
|
+
metadata: input.metadata && typeof input.metadata === 'object' ? input.metadata : {},
|
|
126
188
|
eventId: event.id,
|
|
127
189
|
sessionId: input.sessionId || null,
|
|
128
190
|
turnId: input.turnId || null,
|
|
@@ -138,32 +200,8 @@ export function createEncodeEngine({
|
|
|
138
200
|
sensitivity: sensitivity,
|
|
139
201
|
});
|
|
140
202
|
|
|
141
|
-
const resolvedFacts =
|
|
142
|
-
|
|
143
|
-
for (const raw of input.facts) {
|
|
144
|
-
if (!raw || !raw.subject || !raw.predicate) continue;
|
|
145
|
-
const subject = entities.upsert({ scopeId, name: raw.subject, type: raw.subjectType || 'concept' });
|
|
146
|
-
let objectId = null;
|
|
147
|
-
let objectLiteral = raw.objectLiteral ?? null;
|
|
148
|
-
if (raw.object && !raw.objectLiteral) {
|
|
149
|
-
const object = entities.upsert({ scopeId, name: raw.object, type: raw.objectType || 'concept' });
|
|
150
|
-
objectId = object.id;
|
|
151
|
-
}
|
|
152
|
-
const resolution = facts.resolve({
|
|
153
|
-
scopeId,
|
|
154
|
-
subjectId: subject.id,
|
|
155
|
-
predicate: raw.predicate,
|
|
156
|
-
objectId,
|
|
157
|
-
objectLiteral,
|
|
158
|
-
validFrom: raw.validFrom || recordedAt,
|
|
159
|
-
validTo: raw.validTo || null,
|
|
160
|
-
recordedAt,
|
|
161
|
-
confidence: raw.confidence ?? 0.7,
|
|
162
|
-
sourceEventId: event.id,
|
|
163
|
-
});
|
|
164
|
-
resolvedFacts.push(resolution);
|
|
165
|
-
}
|
|
166
|
-
}
|
|
203
|
+
const resolvedFacts = applyFacts(input.facts, { scopeId, recordedAt, sourceEventId: event.id });
|
|
204
|
+
const edgeCandidates = queueFactEdges(resolvedFacts, { scopeId, sourceEventId: event.id, sourceTraceId: trace.id });
|
|
167
205
|
|
|
168
206
|
audit?.record?.({
|
|
169
207
|
actor: input.actor || 'user',
|
|
@@ -192,7 +230,7 @@ export function createEncodeEngine({
|
|
|
192
230
|
}
|
|
193
231
|
}
|
|
194
232
|
|
|
195
|
-
return { skipped: false, event, trace, facts: resolvedFacts, compression, score: scored };
|
|
233
|
+
return { skipped: false, event, trace, facts: resolvedFacts, edgeCandidates, compression, score: scored };
|
|
196
234
|
}
|
|
197
235
|
|
|
198
236
|
return { encode, scoreTraceText, compressionValue };
|