@pyxmate/memory 1.17.14 → 1.17.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-34MTVIYK.mjs +109 -0
- package/dist/{chunk-3QDXACBV.mjs → chunk-MDFUZ3V2.mjs} +365 -194
- package/dist/{chunk-X3QODOJV.mjs → chunk-ZVI7DCB4.mjs} +1 -1
- package/dist/cli/pyx-mem.mjs +1 -1
- package/dist/dashboard.mjs +3 -3
- package/dist/data-plane-contract.mjs +1 -1
- package/dist/index.d.ts +167 -7
- package/dist/index.mjs +12 -4
- package/dist/react.mjs +3 -3
- package/package.json +1 -1
- package/dist/chunk-A3L46P2G.mjs +0 -57
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
// ../shared/src/graph/extraction.ts
|
|
2
|
+
function normalizeGraphLabel(value, fallback) {
|
|
3
|
+
const normalized = value.trim().toUpperCase().replace(/[^A-Z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
4
|
+
return normalized.length > 0 ? normalized : fallback;
|
|
5
|
+
}
|
|
6
|
+
function normalizeNameKey(name) {
|
|
7
|
+
return name.trim().toLowerCase().replace(/\s+/g, " ");
|
|
8
|
+
}
|
|
9
|
+
function requireGraphRecord(value, field) {
|
|
10
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
11
|
+
throw new Error(`${field} must be an object`);
|
|
12
|
+
}
|
|
13
|
+
return value;
|
|
14
|
+
}
|
|
15
|
+
function requireNonemptyGraphString(value, field) {
|
|
16
|
+
if (typeof value !== "string" || value.trim().length === 0) {
|
|
17
|
+
throw new Error(`${field} must be a non-empty string`);
|
|
18
|
+
}
|
|
19
|
+
return value;
|
|
20
|
+
}
|
|
21
|
+
function assertGraphExtractionPayload(value, field = "graph extraction payload") {
|
|
22
|
+
const record = requireGraphRecord(value, field);
|
|
23
|
+
if (!Array.isArray(record.entities)) throw new Error(`${field}.entities must be an array`);
|
|
24
|
+
if (!Array.isArray(record.relationships)) {
|
|
25
|
+
throw new Error(`${field}.relationships must be an array`);
|
|
26
|
+
}
|
|
27
|
+
const entities = record.entities.map((item, index) => {
|
|
28
|
+
const entity = requireGraphRecord(item, `${field}.entities[${index}]`);
|
|
29
|
+
requireNonemptyGraphString(entity.name, `${field}.entities[${index}].name`);
|
|
30
|
+
requireNonemptyGraphString(entity.type, `${field}.entities[${index}].type`);
|
|
31
|
+
if (entity.properties !== void 0 && (!entity.properties || typeof entity.properties !== "object" || Array.isArray(entity.properties))) {
|
|
32
|
+
throw new Error(`${field}.entities[${index}].properties must be an object`);
|
|
33
|
+
}
|
|
34
|
+
return item;
|
|
35
|
+
});
|
|
36
|
+
const entityNames = new Set(entities.map((entity) => normalizeNameKey(entity.name)));
|
|
37
|
+
const relationships = record.relationships.map((item, index) => {
|
|
38
|
+
const relationship = requireGraphRecord(item, `${field}.relationships[${index}]`);
|
|
39
|
+
const source = requireNonemptyGraphString(
|
|
40
|
+
relationship.source,
|
|
41
|
+
`${field}.relationships[${index}].source`
|
|
42
|
+
);
|
|
43
|
+
const target = requireNonemptyGraphString(
|
|
44
|
+
relationship.target,
|
|
45
|
+
`${field}.relationships[${index}].target`
|
|
46
|
+
);
|
|
47
|
+
requireNonemptyGraphString(relationship.type, `${field}.relationships[${index}].type`);
|
|
48
|
+
if (!entityNames.has(normalizeNameKey(source)) || !entityNames.has(normalizeNameKey(target))) {
|
|
49
|
+
throw new Error(
|
|
50
|
+
`${field}.relationships[${index}] endpoints must reference declared entity names`
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
if (relationship.properties !== void 0 && (!relationship.properties || typeof relationship.properties !== "object" || Array.isArray(relationship.properties))) {
|
|
54
|
+
throw new Error(`${field}.relationships[${index}].properties must be an object`);
|
|
55
|
+
}
|
|
56
|
+
return item;
|
|
57
|
+
});
|
|
58
|
+
return { entities, relationships };
|
|
59
|
+
}
|
|
60
|
+
function relationshipKey(relationship) {
|
|
61
|
+
return [
|
|
62
|
+
relationship.source.trim().toLowerCase(),
|
|
63
|
+
relationship.target.trim().toLowerCase(),
|
|
64
|
+
normalizeGraphLabel(relationship.type, "RELATED_TO")
|
|
65
|
+
].join("\0");
|
|
66
|
+
}
|
|
67
|
+
function mergeExtractedEntities(callerEntities, callerRelationships, extracted) {
|
|
68
|
+
const entities = [...callerEntities ?? []];
|
|
69
|
+
const relationships = [...callerRelationships ?? []];
|
|
70
|
+
const nameByLowercase = /* @__PURE__ */ new Map();
|
|
71
|
+
for (const entity of entities) {
|
|
72
|
+
const key = entity.name.toLowerCase();
|
|
73
|
+
if (!nameByLowercase.has(key)) nameByLowercase.set(key, entity.name);
|
|
74
|
+
}
|
|
75
|
+
for (const entity of extracted.entities) {
|
|
76
|
+
const key = entity.name.toLowerCase();
|
|
77
|
+
if (nameByLowercase.has(key)) continue;
|
|
78
|
+
entities.push({ ...entity, type: normalizeGraphLabel(entity.type, "CONCEPT") });
|
|
79
|
+
nameByLowercase.set(key, entity.name);
|
|
80
|
+
}
|
|
81
|
+
for (const relationship of extracted.relations) {
|
|
82
|
+
const source = nameByLowercase.get(relationship.source.toLowerCase());
|
|
83
|
+
const target = nameByLowercase.get(relationship.target.toLowerCase());
|
|
84
|
+
if (source && target) {
|
|
85
|
+
relationships.push({
|
|
86
|
+
...relationship,
|
|
87
|
+
source,
|
|
88
|
+
target,
|
|
89
|
+
type: normalizeGraphLabel(relationship.type, "RELATED_TO")
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
const seenRelationships = /* @__PURE__ */ new Set();
|
|
94
|
+
const dedupedRelationships = [];
|
|
95
|
+
for (const relationship of relationships) {
|
|
96
|
+
const key = relationshipKey(relationship);
|
|
97
|
+
if (seenRelationships.has(key)) continue;
|
|
98
|
+
seenRelationships.add(key);
|
|
99
|
+
dedupedRelationships.push(relationship);
|
|
100
|
+
}
|
|
101
|
+
return { entities, relationships: dedupedRelationships };
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
export {
|
|
105
|
+
normalizeGraphLabel,
|
|
106
|
+
normalizeNameKey,
|
|
107
|
+
assertGraphExtractionPayload,
|
|
108
|
+
mergeExtractedEntities
|
|
109
|
+
};
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
|
+
assertGraphExtractionPayload,
|
|
2
3
|
mergeExtractedEntities
|
|
3
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-34MTVIYK.mjs";
|
|
4
5
|
|
|
5
6
|
// ../shared/src/constants/defaults.ts
|
|
6
7
|
var DEFAULTS = {
|
|
@@ -62,6 +63,40 @@ function projectSearchResponseForMcp(payload) {
|
|
|
62
63
|
return projectSearchResultRecord(payload);
|
|
63
64
|
}
|
|
64
65
|
|
|
66
|
+
// ../shared/src/mcp/secret-elevation-notice.ts
|
|
67
|
+
function isRecord2(value) {
|
|
68
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
69
|
+
}
|
|
70
|
+
function elevationMessage(credentialTypes) {
|
|
71
|
+
const types = credentialTypes.length > 0 ? credentialTypes.join(", ") : "unspecified";
|
|
72
|
+
return `Auto-classified sensitivity=secret because credential patterns were detected (credentialTypes: ${types}). Secret entries are invisible to hosted MCP reads \u2014 search, get and list all cap MCP callers at internal. When an encryption key is configured the content is also encrypted at rest and embedded as a placeholder, so REST callers lose semantic search recall for it too (REST list/get still return the row). If this is a false positive: delete the entry by id, or store it again under the same id with the credential-shaped notation rephrased \u2014 a same-id re-store re-classifies the content. A canonical entry carrying metadata.dbRef.revision must advance that revision on the re-store, or it fails with revision_reuse_conflict.`;
|
|
73
|
+
}
|
|
74
|
+
function secretElevationNoticeFor(entry) {
|
|
75
|
+
if (!isRecord2(entry) || entry.sensitivity !== "secret") return void 0;
|
|
76
|
+
const metadata = entry.metadata;
|
|
77
|
+
if (!isRecord2(metadata) || metadata.credentialsDetected !== true) return void 0;
|
|
78
|
+
const credentialTypes = Array.isArray(metadata.credentialTypes) ? metadata.credentialTypes.filter((type) => typeof type === "string") : [];
|
|
79
|
+
return { sensitivity: "secret", credentialTypes, message: elevationMessage(credentialTypes) };
|
|
80
|
+
}
|
|
81
|
+
function withSecretElevationNotice(payload) {
|
|
82
|
+
if (!isRecord2(payload)) return payload;
|
|
83
|
+
const entry = isRecord2(payload.data) ? payload.data : payload;
|
|
84
|
+
const notice = secretElevationNoticeFor(entry);
|
|
85
|
+
if (!notice) return payload;
|
|
86
|
+
const elevated = { ...entry, secretElevation: notice };
|
|
87
|
+
return entry === payload ? elevated : { ...payload, data: elevated };
|
|
88
|
+
}
|
|
89
|
+
function secretElevationAggregate(elevated) {
|
|
90
|
+
if (elevated.length === 0) return void 0;
|
|
91
|
+
const credentialTypes = [...new Set(elevated.flatMap((item) => item.notice.credentialTypes))];
|
|
92
|
+
return {
|
|
93
|
+
count: elevated.length,
|
|
94
|
+
entryIds: elevated.map((item) => item.entryId),
|
|
95
|
+
credentialTypes,
|
|
96
|
+
message: `${elevated.length} stored chunk(s) were auto-elevated. ${elevationMessage(credentialTypes)}`
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
|
|
65
100
|
// ../shared/src/types/isolation.ts
|
|
66
101
|
var NamespaceIsolation = {
|
|
67
102
|
SHARED: "shared",
|
|
@@ -390,7 +425,6 @@ var MemoryClient = class {
|
|
|
390
425
|
* collect the terminal event; there is no separate `ingestFile()` Promise
|
|
391
426
|
* method by design (one wire format, one SDK method).
|
|
392
427
|
*/
|
|
393
|
-
// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: NDJSON dispatch + SDK enrichment fan-out is two cooperating paths; documented inline and tested in memory-client-events.test.ts.
|
|
394
428
|
async *ingestFileEvents(file, options) {
|
|
395
429
|
const controller = new AbortController();
|
|
396
430
|
const relayAbort = () => controller.abort(options?.signal?.reason);
|
|
@@ -407,237 +441,371 @@ var MemoryClient = class {
|
|
|
407
441
|
...this._authHeaders
|
|
408
442
|
};
|
|
409
443
|
if (wantsTextWindows) headers["X-Pyx-Enrichment-Capabilities"] = "text_windows_v1";
|
|
410
|
-
|
|
444
|
+
const raw = yield* this.streamNdjsonUpload(
|
|
445
|
+
"/api/memory/ingest/file",
|
|
446
|
+
formData,
|
|
447
|
+
headers,
|
|
448
|
+
controller.signal
|
|
449
|
+
);
|
|
450
|
+
if (!raw) return;
|
|
451
|
+
const serverResult = this.fileIngestResultFromEvent({
|
|
452
|
+
...raw,
|
|
453
|
+
schemaVersion: 1,
|
|
454
|
+
type: "result",
|
|
455
|
+
stage: "complete"
|
|
456
|
+
});
|
|
457
|
+
yield* this.completeIngestFileEvents(file, serverResult, options, controller.signal);
|
|
458
|
+
} finally {
|
|
459
|
+
options?.signal?.removeEventListener("abort", relayAbort);
|
|
460
|
+
controller.abort();
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
/**
|
|
464
|
+
* Graph-only re-enrichment for a document whose chunks are already stored
|
|
465
|
+
* and searchable but whose graph build failed. Uploads the original to
|
|
466
|
+
* `/api/memory/graph/enrich/file` (prepare-only — the server performs zero
|
|
467
|
+
* store/delete before the final graph write), runs the SAME enrichment
|
|
468
|
+
* callback engine as {@link ingestFileEvents}, then finalizes into one
|
|
469
|
+
* stable graph anchor keyed by `documentKey`. Repeating the same key
|
|
470
|
+
* replaces the document's graph references idempotently.
|
|
471
|
+
*
|
|
472
|
+
* The terminal `result` is a {@link GraphEnrichResult} event carrying the
|
|
473
|
+
* ACTUAL persisted graph counts; abort and failures yield a terminal
|
|
474
|
+
* `error` event instead (the server retains the pending session for retry).
|
|
475
|
+
*/
|
|
476
|
+
async *graphEnrichFileEvents(file, options) {
|
|
477
|
+
if (typeof options?.documentKey !== "string" || options.documentKey.trim().length === 0) {
|
|
478
|
+
throw new Error("graphEnrichFileEvents requires a non-empty documentKey");
|
|
479
|
+
}
|
|
480
|
+
if (!options.enrichment?.extractEntitiesV2) {
|
|
481
|
+
throw new Error(
|
|
482
|
+
"graphEnrichFileEvents requires enrichment.extractEntitiesV2 \u2014 graph-only re-enrichment is caller-extraction by definition"
|
|
483
|
+
);
|
|
484
|
+
}
|
|
485
|
+
const controller = new AbortController();
|
|
486
|
+
const relayAbort = () => controller.abort(options.signal?.reason);
|
|
487
|
+
if (options.signal?.aborted) relayAbort();
|
|
488
|
+
options.signal?.addEventListener("abort", relayAbort, { once: true });
|
|
489
|
+
try {
|
|
490
|
+
const formData = new FormData();
|
|
491
|
+
formData.append("file", file);
|
|
492
|
+
formData.append("documentKey", options.documentKey);
|
|
493
|
+
if (options.namespaceId) formData.append("namespaceId", options.namespaceId);
|
|
494
|
+
const headers = {
|
|
495
|
+
Accept: "application/x-ndjson",
|
|
496
|
+
"X-Pyx-Enrichment-Capabilities": "text_windows_v1",
|
|
497
|
+
...this._authHeaders
|
|
498
|
+
};
|
|
499
|
+
const raw = yield* this.streamNdjsonUpload(
|
|
500
|
+
"/api/memory/graph/enrich/file",
|
|
501
|
+
formData,
|
|
502
|
+
headers,
|
|
503
|
+
controller.signal
|
|
504
|
+
);
|
|
505
|
+
if (!raw) return;
|
|
411
506
|
try {
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
yield this.
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
const contentType = res.headers.get("content-type") ?? "";
|
|
426
|
-
if (!contentType.includes("application/x-ndjson")) {
|
|
427
|
-
yield this.ingestErrorEvent(
|
|
428
|
-
new MemoryServerError(
|
|
429
|
-
`Memory server returned ${contentType || "unknown content-type"} instead of application/x-ndjson \u2014 server is older than v0.15.0`,
|
|
430
|
-
res.status
|
|
431
|
-
),
|
|
432
|
-
"parsing"
|
|
433
|
-
);
|
|
434
|
-
return;
|
|
435
|
-
}
|
|
436
|
-
if (!res.body) {
|
|
437
|
-
yield this.ingestErrorEvent(
|
|
438
|
-
new MemoryServerError("Memory server returned an empty stream", res.status),
|
|
439
|
-
"parsing"
|
|
507
|
+
const prepared = raw;
|
|
508
|
+
if (prepared.purpose !== "graph-only" || !prepared.enrichment) {
|
|
509
|
+
throw new MemoryServerError(
|
|
510
|
+
"Graph enrichment prepare returned an unexpected terminal result",
|
|
511
|
+
0
|
|
512
|
+
);
|
|
513
|
+
}
|
|
514
|
+
const outcome = yield* this.runEnrichmentCallbacks(
|
|
515
|
+
file,
|
|
516
|
+
prepared.enrichment,
|
|
517
|
+
options.enrichment,
|
|
518
|
+
controller.signal,
|
|
519
|
+
"graph-only"
|
|
440
520
|
);
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
let serverResult = null;
|
|
448
|
-
try {
|
|
449
|
-
while (true) {
|
|
450
|
-
const { done, value } = await reader.read();
|
|
451
|
-
if (done) break;
|
|
452
|
-
buffer += decoder.decode(value, { stream: true });
|
|
453
|
-
const lines = buffer.split("\n");
|
|
454
|
-
buffer = lines.pop() ?? "";
|
|
455
|
-
for (const line of lines) {
|
|
456
|
-
if (!line.trim()) continue;
|
|
457
|
-
const raw = JSON.parse(line);
|
|
458
|
-
const type = raw.type;
|
|
459
|
-
if (type === "progress" || type === "heartbeat") {
|
|
460
|
-
const stage = this.normalizeActiveIngestStage(raw.stage);
|
|
461
|
-
if (!stage) continue;
|
|
462
|
-
currentStage = stage;
|
|
463
|
-
yield { ...raw, schemaVersion: 1, type, stage };
|
|
464
|
-
continue;
|
|
465
|
-
}
|
|
466
|
-
if (type === "result") {
|
|
467
|
-
serverResult = this.fileIngestResultFromEvent({
|
|
468
|
-
...raw,
|
|
469
|
-
schemaVersion: 1,
|
|
470
|
-
type: "result",
|
|
471
|
-
stage: "complete"
|
|
472
|
-
});
|
|
473
|
-
break;
|
|
474
|
-
}
|
|
475
|
-
if (type === "error") {
|
|
476
|
-
yield {
|
|
477
|
-
schemaVersion: 1,
|
|
478
|
-
type: "error",
|
|
479
|
-
stage: this.normalizeActiveIngestStage(raw.stage) ?? currentStage,
|
|
480
|
-
error: typeof raw.error === "string" ? raw.error : "File ingest failed",
|
|
481
|
-
message: typeof raw.message === "string" ? raw.message : void 0,
|
|
482
|
-
code: typeof raw.code === "string" || typeof raw.code === "number" ? raw.code : void 0,
|
|
483
|
-
status: typeof raw.status === "number" ? raw.status : void 0
|
|
484
|
-
};
|
|
485
|
-
return;
|
|
486
|
-
}
|
|
487
|
-
}
|
|
488
|
-
if (serverResult) break;
|
|
521
|
+
const data = outcome.data;
|
|
522
|
+
if (data?.purpose !== "graph-only" || !Array.isArray(data.entryIds)) {
|
|
523
|
+
throw new MemoryServerError(
|
|
524
|
+
"Graph enrichment finalization returned an unexpected result",
|
|
525
|
+
0
|
|
526
|
+
);
|
|
489
527
|
}
|
|
528
|
+
yield { schemaVersion: 1, type: "result", stage: "complete", ...data };
|
|
490
529
|
} catch (err) {
|
|
491
|
-
yield this.ingestErrorEvent(err,
|
|
492
|
-
return;
|
|
493
|
-
} finally {
|
|
494
|
-
reader.releaseLock();
|
|
530
|
+
yield this.ingestErrorEvent(err, "enrichment");
|
|
495
531
|
}
|
|
496
|
-
if (!serverResult) {
|
|
497
|
-
yield this.ingestErrorEvent(
|
|
498
|
-
new MemoryServerError("File ingest stream ended without a server result", 0),
|
|
499
|
-
currentStage
|
|
500
|
-
);
|
|
501
|
-
return;
|
|
502
|
-
}
|
|
503
|
-
yield* this.completeIngestFileEvents(file, serverResult, options, controller.signal);
|
|
504
532
|
} finally {
|
|
505
|
-
options
|
|
533
|
+
options.signal?.removeEventListener("abort", relayAbort);
|
|
506
534
|
controller.abort();
|
|
507
535
|
}
|
|
508
536
|
}
|
|
509
537
|
/**
|
|
510
|
-
*
|
|
511
|
-
*
|
|
512
|
-
*
|
|
513
|
-
*
|
|
514
|
-
*
|
|
538
|
+
* POST a multipart body to an NDJSON streaming endpoint and relay its
|
|
539
|
+
* progress/heartbeat events. Returns the raw terminal `result` record, or
|
|
540
|
+
* null after yielding a terminal error (transport failure, server error
|
|
541
|
+
* event, non-NDJSON response, stream ending without a result). One
|
|
542
|
+
* implementation for both streaming surfaces so the wire protocol cannot
|
|
543
|
+
* fork.
|
|
544
|
+
*/
|
|
545
|
+
// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: one stateful NDJSON reader validates and relays both ingest and graph-only stream envelopes
|
|
546
|
+
async *streamNdjsonUpload(path, formData, headers, signal) {
|
|
547
|
+
let res;
|
|
548
|
+
try {
|
|
549
|
+
res = await fetch(`${this.baseUrl}${path}`, {
|
|
550
|
+
method: "POST",
|
|
551
|
+
body: formData,
|
|
552
|
+
headers,
|
|
553
|
+
signal
|
|
554
|
+
});
|
|
555
|
+
} catch (err) {
|
|
556
|
+
yield this.ingestErrorEvent(this.translateFetchError(err, path), "parsing");
|
|
557
|
+
return null;
|
|
558
|
+
}
|
|
559
|
+
const contentType = res.headers.get("content-type") ?? "";
|
|
560
|
+
if (!contentType.includes("application/x-ndjson")) {
|
|
561
|
+
yield this.ingestErrorEvent(
|
|
562
|
+
new MemoryServerError(
|
|
563
|
+
`Memory server returned ${contentType || "unknown content-type"} instead of application/x-ndjson \u2014 server is older than v0.15.0`,
|
|
564
|
+
res.status
|
|
565
|
+
),
|
|
566
|
+
"parsing"
|
|
567
|
+
);
|
|
568
|
+
return null;
|
|
569
|
+
}
|
|
570
|
+
if (!res.body) {
|
|
571
|
+
yield this.ingestErrorEvent(
|
|
572
|
+
new MemoryServerError("Memory server returned an empty stream", res.status),
|
|
573
|
+
"parsing"
|
|
574
|
+
);
|
|
575
|
+
return null;
|
|
576
|
+
}
|
|
577
|
+
const reader = res.body.getReader();
|
|
578
|
+
const decoder = new TextDecoder();
|
|
579
|
+
let buffer = "";
|
|
580
|
+
let currentStage = "parsing";
|
|
581
|
+
let serverResult = null;
|
|
582
|
+
try {
|
|
583
|
+
while (true) {
|
|
584
|
+
const { done, value } = await reader.read();
|
|
585
|
+
if (done) break;
|
|
586
|
+
buffer += decoder.decode(value, { stream: true });
|
|
587
|
+
const lines = buffer.split("\n");
|
|
588
|
+
buffer = lines.pop() ?? "";
|
|
589
|
+
for (const line of lines) {
|
|
590
|
+
if (!line.trim()) continue;
|
|
591
|
+
const raw = JSON.parse(line);
|
|
592
|
+
const type = raw.type;
|
|
593
|
+
if (type === "progress" || type === "heartbeat") {
|
|
594
|
+
const stage = this.normalizeActiveIngestStage(raw.stage);
|
|
595
|
+
if (!stage) continue;
|
|
596
|
+
currentStage = stage;
|
|
597
|
+
yield { ...raw, schemaVersion: 1, type, stage };
|
|
598
|
+
continue;
|
|
599
|
+
}
|
|
600
|
+
if (type === "result") {
|
|
601
|
+
serverResult = raw;
|
|
602
|
+
break;
|
|
603
|
+
}
|
|
604
|
+
if (type === "error") {
|
|
605
|
+
yield {
|
|
606
|
+
schemaVersion: 1,
|
|
607
|
+
type: "error",
|
|
608
|
+
stage: this.normalizeActiveIngestStage(raw.stage) ?? currentStage,
|
|
609
|
+
error: typeof raw.error === "string" ? raw.error : "File ingest failed",
|
|
610
|
+
message: typeof raw.message === "string" ? raw.message : void 0,
|
|
611
|
+
code: typeof raw.code === "string" || typeof raw.code === "number" ? raw.code : void 0,
|
|
612
|
+
status: typeof raw.status === "number" ? raw.status : void 0
|
|
613
|
+
};
|
|
614
|
+
return null;
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
if (serverResult) break;
|
|
618
|
+
}
|
|
619
|
+
} catch (err) {
|
|
620
|
+
yield this.ingestErrorEvent(err, currentStage);
|
|
621
|
+
return null;
|
|
622
|
+
} finally {
|
|
623
|
+
reader.releaseLock();
|
|
624
|
+
}
|
|
625
|
+
if (!serverResult) {
|
|
626
|
+
yield this.ingestErrorEvent(
|
|
627
|
+
new MemoryServerError("File ingest stream ended without a server result", 0),
|
|
628
|
+
currentStage
|
|
629
|
+
);
|
|
630
|
+
return null;
|
|
631
|
+
}
|
|
632
|
+
return serverResult;
|
|
633
|
+
}
|
|
634
|
+
/**
|
|
635
|
+
* Run the SDK-side enrichment phase for an ingest result and yield the
|
|
636
|
+
* single terminal {@link IngestResultEvent} at the end. Skips work cleanly
|
|
637
|
+
* when the server emitted no enrichment block or the caller wired no
|
|
638
|
+
* callbacks. The callback work itself lives in
|
|
639
|
+
* {@link runEnrichmentCallbacks} — shared with the graph-only surface.
|
|
515
640
|
*/
|
|
516
|
-
// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: image-describe / extract-entities-v2 / no-op are documented decision branches; the fan-out covers the v0.15.0 enrichment phase + heartbeat plumbing.
|
|
517
641
|
async *completeIngestFileEvents(file, result, options, signal) {
|
|
518
642
|
try {
|
|
519
643
|
if (!result.enrichment || !options?.enrichment) {
|
|
520
644
|
yield { schemaVersion: 1, type: "result", stage: "complete", ...result };
|
|
521
645
|
return;
|
|
522
646
|
}
|
|
523
|
-
const
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
};
|
|
539
|
-
const CONCURRENCY = 5;
|
|
540
|
-
for (let i = 0; i < images.length; i += CONCURRENCY) {
|
|
541
|
-
const batch = images.slice(i, i + CONCURRENCY);
|
|
542
|
-
const batchResults = yield* this.withSdkHeartbeats(
|
|
543
|
-
"enrichment",
|
|
544
|
-
Promise.all(
|
|
545
|
-
batch.map(async (imageMeta) => {
|
|
546
|
-
const imageRes = await fetch(
|
|
547
|
-
`${this.baseUrl}/api/memory/files/${fileId}/images/${imageMeta.imageId}?token=${encodeURIComponent(token)}`,
|
|
548
|
-
{ headers: this._authHeaders, signal }
|
|
549
|
-
);
|
|
550
|
-
if (!imageRes.ok) {
|
|
551
|
-
throw new MemoryServerError(
|
|
552
|
-
`Failed to fetch image ${imageMeta.imageId}: ${imageRes.status}`,
|
|
553
|
-
imageRes.status
|
|
554
|
-
);
|
|
555
|
-
}
|
|
556
|
-
const imageBuffer = await imageRes.arrayBuffer();
|
|
557
|
-
const description = await describeImage(imageBuffer, imageMeta);
|
|
558
|
-
return { imageId: imageMeta.imageId, description };
|
|
559
|
-
})
|
|
560
|
-
),
|
|
561
|
-
signal
|
|
562
|
-
);
|
|
563
|
-
descriptions.push(...batchResults.filter((d) => d.description.trim().length > 0));
|
|
564
|
-
yield {
|
|
565
|
-
schemaVersion: 1,
|
|
566
|
-
type: "progress",
|
|
567
|
-
stage: "enrichment",
|
|
568
|
-
filename: file.name,
|
|
569
|
-
imagesTotal: images.length,
|
|
570
|
-
imagesDescribed: descriptions.length,
|
|
571
|
-
message: `Described ${descriptions.length}/${images.length} images`
|
|
572
|
-
};
|
|
573
|
-
}
|
|
647
|
+
const outcome = yield* this.runEnrichmentCallbacks(
|
|
648
|
+
file,
|
|
649
|
+
result.enrichment,
|
|
650
|
+
options.enrichment,
|
|
651
|
+
signal,
|
|
652
|
+
"ingest"
|
|
653
|
+
);
|
|
654
|
+
if (outcome.posted) {
|
|
655
|
+
const enrichData = outcome.data;
|
|
656
|
+
result.entryIds.push(...enrichData.entryIds);
|
|
657
|
+
result.chunks += outcome.descriptions.length;
|
|
658
|
+
result.totalCharacters += outcome.descriptions.reduce(
|
|
659
|
+
(sum, d) => sum + d.description.length,
|
|
660
|
+
0
|
|
661
|
+
);
|
|
574
662
|
}
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
663
|
+
yield { schemaVersion: 1, type: "result", stage: "complete", ...result };
|
|
664
|
+
} catch (err) {
|
|
665
|
+
yield this.ingestErrorEvent(err, "enrichment", result);
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
/**
|
|
669
|
+
* The ONE enrichment callback engine, shared by {@link ingestFileEvents}
|
|
670
|
+
* and {@link graphEnrichFileEvents}: fetches extracted images, invokes
|
|
671
|
+
* `describeImage` with bounded concurrency, invokes `extractEntitiesV2`
|
|
672
|
+
* exactly once, and POSTs `/files/{fileId}/enrich` — emitting progress and
|
|
673
|
+
* heartbeat events around each slow step. Throws on any failure; the
|
|
674
|
+
* purpose-specific wrappers translate that into their terminal error.
|
|
675
|
+
*
|
|
676
|
+
* The purpose controls empty-output semantics: ingest skips an optional
|
|
677
|
+
* empty add-on; graph-only finalizes an extractor that ran and found zero
|
|
678
|
+
* entities, but refuses to erase prior graph state when no extractable input
|
|
679
|
+
* reached the callback at all.
|
|
680
|
+
*/
|
|
681
|
+
// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: image-describe / extract-entities-v2 / no-op are documented decision branches; the fan-out covers the v0.15.0 enrichment phase + heartbeat plumbing.
|
|
682
|
+
async *runEnrichmentCallbacks(file, enrichment, callbacks, signal, purpose) {
|
|
683
|
+
const { fileId, token, expiresAt, images } = enrichment;
|
|
684
|
+
const isV2 = "version" in enrichment;
|
|
685
|
+
const textWindows = isV2 ? enrichment.textWindows : [];
|
|
686
|
+
const descriptions = [];
|
|
687
|
+
const describeImage = callbacks.describeImage;
|
|
688
|
+
if (describeImage && images.length > 0) {
|
|
689
|
+
yield {
|
|
690
|
+
schemaVersion: 1,
|
|
691
|
+
type: "progress",
|
|
692
|
+
stage: "enrichment",
|
|
693
|
+
filename: file.name,
|
|
694
|
+
imagesTotal: images.length,
|
|
695
|
+
imagesDescribed: 0,
|
|
696
|
+
message: `Describing extracted images (0/${images.length})`
|
|
697
|
+
};
|
|
698
|
+
const CONCURRENCY = 5;
|
|
699
|
+
for (let i = 0; i < images.length; i += CONCURRENCY) {
|
|
700
|
+
const batch = images.slice(i, i + CONCURRENCY);
|
|
701
|
+
const batchResults = yield* this.withSdkHeartbeats(
|
|
702
|
+
"enrichment",
|
|
703
|
+
Promise.all(
|
|
704
|
+
batch.map(async (imageMeta) => {
|
|
705
|
+
const imageRes = await fetch(
|
|
706
|
+
`${this.baseUrl}/api/memory/files/${fileId}/images/${imageMeta.imageId}?token=${encodeURIComponent(token)}`,
|
|
707
|
+
{ headers: this._authHeaders, signal }
|
|
708
|
+
);
|
|
709
|
+
if (!imageRes.ok) {
|
|
710
|
+
throw new MemoryServerError(
|
|
711
|
+
`Failed to fetch image ${imageMeta.imageId}: ${imageRes.status}`,
|
|
712
|
+
imageRes.status
|
|
713
|
+
);
|
|
714
|
+
}
|
|
715
|
+
const imageBuffer = await imageRes.arrayBuffer();
|
|
716
|
+
const description = await describeImage(imageBuffer, imageMeta);
|
|
717
|
+
return { imageId: imageMeta.imageId, description };
|
|
718
|
+
})
|
|
719
|
+
),
|
|
720
|
+
signal
|
|
721
|
+
);
|
|
722
|
+
descriptions.push(...batchResults.filter((d) => d.description.trim().length > 0));
|
|
579
723
|
yield {
|
|
580
724
|
schemaVersion: 1,
|
|
581
725
|
type: "progress",
|
|
582
726
|
stage: "enrichment",
|
|
583
727
|
filename: file.name,
|
|
584
|
-
|
|
728
|
+
imagesTotal: images.length,
|
|
729
|
+
imagesDescribed: descriptions.length,
|
|
730
|
+
message: `Described ${descriptions.length}/${images.length} images`
|
|
585
731
|
};
|
|
586
|
-
const extracted = yield* this.withSdkHeartbeats(
|
|
587
|
-
"enrichment",
|
|
588
|
-
options.enrichment.extractEntitiesV2({
|
|
589
|
-
textWindows,
|
|
590
|
-
imageDescriptions: imageDescriptionTexts,
|
|
591
|
-
mimeType: file.type,
|
|
592
|
-
filename: file.name
|
|
593
|
-
}),
|
|
594
|
-
signal
|
|
595
|
-
);
|
|
596
|
-
entities = extracted.entities;
|
|
597
|
-
relationships = extracted.relationships;
|
|
598
|
-
}
|
|
599
|
-
const hasGraph = (entities?.length ?? 0) > 0;
|
|
600
|
-
const hasImages = descriptions.length > 0;
|
|
601
|
-
if (!hasGraph && !hasImages) {
|
|
602
|
-
yield { schemaVersion: 1, type: "result", stage: "complete", ...result };
|
|
603
|
-
return;
|
|
604
732
|
}
|
|
733
|
+
}
|
|
734
|
+
let entities;
|
|
735
|
+
let relationships;
|
|
736
|
+
const imageDescriptionTexts = descriptions.map((d) => d.description);
|
|
737
|
+
if (callbacks.extractEntitiesV2 && (textWindows.length > 0 || imageDescriptionTexts.length > 0)) {
|
|
605
738
|
yield {
|
|
606
739
|
schemaVersion: 1,
|
|
607
740
|
type: "progress",
|
|
608
741
|
stage: "enrichment",
|
|
609
742
|
filename: file.name,
|
|
610
|
-
message: "
|
|
743
|
+
message: "Extracting entities"
|
|
611
744
|
};
|
|
612
|
-
const
|
|
745
|
+
const rawExtracted = yield* this.withSdkHeartbeats(
|
|
613
746
|
"enrichment",
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
...this._authHeaders
|
|
620
|
-
},
|
|
621
|
-
signal,
|
|
622
|
-
body: JSON.stringify({ imageDescriptions: descriptions, entities, relationships })
|
|
747
|
+
callbacks.extractEntitiesV2({
|
|
748
|
+
textWindows,
|
|
749
|
+
imageDescriptions: imageDescriptionTexts,
|
|
750
|
+
mimeType: file.type,
|
|
751
|
+
filename: file.name
|
|
623
752
|
}),
|
|
624
753
|
signal
|
|
625
754
|
);
|
|
626
|
-
|
|
627
|
-
|
|
755
|
+
let extracted;
|
|
756
|
+
try {
|
|
757
|
+
extracted = assertGraphExtractionPayload(rawExtracted, "extractEntitiesV2 result");
|
|
758
|
+
} catch (error) {
|
|
628
759
|
throw new MemoryServerError(
|
|
629
|
-
|
|
630
|
-
|
|
760
|
+
error instanceof Error ? error.message : "extractEntitiesV2 returned an invalid result",
|
|
761
|
+
422
|
|
631
762
|
);
|
|
632
763
|
}
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
764
|
+
entities = extracted.entities;
|
|
765
|
+
relationships = extracted.relationships;
|
|
766
|
+
}
|
|
767
|
+
const hadExtractionInput = textWindows.length > 0 || imageDescriptionTexts.length > 0;
|
|
768
|
+
if (purpose === "graph-only" && !hadExtractionInput) {
|
|
769
|
+
throw new MemoryServerError(
|
|
770
|
+
"Graph enrichment produced no extractable text or image descriptions; prior graph state was preserved",
|
|
771
|
+
422
|
|
772
|
+
);
|
|
773
|
+
}
|
|
774
|
+
const hasGraph = (entities?.length ?? 0) > 0;
|
|
775
|
+
const hasImages = descriptions.length > 0;
|
|
776
|
+
if (!hasGraph && !hasImages && purpose === "ingest") {
|
|
777
|
+
return { posted: false, data: null, descriptions };
|
|
778
|
+
}
|
|
779
|
+
yield {
|
|
780
|
+
schemaVersion: 1,
|
|
781
|
+
type: "progress",
|
|
782
|
+
stage: "enrichment",
|
|
783
|
+
filename: file.name,
|
|
784
|
+
message: "Persisting enrichment"
|
|
785
|
+
};
|
|
786
|
+
const enrichRes = yield* this.withSdkHeartbeats(
|
|
787
|
+
"enrichment",
|
|
788
|
+
fetch(`${this.baseUrl}/api/memory/files/${fileId}/enrich`, {
|
|
789
|
+
method: "POST",
|
|
790
|
+
headers: {
|
|
791
|
+
"Content-Type": "application/json",
|
|
792
|
+
"X-Enrichment-Token": `${token}:${expiresAt}`,
|
|
793
|
+
...this._authHeaders
|
|
794
|
+
},
|
|
795
|
+
signal,
|
|
796
|
+
body: JSON.stringify({ imageDescriptions: descriptions, entities, relationships })
|
|
797
|
+
}),
|
|
798
|
+
signal
|
|
799
|
+
);
|
|
800
|
+
if (!enrichRes.ok) {
|
|
801
|
+
const body = await enrichRes.json().catch(() => ({}));
|
|
802
|
+
throw new MemoryServerError(
|
|
803
|
+
body.error ?? `Enrichment failed: ${enrichRes.status}`,
|
|
804
|
+
enrichRes.status
|
|
805
|
+
);
|
|
640
806
|
}
|
|
807
|
+
const data = await this.parseApiResponse(enrichRes);
|
|
808
|
+
return { posted: true, data, descriptions };
|
|
641
809
|
}
|
|
642
810
|
/**
|
|
643
811
|
* Race a Promise against a periodic heartbeat tick. Yields a heartbeat
|
|
@@ -1022,6 +1190,9 @@ export {
|
|
|
1022
1190
|
DEFAULTS,
|
|
1023
1191
|
TAXONOMY_MAX_CATEGORIES,
|
|
1024
1192
|
projectSearchResponseForMcp,
|
|
1193
|
+
secretElevationNoticeFor,
|
|
1194
|
+
withSecretElevationNotice,
|
|
1195
|
+
secretElevationAggregate,
|
|
1025
1196
|
NamespaceIsolation,
|
|
1026
1197
|
MemoryType,
|
|
1027
1198
|
SensitivityLevel,
|
package/dist/cli/pyx-mem.mjs
CHANGED
|
@@ -835,7 +835,7 @@ function createProxyServer(client, version, uploadLocalFile) {
|
|
|
835
835
|
return server;
|
|
836
836
|
}
|
|
837
837
|
async function runMcpProxyServer(opts) {
|
|
838
|
-
const version = opts.version ?? (true ? "1.17.
|
|
838
|
+
const version = opts.version ?? (true ? "1.17.16" : "0.0.0-dev");
|
|
839
839
|
const read = await opts.readCredentials();
|
|
840
840
|
if (!read.ok) {
|
|
841
841
|
const text = read.result.content.map((c) => c.type === "text" ? c.text : "").join(" ").trim();
|
package/dist/dashboard.mjs
CHANGED
|
@@ -11,9 +11,9 @@ import {
|
|
|
11
11
|
toGraphologyFormat,
|
|
12
12
|
transformGraphData,
|
|
13
13
|
unreachableHealth
|
|
14
|
-
} from "./chunk-
|
|
15
|
-
import "./chunk-
|
|
16
|
-
import "./chunk-
|
|
14
|
+
} from "./chunk-ZVI7DCB4.mjs";
|
|
15
|
+
import "./chunk-MDFUZ3V2.mjs";
|
|
16
|
+
import "./chunk-34MTVIYK.mjs";
|
|
17
17
|
export {
|
|
18
18
|
DashboardClient,
|
|
19
19
|
Poller,
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { StoreInput as StoreInput$1, MemoryEntry as MemoryEntry$1, MemorySearchParams as MemorySearchParams$1, MemorySearchResult as MemorySearchResult$1, MemoryType as MemoryType$1, PrincipalContext as PrincipalContext$1, SensitivityLevel as SensitivityLevel$1, MemoryStats as MemoryStats$1, LineageParams as LineageParams$1, LineageResult as LineageResult$1, ReinforceParams as ReinforceParams$1, ReinforceResult as ReinforceResult$1, WikiLintReport as WikiLintReport$1, GraphRepairResult as GraphRepairResult$1, ExtractedImageMeta as ExtractedImageMeta$1, IngestEntity as IngestEntity$1, IngestRelationship as IngestRelationship$1, EntityExtractionResult as EntityExtractionResult$1, Topology as Topology$1, IngestEvent as IngestEvent$1, GraphNode as GraphNode$1, GraphTraversalResult as GraphTraversalResult$1, CorrectionRecord as CorrectionRecord$1 } from '@pyx-memory/shared';
|
|
1
|
+
import { StoreInput as StoreInput$1, MemoryEntry as MemoryEntry$1, MemorySearchParams as MemorySearchParams$1, MemorySearchResult as MemorySearchResult$1, MemoryType as MemoryType$1, PrincipalContext as PrincipalContext$1, SensitivityLevel as SensitivityLevel$1, MemoryStats as MemoryStats$1, LineageParams as LineageParams$1, LineageResult as LineageResult$1, ReinforceParams as ReinforceParams$1, ReinforceResult as ReinforceResult$1, WikiLintReport as WikiLintReport$1, GraphRepairResult as GraphRepairResult$1, ExtractedImageMeta as ExtractedImageMeta$1, IngestEntity as IngestEntity$1, IngestRelationship as IngestRelationship$1, EntityExtractionResult as EntityExtractionResult$1, Topology as Topology$1, IngestEvent as IngestEvent$1, GraphEnrichEvent as GraphEnrichEvent$1, GraphNode as GraphNode$1, GraphTraversalResult as GraphTraversalResult$1, CorrectionRecord as CorrectionRecord$1 } from '@pyx-memory/shared';
|
|
2
2
|
|
|
3
3
|
/** Parameters for paginated entry listing. */
|
|
4
4
|
interface MemoryListParams {
|
|
@@ -247,6 +247,22 @@ interface IngestFileOptions {
|
|
|
247
247
|
signal?: AbortSignal;
|
|
248
248
|
namespaceId?: string;
|
|
249
249
|
}
|
|
250
|
+
/**
|
|
251
|
+
* Options for {@link MemoryClient.graphEnrichFileEvents}. `documentKey` is the
|
|
252
|
+
* caller's stable logical identity for the document whose graph is being
|
|
253
|
+
* rebuilt — the server derives the graph anchor id from
|
|
254
|
+
* (tenant, namespace, documentKey), so repeating the same key replaces the
|
|
255
|
+
* document's graph references instead of multiplying them.
|
|
256
|
+
*
|
|
257
|
+
* `enrichment.extractEntitiesV2` is required: graph-only re-enrichment is
|
|
258
|
+
* caller-extraction by definition (there is no server-side fallback).
|
|
259
|
+
*/
|
|
260
|
+
interface GraphEnrichFileOptions {
|
|
261
|
+
documentKey: string;
|
|
262
|
+
namespaceId?: string;
|
|
263
|
+
signal?: AbortSignal;
|
|
264
|
+
enrichment: EnrichmentCallbacks;
|
|
265
|
+
}
|
|
250
266
|
/**
|
|
251
267
|
* Caller-supplied enrichment for the per-call store path. Mirrors
|
|
252
268
|
* {@link EnrichmentCallbacks} for file ingest. When supplied, the SDK invokes
|
|
@@ -360,13 +376,50 @@ declare class MemoryClient implements ExtendedMemoryInterface {
|
|
|
360
376
|
*/
|
|
361
377
|
ingestFileEvents(file: File, options?: IngestFileOptions): AsyncIterable<IngestEvent$1>;
|
|
362
378
|
/**
|
|
363
|
-
*
|
|
364
|
-
*
|
|
365
|
-
*
|
|
366
|
-
*
|
|
367
|
-
*
|
|
379
|
+
* Graph-only re-enrichment for a document whose chunks are already stored
|
|
380
|
+
* and searchable but whose graph build failed. Uploads the original to
|
|
381
|
+
* `/api/memory/graph/enrich/file` (prepare-only — the server performs zero
|
|
382
|
+
* store/delete before the final graph write), runs the SAME enrichment
|
|
383
|
+
* callback engine as {@link ingestFileEvents}, then finalizes into one
|
|
384
|
+
* stable graph anchor keyed by `documentKey`. Repeating the same key
|
|
385
|
+
* replaces the document's graph references idempotently.
|
|
386
|
+
*
|
|
387
|
+
* The terminal `result` is a {@link GraphEnrichResult} event carrying the
|
|
388
|
+
* ACTUAL persisted graph counts; abort and failures yield a terminal
|
|
389
|
+
* `error` event instead (the server retains the pending session for retry).
|
|
390
|
+
*/
|
|
391
|
+
graphEnrichFileEvents(file: File, options: GraphEnrichFileOptions): AsyncIterable<GraphEnrichEvent$1>;
|
|
392
|
+
/**
|
|
393
|
+
* POST a multipart body to an NDJSON streaming endpoint and relay its
|
|
394
|
+
* progress/heartbeat events. Returns the raw terminal `result` record, or
|
|
395
|
+
* null after yielding a terminal error (transport failure, server error
|
|
396
|
+
* event, non-NDJSON response, stream ending without a result). One
|
|
397
|
+
* implementation for both streaming surfaces so the wire protocol cannot
|
|
398
|
+
* fork.
|
|
399
|
+
*/
|
|
400
|
+
private streamNdjsonUpload;
|
|
401
|
+
/**
|
|
402
|
+
* Run the SDK-side enrichment phase for an ingest result and yield the
|
|
403
|
+
* single terminal {@link IngestResultEvent} at the end. Skips work cleanly
|
|
404
|
+
* when the server emitted no enrichment block or the caller wired no
|
|
405
|
+
* callbacks. The callback work itself lives in
|
|
406
|
+
* {@link runEnrichmentCallbacks} — shared with the graph-only surface.
|
|
368
407
|
*/
|
|
369
408
|
private completeIngestFileEvents;
|
|
409
|
+
/**
|
|
410
|
+
* The ONE enrichment callback engine, shared by {@link ingestFileEvents}
|
|
411
|
+
* and {@link graphEnrichFileEvents}: fetches extracted images, invokes
|
|
412
|
+
* `describeImage` with bounded concurrency, invokes `extractEntitiesV2`
|
|
413
|
+
* exactly once, and POSTs `/files/{fileId}/enrich` — emitting progress and
|
|
414
|
+
* heartbeat events around each slow step. Throws on any failure; the
|
|
415
|
+
* purpose-specific wrappers translate that into their terminal error.
|
|
416
|
+
*
|
|
417
|
+
* The purpose controls empty-output semantics: ingest skips an optional
|
|
418
|
+
* empty add-on; graph-only finalizes an extractor that ran and found zero
|
|
419
|
+
* entities, but refuses to erase prior graph state when no extractable input
|
|
420
|
+
* reached the callback at all.
|
|
421
|
+
*/
|
|
422
|
+
private runEnrichmentCallbacks;
|
|
370
423
|
/**
|
|
371
424
|
* Race a Promise against a periodic heartbeat tick. Yields a heartbeat
|
|
372
425
|
* IngestEvent every {@link INGEST_EVENT_HEARTBEAT_MS} until the promise
|
|
@@ -1274,6 +1327,10 @@ interface EntityExtractionResult {
|
|
|
1274
1327
|
type: string;
|
|
1275
1328
|
}>;
|
|
1276
1329
|
}
|
|
1330
|
+
interface GraphExtractionPayload {
|
|
1331
|
+
entities: IngestEntity[];
|
|
1332
|
+
relationships: IngestRelationship[];
|
|
1333
|
+
}
|
|
1277
1334
|
declare function normalizeGraphLabel(value: string, fallback: string): string;
|
|
1278
1335
|
/**
|
|
1279
1336
|
* Identity key for a graph node: the normalized name alone. `type` is a node
|
|
@@ -1287,6 +1344,16 @@ declare function normalizeGraphLabel(value: string, fallback: string): string;
|
|
|
1287
1344
|
* edge resolvability with it — all three MUST agree, so they import this one fn.
|
|
1288
1345
|
*/
|
|
1289
1346
|
declare function normalizeNameKey(name: string): string;
|
|
1347
|
+
/**
|
|
1348
|
+
* Runtime contract for destructive graph replacement payloads. TypeScript
|
|
1349
|
+
* callback types do not protect JavaScript callers or malformed LLM adapter
|
|
1350
|
+
* results; accepting a missing `entities` field as an empty extraction would
|
|
1351
|
+
* erase the prior graph. Both arrays are therefore required, every item is
|
|
1352
|
+
* structurally validated, and every relationship endpoint must resolve to a
|
|
1353
|
+
* declared entity under the graph store's exact name identity normalizer.
|
|
1354
|
+
* A genuinely empty `{ entities: [], relationships: [] }` remains valid.
|
|
1355
|
+
*/
|
|
1356
|
+
declare function assertGraphExtractionPayload(value: unknown, field?: string): GraphExtractionPayload;
|
|
1290
1357
|
/**
|
|
1291
1358
|
* Merge caller-provided entities/relationships with LLM-extracted ones.
|
|
1292
1359
|
*
|
|
@@ -1307,6 +1374,47 @@ declare function mergeExtractedEntities(callerEntities: IngestEntity[] | undefin
|
|
|
1307
1374
|
|
|
1308
1375
|
declare function projectSearchResponseForMcp(payload: unknown): unknown;
|
|
1309
1376
|
|
|
1377
|
+
/**
|
|
1378
|
+
* Secret-elevation notice — one implementation of the predicate and the notice
|
|
1379
|
+
* text for every store surface that can silently lose an entry to automatic
|
|
1380
|
+
* credential classification: the strict hosted MCP store branch, the self-host
|
|
1381
|
+
* MCP store tool, and file ingestion. Three copies of this text drifting apart
|
|
1382
|
+
* is the failure mode this module exists to prevent.
|
|
1383
|
+
*/
|
|
1384
|
+
/** Notice attached to a single auto-elevated store response. */
|
|
1385
|
+
interface SecretElevationNotice {
|
|
1386
|
+
sensitivity: 'secret';
|
|
1387
|
+
credentialTypes: string[];
|
|
1388
|
+
message: string;
|
|
1389
|
+
}
|
|
1390
|
+
/** Aggregate notice for one file ingest, where many chunks store at once. */
|
|
1391
|
+
interface SecretElevationAggregate {
|
|
1392
|
+
count: number;
|
|
1393
|
+
entryIds: string[];
|
|
1394
|
+
credentialTypes: string[];
|
|
1395
|
+
message: string;
|
|
1396
|
+
}
|
|
1397
|
+
/**
|
|
1398
|
+
* D3 predicate: the entry was elevated by the classifier, not by the caller.
|
|
1399
|
+
* A deliberate `sensitivity: 'secret'` store carries no `credentialsDetected`
|
|
1400
|
+
* flag and gets no notice.
|
|
1401
|
+
*/
|
|
1402
|
+
declare function secretElevationNoticeFor(entry: unknown): SecretElevationNotice | undefined;
|
|
1403
|
+
/**
|
|
1404
|
+
* Attach the notice to a store response, returning the payload untouched when
|
|
1405
|
+
* the entry was not auto-elevated. Accepts either the bare stored entry (strict
|
|
1406
|
+
* hosted path) or the `{ success, data }` envelope the SDK http client surfaces
|
|
1407
|
+
* (`http-client.ts` sets `res.data` to the whole envelope, so the entry sits at
|
|
1408
|
+
* `payload.data`). Never mutates its argument: the stored entry and the
|
|
1409
|
+
* canonical payload the receipt digest covers must stay byte-identical.
|
|
1410
|
+
*/
|
|
1411
|
+
declare function withSecretElevationNotice(payload: unknown): unknown;
|
|
1412
|
+
/** Aggregate one file ingest's per-chunk notices. Undefined when nothing elevated. */
|
|
1413
|
+
declare function secretElevationAggregate(elevated: Array<{
|
|
1414
|
+
entryId: string;
|
|
1415
|
+
notice: SecretElevationNotice;
|
|
1416
|
+
}>): SecretElevationAggregate | undefined;
|
|
1417
|
+
|
|
1310
1418
|
interface ApiResponse<T> {
|
|
1311
1419
|
success: boolean;
|
|
1312
1420
|
data?: T;
|
|
@@ -1419,6 +1527,8 @@ interface FileIngestResult {
|
|
|
1419
1527
|
totalCharacters: number;
|
|
1420
1528
|
/** Present when images were extracted (v1) OR text windows / images were emitted (v2). */
|
|
1421
1529
|
enrichment?: EnrichmentPending;
|
|
1530
|
+
/** Present only when ≥1 stored chunk was auto-classified secret by credential detection. */
|
|
1531
|
+
secretElevation?: SecretElevationAggregate;
|
|
1422
1532
|
}
|
|
1423
1533
|
/** Coarse pipeline stages. Stable vocabulary — finer detail goes in counters/message. */
|
|
1424
1534
|
type IngestStage = 'parsing' | 'storing' | 'enrichment' | 'complete';
|
|
@@ -1479,6 +1589,56 @@ interface IngestErrorEvent {
|
|
|
1479
1589
|
partialResult?: FileIngestResult;
|
|
1480
1590
|
}
|
|
1481
1591
|
type IngestEvent = IngestProgressEvent | IngestHeartbeatEvent | IngestResultEvent | IngestErrorEvent;
|
|
1592
|
+
/** What a pending enrichment session finalizes into. Sessions created before
|
|
1593
|
+
* graph-only support carry no purpose and behave as `'ingest'`. */
|
|
1594
|
+
type EnrichmentPurpose = 'ingest' | 'graph-only';
|
|
1595
|
+
/**
|
|
1596
|
+
* Terminal server event for POST /api/memory/graph/enrich/file. Deliberately
|
|
1597
|
+
* NOT an {@link IngestResultEvent}: the prepare phase stores nothing, so there
|
|
1598
|
+
* are no chunk counters to report — fabricating `chunks: 0` would make the
|
|
1599
|
+
* result indistinguishable from a real empty ingest.
|
|
1600
|
+
*/
|
|
1601
|
+
interface GraphEnrichPreparedEvent {
|
|
1602
|
+
schemaVersion: 1;
|
|
1603
|
+
type: 'result';
|
|
1604
|
+
stage: 'complete';
|
|
1605
|
+
purpose: 'graph-only';
|
|
1606
|
+
filename: string;
|
|
1607
|
+
fileType: string;
|
|
1608
|
+
enrichment: EnrichmentPendingV2;
|
|
1609
|
+
message?: string;
|
|
1610
|
+
}
|
|
1611
|
+
/**
|
|
1612
|
+
* Terminal result of a graph-only re-enrichment run — also the response body
|
|
1613
|
+
* of POST /files/{fileId}/enrich when the pending session's purpose is
|
|
1614
|
+
* `'graph-only'`. Counts reflect ACTUAL graph persistence (mirrors
|
|
1615
|
+
* {@link EnrichResult}), never the submitted payload sizes.
|
|
1616
|
+
*/
|
|
1617
|
+
interface GraphEnrichResult {
|
|
1618
|
+
purpose: 'graph-only';
|
|
1619
|
+
filename: string;
|
|
1620
|
+
fileType: string;
|
|
1621
|
+
/** The single stable anchor entry id, including a successful zero-entity replacement. */
|
|
1622
|
+
entryIds: string[];
|
|
1623
|
+
entitiesStored: number;
|
|
1624
|
+
relationshipsStored: number;
|
|
1625
|
+
/** Present (and >0) only when edges were dropped — see {@link EnrichResult}. */
|
|
1626
|
+
relationshipsDropped?: number;
|
|
1627
|
+
droppedRelationships?: DroppedGraphRelationship[];
|
|
1628
|
+
}
|
|
1629
|
+
/** Terminal success of the SDK's graph-only stream — exactly one per run. */
|
|
1630
|
+
interface GraphEnrichResultEvent extends GraphEnrichResult {
|
|
1631
|
+
schemaVersion: 1;
|
|
1632
|
+
type: 'result';
|
|
1633
|
+
stage: 'complete';
|
|
1634
|
+
message?: string;
|
|
1635
|
+
}
|
|
1636
|
+
/**
|
|
1637
|
+
* Typed view of the SDK's `graphEnrichFileEvents()` stream. Progress /
|
|
1638
|
+
* heartbeat / error envelopes are shared with the ingest stream (same wire
|
|
1639
|
+
* protocol); only the terminal result differs.
|
|
1640
|
+
*/
|
|
1641
|
+
type GraphEnrichEvent = IngestProgressEvent | IngestHeartbeatEvent | GraphEnrichResultEvent | IngestErrorEvent;
|
|
1482
1642
|
|
|
1483
1643
|
/**
|
|
1484
1644
|
* Namespace topology-isolation modes for v0.17.0.
|
|
@@ -1593,4 +1753,4 @@ interface CreatePyxMemoryOptions {
|
|
|
1593
1753
|
}
|
|
1594
1754
|
declare function createPyxMemory(opts?: CreatePyxMemoryOptions): MemoryClient;
|
|
1595
1755
|
|
|
1596
|
-
export { type AgentId, type ApiResponse, type ConsolidationRunResult, type CorrectionInput, type CorrectionRecord, type CreatePyxMemoryOptions, DEFAULTS, DEPRECATED_RAG_STRATEGIES, DisabledMemory, type DroppedGraphRelationship, type DueScanInput, EmbeddingProviderName, type EnrichmentCallbacks, type EntityExtractionResult, type ExtendedMemoryInterface, type FetchCorrectionsInput, type GraphEnrichment, type GraphEnrichmentStatus, type GraphFailureMode, type GraphNode, type GraphRelationship, type GraphRepairResult, type GraphTelemetrySnapshot, type GraphTraversalResult, type IngestEntity, type IngestErrorEvent, type IngestEvent, type IngestFileOptions, type IngestHeartbeatEvent, type IngestProgressEvent, type IngestRelationship, type IngestResultEvent, type IngestStage, type IngestionResult, type LineageParams, type LineageResult, type LineageVersion, MemoryClient, type MemoryClientOptions, type MemoryEntry, type MemoryIngestRequest, type MemoryInterface, type MemoryListParams, type MemoryListResult, type MemoryLogFilters, type MemorySearchParams, type MemorySearchResult, MemoryServerError, type MemoryStats, MemoryType, type MoveEntriesFilter, MoveFailureReason, type MoveResult, type MoveTarget, NamespaceIsolation, type PrincipalContext, RAGStrategy, type ReinforceParams, type ReinforceResult, type ReinforceSignal, SINGLE_TENANT_ID, SensitivityLevel, type SourceEvidence, type StoreInput, StoreTarget, TAXONOMY_MAX_CATEGORIES, type TemporalQueryFilters, type TenantScopeOptions, type Timestamp, type Topology, type TopologyExtractionProvider, type TopologyServiceVariant, type UsageHygieneSnapshot, VectorProvider, type VectorStatus, type WikiLintReport, createPyxMemory, mergeExtractedEntities, normalizeGraphLabel, normalizeNameKey, projectSearchResponseForMcp };
|
|
1756
|
+
export { type AgentId, type ApiResponse, type ConsolidationRunResult, type CorrectionInput, type CorrectionRecord, type CreatePyxMemoryOptions, DEFAULTS, DEPRECATED_RAG_STRATEGIES, DisabledMemory, type DroppedGraphRelationship, type DueScanInput, EmbeddingProviderName, type EnrichmentCallbacks, type EnrichmentPurpose, type EntityExtractionResult, type ExtendedMemoryInterface, type FetchCorrectionsInput, type GraphEnrichEvent, type GraphEnrichFileOptions, type GraphEnrichPreparedEvent, type GraphEnrichResult, type GraphEnrichResultEvent, type GraphEnrichment, type GraphEnrichmentStatus, type GraphExtractionPayload, type GraphFailureMode, type GraphNode, type GraphRelationship, type GraphRepairResult, type GraphTelemetrySnapshot, type GraphTraversalResult, type IngestEntity, type IngestErrorEvent, type IngestEvent, type IngestFileOptions, type IngestHeartbeatEvent, type IngestProgressEvent, type IngestRelationship, type IngestResultEvent, type IngestStage, type IngestionResult, type LineageParams, type LineageResult, type LineageVersion, MemoryClient, type MemoryClientOptions, type MemoryEntry, type MemoryIngestRequest, type MemoryInterface, type MemoryListParams, type MemoryListResult, type MemoryLogFilters, type MemorySearchParams, type MemorySearchResult, MemoryServerError, type MemoryStats, MemoryType, type MoveEntriesFilter, MoveFailureReason, type MoveResult, type MoveTarget, NamespaceIsolation, type PrincipalContext, RAGStrategy, type ReinforceParams, type ReinforceResult, type ReinforceSignal, SINGLE_TENANT_ID, type SecretElevationAggregate, type SecretElevationNotice, SensitivityLevel, type SourceEvidence, type StoreInput, StoreTarget, TAXONOMY_MAX_CATEGORIES, type TemporalQueryFilters, type TenantScopeOptions, type Timestamp, type Topology, type TopologyExtractionProvider, type TopologyServiceVariant, type UsageHygieneSnapshot, VectorProvider, type VectorStatus, type WikiLintReport, assertGraphExtractionPayload, createPyxMemory, mergeExtractedEntities, normalizeGraphLabel, normalizeNameKey, projectSearchResponseForMcp, secretElevationAggregate, secretElevationNoticeFor, withSecretElevationNotice };
|
package/dist/index.mjs
CHANGED
|
@@ -14,13 +14,17 @@ import {
|
|
|
14
14
|
StoreTarget,
|
|
15
15
|
TAXONOMY_MAX_CATEGORIES,
|
|
16
16
|
VectorProvider,
|
|
17
|
-
projectSearchResponseForMcp
|
|
18
|
-
|
|
17
|
+
projectSearchResponseForMcp,
|
|
18
|
+
secretElevationAggregate,
|
|
19
|
+
secretElevationNoticeFor,
|
|
20
|
+
withSecretElevationNotice
|
|
21
|
+
} from "./chunk-MDFUZ3V2.mjs";
|
|
19
22
|
import {
|
|
23
|
+
assertGraphExtractionPayload,
|
|
20
24
|
mergeExtractedEntities,
|
|
21
25
|
normalizeGraphLabel,
|
|
22
26
|
normalizeNameKey
|
|
23
|
-
} from "./chunk-
|
|
27
|
+
} from "./chunk-34MTVIYK.mjs";
|
|
24
28
|
|
|
25
29
|
// src/preset.ts
|
|
26
30
|
var DEFAULT_MEMORY_URL = `http://localhost:${DEFAULTS.MEMORY_SERVER_PORT}`;
|
|
@@ -51,9 +55,13 @@ export {
|
|
|
51
55
|
StoreTarget,
|
|
52
56
|
TAXONOMY_MAX_CATEGORIES,
|
|
53
57
|
VectorProvider,
|
|
58
|
+
assertGraphExtractionPayload,
|
|
54
59
|
createPyxMemory,
|
|
55
60
|
mergeExtractedEntities,
|
|
56
61
|
normalizeGraphLabel,
|
|
57
62
|
normalizeNameKey,
|
|
58
|
-
projectSearchResponseForMcp
|
|
63
|
+
projectSearchResponseForMcp,
|
|
64
|
+
secretElevationAggregate,
|
|
65
|
+
secretElevationNoticeFor,
|
|
66
|
+
withSecretElevationNotice
|
|
59
67
|
};
|
package/dist/react.mjs
CHANGED
|
@@ -11,9 +11,9 @@ import {
|
|
|
11
11
|
toGraphologyFormat,
|
|
12
12
|
transformGraphData,
|
|
13
13
|
unreachableHealth
|
|
14
|
-
} from "./chunk-
|
|
15
|
-
import "./chunk-
|
|
16
|
-
import "./chunk-
|
|
14
|
+
} from "./chunk-ZVI7DCB4.mjs";
|
|
15
|
+
import "./chunk-MDFUZ3V2.mjs";
|
|
16
|
+
import "./chunk-34MTVIYK.mjs";
|
|
17
17
|
|
|
18
18
|
// ../dashboard/src/hooks/use-consolidation-log.ts
|
|
19
19
|
import { useCallback as useCallback2, useMemo } from "react";
|
package/package.json
CHANGED
package/dist/chunk-A3L46P2G.mjs
DELETED
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
// ../shared/src/graph/extraction.ts
|
|
2
|
-
function normalizeGraphLabel(value, fallback) {
|
|
3
|
-
const normalized = value.trim().toUpperCase().replace(/[^A-Z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
4
|
-
return normalized.length > 0 ? normalized : fallback;
|
|
5
|
-
}
|
|
6
|
-
function normalizeNameKey(name) {
|
|
7
|
-
return name.trim().toLowerCase().replace(/\s+/g, " ");
|
|
8
|
-
}
|
|
9
|
-
function relationshipKey(relationship) {
|
|
10
|
-
return [
|
|
11
|
-
relationship.source.trim().toLowerCase(),
|
|
12
|
-
relationship.target.trim().toLowerCase(),
|
|
13
|
-
normalizeGraphLabel(relationship.type, "RELATED_TO")
|
|
14
|
-
].join("\0");
|
|
15
|
-
}
|
|
16
|
-
function mergeExtractedEntities(callerEntities, callerRelationships, extracted) {
|
|
17
|
-
const entities = [...callerEntities ?? []];
|
|
18
|
-
const relationships = [...callerRelationships ?? []];
|
|
19
|
-
const nameByLowercase = /* @__PURE__ */ new Map();
|
|
20
|
-
for (const entity of entities) {
|
|
21
|
-
const key = entity.name.toLowerCase();
|
|
22
|
-
if (!nameByLowercase.has(key)) nameByLowercase.set(key, entity.name);
|
|
23
|
-
}
|
|
24
|
-
for (const entity of extracted.entities) {
|
|
25
|
-
const key = entity.name.toLowerCase();
|
|
26
|
-
if (nameByLowercase.has(key)) continue;
|
|
27
|
-
entities.push({ ...entity, type: normalizeGraphLabel(entity.type, "CONCEPT") });
|
|
28
|
-
nameByLowercase.set(key, entity.name);
|
|
29
|
-
}
|
|
30
|
-
for (const relationship of extracted.relations) {
|
|
31
|
-
const source = nameByLowercase.get(relationship.source.toLowerCase());
|
|
32
|
-
const target = nameByLowercase.get(relationship.target.toLowerCase());
|
|
33
|
-
if (source && target) {
|
|
34
|
-
relationships.push({
|
|
35
|
-
...relationship,
|
|
36
|
-
source,
|
|
37
|
-
target,
|
|
38
|
-
type: normalizeGraphLabel(relationship.type, "RELATED_TO")
|
|
39
|
-
});
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
const seenRelationships = /* @__PURE__ */ new Set();
|
|
43
|
-
const dedupedRelationships = [];
|
|
44
|
-
for (const relationship of relationships) {
|
|
45
|
-
const key = relationshipKey(relationship);
|
|
46
|
-
if (seenRelationships.has(key)) continue;
|
|
47
|
-
seenRelationships.add(key);
|
|
48
|
-
dedupedRelationships.push(relationship);
|
|
49
|
-
}
|
|
50
|
-
return { entities, relationships: dedupedRelationships };
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
export {
|
|
54
|
-
normalizeGraphLabel,
|
|
55
|
-
normalizeNameKey,
|
|
56
|
-
mergeExtractedEntities
|
|
57
|
-
};
|