@semiont/make-meaning 0.5.33 → 0.5.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/archivist-main.js +125 -102
- package/dist/archivist-main.js.map +1 -1
- package/dist/index.d.ts +3 -0
- package/dist/index.js +48 -38
- package/dist/index.js.map +1 -1
- package/dist/librarian-main.js +27 -9
- package/dist/librarian-main.js.map +1 -1
- package/dist/smelter-main.js +79 -44
- package/dist/smelter-main.js.map +1 -1
- package/dist/weaver-main.js +34 -12
- package/dist/weaver-main.js.map +1 -1
- package/package.json +12 -12
package/dist/archivist-main.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { HttpTransport } from '@semiont/http-transport';
|
|
2
|
-
import { replyChannelsFor, accessToken, EventBus, baseUrl as baseUrl$1, errField, PERSISTED_EVENT_TYPES, retryWithBackoff, isTransientFetchError, STARTUP_FETCH_RETRY, userId, resourceId, generateUuid, annotationId, getBodySource, getTargetSource, getTargetSelector, getExactText, getStorageUri, cloneToken, getResourceEntityTypes, didToAgent, assembleAnnotation, busRequest, getPrimaryRepresentation, decodeRepresentation, getResourceId, deriveViews, getTextPositionSelector, isAnnotatable, softwareToAgent } from '@semiont/core';
|
|
2
|
+
import { replyChannelsFor, accessToken, EventBus, withDeadline, baseUrl as baseUrl$1, errField, PERSISTED_EVENT_TYPES, retryWithBackoff, isTransientFetchError, STARTUP_FETCH_RETRY, userId, resourceId, generateUuid, annotationId, getBodySource, getTargetSource, getTargetSelector, getExactText, getStorageUri, cloneToken, getResourceEntityTypes, didToAgent, assembleAnnotation, busRequest, getPrimaryRepresentation, decodeRepresentation, getResourceId, deriveViews, getTextPositionSelector, isAnnotatable, softwareToAgent } from '@semiont/core';
|
|
3
3
|
import { loadEnvironmentConfig, SemiontProject } from '@semiont/core/node';
|
|
4
4
|
import { createEventStore, resolveStorageUri, EventQuery } from '@semiont/event-sourcing';
|
|
5
5
|
import { WorkingTreeStore, createAnchoredTextStore, RepresentationMissing, ChecksumMismatchError } from '@semiont/content';
|
|
@@ -13,6 +13,7 @@ import { createInferenceClient } from '@semiont/inference';
|
|
|
13
13
|
import { createServer } from 'http';
|
|
14
14
|
import { pipeline } from 'stream/promises';
|
|
15
15
|
import { createProcessLogger } from '@semiont/observability/process-logger';
|
|
16
|
+
import '@semiont/jobs';
|
|
16
17
|
|
|
17
18
|
var __create = Object.create;
|
|
18
19
|
var __defProp = Object.defineProperty;
|
|
@@ -10266,22 +10267,22 @@ var Stower = class {
|
|
|
10266
10267
|
throw new Error(entityTypesNotRegisteredMessage(result.unknown));
|
|
10267
10268
|
}
|
|
10268
10269
|
}
|
|
10269
|
-
for (const
|
|
10270
|
+
for (const entityType2 of added) {
|
|
10270
10271
|
await this.stores.eventStore.appendEvent({
|
|
10271
10272
|
type: "mark:entity-tag-added",
|
|
10272
10273
|
resourceId: resourceId(event.resourceId),
|
|
10273
10274
|
userId: uid,
|
|
10274
10275
|
version: 1,
|
|
10275
|
-
payload: { entityType }
|
|
10276
|
+
payload: { entityType: entityType2 }
|
|
10276
10277
|
});
|
|
10277
10278
|
}
|
|
10278
|
-
for (const
|
|
10279
|
+
for (const entityType2 of removed) {
|
|
10279
10280
|
await this.stores.eventStore.appendEvent({
|
|
10280
10281
|
type: "mark:entity-tag-removed",
|
|
10281
10282
|
resourceId: resourceId(event.resourceId),
|
|
10282
10283
|
userId: uid,
|
|
10283
10284
|
version: 1,
|
|
10284
|
-
payload: { entityType }
|
|
10285
|
+
payload: { entityType: entityType2 }
|
|
10285
10286
|
});
|
|
10286
10287
|
}
|
|
10287
10288
|
this.eventBus.get("mark:update-entity-types-ok").next({ correlationId: event.correlationId });
|
|
@@ -10372,8 +10373,8 @@ var import_operators2 = __toESM(require_operators());
|
|
|
10372
10373
|
|
|
10373
10374
|
// src/smelt-progress.ts
|
|
10374
10375
|
var SmeltProgressTimeout = class extends Error {
|
|
10375
|
-
constructor(
|
|
10376
|
-
super(`smelt:settled not observed for ${
|
|
10376
|
+
constructor(resourceId10, contentChecksum, timeoutMs) {
|
|
10377
|
+
super(`smelt:settled not observed for ${resourceId10} (checksum ${contentChecksum.slice(0, 12)}\u2026) within ${timeoutMs}ms`);
|
|
10377
10378
|
this.name = "SmeltProgressTimeout";
|
|
10378
10379
|
}
|
|
10379
10380
|
};
|
|
@@ -10384,7 +10385,7 @@ function createSmeltProgress(eventBus) {
|
|
|
10384
10385
|
const waiters = /* @__PURE__ */ new Set();
|
|
10385
10386
|
let disposed = false;
|
|
10386
10387
|
let lastSweep = Date.now();
|
|
10387
|
-
const subscription = eventBus.get("smelt:settled").subscribe(({ resourceId:
|
|
10388
|
+
const subscription = eventBus.get("smelt:settled").subscribe(({ resourceId: resourceId10, contentChecksum, outcome }) => {
|
|
10388
10389
|
const now = Date.now();
|
|
10389
10390
|
if (now - lastSweep >= SWEEP_INTERVAL_MS) {
|
|
10390
10391
|
lastSweep = now;
|
|
@@ -10392,9 +10393,9 @@ function createSmeltProgress(eventBus) {
|
|
|
10392
10393
|
if (now - entry.at >= SETTLED_TTL_MS) settled.delete(rid);
|
|
10393
10394
|
}
|
|
10394
10395
|
}
|
|
10395
|
-
settled.set(
|
|
10396
|
+
settled.set(resourceId10, { contentChecksum, outcome, at: now });
|
|
10396
10397
|
for (const waiter of waiters) {
|
|
10397
|
-
if (waiter.resourceId ===
|
|
10398
|
+
if (waiter.resourceId === resourceId10 && waiter.contentChecksum === contentChecksum) {
|
|
10398
10399
|
clearTimeout(waiter.timer);
|
|
10399
10400
|
waiters.delete(waiter);
|
|
10400
10401
|
waiter.resolve(outcome);
|
|
@@ -10402,24 +10403,24 @@ function createSmeltProgress(eventBus) {
|
|
|
10402
10403
|
}
|
|
10403
10404
|
});
|
|
10404
10405
|
return {
|
|
10405
|
-
settledAt: (
|
|
10406
|
-
const entry = settled.get(
|
|
10406
|
+
settledAt: (resourceId10) => {
|
|
10407
|
+
const entry = settled.get(resourceId10);
|
|
10407
10408
|
return entry ? { contentChecksum: entry.contentChecksum, outcome: entry.outcome } : void 0;
|
|
10408
10409
|
},
|
|
10409
|
-
whenSettled: (
|
|
10410
|
+
whenSettled: (resourceId10, contentChecksum, timeoutMs) => {
|
|
10410
10411
|
if (disposed) return Promise.resolve("inert");
|
|
10411
|
-
const current = settled.get(
|
|
10412
|
+
const current = settled.get(resourceId10);
|
|
10412
10413
|
if (current && current.contentChecksum === contentChecksum) {
|
|
10413
10414
|
return Promise.resolve(current.outcome);
|
|
10414
10415
|
}
|
|
10415
10416
|
return new Promise((resolve2, reject) => {
|
|
10416
10417
|
const waiter = {
|
|
10417
|
-
resourceId:
|
|
10418
|
+
resourceId: resourceId10,
|
|
10418
10419
|
contentChecksum,
|
|
10419
10420
|
resolve: resolve2,
|
|
10420
10421
|
timer: setTimeout(() => {
|
|
10421
10422
|
waiters.delete(waiter);
|
|
10422
|
-
reject(new SmeltProgressTimeout(
|
|
10423
|
+
reject(new SmeltProgressTimeout(resourceId10, contentChecksum, timeoutMs));
|
|
10423
10424
|
}, timeoutMs)
|
|
10424
10425
|
};
|
|
10425
10426
|
waiters.add(waiter);
|
|
@@ -10441,14 +10442,14 @@ function createSmeltProgress(eventBus) {
|
|
|
10441
10442
|
|
|
10442
10443
|
// src/read-anchored-text.ts
|
|
10443
10444
|
var ANCHORED_TEXT_SETTLE_TIMEOUT_MS = 15e3;
|
|
10444
|
-
async function readAnchoredText(kb,
|
|
10445
|
-
const view = await kb.views.get(resourceId(
|
|
10445
|
+
async function readAnchoredText(kb, resourceId10, settleTimeoutMs = ANCHORED_TEXT_SETTLE_TIMEOUT_MS) {
|
|
10446
|
+
const view = await kb.views.get(resourceId(resourceId10));
|
|
10446
10447
|
const checksum = getPrimaryRepresentation(view?.resource)?.checksum;
|
|
10447
10448
|
if (!checksum) return { kind: "unknown" };
|
|
10448
10449
|
const hit = await kb.anchoredText.read(checksum);
|
|
10449
10450
|
if (hit) return hit;
|
|
10450
10451
|
try {
|
|
10451
|
-
const outcome = await kb.smeltProgress.whenSettled(
|
|
10452
|
+
const outcome = await kb.smeltProgress.whenSettled(resourceId10, checksum, settleTimeoutMs);
|
|
10452
10453
|
if (outcome === "skipped") return { kind: "no-map" };
|
|
10453
10454
|
if (outcome === "inert") return { kind: "not-yet" };
|
|
10454
10455
|
return await kb.anchoredText.read(checksum) ?? { kind: "not-yet" };
|
|
@@ -10505,11 +10506,11 @@ function representationSource(resource) {
|
|
|
10505
10506
|
mediaType: primary.mediaType
|
|
10506
10507
|
};
|
|
10507
10508
|
}
|
|
10508
|
-
async function resolveRepresentation(deps,
|
|
10509
|
-
const view = await deps.views.get(
|
|
10510
|
-
if (!view?.resource) throw new RepresentationMissing(String(
|
|
10509
|
+
async function resolveRepresentation(deps, resourceId10) {
|
|
10510
|
+
const view = await deps.views.get(resourceId10);
|
|
10511
|
+
if (!view?.resource) throw new RepresentationMissing(String(resourceId10), "resource");
|
|
10511
10512
|
const source = representationSource(view.resource);
|
|
10512
|
-
if (!source) throw new RepresentationMissing(String(
|
|
10513
|
+
if (!source) throw new RepresentationMissing(String(resourceId10), "representation");
|
|
10513
10514
|
return { stream: deps.content.retrieveStream(source.storageUri), mediaType: source.mediaType };
|
|
10514
10515
|
}
|
|
10515
10516
|
var SEMANTIC_OVER_FETCH = 4;
|
|
@@ -10517,8 +10518,8 @@ var ResourceContext = class _ResourceContext {
|
|
|
10517
10518
|
/**
|
|
10518
10519
|
* Get resource metadata from view storage
|
|
10519
10520
|
*/
|
|
10520
|
-
static async getResourceMetadata(
|
|
10521
|
-
const view = await kb.views.get(
|
|
10521
|
+
static async getResourceMetadata(resourceId10, kb) {
|
|
10522
|
+
const view = await kb.views.get(resourceId10);
|
|
10522
10523
|
if (!view) {
|
|
10523
10524
|
return null;
|
|
10524
10525
|
}
|
|
@@ -10539,13 +10540,13 @@ var ResourceContext = class _ResourceContext {
|
|
|
10539
10540
|
* where the graph is only eventually consistent.
|
|
10540
10541
|
*/
|
|
10541
10542
|
static async listResources(filters, kb, semantic) {
|
|
10542
|
-
const { search: rawSearch, archived, entityType, offset = 0, limit = 50 } = filters ?? {};
|
|
10543
|
+
const { search: rawSearch, archived, entityType: entityType2, offset = 0, limit = 50 } = filters ?? {};
|
|
10543
10544
|
const search = rawSearch?.trim() || void 0;
|
|
10544
10545
|
if (search) {
|
|
10545
10546
|
const lexical = await kb.graph.listResources({
|
|
10546
10547
|
search,
|
|
10547
10548
|
archived,
|
|
10548
|
-
entityTypes:
|
|
10549
|
+
entityTypes: entityType2 ? [entityType2] : void 0,
|
|
10549
10550
|
offset,
|
|
10550
10551
|
limit
|
|
10551
10552
|
});
|
|
@@ -10553,7 +10554,7 @@ var ResourceContext = class _ResourceContext {
|
|
|
10553
10554
|
return _ResourceContext.semanticFallback(search, limit, kb, semantic);
|
|
10554
10555
|
}
|
|
10555
10556
|
const allViews = await kb.views.getAll();
|
|
10556
|
-
const matches = allViews.map((view) => view.resource).filter((doc) => archived === void 0 || doc.archived === archived).filter((doc) => !
|
|
10557
|
+
const matches = allViews.map((view) => view.resource).filter((doc) => archived === void 0 || doc.archived === archived).filter((doc) => !entityType2 || getResourceEntityTypes(doc).includes(entityType2)).sort(compareByRecencyThenId);
|
|
10557
10558
|
return { resources: matches.slice(offset, offset + limit), total: matches.length, matchKind: "lexical" };
|
|
10558
10559
|
}
|
|
10559
10560
|
/**
|
|
@@ -10641,8 +10642,8 @@ var ResourceContext = class _ResourceContext {
|
|
|
10641
10642
|
|
|
10642
10643
|
// src/weave-progress.ts
|
|
10643
10644
|
var WeaveProgressTimeout = class extends Error {
|
|
10644
|
-
constructor(
|
|
10645
|
-
super(`weave:applied parity not reached for ${
|
|
10645
|
+
constructor(resourceId10, sequenceNumber, timeoutMs) {
|
|
10646
|
+
super(`weave:applied parity not reached for ${resourceId10} (seq ${sequenceNumber}) within ${timeoutMs}ms`);
|
|
10646
10647
|
this.name = "WeaveProgressTimeout";
|
|
10647
10648
|
}
|
|
10648
10649
|
};
|
|
@@ -10666,19 +10667,19 @@ var GraphContext = class {
|
|
|
10666
10667
|
* - annotations on the resource → `annotation` nodes + `annotation-of` edges,
|
|
10667
10668
|
* so siblingEntityTypes = union of those nodes' entityTypes
|
|
10668
10669
|
*/
|
|
10669
|
-
static async buildKnowledgeGraph(
|
|
10670
|
-
let mainDoc = await kb.graph.getResource(
|
|
10670
|
+
static async buildKnowledgeGraph(resourceId10, kb, logger2) {
|
|
10671
|
+
let mainDoc = await kb.graph.getResource(resourceId10);
|
|
10671
10672
|
if (!mainDoc) {
|
|
10672
|
-
const view = await kb.views.get(
|
|
10673
|
+
const view = await kb.views.get(resourceId10);
|
|
10673
10674
|
if (view) {
|
|
10674
10675
|
if (view.lastSequence !== void 0) {
|
|
10675
10676
|
try {
|
|
10676
10677
|
await kb.weaveProgress.whenApplied(
|
|
10677
|
-
String(
|
|
10678
|
+
String(resourceId10),
|
|
10678
10679
|
view.lastSequence,
|
|
10679
10680
|
PROJECTION_BARRIER_TIMEOUT_MS
|
|
10680
10681
|
);
|
|
10681
|
-
mainDoc = await kb.graph.getResource(
|
|
10682
|
+
mainDoc = await kb.graph.getResource(resourceId10);
|
|
10682
10683
|
} catch (error) {
|
|
10683
10684
|
if (!(error instanceof WeaveProgressTimeout)) throw error;
|
|
10684
10685
|
}
|
|
@@ -10686,19 +10687,19 @@ var GraphContext = class {
|
|
|
10686
10687
|
if (!mainDoc) {
|
|
10687
10688
|
for (const delayMs of PROJECTION_LAG_BACKOFF_MS) {
|
|
10688
10689
|
await new Promise((resolve2) => setTimeout(resolve2, delayMs));
|
|
10689
|
-
mainDoc = await kb.graph.getResource(
|
|
10690
|
+
mainDoc = await kb.graph.getResource(resourceId10);
|
|
10690
10691
|
if (mainDoc) break;
|
|
10691
10692
|
}
|
|
10692
10693
|
}
|
|
10693
10694
|
if (!mainDoc) {
|
|
10694
10695
|
recordGatherDegrade("graph");
|
|
10695
10696
|
logger2?.warn("[gather DEGRADED] graph projection did not catch up \u2014 resource present in views, absent in graph", {
|
|
10696
|
-
resourceId: String(
|
|
10697
|
+
resourceId: String(resourceId10),
|
|
10697
10698
|
lastSequence: view.lastSequence,
|
|
10698
10699
|
barrierTimeoutMs: PROJECTION_BARRIER_TIMEOUT_MS
|
|
10699
10700
|
});
|
|
10700
10701
|
throw new Error(
|
|
10701
|
-
`Graph projection did not catch up for ${String(
|
|
10702
|
+
`Graph projection did not catch up for ${String(resourceId10)} \u2014 present in views, absent in graph (Weaver lag, not a missing resource)`
|
|
10702
10703
|
);
|
|
10703
10704
|
}
|
|
10704
10705
|
}
|
|
@@ -10706,11 +10707,11 @@ var GraphContext = class {
|
|
|
10706
10707
|
if (!mainDoc) {
|
|
10707
10708
|
throw new Error("Resource not found");
|
|
10708
10709
|
}
|
|
10709
|
-
const mainId = String(
|
|
10710
|
+
const mainId = String(resourceId10);
|
|
10710
10711
|
const [connections, referencedBy, annotations] = await Promise.all([
|
|
10711
|
-
kb.graph.getResourceConnections(
|
|
10712
|
-
kb.graph.getResourceReferencedBy(
|
|
10713
|
-
kb.graph.getResourceAnnotations(
|
|
10712
|
+
kb.graph.getResourceConnections(resourceId10),
|
|
10713
|
+
kb.graph.getResourceReferencedBy(resourceId10),
|
|
10714
|
+
kb.graph.getResourceAnnotations(resourceId10)
|
|
10714
10715
|
]);
|
|
10715
10716
|
const nodes = [];
|
|
10716
10717
|
const edges = [];
|
|
@@ -10730,7 +10731,7 @@ var GraphContext = class {
|
|
|
10730
10731
|
const citedSeen = /* @__PURE__ */ new Set();
|
|
10731
10732
|
for (const ann of referencedBy) {
|
|
10732
10733
|
const source = getTargetSource(ann.target);
|
|
10733
|
-
if (!source || source === String(
|
|
10734
|
+
if (!source || source === String(resourceId10) || !ann.id || seen.has(ann.id)) continue;
|
|
10734
10735
|
if (!citedSeen.has(source)) {
|
|
10735
10736
|
citedSeen.add(source);
|
|
10736
10737
|
const view = await kb.views.get(resourceId(source));
|
|
@@ -10764,7 +10765,7 @@ var AnnotationContext = class {
|
|
|
10764
10765
|
* @returns Rich context for LLM processing
|
|
10765
10766
|
* @throws Error if annotation or resource not found
|
|
10766
10767
|
*/
|
|
10767
|
-
static async buildLLMContext(
|
|
10768
|
+
static async buildLLMContext(annotationId5, resourceId10, kb, embeddingProvider, options = {}, inferenceClient, logger2) {
|
|
10768
10769
|
const {
|
|
10769
10770
|
includeSourceContext = true,
|
|
10770
10771
|
includeTargetContext = true,
|
|
@@ -10773,34 +10774,34 @@ var AnnotationContext = class {
|
|
|
10773
10774
|
if (contextWindow < 100 || contextWindow > 5e3) {
|
|
10774
10775
|
throw new Error("contextWindow must be between 100 and 5000");
|
|
10775
10776
|
}
|
|
10776
|
-
logger2?.debug("Building LLM context", { annotationId:
|
|
10777
|
-
logger2?.debug("Getting view for resource", { resourceId:
|
|
10777
|
+
logger2?.debug("Building LLM context", { annotationId: annotationId5, resourceId: resourceId10 });
|
|
10778
|
+
logger2?.debug("Getting view for resource", { resourceId: resourceId10 });
|
|
10778
10779
|
let sourceView;
|
|
10779
10780
|
try {
|
|
10780
|
-
sourceView = await kb.views.get(
|
|
10781
|
+
sourceView = await kb.views.get(resourceId10);
|
|
10781
10782
|
logger2?.debug("Retrieved view", { hasView: !!sourceView });
|
|
10782
10783
|
if (!sourceView) {
|
|
10783
10784
|
throw new Error("Source resource not found");
|
|
10784
10785
|
}
|
|
10785
10786
|
} catch (error) {
|
|
10786
|
-
logger2?.error("Error getting view", { resourceId:
|
|
10787
|
+
logger2?.error("Error getting view", { resourceId: resourceId10, error });
|
|
10787
10788
|
throw error;
|
|
10788
10789
|
}
|
|
10789
10790
|
logger2?.debug("Looking for annotation in resource", {
|
|
10790
|
-
annotationId:
|
|
10791
|
-
resourceId:
|
|
10791
|
+
annotationId: annotationId5,
|
|
10792
|
+
resourceId: resourceId10,
|
|
10792
10793
|
totalAnnotations: sourceView.annotations.annotations.length,
|
|
10793
10794
|
firstFiveIds: sourceView.annotations.annotations.slice(0, 5).map((a) => a.id)
|
|
10794
10795
|
});
|
|
10795
|
-
const annotation = sourceView.annotations.annotations.find((a) => a.id ===
|
|
10796
|
+
const annotation = sourceView.annotations.annotations.find((a) => a.id === annotationId5);
|
|
10796
10797
|
logger2?.debug("Annotation search result", { found: !!annotation });
|
|
10797
10798
|
if (!annotation) {
|
|
10798
10799
|
throw new Error("Annotation not found in view");
|
|
10799
10800
|
}
|
|
10800
10801
|
const targetSource = getTargetSource(annotation.target);
|
|
10801
|
-
logger2?.debug("Validating target resource", { targetSource, expectedResourceId:
|
|
10802
|
-
if (targetSource !== String(
|
|
10803
|
-
throw new Error(`Annotation target resource ID (${targetSource}) does not match expected resource ID (${
|
|
10802
|
+
logger2?.debug("Validating target resource", { targetSource, expectedResourceId: resourceId10 });
|
|
10803
|
+
if (targetSource !== String(resourceId10)) {
|
|
10804
|
+
throw new Error(`Annotation target resource ID (${targetSource}) does not match expected resource ID (${resourceId10})`);
|
|
10804
10805
|
}
|
|
10805
10806
|
const sourceDoc = sourceView.resource;
|
|
10806
10807
|
const bodySource = getBodySource(annotation.body);
|
|
@@ -10816,7 +10817,7 @@ var AnnotationContext = class {
|
|
|
10816
10817
|
throw new Error("Source content not found: no storageUri");
|
|
10817
10818
|
}
|
|
10818
10819
|
const primaryRep = getPrimaryRepresentation(sourceDoc);
|
|
10819
|
-
const { data: sourceContent } = await kb.content.getBinary(
|
|
10820
|
+
const { data: sourceContent } = await kb.content.getBinary(resourceId10);
|
|
10820
10821
|
const contentStr = decodeRepresentation(Buffer.from(sourceContent), primaryRep?.mediaType ?? "text/plain");
|
|
10821
10822
|
const targetSelectorRaw = getTargetSelector(annotation.target);
|
|
10822
10823
|
const targetSelector = Array.isArray(targetSelectorRaw) ? targetSelectorRaw[0] : targetSelectorRaw;
|
|
@@ -10863,9 +10864,9 @@ var AnnotationContext = class {
|
|
|
10863
10864
|
};
|
|
10864
10865
|
}
|
|
10865
10866
|
}
|
|
10866
|
-
logger2?.debug("Building knowledge graph", { resourceId:
|
|
10867
|
-
const graph = await GraphContext.buildKnowledgeGraph(
|
|
10868
|
-
const views = deriveViews(graph, String(
|
|
10867
|
+
logger2?.debug("Building knowledge graph", { resourceId: resourceId10 });
|
|
10868
|
+
const graph = await GraphContext.buildKnowledgeGraph(resourceId10, kb, logger2);
|
|
10869
|
+
const views = deriveViews(graph, String(resourceId10), annotationId5);
|
|
10869
10870
|
const entityTypeStats = await kb.graph.getEntityTypeStats();
|
|
10870
10871
|
const entityTypeFrequencies = {};
|
|
10871
10872
|
for (const stat of entityTypeStats) {
|
|
@@ -10902,7 +10903,7 @@ Summary:`;
|
|
|
10902
10903
|
const results = await kb.vectors.searchAnnotations(focalEmbedding, {
|
|
10903
10904
|
limit: 10,
|
|
10904
10905
|
scoreThreshold: 0.5,
|
|
10905
|
-
filter: { excludeResourceId:
|
|
10906
|
+
filter: { excludeResourceId: resourceId10 }
|
|
10906
10907
|
});
|
|
10907
10908
|
const similar = [];
|
|
10908
10909
|
for (const r of results) {
|
|
@@ -10955,10 +10956,10 @@ Summary:`;
|
|
|
10955
10956
|
* Get resource annotations from view storage (fast path)
|
|
10956
10957
|
* Throws if view missing
|
|
10957
10958
|
*/
|
|
10958
|
-
static async getResourceAnnotations(
|
|
10959
|
-
const view = await kb.views.get(
|
|
10959
|
+
static async getResourceAnnotations(resourceId10, kb) {
|
|
10960
|
+
const view = await kb.views.get(resourceId10);
|
|
10960
10961
|
if (!view) {
|
|
10961
|
-
throw new Error(`Resource ${
|
|
10962
|
+
throw new Error(`Resource ${resourceId10} not found in view storage`);
|
|
10962
10963
|
}
|
|
10963
10964
|
return view.annotations;
|
|
10964
10965
|
}
|
|
@@ -10966,8 +10967,8 @@ Summary:`;
|
|
|
10966
10967
|
* Get all annotations
|
|
10967
10968
|
* @returns Array of all annotation objects
|
|
10968
10969
|
*/
|
|
10969
|
-
static async getAllAnnotations(
|
|
10970
|
-
const annotations = await this.getResourceAnnotations(
|
|
10970
|
+
static async getAllAnnotations(resourceId10, kb) {
|
|
10971
|
+
const annotations = await this.getResourceAnnotations(resourceId10, kb);
|
|
10971
10972
|
return this.enrichResolvedReferences(annotations.annotations, kb);
|
|
10972
10973
|
}
|
|
10973
10974
|
/**
|
|
@@ -11036,8 +11037,8 @@ Summary:`;
|
|
|
11036
11037
|
* Get resource stats (version info)
|
|
11037
11038
|
* @returns Version and timestamp info for the annotations
|
|
11038
11039
|
*/
|
|
11039
|
-
static async getResourceStats(
|
|
11040
|
-
const annotations = await this.getResourceAnnotations(
|
|
11040
|
+
static async getResourceStats(resourceId10, kb) {
|
|
11041
|
+
const annotations = await this.getResourceAnnotations(resourceId10, kb);
|
|
11041
11042
|
return {
|
|
11042
11043
|
resourceId: annotations.resourceId,
|
|
11043
11044
|
version: annotations.version,
|
|
@@ -11047,16 +11048,16 @@ Summary:`;
|
|
|
11047
11048
|
/**
|
|
11048
11049
|
* Check if resource exists in view storage
|
|
11049
11050
|
*/
|
|
11050
|
-
static async resourceExists(
|
|
11051
|
-
return kb.views.exists(
|
|
11051
|
+
static async resourceExists(resourceId10, kb) {
|
|
11052
|
+
return kb.views.exists(resourceId10);
|
|
11052
11053
|
}
|
|
11053
11054
|
/**
|
|
11054
11055
|
* Get a single annotation by ID
|
|
11055
11056
|
* O(1) lookup using resource ID to access view storage
|
|
11056
11057
|
*/
|
|
11057
|
-
static async getAnnotation(
|
|
11058
|
-
const annotations = await this.getResourceAnnotations(
|
|
11059
|
-
return annotations.annotations.find((a) => a.id ===
|
|
11058
|
+
static async getAnnotation(annotationId5, resourceId10, kb) {
|
|
11059
|
+
const annotations = await this.getResourceAnnotations(resourceId10, kb);
|
|
11060
|
+
return annotations.annotations.find((a) => a.id === annotationId5) || null;
|
|
11060
11061
|
}
|
|
11061
11062
|
/**
|
|
11062
11063
|
* List annotations with optional filtering
|
|
@@ -11072,8 +11073,8 @@ Summary:`;
|
|
|
11072
11073
|
/**
|
|
11073
11074
|
* Get annotation context (selected text with surrounding context)
|
|
11074
11075
|
*/
|
|
11075
|
-
static async getAnnotationContext(
|
|
11076
|
-
const annotation = await this.getAnnotation(
|
|
11076
|
+
static async getAnnotationContext(annotationId5, resourceId10, contextBefore, contextAfter, kb) {
|
|
11077
|
+
const annotation = await this.getAnnotation(annotationId5, resourceId10, kb);
|
|
11077
11078
|
if (!annotation) {
|
|
11078
11079
|
throw new Error("Annotation not found");
|
|
11079
11080
|
}
|
|
@@ -11104,8 +11105,8 @@ Summary:`;
|
|
|
11104
11105
|
/**
|
|
11105
11106
|
* Generate AI summary of annotation in context
|
|
11106
11107
|
*/
|
|
11107
|
-
static async generateAnnotationSummary(
|
|
11108
|
-
const annotation = await this.getAnnotation(
|
|
11108
|
+
static async generateAnnotationSummary(annotationId5, resourceId10, kb, inferenceClient) {
|
|
11109
|
+
const annotation = await this.getAnnotation(annotationId5, resourceId10, kb);
|
|
11109
11110
|
if (!annotation) {
|
|
11110
11111
|
throw new Error("Annotation not found");
|
|
11111
11112
|
}
|
|
@@ -11186,10 +11187,10 @@ Entity types: ${entityTypes.join(", ")}`;
|
|
|
11186
11187
|
return inferenceClient.generateText(summaryPrompt, 500, 0.5);
|
|
11187
11188
|
}
|
|
11188
11189
|
};
|
|
11189
|
-
async function assembleResourceGraph(kb,
|
|
11190
|
+
async function assembleResourceGraph(kb, resourceId10) {
|
|
11190
11191
|
const eventQuery = new EventQuery(kb.eventStore.log.storage);
|
|
11191
|
-
const events = await eventQuery.getResourceEvents(
|
|
11192
|
-
const stored = await kb.eventStore.views.materializer.materialize(events,
|
|
11192
|
+
const events = await eventQuery.getResourceEvents(resourceId10);
|
|
11193
|
+
const stored = await kb.eventStore.views.materializer.materialize(events, resourceId10);
|
|
11193
11194
|
if (!stored) return null;
|
|
11194
11195
|
const annotations = stored.annotations.annotations;
|
|
11195
11196
|
const entityReferences = annotations.filter((a) => {
|
|
@@ -11818,7 +11819,7 @@ var ResourceOperations = class {
|
|
|
11818
11819
|
/**
|
|
11819
11820
|
* Create a new resource via EventBus → Stower
|
|
11820
11821
|
*/
|
|
11821
|
-
static async createResource(input,
|
|
11822
|
+
static async createResource(input, userId3, eventBus) {
|
|
11822
11823
|
const { resourceId: rId } = await busRequest(
|
|
11823
11824
|
asBusRequestPrimitive(eventBus),
|
|
11824
11825
|
"yield:create",
|
|
@@ -11828,7 +11829,7 @@ var ResourceOperations = class {
|
|
|
11828
11829
|
contentChecksum: input.contentChecksum,
|
|
11829
11830
|
byteSize: input.byteSize,
|
|
11830
11831
|
format: input.format,
|
|
11831
|
-
_userId:
|
|
11832
|
+
_userId: userId3,
|
|
11832
11833
|
language: input.language,
|
|
11833
11834
|
entityTypes: input.entityTypes,
|
|
11834
11835
|
generatedFrom: input.generatedFrom,
|
|
@@ -11848,7 +11849,7 @@ var ResourceOperations = class {
|
|
|
11848
11849
|
* this only after the CloneTokenManager has validated the token — it is the
|
|
11849
11850
|
* inner half of the flow, not a public entry point.
|
|
11850
11851
|
*/
|
|
11851
|
-
static async persistClone(input,
|
|
11852
|
+
static async persistClone(input, userId3, eventBus) {
|
|
11852
11853
|
const { resourceId: rId } = await busRequest(
|
|
11853
11854
|
asBusRequestPrimitive(eventBus),
|
|
11854
11855
|
"yield:clone-persist",
|
|
@@ -11861,7 +11862,7 @@ var ResourceOperations = class {
|
|
|
11861
11862
|
parentResourceId: input.parentResourceId,
|
|
11862
11863
|
entityTypes: input.entityTypes,
|
|
11863
11864
|
language: input.language,
|
|
11864
|
-
_userId:
|
|
11865
|
+
_userId: userId3
|
|
11865
11866
|
},
|
|
11866
11867
|
3e4
|
|
11867
11868
|
);
|
|
@@ -11873,7 +11874,7 @@ var ResourceOperations = class {
|
|
|
11873
11874
|
* Archivist's register does the one `git add`, D4b); the command carries
|
|
11874
11875
|
* storage coordinates only (EXTRACT-ARCHIVIST P3, D4a).
|
|
11875
11876
|
*/
|
|
11876
|
-
static async createFromCloneToken(input,
|
|
11877
|
+
static async createFromCloneToken(input, userId3, eventBus) {
|
|
11877
11878
|
const { resourceId: rId } = await busRequest(
|
|
11878
11879
|
asBusRequestPrimitive(eventBus),
|
|
11879
11880
|
"yield:clone-create",
|
|
@@ -11885,7 +11886,7 @@ var ResourceOperations = class {
|
|
|
11885
11886
|
byteSize: input.byteSize,
|
|
11886
11887
|
format: input.format,
|
|
11887
11888
|
archiveOriginal: input.archiveOriginal,
|
|
11888
|
-
_userId:
|
|
11889
|
+
_userId: userId3
|
|
11889
11890
|
},
|
|
11890
11891
|
3e4
|
|
11891
11892
|
);
|
|
@@ -12417,8 +12418,8 @@ function registerAnnotationContextHandler(eventBus, kb, parentLogger) {
|
|
|
12417
12418
|
}
|
|
12418
12419
|
function workingTreeContentReads(views, content) {
|
|
12419
12420
|
return {
|
|
12420
|
-
getBinary: async (
|
|
12421
|
-
const { stream, mediaType } = await resolveRepresentation({ views, content },
|
|
12421
|
+
getBinary: async (resourceId10) => {
|
|
12422
|
+
const { stream, mediaType } = await resolveRepresentation({ views, content }, resourceId10);
|
|
12422
12423
|
const chunks = [];
|
|
12423
12424
|
for await (const chunk of stream) chunks.push(chunk);
|
|
12424
12425
|
const buf = Buffer.concat(chunks);
|
|
@@ -12440,12 +12441,12 @@ async function bootstrapEntityTypes(eventBus, eventStore, logger2) {
|
|
|
12440
12441
|
}
|
|
12441
12442
|
logger2?.info("Bootstrapping missing entity types", { missing: missing.length, existing: existingTypes.size });
|
|
12442
12443
|
const SYSTEM_USER_ID = userId("00000000-0000-0000-0000-000000000000");
|
|
12443
|
-
for (const
|
|
12444
|
-
logger2?.debug("Adding entity type via EventBus", { entityType });
|
|
12444
|
+
for (const entityType2 of missing) {
|
|
12445
|
+
logger2?.debug("Adding entity type via EventBus", { entityType: entityType2 });
|
|
12445
12446
|
await busRequest(
|
|
12446
12447
|
asBusRequestPrimitive(eventBus),
|
|
12447
12448
|
"frame:add-entity-type",
|
|
12448
|
-
{ tag:
|
|
12449
|
+
{ tag: entityType2, _userId: SYSTEM_USER_ID },
|
|
12449
12450
|
1e4
|
|
12450
12451
|
);
|
|
12451
12452
|
}
|
|
@@ -12465,19 +12466,25 @@ function eventAnnotationId(event) {
|
|
|
12465
12466
|
return null;
|
|
12466
12467
|
}
|
|
12467
12468
|
}
|
|
12468
|
-
async function readAnnotationFromView(kb,
|
|
12469
|
-
const allAnnotations = await AnnotationContext.getAllAnnotations(
|
|
12470
|
-
return allAnnotations.find((a) => a.id ===
|
|
12469
|
+
async function readAnnotationFromView(kb, resourceId10, annotationId5) {
|
|
12470
|
+
const allAnnotations = await AnnotationContext.getAllAnnotations(resourceId10, kb);
|
|
12471
|
+
return allAnnotations.find((a) => a.id === annotationId5) ?? null;
|
|
12471
12472
|
}
|
|
12472
12473
|
function wireEnrichment(eventStore, kb) {
|
|
12473
|
-
eventStore.setEnrichEvent(async (event,
|
|
12474
|
+
eventStore.setEnrichEvent(async (event, resourceId10) => {
|
|
12474
12475
|
const annId = eventAnnotationId(event);
|
|
12475
12476
|
if (annId === null) return event;
|
|
12476
|
-
const annotation = await readAnnotationFromView(kb,
|
|
12477
|
+
const annotation = await readAnnotationFromView(kb, resourceId10, annId);
|
|
12477
12478
|
if (annotation === null) return event;
|
|
12478
12479
|
return { ...event, annotation };
|
|
12479
12480
|
});
|
|
12480
12481
|
}
|
|
12482
|
+
|
|
12483
|
+
// src/service.ts
|
|
12484
|
+
var STARTUP_CONNECT_TIMEOUT_MS = 6e4;
|
|
12485
|
+
var RESTART_HINT = "Exiting so the container restart policy can retry \u2014 it is normal for a dependency to be slow when every service restarts at once.";
|
|
12486
|
+
|
|
12487
|
+
// src/archivist-main.ts
|
|
12481
12488
|
var maybeRoot = process.env.SEMIONT_ROOT;
|
|
12482
12489
|
if (!maybeRoot) {
|
|
12483
12490
|
throw new Error("SEMIONT_ROOT environment variable is not set");
|
|
@@ -12577,18 +12584,34 @@ async function main() {
|
|
|
12577
12584
|
const anchoredText = createAnchoredTextStore(anchoredTextDir, logger.child({ component: "anchored-text-store" }));
|
|
12578
12585
|
const smeltProgress = createSmeltProgress(localBus);
|
|
12579
12586
|
logger.info("Connecting to graph database", { type: graphConfig.type });
|
|
12580
|
-
const graphDb = await
|
|
12587
|
+
const graphDb = await withDeadline(
|
|
12588
|
+
"Graph database",
|
|
12589
|
+
STARTUP_CONNECT_TIMEOUT_MS,
|
|
12590
|
+
() => getGraphDatabase(graphConfig),
|
|
12591
|
+
RESTART_HINT
|
|
12592
|
+
);
|
|
12581
12593
|
const embeddingConfig = config.services.embedding;
|
|
12582
12594
|
logger.info("Connecting to embedding provider", { type: embeddingConfig.type, model: embeddingConfig.model });
|
|
12583
|
-
const embeddingProvider = await
|
|
12595
|
+
const embeddingProvider = await withDeadline(
|
|
12596
|
+
"Embedding provider",
|
|
12597
|
+
STARTUP_CONNECT_TIMEOUT_MS,
|
|
12598
|
+
() => createEmbeddingProvider(embeddingConfig),
|
|
12599
|
+
RESTART_HINT
|
|
12600
|
+
);
|
|
12584
12601
|
const vectorsConfig = config.services.vectors;
|
|
12585
12602
|
logger.info("Connecting to vector store", { type: vectorsConfig.type });
|
|
12586
|
-
const vectorStore = await
|
|
12587
|
-
|
|
12588
|
-
|
|
12589
|
-
|
|
12590
|
-
|
|
12591
|
-
|
|
12603
|
+
const vectorStore = await withDeadline(
|
|
12604
|
+
"Vector store",
|
|
12605
|
+
STARTUP_CONNECT_TIMEOUT_MS,
|
|
12606
|
+
(signal) => createVectorStore({
|
|
12607
|
+
signal,
|
|
12608
|
+
type: vectorsConfig.type,
|
|
12609
|
+
host: vectorsConfig.host,
|
|
12610
|
+
port: vectorsConfig.port,
|
|
12611
|
+
dimensions: () => embeddingProvider.dimensions()
|
|
12612
|
+
}),
|
|
12613
|
+
RESTART_HINT
|
|
12614
|
+
);
|
|
12592
12615
|
const stower = new Stower(
|
|
12593
12616
|
{ content, eventStore },
|
|
12594
12617
|
localBus,
|