@semiont/make-meaning 0.5.32 → 0.5.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/archivist-main.js +174 -124
- package/dist/archivist-main.js.map +1 -1
- package/dist/index.d.ts +36 -8
- package/dist/index.js +95 -49
- package/dist/index.js.map +1 -1
- package/dist/librarian-main.js +29 -10
- package/dist/librarian-main.js.map +1 -1
- package/dist/smelter-main.js +81 -45
- package/dist/smelter-main.js.map +1 -1
- package/dist/weaver-main.js +36 -13
- package/dist/weaver-main.js.map +1 -1
- package/package.json +12 -12
package/dist/archivist-main.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { HttpTransport } from '@semiont/http-transport';
|
|
2
|
-
import { replyChannelsFor, accessToken, EventBus, baseUrl as baseUrl$1, errField, PERSISTED_EVENT_TYPES, retryWithBackoff, isTransientFetchError, STARTUP_FETCH_RETRY, userId, resourceId, generateUuid, annotationId, getBodySource, getTargetSource, getTargetSelector, getExactText, getStorageUri, cloneToken, getResourceEntityTypes, didToAgent, assembleAnnotation, busRequest, getPrimaryRepresentation, decodeRepresentation, getResourceId, deriveViews, getTextPositionSelector, isAnnotatable, softwareToAgent } from '@semiont/core';
|
|
2
|
+
import { replyChannelsFor, accessToken, EventBus, withDeadline, baseUrl as baseUrl$1, errField, PERSISTED_EVENT_TYPES, retryWithBackoff, isTransientFetchError, STARTUP_FETCH_RETRY, userId, resourceId, generateUuid, annotationId, getBodySource, getTargetSource, getTargetSelector, getExactText, getStorageUri, cloneToken, getResourceEntityTypes, didToAgent, assembleAnnotation, busRequest, getPrimaryRepresentation, decodeRepresentation, getResourceId, deriveViews, getTextPositionSelector, isAnnotatable, softwareToAgent } from '@semiont/core';
|
|
3
3
|
import { loadEnvironmentConfig, SemiontProject } from '@semiont/core/node';
|
|
4
4
|
import { createEventStore, resolveStorageUri, EventQuery } from '@semiont/event-sourcing';
|
|
5
5
|
import { WorkingTreeStore, createAnchoredTextStore, RepresentationMissing, ChecksumMismatchError } from '@semiont/content';
|
|
@@ -13,6 +13,7 @@ import { createInferenceClient } from '@semiont/inference';
|
|
|
13
13
|
import { createServer } from 'http';
|
|
14
14
|
import { pipeline } from 'stream/promises';
|
|
15
15
|
import { createProcessLogger } from '@semiont/observability/process-logger';
|
|
16
|
+
import '@semiont/jobs';
|
|
16
17
|
|
|
17
18
|
var __create = Object.create;
|
|
18
19
|
var __defProp = Object.defineProperty;
|
|
@@ -10047,36 +10048,56 @@ var Stower = class {
|
|
|
10047
10048
|
*
|
|
10048
10049
|
* Appends are sequential, not concurrent: the event log is the system of
|
|
10049
10050
|
* record and a batch that half-lands under concurrency is harder to reason
|
|
10050
|
-
* about than one that stops at the first failure.
|
|
10051
|
-
*
|
|
10052
|
-
*
|
|
10053
|
-
*
|
|
10051
|
+
* about than one that stops at the first failure.
|
|
10052
|
+
*
|
|
10053
|
+
* This channel is AT-LEAST-ONCE, and the log must not grow on a repeat
|
|
10054
|
+
* (COMMIT-ACK-FALSE-FAILURE F3). Two paths re-send a batch that already
|
|
10055
|
+
* landed: an acknowledgement lost after a successful append (the unit is
|
|
10056
|
+
* never checkpointed, so the retry re-runs exactly the unit that landed), and
|
|
10057
|
+
* a partial batch, reported as a failure and retried whole. Deterministic ids
|
|
10058
|
+
* (JOB-RESTART-SAFETY P3) made those safe for the PROJECTIONS — the resource
|
|
10059
|
+
* view and the graph both refuse a duplicate id — but a projection's guard
|
|
10060
|
+
* says nothing about the log, which appends whatever it is handed. The result
|
|
10061
|
+
* was a green graph over a doubled log: silent, and not undoable.
|
|
10062
|
+
*
|
|
10063
|
+
* So the batch is diffed against what the resource already holds. ONE view
|
|
10064
|
+
* read per commit, never per annotation: the view for a 1,673-annotation
|
|
10065
|
+
* resource is ~3 MB, and re-reading it per append would cost gigabytes of
|
|
10066
|
+
* parsing for a single job.
|
|
10054
10067
|
*/
|
|
10055
10068
|
async handleMarkCommit(event) {
|
|
10056
10069
|
if (!event._userId) {
|
|
10057
10070
|
throw new Error("mark:commit missing _userId (gateway injection)");
|
|
10058
10071
|
}
|
|
10059
10072
|
const annotations = event.annotations ?? [];
|
|
10073
|
+
const rid = resourceId(event.resourceId);
|
|
10060
10074
|
try {
|
|
10061
|
-
|
|
10075
|
+
const view = await this.stores.eventStore.viewStorage.get(rid);
|
|
10076
|
+
const present = new Set((view?.annotations.annotations ?? []).map((a) => String(a.id)));
|
|
10062
10077
|
for (const annotation of annotations) {
|
|
10078
|
+
if (present.has(String(annotation.id))) continue;
|
|
10063
10079
|
await this.stores.eventStore.appendEvent({
|
|
10064
10080
|
type: "mark:added",
|
|
10065
|
-
resourceId:
|
|
10081
|
+
resourceId: rid,
|
|
10066
10082
|
userId: userId(event._userId),
|
|
10067
10083
|
version: 1,
|
|
10068
10084
|
payload: { annotation }
|
|
10069
10085
|
});
|
|
10070
|
-
|
|
10086
|
+
present.add(String(annotation.id));
|
|
10071
10087
|
}
|
|
10072
10088
|
this.logger.debug("Committed annotation batch", {
|
|
10073
10089
|
correlationId: event.correlationId,
|
|
10074
10090
|
resourceId: event.resourceId,
|
|
10075
|
-
persisted
|
|
10091
|
+
persisted: annotations.length
|
|
10076
10092
|
});
|
|
10077
10093
|
this.eventBus.get("mark:commit-ok").next({
|
|
10078
10094
|
correlationId: event.correlationId,
|
|
10079
|
-
|
|
10095
|
+
// The DURABLE count, which is what the acknowledgement means ("every
|
|
10096
|
+
// annotation named by the command is in the event log"). Not an append
|
|
10097
|
+
// tally: a retry whose annotations are all already present has
|
|
10098
|
+
// succeeded, and must be indistinguishable from the first commit or the
|
|
10099
|
+
// caller would have to interpret a 0 that means "all good".
|
|
10100
|
+
response: { persisted: annotations.length, annotationIds: annotations.map((a) => String(a.id)) }
|
|
10080
10101
|
});
|
|
10081
10102
|
} catch (error) {
|
|
10082
10103
|
this.logger.error("Failed to commit annotation batch", {
|
|
@@ -10246,22 +10267,22 @@ var Stower = class {
|
|
|
10246
10267
|
throw new Error(entityTypesNotRegisteredMessage(result.unknown));
|
|
10247
10268
|
}
|
|
10248
10269
|
}
|
|
10249
|
-
for (const
|
|
10270
|
+
for (const entityType2 of added) {
|
|
10250
10271
|
await this.stores.eventStore.appendEvent({
|
|
10251
10272
|
type: "mark:entity-tag-added",
|
|
10252
10273
|
resourceId: resourceId(event.resourceId),
|
|
10253
10274
|
userId: uid,
|
|
10254
10275
|
version: 1,
|
|
10255
|
-
payload: { entityType }
|
|
10276
|
+
payload: { entityType: entityType2 }
|
|
10256
10277
|
});
|
|
10257
10278
|
}
|
|
10258
|
-
for (const
|
|
10279
|
+
for (const entityType2 of removed) {
|
|
10259
10280
|
await this.stores.eventStore.appendEvent({
|
|
10260
10281
|
type: "mark:entity-tag-removed",
|
|
10261
10282
|
resourceId: resourceId(event.resourceId),
|
|
10262
10283
|
userId: uid,
|
|
10263
10284
|
version: 1,
|
|
10264
|
-
payload: { entityType }
|
|
10285
|
+
payload: { entityType: entityType2 }
|
|
10265
10286
|
});
|
|
10266
10287
|
}
|
|
10267
10288
|
this.eventBus.get("mark:update-entity-types-ok").next({ correlationId: event.correlationId });
|
|
@@ -10302,7 +10323,11 @@ var Stower = class {
|
|
|
10302
10323
|
jobId: event.jobId,
|
|
10303
10324
|
jobType: event.jobType,
|
|
10304
10325
|
...event.annotationId ? { annotationId: event.annotationId } : {},
|
|
10305
|
-
result: event.result
|
|
10326
|
+
result: event.result,
|
|
10327
|
+
// How durability was ESTABLISHED (COMMIT-ACK-FALSE-FAILURE). An
|
|
10328
|
+
// acknowledged batch and one inferred from a probe are different
|
|
10329
|
+
// claims; absent means the question never arose.
|
|
10330
|
+
...event.durability !== void 0 ? { durability: event.durability } : {}
|
|
10306
10331
|
}
|
|
10307
10332
|
});
|
|
10308
10333
|
}
|
|
@@ -10319,7 +10344,19 @@ var Stower = class {
|
|
|
10319
10344
|
jobId: event.jobId,
|
|
10320
10345
|
jobType: event.jobType,
|
|
10321
10346
|
...event.annotationId ? { annotationId: event.annotationId } : {},
|
|
10322
|
-
error: event.error
|
|
10347
|
+
error: event.error,
|
|
10348
|
+
// The worker's JUDGMENTS, not just its message. Both are computed where
|
|
10349
|
+
// the error is still typed and are unrecoverable here — the only other
|
|
10350
|
+
// witness in the log is `error`, a flattened English string. Spread
|
|
10351
|
+
// conditionally: absent `failureClass` means UNRECOGNISED, a different
|
|
10352
|
+
// claim from 'transient', and defaulting either would write a judgment
|
|
10353
|
+
// nobody made into a log nobody can rewrite.
|
|
10354
|
+
...event.failureClass !== void 0 ? { failureClass: event.failureClass } : {},
|
|
10355
|
+
...event.willRetry !== void 0 ? { willRetry: event.willRetry } : {},
|
|
10356
|
+
// How durability was ESTABLISHED (COMMIT-ACK-FALSE-FAILURE). An
|
|
10357
|
+
// acknowledged batch and one inferred from a probe are different
|
|
10358
|
+
// claims; absent means the question never arose.
|
|
10359
|
+
...event.durability !== void 0 ? { durability: event.durability } : {}
|
|
10323
10360
|
}
|
|
10324
10361
|
});
|
|
10325
10362
|
}
|
|
@@ -10336,8 +10373,8 @@ var import_operators2 = __toESM(require_operators());
|
|
|
10336
10373
|
|
|
10337
10374
|
// src/smelt-progress.ts
|
|
10338
10375
|
var SmeltProgressTimeout = class extends Error {
|
|
10339
|
-
constructor(
|
|
10340
|
-
super(`smelt:settled not observed for ${
|
|
10376
|
+
constructor(resourceId10, contentChecksum, timeoutMs) {
|
|
10377
|
+
super(`smelt:settled not observed for ${resourceId10} (checksum ${contentChecksum.slice(0, 12)}\u2026) within ${timeoutMs}ms`);
|
|
10341
10378
|
this.name = "SmeltProgressTimeout";
|
|
10342
10379
|
}
|
|
10343
10380
|
};
|
|
@@ -10348,7 +10385,7 @@ function createSmeltProgress(eventBus) {
|
|
|
10348
10385
|
const waiters = /* @__PURE__ */ new Set();
|
|
10349
10386
|
let disposed = false;
|
|
10350
10387
|
let lastSweep = Date.now();
|
|
10351
|
-
const subscription = eventBus.get("smelt:settled").subscribe(({ resourceId:
|
|
10388
|
+
const subscription = eventBus.get("smelt:settled").subscribe(({ resourceId: resourceId10, contentChecksum, outcome }) => {
|
|
10352
10389
|
const now = Date.now();
|
|
10353
10390
|
if (now - lastSweep >= SWEEP_INTERVAL_MS) {
|
|
10354
10391
|
lastSweep = now;
|
|
@@ -10356,9 +10393,9 @@ function createSmeltProgress(eventBus) {
|
|
|
10356
10393
|
if (now - entry.at >= SETTLED_TTL_MS) settled.delete(rid);
|
|
10357
10394
|
}
|
|
10358
10395
|
}
|
|
10359
|
-
settled.set(
|
|
10396
|
+
settled.set(resourceId10, { contentChecksum, outcome, at: now });
|
|
10360
10397
|
for (const waiter of waiters) {
|
|
10361
|
-
if (waiter.resourceId ===
|
|
10398
|
+
if (waiter.resourceId === resourceId10 && waiter.contentChecksum === contentChecksum) {
|
|
10362
10399
|
clearTimeout(waiter.timer);
|
|
10363
10400
|
waiters.delete(waiter);
|
|
10364
10401
|
waiter.resolve(outcome);
|
|
@@ -10366,24 +10403,24 @@ function createSmeltProgress(eventBus) {
|
|
|
10366
10403
|
}
|
|
10367
10404
|
});
|
|
10368
10405
|
return {
|
|
10369
|
-
settledAt: (
|
|
10370
|
-
const entry = settled.get(
|
|
10406
|
+
settledAt: (resourceId10) => {
|
|
10407
|
+
const entry = settled.get(resourceId10);
|
|
10371
10408
|
return entry ? { contentChecksum: entry.contentChecksum, outcome: entry.outcome } : void 0;
|
|
10372
10409
|
},
|
|
10373
|
-
whenSettled: (
|
|
10410
|
+
whenSettled: (resourceId10, contentChecksum, timeoutMs) => {
|
|
10374
10411
|
if (disposed) return Promise.resolve("inert");
|
|
10375
|
-
const current = settled.get(
|
|
10412
|
+
const current = settled.get(resourceId10);
|
|
10376
10413
|
if (current && current.contentChecksum === contentChecksum) {
|
|
10377
10414
|
return Promise.resolve(current.outcome);
|
|
10378
10415
|
}
|
|
10379
10416
|
return new Promise((resolve2, reject) => {
|
|
10380
10417
|
const waiter = {
|
|
10381
|
-
resourceId:
|
|
10418
|
+
resourceId: resourceId10,
|
|
10382
10419
|
contentChecksum,
|
|
10383
10420
|
resolve: resolve2,
|
|
10384
10421
|
timer: setTimeout(() => {
|
|
10385
10422
|
waiters.delete(waiter);
|
|
10386
|
-
reject(new SmeltProgressTimeout(
|
|
10423
|
+
reject(new SmeltProgressTimeout(resourceId10, contentChecksum, timeoutMs));
|
|
10387
10424
|
}, timeoutMs)
|
|
10388
10425
|
};
|
|
10389
10426
|
waiters.add(waiter);
|
|
@@ -10405,14 +10442,14 @@ function createSmeltProgress(eventBus) {
|
|
|
10405
10442
|
|
|
10406
10443
|
// src/read-anchored-text.ts
|
|
10407
10444
|
var ANCHORED_TEXT_SETTLE_TIMEOUT_MS = 15e3;
|
|
10408
|
-
async function readAnchoredText(kb,
|
|
10409
|
-
const view = await kb.views.get(resourceId(
|
|
10445
|
+
async function readAnchoredText(kb, resourceId10, settleTimeoutMs = ANCHORED_TEXT_SETTLE_TIMEOUT_MS) {
|
|
10446
|
+
const view = await kb.views.get(resourceId(resourceId10));
|
|
10410
10447
|
const checksum = getPrimaryRepresentation(view?.resource)?.checksum;
|
|
10411
10448
|
if (!checksum) return { kind: "unknown" };
|
|
10412
10449
|
const hit = await kb.anchoredText.read(checksum);
|
|
10413
10450
|
if (hit) return hit;
|
|
10414
10451
|
try {
|
|
10415
|
-
const outcome = await kb.smeltProgress.whenSettled(
|
|
10452
|
+
const outcome = await kb.smeltProgress.whenSettled(resourceId10, checksum, settleTimeoutMs);
|
|
10416
10453
|
if (outcome === "skipped") return { kind: "no-map" };
|
|
10417
10454
|
if (outcome === "inert") return { kind: "not-yet" };
|
|
10418
10455
|
return await kb.anchoredText.read(checksum) ?? { kind: "not-yet" };
|
|
@@ -10469,11 +10506,11 @@ function representationSource(resource) {
|
|
|
10469
10506
|
mediaType: primary.mediaType
|
|
10470
10507
|
};
|
|
10471
10508
|
}
|
|
10472
|
-
async function resolveRepresentation(deps,
|
|
10473
|
-
const view = await deps.views.get(
|
|
10474
|
-
if (!view?.resource) throw new RepresentationMissing(String(
|
|
10509
|
+
async function resolveRepresentation(deps, resourceId10) {
|
|
10510
|
+
const view = await deps.views.get(resourceId10);
|
|
10511
|
+
if (!view?.resource) throw new RepresentationMissing(String(resourceId10), "resource");
|
|
10475
10512
|
const source = representationSource(view.resource);
|
|
10476
|
-
if (!source) throw new RepresentationMissing(String(
|
|
10513
|
+
if (!source) throw new RepresentationMissing(String(resourceId10), "representation");
|
|
10477
10514
|
return { stream: deps.content.retrieveStream(source.storageUri), mediaType: source.mediaType };
|
|
10478
10515
|
}
|
|
10479
10516
|
var SEMANTIC_OVER_FETCH = 4;
|
|
@@ -10481,8 +10518,8 @@ var ResourceContext = class _ResourceContext {
|
|
|
10481
10518
|
/**
|
|
10482
10519
|
* Get resource metadata from view storage
|
|
10483
10520
|
*/
|
|
10484
|
-
static async getResourceMetadata(
|
|
10485
|
-
const view = await kb.views.get(
|
|
10521
|
+
static async getResourceMetadata(resourceId10, kb) {
|
|
10522
|
+
const view = await kb.views.get(resourceId10);
|
|
10486
10523
|
if (!view) {
|
|
10487
10524
|
return null;
|
|
10488
10525
|
}
|
|
@@ -10503,13 +10540,13 @@ var ResourceContext = class _ResourceContext {
|
|
|
10503
10540
|
* where the graph is only eventually consistent.
|
|
10504
10541
|
*/
|
|
10505
10542
|
static async listResources(filters, kb, semantic) {
|
|
10506
|
-
const { search: rawSearch, archived, entityType, offset = 0, limit = 50 } = filters ?? {};
|
|
10543
|
+
const { search: rawSearch, archived, entityType: entityType2, offset = 0, limit = 50 } = filters ?? {};
|
|
10507
10544
|
const search = rawSearch?.trim() || void 0;
|
|
10508
10545
|
if (search) {
|
|
10509
10546
|
const lexical = await kb.graph.listResources({
|
|
10510
10547
|
search,
|
|
10511
10548
|
archived,
|
|
10512
|
-
entityTypes:
|
|
10549
|
+
entityTypes: entityType2 ? [entityType2] : void 0,
|
|
10513
10550
|
offset,
|
|
10514
10551
|
limit
|
|
10515
10552
|
});
|
|
@@ -10517,7 +10554,7 @@ var ResourceContext = class _ResourceContext {
|
|
|
10517
10554
|
return _ResourceContext.semanticFallback(search, limit, kb, semantic);
|
|
10518
10555
|
}
|
|
10519
10556
|
const allViews = await kb.views.getAll();
|
|
10520
|
-
const matches = allViews.map((view) => view.resource).filter((doc) => archived === void 0 || doc.archived === archived).filter((doc) => !
|
|
10557
|
+
const matches = allViews.map((view) => view.resource).filter((doc) => archived === void 0 || doc.archived === archived).filter((doc) => !entityType2 || getResourceEntityTypes(doc).includes(entityType2)).sort(compareByRecencyThenId);
|
|
10521
10558
|
return { resources: matches.slice(offset, offset + limit), total: matches.length, matchKind: "lexical" };
|
|
10522
10559
|
}
|
|
10523
10560
|
/**
|
|
@@ -10605,8 +10642,8 @@ var ResourceContext = class _ResourceContext {
|
|
|
10605
10642
|
|
|
10606
10643
|
// src/weave-progress.ts
|
|
10607
10644
|
var WeaveProgressTimeout = class extends Error {
|
|
10608
|
-
constructor(
|
|
10609
|
-
super(`weave:applied parity not reached for ${
|
|
10645
|
+
constructor(resourceId10, sequenceNumber, timeoutMs) {
|
|
10646
|
+
super(`weave:applied parity not reached for ${resourceId10} (seq ${sequenceNumber}) within ${timeoutMs}ms`);
|
|
10610
10647
|
this.name = "WeaveProgressTimeout";
|
|
10611
10648
|
}
|
|
10612
10649
|
};
|
|
@@ -10630,19 +10667,19 @@ var GraphContext = class {
|
|
|
10630
10667
|
* - annotations on the resource → `annotation` nodes + `annotation-of` edges,
|
|
10631
10668
|
* so siblingEntityTypes = union of those nodes' entityTypes
|
|
10632
10669
|
*/
|
|
10633
|
-
static async buildKnowledgeGraph(
|
|
10634
|
-
let mainDoc = await kb.graph.getResource(
|
|
10670
|
+
static async buildKnowledgeGraph(resourceId10, kb, logger2) {
|
|
10671
|
+
let mainDoc = await kb.graph.getResource(resourceId10);
|
|
10635
10672
|
if (!mainDoc) {
|
|
10636
|
-
const view = await kb.views.get(
|
|
10673
|
+
const view = await kb.views.get(resourceId10);
|
|
10637
10674
|
if (view) {
|
|
10638
10675
|
if (view.lastSequence !== void 0) {
|
|
10639
10676
|
try {
|
|
10640
10677
|
await kb.weaveProgress.whenApplied(
|
|
10641
|
-
String(
|
|
10678
|
+
String(resourceId10),
|
|
10642
10679
|
view.lastSequence,
|
|
10643
10680
|
PROJECTION_BARRIER_TIMEOUT_MS
|
|
10644
10681
|
);
|
|
10645
|
-
mainDoc = await kb.graph.getResource(
|
|
10682
|
+
mainDoc = await kb.graph.getResource(resourceId10);
|
|
10646
10683
|
} catch (error) {
|
|
10647
10684
|
if (!(error instanceof WeaveProgressTimeout)) throw error;
|
|
10648
10685
|
}
|
|
@@ -10650,19 +10687,19 @@ var GraphContext = class {
|
|
|
10650
10687
|
if (!mainDoc) {
|
|
10651
10688
|
for (const delayMs of PROJECTION_LAG_BACKOFF_MS) {
|
|
10652
10689
|
await new Promise((resolve2) => setTimeout(resolve2, delayMs));
|
|
10653
|
-
mainDoc = await kb.graph.getResource(
|
|
10690
|
+
mainDoc = await kb.graph.getResource(resourceId10);
|
|
10654
10691
|
if (mainDoc) break;
|
|
10655
10692
|
}
|
|
10656
10693
|
}
|
|
10657
10694
|
if (!mainDoc) {
|
|
10658
10695
|
recordGatherDegrade("graph");
|
|
10659
10696
|
logger2?.warn("[gather DEGRADED] graph projection did not catch up \u2014 resource present in views, absent in graph", {
|
|
10660
|
-
resourceId: String(
|
|
10697
|
+
resourceId: String(resourceId10),
|
|
10661
10698
|
lastSequence: view.lastSequence,
|
|
10662
10699
|
barrierTimeoutMs: PROJECTION_BARRIER_TIMEOUT_MS
|
|
10663
10700
|
});
|
|
10664
10701
|
throw new Error(
|
|
10665
|
-
`Graph projection did not catch up for ${String(
|
|
10702
|
+
`Graph projection did not catch up for ${String(resourceId10)} \u2014 present in views, absent in graph (Weaver lag, not a missing resource)`
|
|
10666
10703
|
);
|
|
10667
10704
|
}
|
|
10668
10705
|
}
|
|
@@ -10670,11 +10707,11 @@ var GraphContext = class {
|
|
|
10670
10707
|
if (!mainDoc) {
|
|
10671
10708
|
throw new Error("Resource not found");
|
|
10672
10709
|
}
|
|
10673
|
-
const mainId = String(
|
|
10710
|
+
const mainId = String(resourceId10);
|
|
10674
10711
|
const [connections, referencedBy, annotations] = await Promise.all([
|
|
10675
|
-
kb.graph.getResourceConnections(
|
|
10676
|
-
kb.graph.getResourceReferencedBy(
|
|
10677
|
-
kb.graph.getResourceAnnotations(
|
|
10712
|
+
kb.graph.getResourceConnections(resourceId10),
|
|
10713
|
+
kb.graph.getResourceReferencedBy(resourceId10),
|
|
10714
|
+
kb.graph.getResourceAnnotations(resourceId10)
|
|
10678
10715
|
]);
|
|
10679
10716
|
const nodes = [];
|
|
10680
10717
|
const edges = [];
|
|
@@ -10694,7 +10731,7 @@ var GraphContext = class {
|
|
|
10694
10731
|
const citedSeen = /* @__PURE__ */ new Set();
|
|
10695
10732
|
for (const ann of referencedBy) {
|
|
10696
10733
|
const source = getTargetSource(ann.target);
|
|
10697
|
-
if (!source || source === String(
|
|
10734
|
+
if (!source || source === String(resourceId10) || !ann.id || seen.has(ann.id)) continue;
|
|
10698
10735
|
if (!citedSeen.has(source)) {
|
|
10699
10736
|
citedSeen.add(source);
|
|
10700
10737
|
const view = await kb.views.get(resourceId(source));
|
|
@@ -10728,7 +10765,7 @@ var AnnotationContext = class {
|
|
|
10728
10765
|
* @returns Rich context for LLM processing
|
|
10729
10766
|
* @throws Error if annotation or resource not found
|
|
10730
10767
|
*/
|
|
10731
|
-
static async buildLLMContext(
|
|
10768
|
+
static async buildLLMContext(annotationId5, resourceId10, kb, embeddingProvider, options = {}, inferenceClient, logger2) {
|
|
10732
10769
|
const {
|
|
10733
10770
|
includeSourceContext = true,
|
|
10734
10771
|
includeTargetContext = true,
|
|
@@ -10737,34 +10774,34 @@ var AnnotationContext = class {
|
|
|
10737
10774
|
if (contextWindow < 100 || contextWindow > 5e3) {
|
|
10738
10775
|
throw new Error("contextWindow must be between 100 and 5000");
|
|
10739
10776
|
}
|
|
10740
|
-
logger2?.debug("Building LLM context", { annotationId:
|
|
10741
|
-
logger2?.debug("Getting view for resource", { resourceId:
|
|
10777
|
+
logger2?.debug("Building LLM context", { annotationId: annotationId5, resourceId: resourceId10 });
|
|
10778
|
+
logger2?.debug("Getting view for resource", { resourceId: resourceId10 });
|
|
10742
10779
|
let sourceView;
|
|
10743
10780
|
try {
|
|
10744
|
-
sourceView = await kb.views.get(
|
|
10781
|
+
sourceView = await kb.views.get(resourceId10);
|
|
10745
10782
|
logger2?.debug("Retrieved view", { hasView: !!sourceView });
|
|
10746
10783
|
if (!sourceView) {
|
|
10747
10784
|
throw new Error("Source resource not found");
|
|
10748
10785
|
}
|
|
10749
10786
|
} catch (error) {
|
|
10750
|
-
logger2?.error("Error getting view", { resourceId:
|
|
10787
|
+
logger2?.error("Error getting view", { resourceId: resourceId10, error });
|
|
10751
10788
|
throw error;
|
|
10752
10789
|
}
|
|
10753
10790
|
logger2?.debug("Looking for annotation in resource", {
|
|
10754
|
-
annotationId:
|
|
10755
|
-
resourceId:
|
|
10791
|
+
annotationId: annotationId5,
|
|
10792
|
+
resourceId: resourceId10,
|
|
10756
10793
|
totalAnnotations: sourceView.annotations.annotations.length,
|
|
10757
10794
|
firstFiveIds: sourceView.annotations.annotations.slice(0, 5).map((a) => a.id)
|
|
10758
10795
|
});
|
|
10759
|
-
const annotation = sourceView.annotations.annotations.find((a) => a.id ===
|
|
10796
|
+
const annotation = sourceView.annotations.annotations.find((a) => a.id === annotationId5);
|
|
10760
10797
|
logger2?.debug("Annotation search result", { found: !!annotation });
|
|
10761
10798
|
if (!annotation) {
|
|
10762
10799
|
throw new Error("Annotation not found in view");
|
|
10763
10800
|
}
|
|
10764
10801
|
const targetSource = getTargetSource(annotation.target);
|
|
10765
|
-
logger2?.debug("Validating target resource", { targetSource, expectedResourceId:
|
|
10766
|
-
if (targetSource !== String(
|
|
10767
|
-
throw new Error(`Annotation target resource ID (${targetSource}) does not match expected resource ID (${
|
|
10802
|
+
logger2?.debug("Validating target resource", { targetSource, expectedResourceId: resourceId10 });
|
|
10803
|
+
if (targetSource !== String(resourceId10)) {
|
|
10804
|
+
throw new Error(`Annotation target resource ID (${targetSource}) does not match expected resource ID (${resourceId10})`);
|
|
10768
10805
|
}
|
|
10769
10806
|
const sourceDoc = sourceView.resource;
|
|
10770
10807
|
const bodySource = getBodySource(annotation.body);
|
|
@@ -10780,7 +10817,7 @@ var AnnotationContext = class {
|
|
|
10780
10817
|
throw new Error("Source content not found: no storageUri");
|
|
10781
10818
|
}
|
|
10782
10819
|
const primaryRep = getPrimaryRepresentation(sourceDoc);
|
|
10783
|
-
const { data: sourceContent } = await kb.content.getBinary(
|
|
10820
|
+
const { data: sourceContent } = await kb.content.getBinary(resourceId10);
|
|
10784
10821
|
const contentStr = decodeRepresentation(Buffer.from(sourceContent), primaryRep?.mediaType ?? "text/plain");
|
|
10785
10822
|
const targetSelectorRaw = getTargetSelector(annotation.target);
|
|
10786
10823
|
const targetSelector = Array.isArray(targetSelectorRaw) ? targetSelectorRaw[0] : targetSelectorRaw;
|
|
@@ -10827,9 +10864,9 @@ var AnnotationContext = class {
|
|
|
10827
10864
|
};
|
|
10828
10865
|
}
|
|
10829
10866
|
}
|
|
10830
|
-
logger2?.debug("Building knowledge graph", { resourceId:
|
|
10831
|
-
const graph = await GraphContext.buildKnowledgeGraph(
|
|
10832
|
-
const views = deriveViews(graph, String(
|
|
10867
|
+
logger2?.debug("Building knowledge graph", { resourceId: resourceId10 });
|
|
10868
|
+
const graph = await GraphContext.buildKnowledgeGraph(resourceId10, kb, logger2);
|
|
10869
|
+
const views = deriveViews(graph, String(resourceId10), annotationId5);
|
|
10833
10870
|
const entityTypeStats = await kb.graph.getEntityTypeStats();
|
|
10834
10871
|
const entityTypeFrequencies = {};
|
|
10835
10872
|
for (const stat of entityTypeStats) {
|
|
@@ -10866,7 +10903,7 @@ Summary:`;
|
|
|
10866
10903
|
const results = await kb.vectors.searchAnnotations(focalEmbedding, {
|
|
10867
10904
|
limit: 10,
|
|
10868
10905
|
scoreThreshold: 0.5,
|
|
10869
|
-
filter: { excludeResourceId:
|
|
10906
|
+
filter: { excludeResourceId: resourceId10 }
|
|
10870
10907
|
});
|
|
10871
10908
|
const similar = [];
|
|
10872
10909
|
for (const r of results) {
|
|
@@ -10919,10 +10956,10 @@ Summary:`;
|
|
|
10919
10956
|
* Get resource annotations from view storage (fast path)
|
|
10920
10957
|
* Throws if view missing
|
|
10921
10958
|
*/
|
|
10922
|
-
static async getResourceAnnotations(
|
|
10923
|
-
const view = await kb.views.get(
|
|
10959
|
+
static async getResourceAnnotations(resourceId10, kb) {
|
|
10960
|
+
const view = await kb.views.get(resourceId10);
|
|
10924
10961
|
if (!view) {
|
|
10925
|
-
throw new Error(`Resource ${
|
|
10962
|
+
throw new Error(`Resource ${resourceId10} not found in view storage`);
|
|
10926
10963
|
}
|
|
10927
10964
|
return view.annotations;
|
|
10928
10965
|
}
|
|
@@ -10930,8 +10967,8 @@ Summary:`;
|
|
|
10930
10967
|
* Get all annotations
|
|
10931
10968
|
* @returns Array of all annotation objects
|
|
10932
10969
|
*/
|
|
10933
|
-
static async getAllAnnotations(
|
|
10934
|
-
const annotations = await this.getResourceAnnotations(
|
|
10970
|
+
static async getAllAnnotations(resourceId10, kb) {
|
|
10971
|
+
const annotations = await this.getResourceAnnotations(resourceId10, kb);
|
|
10935
10972
|
return this.enrichResolvedReferences(annotations.annotations, kb);
|
|
10936
10973
|
}
|
|
10937
10974
|
/**
|
|
@@ -11000,8 +11037,8 @@ Summary:`;
|
|
|
11000
11037
|
* Get resource stats (version info)
|
|
11001
11038
|
* @returns Version and timestamp info for the annotations
|
|
11002
11039
|
*/
|
|
11003
|
-
static async getResourceStats(
|
|
11004
|
-
const annotations = await this.getResourceAnnotations(
|
|
11040
|
+
static async getResourceStats(resourceId10, kb) {
|
|
11041
|
+
const annotations = await this.getResourceAnnotations(resourceId10, kb);
|
|
11005
11042
|
return {
|
|
11006
11043
|
resourceId: annotations.resourceId,
|
|
11007
11044
|
version: annotations.version,
|
|
@@ -11011,16 +11048,16 @@ Summary:`;
|
|
|
11011
11048
|
/**
|
|
11012
11049
|
* Check if resource exists in view storage
|
|
11013
11050
|
*/
|
|
11014
|
-
static async resourceExists(
|
|
11015
|
-
return kb.views.exists(
|
|
11051
|
+
static async resourceExists(resourceId10, kb) {
|
|
11052
|
+
return kb.views.exists(resourceId10);
|
|
11016
11053
|
}
|
|
11017
11054
|
/**
|
|
11018
11055
|
* Get a single annotation by ID
|
|
11019
11056
|
* O(1) lookup using resource ID to access view storage
|
|
11020
11057
|
*/
|
|
11021
|
-
static async getAnnotation(
|
|
11022
|
-
const annotations = await this.getResourceAnnotations(
|
|
11023
|
-
return annotations.annotations.find((a) => a.id ===
|
|
11058
|
+
static async getAnnotation(annotationId5, resourceId10, kb) {
|
|
11059
|
+
const annotations = await this.getResourceAnnotations(resourceId10, kb);
|
|
11060
|
+
return annotations.annotations.find((a) => a.id === annotationId5) || null;
|
|
11024
11061
|
}
|
|
11025
11062
|
/**
|
|
11026
11063
|
* List annotations with optional filtering
|
|
@@ -11036,8 +11073,8 @@ Summary:`;
|
|
|
11036
11073
|
/**
|
|
11037
11074
|
* Get annotation context (selected text with surrounding context)
|
|
11038
11075
|
*/
|
|
11039
|
-
static async getAnnotationContext(
|
|
11040
|
-
const annotation = await this.getAnnotation(
|
|
11076
|
+
static async getAnnotationContext(annotationId5, resourceId10, contextBefore, contextAfter, kb) {
|
|
11077
|
+
const annotation = await this.getAnnotation(annotationId5, resourceId10, kb);
|
|
11041
11078
|
if (!annotation) {
|
|
11042
11079
|
throw new Error("Annotation not found");
|
|
11043
11080
|
}
|
|
@@ -11068,8 +11105,8 @@ Summary:`;
|
|
|
11068
11105
|
/**
|
|
11069
11106
|
* Generate AI summary of annotation in context
|
|
11070
11107
|
*/
|
|
11071
|
-
static async generateAnnotationSummary(
|
|
11072
|
-
const annotation = await this.getAnnotation(
|
|
11108
|
+
static async generateAnnotationSummary(annotationId5, resourceId10, kb, inferenceClient) {
|
|
11109
|
+
const annotation = await this.getAnnotation(annotationId5, resourceId10, kb);
|
|
11073
11110
|
if (!annotation) {
|
|
11074
11111
|
throw new Error("Annotation not found");
|
|
11075
11112
|
}
|
|
@@ -11150,10 +11187,10 @@ Entity types: ${entityTypes.join(", ")}`;
|
|
|
11150
11187
|
return inferenceClient.generateText(summaryPrompt, 500, 0.5);
|
|
11151
11188
|
}
|
|
11152
11189
|
};
|
|
11153
|
-
async function assembleResourceGraph(kb,
|
|
11190
|
+
async function assembleResourceGraph(kb, resourceId10) {
|
|
11154
11191
|
const eventQuery = new EventQuery(kb.eventStore.log.storage);
|
|
11155
|
-
const events = await eventQuery.getResourceEvents(
|
|
11156
|
-
const stored = await kb.eventStore.views.materializer.materialize(events,
|
|
11192
|
+
const events = await eventQuery.getResourceEvents(resourceId10);
|
|
11193
|
+
const stored = await kb.eventStore.views.materializer.materialize(events, resourceId10);
|
|
11157
11194
|
if (!stored) return null;
|
|
11158
11195
|
const annotations = stored.annotations.annotations;
|
|
11159
11196
|
const entityReferences = annotations.filter((a) => {
|
|
@@ -11782,7 +11819,7 @@ var ResourceOperations = class {
|
|
|
11782
11819
|
/**
|
|
11783
11820
|
* Create a new resource via EventBus → Stower
|
|
11784
11821
|
*/
|
|
11785
|
-
static async createResource(input,
|
|
11822
|
+
static async createResource(input, userId3, eventBus) {
|
|
11786
11823
|
const { resourceId: rId } = await busRequest(
|
|
11787
11824
|
asBusRequestPrimitive(eventBus),
|
|
11788
11825
|
"yield:create",
|
|
@@ -11792,7 +11829,7 @@ var ResourceOperations = class {
|
|
|
11792
11829
|
contentChecksum: input.contentChecksum,
|
|
11793
11830
|
byteSize: input.byteSize,
|
|
11794
11831
|
format: input.format,
|
|
11795
|
-
_userId:
|
|
11832
|
+
_userId: userId3,
|
|
11796
11833
|
language: input.language,
|
|
11797
11834
|
entityTypes: input.entityTypes,
|
|
11798
11835
|
generatedFrom: input.generatedFrom,
|
|
@@ -11812,7 +11849,7 @@ var ResourceOperations = class {
|
|
|
11812
11849
|
* this only after the CloneTokenManager has validated the token — it is the
|
|
11813
11850
|
* inner half of the flow, not a public entry point.
|
|
11814
11851
|
*/
|
|
11815
|
-
static async persistClone(input,
|
|
11852
|
+
static async persistClone(input, userId3, eventBus) {
|
|
11816
11853
|
const { resourceId: rId } = await busRequest(
|
|
11817
11854
|
asBusRequestPrimitive(eventBus),
|
|
11818
11855
|
"yield:clone-persist",
|
|
@@ -11825,7 +11862,7 @@ var ResourceOperations = class {
|
|
|
11825
11862
|
parentResourceId: input.parentResourceId,
|
|
11826
11863
|
entityTypes: input.entityTypes,
|
|
11827
11864
|
language: input.language,
|
|
11828
|
-
_userId:
|
|
11865
|
+
_userId: userId3
|
|
11829
11866
|
},
|
|
11830
11867
|
3e4
|
|
11831
11868
|
);
|
|
@@ -11837,7 +11874,7 @@ var ResourceOperations = class {
|
|
|
11837
11874
|
* Archivist's register does the one `git add`, D4b); the command carries
|
|
11838
11875
|
* storage coordinates only (EXTRACT-ARCHIVIST P3, D4a).
|
|
11839
11876
|
*/
|
|
11840
|
-
static async createFromCloneToken(input,
|
|
11877
|
+
static async createFromCloneToken(input, userId3, eventBus) {
|
|
11841
11878
|
const { resourceId: rId } = await busRequest(
|
|
11842
11879
|
asBusRequestPrimitive(eventBus),
|
|
11843
11880
|
"yield:clone-create",
|
|
@@ -11849,7 +11886,7 @@ var ResourceOperations = class {
|
|
|
11849
11886
|
byteSize: input.byteSize,
|
|
11850
11887
|
format: input.format,
|
|
11851
11888
|
archiveOriginal: input.archiveOriginal,
|
|
11852
|
-
_userId:
|
|
11889
|
+
_userId: userId3
|
|
11853
11890
|
},
|
|
11854
11891
|
3e4
|
|
11855
11892
|
);
|
|
@@ -12381,8 +12418,8 @@ function registerAnnotationContextHandler(eventBus, kb, parentLogger) {
|
|
|
12381
12418
|
}
|
|
12382
12419
|
function workingTreeContentReads(views, content) {
|
|
12383
12420
|
return {
|
|
12384
|
-
getBinary: async (
|
|
12385
|
-
const { stream, mediaType } = await resolveRepresentation({ views, content },
|
|
12421
|
+
getBinary: async (resourceId10) => {
|
|
12422
|
+
const { stream, mediaType } = await resolveRepresentation({ views, content }, resourceId10);
|
|
12386
12423
|
const chunks = [];
|
|
12387
12424
|
for await (const chunk of stream) chunks.push(chunk);
|
|
12388
12425
|
const buf = Buffer.concat(chunks);
|
|
@@ -12404,12 +12441,12 @@ async function bootstrapEntityTypes(eventBus, eventStore, logger2) {
|
|
|
12404
12441
|
}
|
|
12405
12442
|
logger2?.info("Bootstrapping missing entity types", { missing: missing.length, existing: existingTypes.size });
|
|
12406
12443
|
const SYSTEM_USER_ID = userId("00000000-0000-0000-0000-000000000000");
|
|
12407
|
-
for (const
|
|
12408
|
-
logger2?.debug("Adding entity type via EventBus", { entityType });
|
|
12444
|
+
for (const entityType2 of missing) {
|
|
12445
|
+
logger2?.debug("Adding entity type via EventBus", { entityType: entityType2 });
|
|
12409
12446
|
await busRequest(
|
|
12410
12447
|
asBusRequestPrimitive(eventBus),
|
|
12411
12448
|
"frame:add-entity-type",
|
|
12412
|
-
{ tag:
|
|
12449
|
+
{ tag: entityType2, _userId: SYSTEM_USER_ID },
|
|
12413
12450
|
1e4
|
|
12414
12451
|
);
|
|
12415
12452
|
}
|
|
@@ -12429,19 +12466,25 @@ function eventAnnotationId(event) {
|
|
|
12429
12466
|
return null;
|
|
12430
12467
|
}
|
|
12431
12468
|
}
|
|
12432
|
-
async function readAnnotationFromView(kb,
|
|
12433
|
-
const allAnnotations = await AnnotationContext.getAllAnnotations(
|
|
12434
|
-
return allAnnotations.find((a) => a.id ===
|
|
12469
|
+
async function readAnnotationFromView(kb, resourceId10, annotationId5) {
|
|
12470
|
+
const allAnnotations = await AnnotationContext.getAllAnnotations(resourceId10, kb);
|
|
12471
|
+
return allAnnotations.find((a) => a.id === annotationId5) ?? null;
|
|
12435
12472
|
}
|
|
12436
12473
|
function wireEnrichment(eventStore, kb) {
|
|
12437
|
-
eventStore.setEnrichEvent(async (event,
|
|
12474
|
+
eventStore.setEnrichEvent(async (event, resourceId10) => {
|
|
12438
12475
|
const annId = eventAnnotationId(event);
|
|
12439
12476
|
if (annId === null) return event;
|
|
12440
|
-
const annotation = await readAnnotationFromView(kb,
|
|
12477
|
+
const annotation = await readAnnotationFromView(kb, resourceId10, annId);
|
|
12441
12478
|
if (annotation === null) return event;
|
|
12442
12479
|
return { ...event, annotation };
|
|
12443
12480
|
});
|
|
12444
12481
|
}
|
|
12482
|
+
|
|
12483
|
+
// src/service.ts
|
|
12484
|
+
var STARTUP_CONNECT_TIMEOUT_MS = 6e4;
|
|
12485
|
+
var RESTART_HINT = "Exiting so the container restart policy can retry \u2014 it is normal for a dependency to be slow when every service restarts at once.";
|
|
12486
|
+
|
|
12487
|
+
// src/archivist-main.ts
|
|
12445
12488
|
var maybeRoot = process.env.SEMIONT_ROOT;
|
|
12446
12489
|
if (!maybeRoot) {
|
|
12447
12490
|
throw new Error("SEMIONT_ROOT environment variable is not set");
|
|
@@ -12511,17 +12554,8 @@ async function authenticate() {
|
|
|
12511
12554
|
async function main() {
|
|
12512
12555
|
const { initObservabilityNode } = await import('@semiont/observability/node');
|
|
12513
12556
|
initObservabilityNode({ serviceName: "semiont-archivist" });
|
|
12514
|
-
const {
|
|
12515
|
-
|
|
12516
|
-
registerRestartCountProvider(async () => {
|
|
12517
|
-
try {
|
|
12518
|
-
const { readFile } = await import('fs/promises');
|
|
12519
|
-
const log = await readFile(supervisorEvents, "utf-8");
|
|
12520
|
-
return Math.max(0, log.split("\n").filter((l) => l.includes("starting archivist")).length - 1);
|
|
12521
|
-
} catch {
|
|
12522
|
-
return 0;
|
|
12523
|
-
}
|
|
12524
|
-
});
|
|
12557
|
+
const { registerSupervisorRestartCount } = await import('@semiont/observability/node');
|
|
12558
|
+
registerSupervisorRestartCount();
|
|
12525
12559
|
logger.info("Authenticating", { baseUrl });
|
|
12526
12560
|
const tokenSubject = new import_rxjs6.BehaviorSubject(accessToken(await authenticate()));
|
|
12527
12561
|
logger.info("Authenticated");
|
|
@@ -12550,18 +12584,34 @@ async function main() {
|
|
|
12550
12584
|
const anchoredText = createAnchoredTextStore(anchoredTextDir, logger.child({ component: "anchored-text-store" }));
|
|
12551
12585
|
const smeltProgress = createSmeltProgress(localBus);
|
|
12552
12586
|
logger.info("Connecting to graph database", { type: graphConfig.type });
|
|
12553
|
-
const graphDb = await
|
|
12587
|
+
const graphDb = await withDeadline(
|
|
12588
|
+
"Graph database",
|
|
12589
|
+
STARTUP_CONNECT_TIMEOUT_MS,
|
|
12590
|
+
() => getGraphDatabase(graphConfig),
|
|
12591
|
+
RESTART_HINT
|
|
12592
|
+
);
|
|
12554
12593
|
const embeddingConfig = config.services.embedding;
|
|
12555
12594
|
logger.info("Connecting to embedding provider", { type: embeddingConfig.type, model: embeddingConfig.model });
|
|
12556
|
-
const embeddingProvider = await
|
|
12595
|
+
const embeddingProvider = await withDeadline(
|
|
12596
|
+
"Embedding provider",
|
|
12597
|
+
STARTUP_CONNECT_TIMEOUT_MS,
|
|
12598
|
+
() => createEmbeddingProvider(embeddingConfig),
|
|
12599
|
+
RESTART_HINT
|
|
12600
|
+
);
|
|
12557
12601
|
const vectorsConfig = config.services.vectors;
|
|
12558
12602
|
logger.info("Connecting to vector store", { type: vectorsConfig.type });
|
|
12559
|
-
const vectorStore = await
|
|
12560
|
-
|
|
12561
|
-
|
|
12562
|
-
|
|
12563
|
-
|
|
12564
|
-
|
|
12603
|
+
const vectorStore = await withDeadline(
|
|
12604
|
+
"Vector store",
|
|
12605
|
+
STARTUP_CONNECT_TIMEOUT_MS,
|
|
12606
|
+
(signal) => createVectorStore({
|
|
12607
|
+
signal,
|
|
12608
|
+
type: vectorsConfig.type,
|
|
12609
|
+
host: vectorsConfig.host,
|
|
12610
|
+
port: vectorsConfig.port,
|
|
12611
|
+
dimensions: () => embeddingProvider.dimensions()
|
|
12612
|
+
}),
|
|
12613
|
+
RESTART_HINT
|
|
12614
|
+
);
|
|
12565
12615
|
const stower = new Stower(
|
|
12566
12616
|
{ content, eventStore },
|
|
12567
12617
|
localBus,
|