@semiont/make-meaning 0.5.27 → 0.5.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -5
- package/dist/archivist-main.d.ts +2 -0
- package/dist/archivist-main.js +12568 -0
- package/dist/archivist-main.js.map +1 -0
- package/dist/index.d.ts +649 -460
- package/dist/index.js +633 -1086
- package/dist/index.js.map +1 -1
- package/dist/librarian-main.d.ts +2 -0
- package/dist/librarian-main.js +11449 -0
- package/dist/librarian-main.js.map +1 -0
- package/dist/smelter-main.js +24 -18
- package/dist/smelter-main.js.map +1 -1
- package/dist/weaver-main.d.ts +2 -0
- package/dist/weaver-main.js +15 -9
- package/dist/weaver-main.js.map +1 -1
- package/package.json +28 -17
package/dist/index.js
CHANGED
|
@@ -1,17 +1,16 @@
|
|
|
1
1
|
import { STALL_THRESHOLD_MS, FsJobQueue } from '@semiont/jobs';
|
|
2
2
|
import { FilesystemViewStorage, resolveStorageUri, EventQuery, createEventStore } from '@semiont/event-sourcing';
|
|
3
|
-
import { getResourceEntityTypes, getResourceId, getTargetSource, resourceId,
|
|
3
|
+
import { getResourceEntityTypes, getResourceId, getTargetSource, resourceId, decodeRepresentation, getBodySource, getStorageUri, getPrimaryRepresentation, getTargetSelector, deriveViews, getTextPositionSelector, annotationId, errField, userId, generateUuid, getExactText, busRequest, cloneToken, assembleAnnotation, applyBodyOperations, didToAgent, isGenerationJobParams, jobId, entityType, baseUrl, busLog, BRIDGED_CHANNELS, burstBuffer, textExtractionOf, chunkText, getPrimaryMediaType, isAnnotatable, softwareToAgent } from '@semiont/core';
|
|
4
4
|
import { recordGatherDegrade, withActorSpan, recordBusEmit, withSpan, SpanKind, registerJobQueueProvider, registerVectorIndexSizeProvider } from '@semiont/observability';
|
|
5
5
|
import { createInferenceClient } from '@semiont/inference';
|
|
6
6
|
import { compareByRecencyThenId, getGraphDatabase } from '@semiont/graph';
|
|
7
|
-
import { WorkingTreeStore, createAnchoredTextStore,
|
|
7
|
+
import { WorkingTreeStore, createAnchoredTextStore, RepresentationMissing, ChecksumMismatchError, EXTRACTORS, calculateChecksum } from '@semiont/content';
|
|
8
8
|
import { getEntityTypes, DEFAULT_ENTITY_TYPES } from '@semiont/ontology';
|
|
9
9
|
import { mergeByResource } from '@semiont/vectors';
|
|
10
10
|
import { promises } from 'fs';
|
|
11
11
|
import * as path3 from 'path';
|
|
12
|
-
import {
|
|
13
|
-
import { pipeline
|
|
14
|
-
import { promisify } from 'util';
|
|
12
|
+
import { createServer } from 'http';
|
|
13
|
+
import { pipeline } from 'stream/promises';
|
|
15
14
|
|
|
16
15
|
var __create = Object.create;
|
|
17
16
|
var __defProp = Object.defineProperty;
|
|
@@ -3568,7 +3567,7 @@ var require_firstValueFrom = __commonJS({
|
|
|
3568
3567
|
exports.firstValueFrom = void 0;
|
|
3569
3568
|
var EmptyError_1 = require_EmptyError();
|
|
3570
3569
|
var Subscriber_1 = require_Subscriber();
|
|
3571
|
-
function
|
|
3570
|
+
function firstValueFrom(source, config) {
|
|
3572
3571
|
var hasConfig = typeof config === "object";
|
|
3573
3572
|
return new Promise(function(resolve2, reject) {
|
|
3574
3573
|
var subscriber = new Subscriber_1.SafeSubscriber({
|
|
@@ -3588,7 +3587,7 @@ var require_firstValueFrom = __commonJS({
|
|
|
3588
3587
|
source.subscribe(subscriber);
|
|
3589
3588
|
});
|
|
3590
3589
|
}
|
|
3591
|
-
exports.firstValueFrom =
|
|
3590
|
+
exports.firstValueFrom = firstValueFrom;
|
|
3592
3591
|
}
|
|
3593
3592
|
});
|
|
3594
3593
|
|
|
@@ -3727,7 +3726,7 @@ var require_map = __commonJS({
|
|
|
3727
3726
|
exports.map = void 0;
|
|
3728
3727
|
var lift_1 = require_lift();
|
|
3729
3728
|
var OperatorSubscriber_1 = require_OperatorSubscriber();
|
|
3730
|
-
function
|
|
3729
|
+
function map2(project, thisArg) {
|
|
3731
3730
|
return lift_1.operate(function(source, subscriber) {
|
|
3732
3731
|
var index = 0;
|
|
3733
3732
|
source.subscribe(OperatorSubscriber_1.createOperatorSubscriber(subscriber, function(value) {
|
|
@@ -3735,7 +3734,7 @@ var require_map = __commonJS({
|
|
|
3735
3734
|
}));
|
|
3736
3735
|
});
|
|
3737
3736
|
}
|
|
3738
|
-
exports.map =
|
|
3737
|
+
exports.map = map2;
|
|
3739
3738
|
}
|
|
3740
3739
|
});
|
|
3741
3740
|
|
|
@@ -4565,7 +4564,7 @@ var require_timer = __commonJS({
|
|
|
4565
4564
|
var async_1 = require_async();
|
|
4566
4565
|
var isScheduler_1 = require_isScheduler();
|
|
4567
4566
|
var isDate_1 = require_isDate();
|
|
4568
|
-
function
|
|
4567
|
+
function timer(dueTime, intervalOrScheduler, scheduler) {
|
|
4569
4568
|
if (dueTime === void 0) {
|
|
4570
4569
|
dueTime = 0;
|
|
4571
4570
|
}
|
|
@@ -4598,7 +4597,7 @@ var require_timer = __commonJS({
|
|
|
4598
4597
|
}, due);
|
|
4599
4598
|
});
|
|
4600
4599
|
}
|
|
4601
|
-
exports.timer =
|
|
4600
|
+
exports.timer = timer;
|
|
4602
4601
|
}
|
|
4603
4602
|
});
|
|
4604
4603
|
|
|
@@ -4788,7 +4787,7 @@ var require_race = __commonJS({
|
|
|
4788
4787
|
var innerFrom_1 = require_innerFrom();
|
|
4789
4788
|
var argsOrArgArray_1 = require_argsOrArgArray();
|
|
4790
4789
|
var OperatorSubscriber_1 = require_OperatorSubscriber();
|
|
4791
|
-
function
|
|
4790
|
+
function race() {
|
|
4792
4791
|
var sources = [];
|
|
4793
4792
|
for (var _i = 0; _i < arguments.length; _i++) {
|
|
4794
4793
|
sources[_i] = arguments[_i];
|
|
@@ -4796,7 +4795,7 @@ var require_race = __commonJS({
|
|
|
4796
4795
|
sources = argsOrArgArray_1.argsOrArgArray(sources);
|
|
4797
4796
|
return sources.length === 1 ? innerFrom_1.innerFrom(sources[0]) : new Observable_1.Observable(raceInit(sources));
|
|
4798
4797
|
}
|
|
4799
|
-
exports.race =
|
|
4798
|
+
exports.race = race;
|
|
4800
4799
|
function raceInit(sources) {
|
|
4801
4800
|
return function(subscriber) {
|
|
4802
4801
|
var subscriptions = [];
|
|
@@ -9252,14 +9251,14 @@ var require_race2 = __commonJS({
|
|
|
9252
9251
|
exports.race = void 0;
|
|
9253
9252
|
var argsOrArgArray_1 = require_argsOrArgArray();
|
|
9254
9253
|
var raceWith_1 = require_raceWith();
|
|
9255
|
-
function
|
|
9254
|
+
function race() {
|
|
9256
9255
|
var args = [];
|
|
9257
9256
|
for (var _i = 0; _i < arguments.length; _i++) {
|
|
9258
9257
|
args[_i] = arguments[_i];
|
|
9259
9258
|
}
|
|
9260
9259
|
return raceWith_1.raceWith.apply(void 0, __spreadArray([], __read(argsOrArgArray_1.argsOrArgArray(args))));
|
|
9261
9260
|
}
|
|
9262
|
-
exports.race =
|
|
9261
|
+
exports.race = race;
|
|
9263
9262
|
}
|
|
9264
9263
|
});
|
|
9265
9264
|
|
|
@@ -9726,6 +9725,45 @@ var require_operators = __commonJS({
|
|
|
9726
9725
|
});
|
|
9727
9726
|
|
|
9728
9727
|
// src/config.ts
|
|
9728
|
+
function requireKBName(config) {
|
|
9729
|
+
const name = config.kb?.name;
|
|
9730
|
+
if (!name) {
|
|
9731
|
+
throw new Error(
|
|
9732
|
+
"[kb] name is missing from the environment config. The launcher stages the KB's committed identity into each service's config (SINGLE-KB-MOUNT D4); without it this service cannot locate the state tree."
|
|
9733
|
+
);
|
|
9734
|
+
}
|
|
9735
|
+
return name;
|
|
9736
|
+
}
|
|
9737
|
+
function makeMeaningConfigFrom(config) {
|
|
9738
|
+
const meta = config._metadata;
|
|
9739
|
+
const gather = meta?.gather;
|
|
9740
|
+
if (!gather) {
|
|
9741
|
+
throw new Error("make-meaning gather config missing \u2014 load config via loadEnvironmentConfig (the TOML loader owns the settleTimeoutMs default)");
|
|
9742
|
+
}
|
|
9743
|
+
const search = meta?.search;
|
|
9744
|
+
if (!search) {
|
|
9745
|
+
throw new Error("make-meaning search config missing \u2014 load config via loadEnvironmentConfig (the TOML loader owns the semanticFloor default)");
|
|
9746
|
+
}
|
|
9747
|
+
return {
|
|
9748
|
+
gather,
|
|
9749
|
+
search,
|
|
9750
|
+
services: {
|
|
9751
|
+
// vectors/embedding are required on both sides (MANDATORY-EMBEDDING P3):
|
|
9752
|
+
// core's ServicesConfig requires the pair, so a config missing either
|
|
9753
|
+
// already refused at the TOML loader — nothing to re-check here.
|
|
9754
|
+
graph: config.services.graph,
|
|
9755
|
+
vectors: config.services.vectors,
|
|
9756
|
+
embedding: config.services.embedding,
|
|
9757
|
+
archivist: config.services.archivist
|
|
9758
|
+
},
|
|
9759
|
+
// The KB's canonical identity — the agent roster mints DIDs from this,
|
|
9760
|
+
// the SAME value /api/tokens/agent uses (agent-did-host-skew fix). The
|
|
9761
|
+
// value, not JWTService, so make-meaning stays gateway-agnostic.
|
|
9762
|
+
...config.site?.domain ? { site: { domain: config.site.domain } } : {},
|
|
9763
|
+
actors: meta?.actors,
|
|
9764
|
+
workers: meta?.workers
|
|
9765
|
+
};
|
|
9766
|
+
}
|
|
9729
9767
|
function resolveActorInference(config, actor) {
|
|
9730
9768
|
const specific = config.actors?.[actor];
|
|
9731
9769
|
if (specific) return specific;
|
|
@@ -9746,6 +9784,21 @@ function resolveWorkerInference(config, workerType) {
|
|
|
9746
9784
|
// src/service.ts
|
|
9747
9785
|
var import_rxjs7 = __toESM(require_cjs());
|
|
9748
9786
|
var import_operators6 = __toESM(require_operators());
|
|
9787
|
+
function representationSource(resource) {
|
|
9788
|
+
const primary = getPrimaryRepresentation(resource);
|
|
9789
|
+
if (!primary?.storageUri) return null;
|
|
9790
|
+
return {
|
|
9791
|
+
storageUri: primary.storageUri,
|
|
9792
|
+
mediaType: primary.mediaType
|
|
9793
|
+
};
|
|
9794
|
+
}
|
|
9795
|
+
async function resolveRepresentation(deps, resourceId10) {
|
|
9796
|
+
const view = await deps.views.get(resourceId10);
|
|
9797
|
+
if (!view?.resource) throw new RepresentationMissing(String(resourceId10), "resource");
|
|
9798
|
+
const source = representationSource(view.resource);
|
|
9799
|
+
if (!source) throw new RepresentationMissing(String(resourceId10), "representation");
|
|
9800
|
+
return { stream: deps.content.retrieveStream(source.storageUri), mediaType: source.mediaType };
|
|
9801
|
+
}
|
|
9749
9802
|
|
|
9750
9803
|
// src/weave-progress.ts
|
|
9751
9804
|
var WeaveProgressTimeout = class extends Error {
|
|
@@ -9883,6 +9936,19 @@ function createSmeltProgress(eventBus) {
|
|
|
9883
9936
|
}
|
|
9884
9937
|
|
|
9885
9938
|
// src/knowledge-base.ts
|
|
9939
|
+
function workingTreeContentReads(views, content) {
|
|
9940
|
+
return {
|
|
9941
|
+
getBinary: async (resourceId10) => {
|
|
9942
|
+
const { stream, mediaType } = await resolveRepresentation({ views, content }, resourceId10);
|
|
9943
|
+
const chunks = [];
|
|
9944
|
+
for await (const chunk of stream) chunks.push(chunk);
|
|
9945
|
+
const buf = Buffer.concat(chunks);
|
|
9946
|
+
const data = new ArrayBuffer(buf.byteLength);
|
|
9947
|
+
new Uint8Array(data).set(buf);
|
|
9948
|
+
return { data, contentType: mediaType };
|
|
9949
|
+
}
|
|
9950
|
+
};
|
|
9951
|
+
}
|
|
9886
9952
|
async function createKnowledgeBase(eventStore, project, graphDb, eventBus, logger, options) {
|
|
9887
9953
|
const views = new FilesystemViewStorage(project, logger.child({ component: "view-storage" }));
|
|
9888
9954
|
const content = new WorkingTreeStore(
|
|
@@ -9915,27 +9981,6 @@ var PROJECTION_LAG_BACKOFF_MS = [25, 50, 100, 200];
|
|
|
9915
9981
|
var PROJECTION_BARRIER_TIMEOUT_MS = 500;
|
|
9916
9982
|
var GRAPH_BARRIER_BUDGET_MS = PROJECTION_BARRIER_TIMEOUT_MS + PROJECTION_LAG_BACKOFF_MS.reduce((a, b) => a + b, 0);
|
|
9917
9983
|
var GraphContext = class {
|
|
9918
|
-
/**
|
|
9919
|
-
* Get all resources referencing this resource (backlinks)
|
|
9920
|
-
* Requires graph traversal - must use graph database
|
|
9921
|
-
*/
|
|
9922
|
-
static async getBacklinks(resourceId10, kb) {
|
|
9923
|
-
return kb.graph.getResourceReferencedBy(resourceId10);
|
|
9924
|
-
}
|
|
9925
|
-
/**
|
|
9926
|
-
* Find shortest path between two resources
|
|
9927
|
-
* Requires graph traversal - must use graph database
|
|
9928
|
-
*/
|
|
9929
|
-
static async findPath(fromResourceId, toResourceId, kb, maxDepth) {
|
|
9930
|
-
return kb.graph.findPath(fromResourceId, toResourceId, maxDepth);
|
|
9931
|
-
}
|
|
9932
|
-
/**
|
|
9933
|
-
* Get resource connections (graph edges)
|
|
9934
|
-
* Requires graph traversal - must use graph database
|
|
9935
|
-
*/
|
|
9936
|
-
static async getResourceConnections(resourceId10, kb) {
|
|
9937
|
-
return kb.graph.getResourceConnections(resourceId10);
|
|
9938
|
-
}
|
|
9939
9984
|
/**
|
|
9940
9985
|
* Build the unified knowledge graph for a resource's neighborhood:
|
|
9941
9986
|
* resources AND annotations as typed nodes, typed/directional edges.
|
|
@@ -10014,15 +10059,20 @@ var GraphContext = class {
|
|
|
10014
10059
|
const citedSeen = /* @__PURE__ */ new Set();
|
|
10015
10060
|
for (const ann of referencedBy) {
|
|
10016
10061
|
const source = getTargetSource(ann.target);
|
|
10017
|
-
if (!source || source === String(resourceId10) ||
|
|
10018
|
-
citedSeen.
|
|
10019
|
-
|
|
10020
|
-
|
|
10021
|
-
|
|
10062
|
+
if (!source || source === String(resourceId10) || !ann.id || seen.has(ann.id)) continue;
|
|
10063
|
+
if (!citedSeen.has(source)) {
|
|
10064
|
+
citedSeen.add(source);
|
|
10065
|
+
const view = await kb.views.get(resourceId(source));
|
|
10066
|
+
addResourceNode(source, view?.resource?.name ?? source, view?.resource ? getResourceEntityTypes(view.resource) : []);
|
|
10067
|
+
}
|
|
10068
|
+
nodes.push({ id: ann.id, type: "annotation", label: ann.motivation ?? "annotation", entityTypes: getEntityTypes(ann), annotation: ann });
|
|
10069
|
+
seen.add(ann.id);
|
|
10070
|
+
edges.push({ source: ann.id, target: source, type: "annotation-of" });
|
|
10071
|
+
edges.push({ source: ann.id, target: mainId, type: "cites" });
|
|
10022
10072
|
}
|
|
10023
10073
|
for (const ann of annotations) {
|
|
10024
10074
|
if (!ann.id || seen.has(ann.id)) continue;
|
|
10025
|
-
nodes.push({ id: ann.id, type: "annotation", label: ann.motivation ?? "annotation", entityTypes: getEntityTypes(ann) });
|
|
10075
|
+
nodes.push({ id: ann.id, type: "annotation", label: ann.motivation ?? "annotation", entityTypes: getEntityTypes(ann), annotation: ann });
|
|
10026
10076
|
seen.add(ann.id);
|
|
10027
10077
|
edges.push({ source: ann.id, target: mainId, type: "annotation-of" });
|
|
10028
10078
|
}
|
|
@@ -10163,10 +10213,10 @@ var ResourceContext = class _ResourceContext {
|
|
|
10163
10213
|
return Promise.all(
|
|
10164
10214
|
resources.map(async (doc) => {
|
|
10165
10215
|
try {
|
|
10166
|
-
|
|
10167
|
-
|
|
10168
|
-
const
|
|
10169
|
-
const contentPreview = decodeRepresentation(contentBuffer,
|
|
10216
|
+
const source = representationSource(doc);
|
|
10217
|
+
if (source) {
|
|
10218
|
+
const contentBuffer = await kb.content.retrieve(source.storageUri);
|
|
10219
|
+
const contentPreview = decodeRepresentation(contentBuffer, source.mediaType).slice(0, 200);
|
|
10170
10220
|
return { ...doc, content: contentPreview };
|
|
10171
10221
|
}
|
|
10172
10222
|
return { ...doc, content: "" };
|
|
@@ -10178,13 +10228,17 @@ var ResourceContext = class _ResourceContext {
|
|
|
10178
10228
|
}
|
|
10179
10229
|
/**
|
|
10180
10230
|
* Get full content for a resource
|
|
10181
|
-
* Retrieves and decodes the primary representation
|
|
10231
|
+
* Retrieves and decodes the primary representation. ResourceId-keyed
|
|
10232
|
+
* (EXTRACT-LIBRARIAN P3, D-CONTENT b): the descriptor's `storageUri` is the
|
|
10233
|
+
* has-content signal; the fetch itself goes by id, so the standalone
|
|
10234
|
+
* Librarian serves it over the transport.
|
|
10182
10235
|
*/
|
|
10183
10236
|
static async getResourceContent(resource, kb) {
|
|
10184
|
-
|
|
10185
|
-
|
|
10186
|
-
|
|
10187
|
-
|
|
10237
|
+
const id = getResourceId(resource);
|
|
10238
|
+
const source = representationSource(resource);
|
|
10239
|
+
if (source && id) {
|
|
10240
|
+
const { data, contentType } = await kb.content.getBinary(resourceId(id));
|
|
10241
|
+
return decodeRepresentation(Buffer.from(data), contentType);
|
|
10188
10242
|
}
|
|
10189
10243
|
return void 0;
|
|
10190
10244
|
}
|
|
@@ -10251,12 +10305,12 @@ var AnnotationContext = class {
|
|
|
10251
10305
|
}
|
|
10252
10306
|
let sourceContext;
|
|
10253
10307
|
if (includeSourceContext) {
|
|
10254
|
-
if (!sourceDoc
|
|
10308
|
+
if (!getStorageUri(sourceDoc)) {
|
|
10255
10309
|
throw new Error("Source content not found: no storageUri");
|
|
10256
10310
|
}
|
|
10257
10311
|
const primaryRep = getPrimaryRepresentation(sourceDoc);
|
|
10258
|
-
const sourceContent = await kb.content.
|
|
10259
|
-
const contentStr = decodeRepresentation(sourceContent, primaryRep?.mediaType ?? "text/plain");
|
|
10312
|
+
const { data: sourceContent } = await kb.content.getBinary(resourceId10);
|
|
10313
|
+
const contentStr = decodeRepresentation(Buffer.from(sourceContent), primaryRep?.mediaType ?? "text/plain");
|
|
10260
10314
|
const targetSelectorRaw = getTargetSelector(annotation.target);
|
|
10261
10315
|
const targetSelector = Array.isArray(targetSelectorRaw) ? targetSelectorRaw[0] : targetSelectorRaw;
|
|
10262
10316
|
logger?.debug("Target selector", { type: targetSelector?.type });
|
|
@@ -10292,10 +10346,10 @@ var AnnotationContext = class {
|
|
|
10292
10346
|
}
|
|
10293
10347
|
let targetContext;
|
|
10294
10348
|
if (includeTargetContext && targetDoc) {
|
|
10295
|
-
if (targetDoc
|
|
10349
|
+
if (getStorageUri(targetDoc) && bodySource) {
|
|
10296
10350
|
const targetRep = getPrimaryRepresentation(targetDoc);
|
|
10297
|
-
const targetContent = await kb.content.
|
|
10298
|
-
const contentStr = decodeRepresentation(targetContent, targetRep?.mediaType ?? "text/plain");
|
|
10351
|
+
const { data: targetContent } = await kb.content.getBinary(resourceId(bodySource));
|
|
10352
|
+
const contentStr = decodeRepresentation(Buffer.from(targetContent), targetRep?.mediaType ?? "text/plain");
|
|
10299
10353
|
targetContext = {
|
|
10300
10354
|
content: contentStr.slice(0, contextWindow * 2),
|
|
10301
10355
|
summary: inferenceClient ? await generateResourceSummary(targetDoc.name, contentStr, getResourceEntityTypes(targetDoc), inferenceClient) : void 0
|
|
@@ -10343,18 +10397,27 @@ Summary:`;
|
|
|
10343
10397
|
scoreThreshold: 0.5,
|
|
10344
10398
|
filter: { excludeResourceId: resourceId10 }
|
|
10345
10399
|
});
|
|
10346
|
-
|
|
10347
|
-
|
|
10348
|
-
|
|
10349
|
-
|
|
10350
|
-
|
|
10351
|
-
|
|
10352
|
-
|
|
10353
|
-
|
|
10354
|
-
|
|
10355
|
-
|
|
10356
|
-
|
|
10357
|
-
|
|
10400
|
+
const similar = [];
|
|
10401
|
+
for (const r of results) {
|
|
10402
|
+
const matchView = await kb.views.get(r.resourceId);
|
|
10403
|
+
const resourceName = matchView?.resource?.name;
|
|
10404
|
+
if (!resourceName) {
|
|
10405
|
+
logger?.debug("Semantic match dropped \u2014 no view for source resource", { resourceId: String(r.resourceId) });
|
|
10406
|
+
continue;
|
|
10407
|
+
}
|
|
10408
|
+
similar.push({
|
|
10409
|
+
text: r.text,
|
|
10410
|
+
resourceId: r.resourceId,
|
|
10411
|
+
resourceName,
|
|
10412
|
+
annotationId: r.annotationId,
|
|
10413
|
+
score: r.score,
|
|
10414
|
+
entityTypes: r.entityTypes,
|
|
10415
|
+
...r.machineRead ? { machineRead: true } : {}
|
|
10416
|
+
});
|
|
10417
|
+
}
|
|
10418
|
+
if (similar.length > 0) {
|
|
10419
|
+
semanticContext = { similar };
|
|
10420
|
+
logger?.debug("Semantic context found", { matches: similar.length });
|
|
10358
10421
|
}
|
|
10359
10422
|
} catch (error) {
|
|
10360
10423
|
logger?.warn("Semantic context search failed", { error });
|
|
@@ -10568,15 +10631,18 @@ Summary:`;
|
|
|
10568
10631
|
};
|
|
10569
10632
|
}
|
|
10570
10633
|
/**
|
|
10571
|
-
* Get resource content as string
|
|
10634
|
+
* Get resource content as string. ResourceId-keyed (D-CONTENT b): the
|
|
10635
|
+
* primary representation's `storageUri` stays the has-content signal, the
|
|
10636
|
+
* fetch goes by id.
|
|
10572
10637
|
*/
|
|
10573
10638
|
static async getResourceContent(resource, content) {
|
|
10574
|
-
|
|
10639
|
+
const id = getResourceId(resource);
|
|
10640
|
+
if (!getStorageUri(resource) || !id) {
|
|
10575
10641
|
throw new Error("Resource content not found: no storageUri");
|
|
10576
10642
|
}
|
|
10577
10643
|
const primaryRep = getPrimaryRepresentation(resource);
|
|
10578
|
-
const
|
|
10579
|
-
return decodeRepresentation(
|
|
10644
|
+
const { data } = await content.getBinary(resourceId(id));
|
|
10645
|
+
return decodeRepresentation(Buffer.from(data), primaryRep?.mediaType ?? "text/plain");
|
|
10580
10646
|
}
|
|
10581
10647
|
/**
|
|
10582
10648
|
* Extract annotation context from resource content
|
|
@@ -10686,17 +10752,30 @@ var LLMContext = class {
|
|
|
10686
10752
|
}
|
|
10687
10753
|
}
|
|
10688
10754
|
if (matches.length > 0) {
|
|
10689
|
-
|
|
10690
|
-
|
|
10755
|
+
const similar = [];
|
|
10756
|
+
for (const m of matches) {
|
|
10757
|
+
const matchView = await kb.views.get(m.resourceId);
|
|
10758
|
+
const resourceName = matchView?.resource?.name;
|
|
10759
|
+
if (!resourceName) {
|
|
10760
|
+
logger?.debug("Semantic match dropped \u2014 no view for source resource", { resourceId: String(m.resourceId) });
|
|
10761
|
+
continue;
|
|
10762
|
+
}
|
|
10763
|
+
similar.push({
|
|
10691
10764
|
text: m.text,
|
|
10692
10765
|
resourceId: m.resourceId,
|
|
10766
|
+
resourceName,
|
|
10693
10767
|
...m.annotationId ? { annotationId: m.annotationId } : {},
|
|
10694
10768
|
score: m.score,
|
|
10695
10769
|
...m.entityTypes ? { entityTypes: m.entityTypes } : {},
|
|
10696
10770
|
...m.machineRead ? { machineRead: true } : {}
|
|
10697
|
-
})
|
|
10698
|
-
|
|
10699
|
-
|
|
10771
|
+
});
|
|
10772
|
+
}
|
|
10773
|
+
if (similar.length > 0) {
|
|
10774
|
+
semanticContext = {
|
|
10775
|
+
similar,
|
|
10776
|
+
...excludeEntityTypes.length ? { excludedEntityTypes: excludeEntityTypes } : {}
|
|
10777
|
+
};
|
|
10778
|
+
}
|
|
10700
10779
|
}
|
|
10701
10780
|
return {
|
|
10702
10781
|
focus: {
|
|
@@ -10718,15 +10797,15 @@ var LLMContext = class {
|
|
|
10718
10797
|
|
|
10719
10798
|
// src/gatherer.ts
|
|
10720
10799
|
var Gatherer = class {
|
|
10721
|
-
constructor(
|
|
10722
|
-
this.
|
|
10800
|
+
constructor(stores, eventBus, inferenceClient, settleTimeoutMs, logger, embeddingProvider) {
|
|
10801
|
+
this.stores = stores;
|
|
10723
10802
|
this.eventBus = eventBus;
|
|
10724
10803
|
this.inferenceClient = inferenceClient;
|
|
10725
10804
|
this.settleTimeoutMs = settleTimeoutMs;
|
|
10726
10805
|
this.embeddingProvider = embeddingProvider;
|
|
10727
10806
|
this.logger = logger;
|
|
10728
10807
|
}
|
|
10729
|
-
|
|
10808
|
+
stores;
|
|
10730
10809
|
eventBus;
|
|
10731
10810
|
inferenceClient;
|
|
10732
10811
|
settleTimeoutMs;
|
|
@@ -10773,7 +10852,7 @@ var Gatherer = class {
|
|
|
10773
10852
|
const response = await AnnotationContext.buildLLMContext(
|
|
10774
10853
|
annotationId(event.annotationId),
|
|
10775
10854
|
resourceId(event.resourceId),
|
|
10776
|
-
this.
|
|
10855
|
+
this.stores,
|
|
10777
10856
|
this.embeddingProvider,
|
|
10778
10857
|
event.options ?? {},
|
|
10779
10858
|
this.inferenceClient,
|
|
@@ -10804,7 +10883,7 @@ var Gatherer = class {
|
|
|
10804
10883
|
const result = await LLMContext.getResourceContext(
|
|
10805
10884
|
resourceId(event.resourceId),
|
|
10806
10885
|
event.options,
|
|
10807
|
-
this.
|
|
10886
|
+
this.stores,
|
|
10808
10887
|
this.inferenceClient,
|
|
10809
10888
|
this.settleTimeoutMs,
|
|
10810
10889
|
this.logger
|
|
@@ -10830,7 +10909,7 @@ var Gatherer = class {
|
|
|
10830
10909
|
return AnnotationContext.generateAnnotationSummary(
|
|
10831
10910
|
annotationId5,
|
|
10832
10911
|
resourceId10,
|
|
10833
|
-
this.
|
|
10912
|
+
this.stores,
|
|
10834
10913
|
this.inferenceClient
|
|
10835
10914
|
);
|
|
10836
10915
|
}
|
|
@@ -10847,14 +10926,14 @@ var Gatherer = class {
|
|
|
10847
10926
|
var import_rxjs2 = __toESM(require_cjs());
|
|
10848
10927
|
var import_operators2 = __toESM(require_operators());
|
|
10849
10928
|
var Matcher = class {
|
|
10850
|
-
constructor(
|
|
10851
|
-
this.
|
|
10929
|
+
constructor(stores, eventBus, logger, inferenceClient, embeddingProvider) {
|
|
10930
|
+
this.stores = stores;
|
|
10852
10931
|
this.eventBus = eventBus;
|
|
10853
10932
|
this.inferenceClient = inferenceClient;
|
|
10854
10933
|
this.embeddingProvider = embeddingProvider;
|
|
10855
10934
|
this.logger = logger;
|
|
10856
10935
|
}
|
|
10857
|
-
|
|
10936
|
+
stores;
|
|
10858
10937
|
eventBus;
|
|
10859
10938
|
inferenceClient;
|
|
10860
10939
|
embeddingProvider;
|
|
@@ -10934,13 +11013,13 @@ var Matcher = class {
|
|
|
10934
11013
|
const views = deriveViews(context.graph, mainResourceId);
|
|
10935
11014
|
const connections = views.connections;
|
|
10936
11015
|
const [nameMatches, entityTypeMatches, semanticMatches] = await Promise.all([
|
|
10937
|
-
this.
|
|
10938
|
-
annotationEntityTypes.length > 0 ? this.
|
|
11016
|
+
this.stores.graph.listResources({ search: searchTerm, limit: 20 }).then((r) => r.resources),
|
|
11017
|
+
annotationEntityTypes.length > 0 ? this.stores.graph.listResources({ entityTypes: annotationEntityTypes, limit: 50 }).then((r) => r.resources) : Promise.resolve([]),
|
|
10939
11018
|
// 4. Semantic match — vector similarity search (if vectors configured)
|
|
10940
11019
|
this.searchVectors(searchTerm)
|
|
10941
11020
|
]);
|
|
10942
11021
|
const neighborResolved = await Promise.all(
|
|
10943
|
-
connections.map((conn) => resourceWithViewGrace(this.
|
|
11022
|
+
connections.map((conn) => resourceWithViewGrace(this.stores, resourceId(conn.resourceId)))
|
|
10944
11023
|
);
|
|
10945
11024
|
const laggedNeighbors = neighborResolved.filter((r) => r.laggedBehindView).length;
|
|
10946
11025
|
if (laggedNeighbors > 0) {
|
|
@@ -10967,7 +11046,7 @@ var Matcher = class {
|
|
|
10967
11046
|
let laggedSemantic = 0;
|
|
10968
11047
|
for (const sm of semanticMatches) {
|
|
10969
11048
|
semanticScores.set(sm.resourceId, sm.score);
|
|
10970
|
-
const { resource, laggedBehindView } = await resourceWithViewGrace(this.
|
|
11049
|
+
const { resource, laggedBehindView } = await resourceWithViewGrace(this.stores, resourceId(sm.resourceId));
|
|
10971
11050
|
if (laggedBehindView) laggedSemantic++;
|
|
10972
11051
|
if (resource) addCandidate(resource, "semantic");
|
|
10973
11052
|
}
|
|
@@ -11163,7 +11242,7 @@ For each candidate, output a line with the number and score, like:
|
|
|
11163
11242
|
if (!searchTerm.trim()) return [];
|
|
11164
11243
|
try {
|
|
11165
11244
|
const embedding = await this.embeddingProvider.embed(searchTerm);
|
|
11166
|
-
const results = await this.
|
|
11245
|
+
const results = await this.stores.vectors.searchResources(embedding, {
|
|
11167
11246
|
limit: 40,
|
|
11168
11247
|
scoreThreshold: 0.4
|
|
11169
11248
|
});
|
|
@@ -11188,9 +11267,9 @@ For each candidate, output a line with the number and score, like:
|
|
|
11188
11267
|
// src/stower.ts
|
|
11189
11268
|
var import_rxjs3 = __toESM(require_cjs());
|
|
11190
11269
|
var import_operators3 = __toESM(require_operators());
|
|
11191
|
-
async function readEntityTypesProjection(
|
|
11270
|
+
async function readEntityTypesProjection(state) {
|
|
11192
11271
|
const entityTypesPath = path3.join(
|
|
11193
|
-
|
|
11272
|
+
state.stateDir,
|
|
11194
11273
|
"projections",
|
|
11195
11274
|
"__system__",
|
|
11196
11275
|
"entitytypes.json"
|
|
@@ -11232,13 +11311,13 @@ function entityTypesNotRegisteredMessage(unknown) {
|
|
|
11232
11311
|
|
|
11233
11312
|
// src/stower.ts
|
|
11234
11313
|
var Stower = class {
|
|
11235
|
-
constructor(
|
|
11236
|
-
this.
|
|
11314
|
+
constructor(stores, eventBus, project, logger) {
|
|
11315
|
+
this.stores = stores;
|
|
11237
11316
|
this.eventBus = eventBus;
|
|
11238
11317
|
this.project = project;
|
|
11239
11318
|
this.logger = logger;
|
|
11240
11319
|
}
|
|
11241
|
-
|
|
11320
|
+
stores;
|
|
11242
11321
|
eventBus;
|
|
11243
11322
|
project;
|
|
11244
11323
|
subscription = null;
|
|
@@ -11252,6 +11331,7 @@ var Stower = class {
|
|
|
11252
11331
|
);
|
|
11253
11332
|
this.subscription = (0, import_rxjs3.merge)(
|
|
11254
11333
|
pipe("yield:create", (e) => this.handleYieldCreate(e)),
|
|
11334
|
+
pipe("yield:clone-persist", (e) => this.handleYieldClonePersist(e)),
|
|
11255
11335
|
pipe("yield:update", (e) => this.handleYieldUpdate(e)),
|
|
11256
11336
|
pipe("yield:mv", (e) => this.handleYieldMv(e)),
|
|
11257
11337
|
pipe("mark:create", (e) => this.handleMarkCreate(e)),
|
|
@@ -11279,14 +11359,14 @@ var Stower = class {
|
|
|
11279
11359
|
const uid = userId(event._userId);
|
|
11280
11360
|
try {
|
|
11281
11361
|
const rId = resourceId(generateUuid());
|
|
11282
|
-
const stored = await this.
|
|
11362
|
+
const stored = await this.stores.content.register(event.storageUri, event.contentChecksum, { noGit: event.noGit });
|
|
11283
11363
|
const checksum = stored.checksum;
|
|
11284
11364
|
const byteSize = event.byteSize;
|
|
11285
11365
|
const generatedFrom = event.generatedFrom?.resourceId && event.generatedFrom?.annotationId ? {
|
|
11286
11366
|
resourceId: resourceId(event.generatedFrom.resourceId),
|
|
11287
11367
|
annotationId: annotationId(event.generatedFrom.annotationId)
|
|
11288
11368
|
} : void 0;
|
|
11289
|
-
await this.
|
|
11369
|
+
await this.stores.eventStore.appendEvent({
|
|
11290
11370
|
type: "yield:created",
|
|
11291
11371
|
resourceId: rId,
|
|
11292
11372
|
userId: uid,
|
|
@@ -11339,13 +11419,66 @@ var Stower = class {
|
|
|
11339
11419
|
});
|
|
11340
11420
|
}
|
|
11341
11421
|
}
|
|
11422
|
+
/**
|
|
11423
|
+
* Persist a clone (the CloneTokenManager's second half).
|
|
11424
|
+
*
|
|
11425
|
+
* A clone is its own domain fact, not a creation with an extra field: it
|
|
11426
|
+
* NAMES the resource it came from, and `ResourceClonedPayload` requires
|
|
11427
|
+
* that name. So it gets its own command and appends its own event, rather
|
|
11428
|
+
* than an optional parent turning `yield:created` into something else at
|
|
11429
|
+
* runtime.
|
|
11430
|
+
*
|
|
11431
|
+
* The token was already validated and the source's entity types already
|
|
11432
|
+
* read by the CloneTokenManager — the one party that can do either. What
|
|
11433
|
+
* happens here is what only the Stower may do: register the bytes and
|
|
11434
|
+
* append the event (single-writer, GATEWAY.md D4b).
|
|
11435
|
+
*
|
|
11436
|
+
* Generated resources do NOT come through here. Their provenance is
|
|
11437
|
+
* `generatedFrom` on `yield:created`, which is a different relation: a
|
|
11438
|
+
* generated resource is derived from a source, a clone IS a copy of one.
|
|
11439
|
+
*/
|
|
11440
|
+
async handleYieldClonePersist(event) {
|
|
11441
|
+
if (!event._userId) {
|
|
11442
|
+
throw new Error("yield:clone-persist missing _userId (gateway injection)");
|
|
11443
|
+
}
|
|
11444
|
+
try {
|
|
11445
|
+
const rId = resourceId(generateUuid());
|
|
11446
|
+
const stored = await this.stores.content.register(event.storageUri, event.contentChecksum, { noGit: event.noGit });
|
|
11447
|
+
await this.stores.eventStore.appendEvent({
|
|
11448
|
+
type: "yield:cloned",
|
|
11449
|
+
resourceId: rId,
|
|
11450
|
+
userId: userId(event._userId),
|
|
11451
|
+
version: 1,
|
|
11452
|
+
payload: {
|
|
11453
|
+
name: event.name,
|
|
11454
|
+
format: event.format,
|
|
11455
|
+
contentChecksum: stored.checksum,
|
|
11456
|
+
contentByteSize: event.byteSize,
|
|
11457
|
+
storageUri: event.storageUri,
|
|
11458
|
+
parentResourceId: event.parentResourceId,
|
|
11459
|
+
entityTypes: event.entityTypes || [],
|
|
11460
|
+
language: event.language || void 0
|
|
11461
|
+
}
|
|
11462
|
+
});
|
|
11463
|
+
this.eventBus.get("yield:clone-persist-ok").next({
|
|
11464
|
+
correlationId: event.correlationId,
|
|
11465
|
+
response: { resourceId: rId }
|
|
11466
|
+
});
|
|
11467
|
+
} catch (error) {
|
|
11468
|
+
this.logger.error("Failed to persist clone", { error: errField(error) });
|
|
11469
|
+
this.eventBus.get("yield:clone-persist-failed").next({
|
|
11470
|
+
correlationId: event.correlationId,
|
|
11471
|
+
message: error instanceof Error ? error.message : String(error)
|
|
11472
|
+
});
|
|
11473
|
+
}
|
|
11474
|
+
}
|
|
11342
11475
|
async handleYieldUpdate(event) {
|
|
11343
11476
|
if (!event._userId) {
|
|
11344
11477
|
throw new Error("yield:update missing _userId (gateway injection)");
|
|
11345
11478
|
}
|
|
11346
11479
|
try {
|
|
11347
|
-
await this.
|
|
11348
|
-
await this.
|
|
11480
|
+
await this.stores.content.register(event.storageUri, event.contentChecksum, { noGit: event.noGit });
|
|
11481
|
+
await this.stores.eventStore.appendEvent({
|
|
11349
11482
|
type: "yield:updated",
|
|
11350
11483
|
resourceId: resourceId(event.resourceId),
|
|
11351
11484
|
userId: userId(event._userId),
|
|
@@ -11370,7 +11503,7 @@ var Stower = class {
|
|
|
11370
11503
|
async handleYieldMv(event) {
|
|
11371
11504
|
let rId;
|
|
11372
11505
|
try {
|
|
11373
|
-
const resolved = await resolveStorageUri(this.
|
|
11506
|
+
const resolved = await resolveStorageUri(this.project.projectionsDir, event.fromUri);
|
|
11374
11507
|
rId = resolved;
|
|
11375
11508
|
} catch (error) {
|
|
11376
11509
|
this.logger.error("Failed to resolve resource for move", { fromUri: event.fromUri, error });
|
|
@@ -11384,8 +11517,8 @@ var Stower = class {
|
|
|
11384
11517
|
throw new Error("yield:mv missing _userId (gateway injection)");
|
|
11385
11518
|
}
|
|
11386
11519
|
try {
|
|
11387
|
-
await this.
|
|
11388
|
-
await this.
|
|
11520
|
+
await this.stores.content.move(event.fromUri, event.toUri, { noGit: event.noGit });
|
|
11521
|
+
await this.stores.eventStore.appendEvent({
|
|
11389
11522
|
type: "yield:moved",
|
|
11390
11523
|
resourceId: rId,
|
|
11391
11524
|
userId: userId(event._userId),
|
|
@@ -11409,7 +11542,7 @@ var Stower = class {
|
|
|
11409
11542
|
}
|
|
11410
11543
|
try {
|
|
11411
11544
|
this.logger.debug("Stowing annotation", { annotationId: event.annotation.id });
|
|
11412
|
-
await this.
|
|
11545
|
+
await this.stores.eventStore.appendEvent(
|
|
11413
11546
|
{
|
|
11414
11547
|
type: "mark:added",
|
|
11415
11548
|
resourceId: resourceId(event.resourceId),
|
|
@@ -11435,7 +11568,7 @@ var Stower = class {
|
|
|
11435
11568
|
throw new Error("mark:delete missing resourceId");
|
|
11436
11569
|
}
|
|
11437
11570
|
try {
|
|
11438
|
-
await this.
|
|
11571
|
+
await this.stores.eventStore.appendEvent({
|
|
11439
11572
|
type: "mark:removed",
|
|
11440
11573
|
resourceId: resourceId(event.resourceId),
|
|
11441
11574
|
userId: userId(event._userId),
|
|
@@ -11456,7 +11589,7 @@ var Stower = class {
|
|
|
11456
11589
|
throw new Error("mark:update-body missing _userId (gateway injection)");
|
|
11457
11590
|
}
|
|
11458
11591
|
try {
|
|
11459
|
-
await this.
|
|
11592
|
+
await this.stores.eventStore.appendEvent(
|
|
11460
11593
|
{
|
|
11461
11594
|
type: "mark:body-updated",
|
|
11462
11595
|
resourceId: resourceId(event.resourceId),
|
|
@@ -11482,9 +11615,9 @@ var Stower = class {
|
|
|
11482
11615
|
}
|
|
11483
11616
|
try {
|
|
11484
11617
|
if (event.storageUri) {
|
|
11485
|
-
await this.
|
|
11618
|
+
await this.stores.content.remove(event.storageUri, { keepFile: event.keepFile, noGit: event.noGit });
|
|
11486
11619
|
}
|
|
11487
|
-
await this.
|
|
11620
|
+
await this.stores.eventStore.appendEvent({
|
|
11488
11621
|
type: "mark:archived",
|
|
11489
11622
|
resourceId: resourceId(event.resourceId),
|
|
11490
11623
|
userId: userId(event._userId),
|
|
@@ -11506,14 +11639,14 @@ var Stower = class {
|
|
|
11506
11639
|
}
|
|
11507
11640
|
try {
|
|
11508
11641
|
if (event.storageUri) {
|
|
11509
|
-
const absPath = this.
|
|
11642
|
+
const absPath = this.stores.content.resolveUri(event.storageUri);
|
|
11510
11643
|
try {
|
|
11511
11644
|
await promises.access(absPath);
|
|
11512
11645
|
} catch {
|
|
11513
11646
|
throw new Error(`Cannot unarchive: file not found at ${event.storageUri}`);
|
|
11514
11647
|
}
|
|
11515
11648
|
}
|
|
11516
|
-
await this.
|
|
11649
|
+
await this.stores.eventStore.appendEvent({
|
|
11517
11650
|
type: "mark:unarchived",
|
|
11518
11651
|
resourceId: resourceId(event.resourceId),
|
|
11519
11652
|
userId: userId(event._userId),
|
|
@@ -11534,7 +11667,7 @@ var Stower = class {
|
|
|
11534
11667
|
throw new Error("frame:add-entity-type missing _userId (gateway injection)");
|
|
11535
11668
|
}
|
|
11536
11669
|
try {
|
|
11537
|
-
await this.
|
|
11670
|
+
await this.stores.eventStore.appendEvent({
|
|
11538
11671
|
type: "frame:entity-type-added",
|
|
11539
11672
|
userId: userId(event._userId),
|
|
11540
11673
|
version: 1,
|
|
@@ -11554,7 +11687,7 @@ var Stower = class {
|
|
|
11554
11687
|
throw new Error("frame:add-tag-schema missing _userId (gateway injection)");
|
|
11555
11688
|
}
|
|
11556
11689
|
try {
|
|
11557
|
-
await this.
|
|
11690
|
+
await this.stores.eventStore.appendEvent({
|
|
11558
11691
|
type: "frame:tag-schema-added",
|
|
11559
11692
|
userId: userId(event._userId),
|
|
11560
11693
|
version: 1,
|
|
@@ -11585,7 +11718,7 @@ var Stower = class {
|
|
|
11585
11718
|
}
|
|
11586
11719
|
}
|
|
11587
11720
|
for (const entityType2 of added) {
|
|
11588
|
-
await this.
|
|
11721
|
+
await this.stores.eventStore.appendEvent({
|
|
11589
11722
|
type: "mark:entity-tag-added",
|
|
11590
11723
|
resourceId: resourceId(event.resourceId),
|
|
11591
11724
|
userId: uid,
|
|
@@ -11594,7 +11727,7 @@ var Stower = class {
|
|
|
11594
11727
|
});
|
|
11595
11728
|
}
|
|
11596
11729
|
for (const entityType2 of removed) {
|
|
11597
|
-
await this.
|
|
11730
|
+
await this.stores.eventStore.appendEvent({
|
|
11598
11731
|
type: "mark:entity-tag-removed",
|
|
11599
11732
|
resourceId: resourceId(event.resourceId),
|
|
11600
11733
|
userId: uid,
|
|
@@ -11615,7 +11748,7 @@ var Stower = class {
|
|
|
11615
11748
|
if (!event._userId) {
|
|
11616
11749
|
throw new Error("job:start missing _userId (gateway injection)");
|
|
11617
11750
|
}
|
|
11618
|
-
await this.
|
|
11751
|
+
await this.stores.eventStore.appendEvent({
|
|
11619
11752
|
type: "job:started",
|
|
11620
11753
|
resourceId: resourceId(event.resourceId),
|
|
11621
11754
|
userId: userId(event._userId),
|
|
@@ -11631,7 +11764,7 @@ var Stower = class {
|
|
|
11631
11764
|
if (!event._userId) {
|
|
11632
11765
|
throw new Error("job:complete missing _userId (gateway injection)");
|
|
11633
11766
|
}
|
|
11634
|
-
await this.
|
|
11767
|
+
await this.stores.eventStore.appendEvent({
|
|
11635
11768
|
type: "job:completed",
|
|
11636
11769
|
resourceId: resourceId(event.resourceId),
|
|
11637
11770
|
userId: userId(event._userId),
|
|
@@ -11648,7 +11781,7 @@ var Stower = class {
|
|
|
11648
11781
|
if (!event._userId) {
|
|
11649
11782
|
throw new Error("job:fail missing _userId (gateway injection)");
|
|
11650
11783
|
}
|
|
11651
|
-
await this.
|
|
11784
|
+
await this.stores.eventStore.appendEvent({
|
|
11652
11785
|
type: "job:failed",
|
|
11653
11786
|
resourceId: resourceId(event.resourceId),
|
|
11654
11787
|
userId: userId(event._userId),
|
|
@@ -11686,9 +11819,9 @@ async function readAnchoredText(kb, resourceId10) {
|
|
|
11686
11819
|
throw error;
|
|
11687
11820
|
}
|
|
11688
11821
|
}
|
|
11689
|
-
async function readTagSchemasProjection(
|
|
11822
|
+
async function readTagSchemasProjection(state) {
|
|
11690
11823
|
const tagSchemasPath = path3.join(
|
|
11691
|
-
|
|
11824
|
+
state.stateDir,
|
|
11692
11825
|
"projections",
|
|
11693
11826
|
"__system__",
|
|
11694
11827
|
"tagschemas.json"
|
|
@@ -11774,8 +11907,7 @@ function deriveAgentRoster(config) {
|
|
|
11774
11907
|
|
|
11775
11908
|
// src/browser.ts
|
|
11776
11909
|
var Browser = class {
|
|
11777
|
-
constructor(
|
|
11778
|
-
this.views = views;
|
|
11910
|
+
constructor(kb, eventBus, project, config, limitsDiscovery, embeddingProvider, logger) {
|
|
11779
11911
|
this.kb = kb;
|
|
11780
11912
|
this.eventBus = eventBus;
|
|
11781
11913
|
this.project = project;
|
|
@@ -11784,7 +11916,6 @@ var Browser = class {
|
|
|
11784
11916
|
this.embeddingProvider = embeddingProvider;
|
|
11785
11917
|
this.logger = logger;
|
|
11786
11918
|
}
|
|
11787
|
-
views;
|
|
11788
11919
|
kb;
|
|
11789
11920
|
eventBus;
|
|
11790
11921
|
project;
|
|
@@ -11816,6 +11947,7 @@ var Browser = class {
|
|
|
11816
11947
|
this.subscriptions.push(
|
|
11817
11948
|
pipe("browse:resource-requested", (e) => this.handleBrowseResource(e)).subscribe({ error: errorHandler }),
|
|
11818
11949
|
pipe("browse:anchored-text-requested", (e) => this.handleAnchoredText(e)).subscribe({ error: errorHandler }),
|
|
11950
|
+
pipe("browse:anchored-text-by-checksum-requested", (e) => this.handleAnchoredTextByChecksum(e)).subscribe({ error: errorHandler }),
|
|
11819
11951
|
pipe("browse:resources-requested", (e) => this.handleBrowseResources(e)).subscribe({ error: errorHandler }),
|
|
11820
11952
|
pipe("browse:annotations-requested", (e) => this.handleBrowseAnnotations(e)).subscribe({ error: errorHandler }),
|
|
11821
11953
|
pipe("browse:annotation-requested", (e) => this.handleBrowseAnnotation(e)).subscribe({ error: errorHandler }),
|
|
@@ -11859,6 +11991,30 @@ var Browser = class {
|
|
|
11859
11991
|
});
|
|
11860
11992
|
}
|
|
11861
11993
|
}
|
|
11994
|
+
/**
|
|
11995
|
+
* Serve the checksum-addressed consult (ANCHORED-TEXT-TO-SMELTER P2, D2) —
|
|
11996
|
+
* the detection workers' read-through cache. The caller computed the key
|
|
11997
|
+
* from bytes it already holds, so unlike the resource-addressed read there
|
|
11998
|
+
* is nothing to resolve and no settle barrier: with no resourceId there is
|
|
11999
|
+
* no content generation to wait on, and a miss means "extract it yourself".
|
|
12000
|
+
* A hit — success or decline — is served whole, so the caller runs neither
|
|
12001
|
+
* parser nor engine. Read-only over the wire: the Smelter is the sole
|
|
12002
|
+
* writer and never answers here.
|
|
12003
|
+
*/
|
|
12004
|
+
async handleAnchoredTextByChecksum(event) {
|
|
12005
|
+
try {
|
|
12006
|
+
this.eventBus.get("browse:anchored-text-by-checksum-result").next({
|
|
12007
|
+
correlationId: event.correlationId,
|
|
12008
|
+
response: await this.kb.anchoredText.read(event.checksum)
|
|
12009
|
+
});
|
|
12010
|
+
} catch (error) {
|
|
12011
|
+
this.logger.error("Browse anchored text by checksum failed", { checksum: event.checksum, error: errField(error) });
|
|
12012
|
+
this.eventBus.get("browse:anchored-text-by-checksum-failed").next({
|
|
12013
|
+
correlationId: event.correlationId,
|
|
12014
|
+
message: error instanceof Error ? error.message : String(error)
|
|
12015
|
+
});
|
|
12016
|
+
}
|
|
12017
|
+
}
|
|
11862
12018
|
async handleBrowseResource(event) {
|
|
11863
12019
|
try {
|
|
11864
12020
|
const response = await assembleResourceGraph(this.kb, resourceId(event.resourceId));
|
|
@@ -12156,10 +12312,13 @@ var Browser = class {
|
|
|
12156
12312
|
return;
|
|
12157
12313
|
}
|
|
12158
12314
|
const visible = dirents.filter((d) => d.name !== ".semiont" && !d.name.startsWith("."));
|
|
12159
|
-
const allViews = await this.views.getAll();
|
|
12315
|
+
const allViews = await this.kb.views.getAll();
|
|
12160
12316
|
const prefix = `file://${resolved}`;
|
|
12161
12317
|
const viewsByUri = new Map(
|
|
12162
|
-
allViews.filter((v) =>
|
|
12318
|
+
allViews.filter((v) => {
|
|
12319
|
+
const uri = getStorageUri(v.resource);
|
|
12320
|
+
return uri?.startsWith(prefix + "/") || uri?.startsWith(prefix + path3.sep);
|
|
12321
|
+
}).map((v) => [getStorageUri(v.resource), v])
|
|
12163
12322
|
);
|
|
12164
12323
|
const entries = [];
|
|
12165
12324
|
for (const dirent of visible) {
|
|
@@ -12248,13 +12407,13 @@ function createLimitsDiscovery(config, logger, options) {
|
|
|
12248
12407
|
}
|
|
12249
12408
|
}
|
|
12250
12409
|
const consult = async (pair, client) => {
|
|
12251
|
-
let
|
|
12410
|
+
let timer;
|
|
12252
12411
|
try {
|
|
12253
12412
|
const discovery = client.limits();
|
|
12254
12413
|
const raced = await Promise.race([
|
|
12255
12414
|
discovery,
|
|
12256
12415
|
new Promise((resolve2) => {
|
|
12257
|
-
|
|
12416
|
+
timer = setTimeout(() => resolve2(void 0), budgetMs);
|
|
12258
12417
|
})
|
|
12259
12418
|
]);
|
|
12260
12419
|
if (raced === void 0) {
|
|
@@ -12271,7 +12430,7 @@ function createLimitsDiscovery(config, logger, options) {
|
|
|
12271
12430
|
});
|
|
12272
12431
|
return void 0;
|
|
12273
12432
|
} finally {
|
|
12274
|
-
clearTimeout(
|
|
12433
|
+
clearTimeout(timer);
|
|
12275
12434
|
}
|
|
12276
12435
|
};
|
|
12277
12436
|
return {
|
|
@@ -12307,6 +12466,15 @@ async function readAnnotationFromView(kb, resourceId10, annotationId5) {
|
|
|
12307
12466
|
const allAnnotations = await AnnotationContext.getAllAnnotations(resourceId10, kb);
|
|
12308
12467
|
return allAnnotations.find((a) => a.id === annotationId5) ?? null;
|
|
12309
12468
|
}
|
|
12469
|
+
function wireEnrichment(eventStore, kb) {
|
|
12470
|
+
eventStore.setEnrichEvent(async (event, resourceId10) => {
|
|
12471
|
+
const annId = eventAnnotationId(event);
|
|
12472
|
+
if (annId === null) return event;
|
|
12473
|
+
const annotation = await readAnnotationFromView(kb, resourceId10, annId);
|
|
12474
|
+
if (annotation === null) return event;
|
|
12475
|
+
return { ...event, annotation };
|
|
12476
|
+
});
|
|
12477
|
+
}
|
|
12310
12478
|
|
|
12311
12479
|
// src/clone-token-manager.ts
|
|
12312
12480
|
var import_rxjs6 = __toESM(require_cjs());
|
|
@@ -12358,16 +12526,67 @@ var ResourceOperations = class {
|
|
|
12358
12526
|
);
|
|
12359
12527
|
return resourceId(rId);
|
|
12360
12528
|
}
|
|
12529
|
+
/**
|
|
12530
|
+
* Persist a CLONE via EventBus → Stower.
|
|
12531
|
+
*
|
|
12532
|
+
* Separate from `createResource` because a clone is a separate fact: it
|
|
12533
|
+
* names its parent, and `yield:cloned` requires that name. Callers reach
|
|
12534
|
+
* this only after the CloneTokenManager has validated the token — it is the
|
|
12535
|
+
* inner half of the flow, not a public entry point.
|
|
12536
|
+
*/
|
|
12537
|
+
static async persistClone(input, userId3, eventBus) {
|
|
12538
|
+
const { resourceId: rId } = await busRequest(
|
|
12539
|
+
asBusRequestPrimitive(eventBus),
|
|
12540
|
+
"yield:clone-persist",
|
|
12541
|
+
{
|
|
12542
|
+
name: input.name,
|
|
12543
|
+
storageUri: input.storageUri,
|
|
12544
|
+
contentChecksum: input.contentChecksum,
|
|
12545
|
+
byteSize: input.byteSize,
|
|
12546
|
+
format: input.format,
|
|
12547
|
+
parentResourceId: input.parentResourceId,
|
|
12548
|
+
entityTypes: input.entityTypes,
|
|
12549
|
+
language: input.language,
|
|
12550
|
+
_userId: userId3
|
|
12551
|
+
},
|
|
12552
|
+
3e4
|
|
12553
|
+
);
|
|
12554
|
+
return resourceId(rId);
|
|
12555
|
+
}
|
|
12556
|
+
/**
|
|
12557
|
+
* Create a resource from a clone token via EventBus → CloneTokenManager.
|
|
12558
|
+
* The bytes are already stored (the gateway's upload path, `noGit` — the
|
|
12559
|
+
* Archivist's register does the one `git add`, D4b); the command carries
|
|
12560
|
+
* storage coordinates only (EXTRACT-ARCHIVIST P3, D4a).
|
|
12561
|
+
*/
|
|
12562
|
+
static async createFromCloneToken(input, userId3, eventBus) {
|
|
12563
|
+
const { resourceId: rId } = await busRequest(
|
|
12564
|
+
asBusRequestPrimitive(eventBus),
|
|
12565
|
+
"yield:clone-create",
|
|
12566
|
+
{
|
|
12567
|
+
token: input.token,
|
|
12568
|
+
name: input.name,
|
|
12569
|
+
storageUri: input.storageUri,
|
|
12570
|
+
contentChecksum: input.contentChecksum,
|
|
12571
|
+
byteSize: input.byteSize,
|
|
12572
|
+
format: input.format,
|
|
12573
|
+
archiveOriginal: input.archiveOriginal,
|
|
12574
|
+
_userId: userId3
|
|
12575
|
+
},
|
|
12576
|
+
3e4
|
|
12577
|
+
);
|
|
12578
|
+
return resourceId(rId);
|
|
12579
|
+
}
|
|
12361
12580
|
};
|
|
12362
12581
|
|
|
12363
12582
|
// src/clone-token-manager.ts
|
|
12364
12583
|
var CloneTokenManager = class {
|
|
12365
|
-
constructor(
|
|
12366
|
-
this.
|
|
12584
|
+
constructor(stores, eventBus, logger) {
|
|
12585
|
+
this.stores = stores;
|
|
12367
12586
|
this.eventBus = eventBus;
|
|
12368
12587
|
this.logger = logger;
|
|
12369
12588
|
}
|
|
12370
|
-
|
|
12589
|
+
stores;
|
|
12371
12590
|
eventBus;
|
|
12372
12591
|
subscriptions = [];
|
|
12373
12592
|
logger;
|
|
@@ -12392,7 +12611,7 @@ var CloneTokenManager = class {
|
|
|
12392
12611
|
}
|
|
12393
12612
|
async handleGenerateToken(event) {
|
|
12394
12613
|
try {
|
|
12395
|
-
const resource = await ResourceContext.getResourceMetadata(resourceId(event.resourceId), this.
|
|
12614
|
+
const resource = await ResourceContext.getResourceMetadata(resourceId(event.resourceId), this.stores);
|
|
12396
12615
|
if (!resource) {
|
|
12397
12616
|
this.eventBus.get("yield:clone-token-failed").next({
|
|
12398
12617
|
correlationId: event.correlationId,
|
|
@@ -12400,7 +12619,8 @@ var CloneTokenManager = class {
|
|
|
12400
12619
|
});
|
|
12401
12620
|
return;
|
|
12402
12621
|
}
|
|
12403
|
-
|
|
12622
|
+
const storageUri = getStorageUri(resource);
|
|
12623
|
+
if (!storageUri) {
|
|
12404
12624
|
this.eventBus.get("yield:clone-token-failed").next({
|
|
12405
12625
|
correlationId: event.correlationId,
|
|
12406
12626
|
message: "Resource content not found"
|
|
@@ -12408,7 +12628,7 @@ var CloneTokenManager = class {
|
|
|
12408
12628
|
return;
|
|
12409
12629
|
}
|
|
12410
12630
|
try {
|
|
12411
|
-
await this.
|
|
12631
|
+
await promises.access(this.stores.content.resolveUri(storageUri));
|
|
12412
12632
|
} catch {
|
|
12413
12633
|
this.eventBus.get("yield:clone-token-failed").next({
|
|
12414
12634
|
correlationId: event.correlationId,
|
|
@@ -12455,7 +12675,7 @@ var CloneTokenManager = class {
|
|
|
12455
12675
|
});
|
|
12456
12676
|
return;
|
|
12457
12677
|
}
|
|
12458
|
-
const sourceResource = await ResourceContext.getResourceMetadata(tokenData.resourceId, this.
|
|
12678
|
+
const sourceResource = await ResourceContext.getResourceMetadata(tokenData.resourceId, this.stores);
|
|
12459
12679
|
if (!sourceResource) {
|
|
12460
12680
|
this.eventBus.get("yield:clone-resource-failed").next({
|
|
12461
12681
|
correlationId: event.correlationId,
|
|
@@ -12504,7 +12724,7 @@ var CloneTokenManager = class {
|
|
|
12504
12724
|
});
|
|
12505
12725
|
return;
|
|
12506
12726
|
}
|
|
12507
|
-
const sourceDoc = await ResourceContext.getResourceMetadata(tokenData.resourceId, this.
|
|
12727
|
+
const sourceDoc = await ResourceContext.getResourceMetadata(tokenData.resourceId, this.stores);
|
|
12508
12728
|
if (!sourceDoc) {
|
|
12509
12729
|
this.eventBus.get("yield:clone-create-failed").next({
|
|
12510
12730
|
correlationId: event.correlationId,
|
|
@@ -12512,17 +12732,14 @@ var CloneTokenManager = class {
|
|
|
12512
12732
|
});
|
|
12513
12733
|
return;
|
|
12514
12734
|
}
|
|
12515
|
-
const
|
|
12516
|
-
const format = isSupportedMediaType(base) && capabilitiesOf(base)?.authorable ? base : "text/plain";
|
|
12517
|
-
const resolvedUri = deriveStorageUri(event.name, format);
|
|
12518
|
-
const stored = await this.kb.content.store(Buffer.from(event.content), resolvedUri);
|
|
12519
|
-
const newResourceId = await ResourceOperations.createResource(
|
|
12735
|
+
const newResourceId = await ResourceOperations.persistClone(
|
|
12520
12736
|
{
|
|
12521
12737
|
name: event.name,
|
|
12522
|
-
storageUri:
|
|
12523
|
-
contentChecksum:
|
|
12524
|
-
byteSize:
|
|
12525
|
-
format,
|
|
12738
|
+
storageUri: event.storageUri,
|
|
12739
|
+
contentChecksum: event.contentChecksum,
|
|
12740
|
+
byteSize: event.byteSize,
|
|
12741
|
+
format: event.format,
|
|
12742
|
+
parentResourceId: String(tokenData.resourceId),
|
|
12526
12743
|
entityTypes: getResourceEntityTypes(sourceDoc)
|
|
12527
12744
|
},
|
|
12528
12745
|
userId(event._userId),
|
|
@@ -12590,42 +12807,115 @@ async function stopKnowledgeSystem(ks) {
|
|
|
12590
12807
|
ks.kb.weaveProgress.dispose();
|
|
12591
12808
|
await ks.kb.graph.disconnect();
|
|
12592
12809
|
}
|
|
12593
|
-
function
|
|
12810
|
+
async function assertAnnotatableTarget(kb, target) {
|
|
12811
|
+
const view = await kb.views.get(resourceId(target));
|
|
12812
|
+
const mediaType = getPrimaryRepresentation(view?.resource)?.mediaType;
|
|
12813
|
+
if (!mediaType || !isAnnotatable(mediaType)) {
|
|
12814
|
+
throw new Error(`"${mediaType ?? "unknown"}" cannot be annotated`);
|
|
12815
|
+
}
|
|
12816
|
+
}
|
|
12817
|
+
var AnnotationOperations = class {
|
|
12818
|
+
/**
|
|
12819
|
+
* Create a new annotation via EventBus → Stower
|
|
12820
|
+
*/
|
|
12821
|
+
static async createAnnotation(request, userId3, creator, eventBus, kb) {
|
|
12822
|
+
await assertAnnotatableTarget(kb, request.target.source);
|
|
12823
|
+
const { annotation } = assembleAnnotation(request, creator);
|
|
12824
|
+
const resId = resourceId(request.target.source);
|
|
12825
|
+
eventBus.get("mark:create").next({
|
|
12826
|
+
annotation,
|
|
12827
|
+
_userId: userId3,
|
|
12828
|
+
resourceId: resId
|
|
12829
|
+
});
|
|
12830
|
+
return { annotation };
|
|
12831
|
+
}
|
|
12832
|
+
/**
|
|
12833
|
+
* Update annotation body via EventBus → Stower
|
|
12834
|
+
*/
|
|
12835
|
+
static async updateAnnotationBody(id, request, userId3, eventBus, kb) {
|
|
12836
|
+
const resId = resourceId(request.resourceId);
|
|
12837
|
+
const annotation = await AnnotationContext.getAnnotation(
|
|
12838
|
+
annotationId(id),
|
|
12839
|
+
resId,
|
|
12840
|
+
kb
|
|
12841
|
+
);
|
|
12842
|
+
if (!annotation) {
|
|
12843
|
+
throw new Error("Annotation not found");
|
|
12844
|
+
}
|
|
12845
|
+
eventBus.get("mark:update-body").next({
|
|
12846
|
+
annotationId: annotationId(id),
|
|
12847
|
+
_userId: userId3,
|
|
12848
|
+
resourceId: resId,
|
|
12849
|
+
operations: request.operations
|
|
12850
|
+
});
|
|
12851
|
+
const updatedBody = applyBodyOperations(annotation.body, request.operations);
|
|
12852
|
+
return {
|
|
12853
|
+
annotation: {
|
|
12854
|
+
...annotation,
|
|
12855
|
+
body: updatedBody
|
|
12856
|
+
}
|
|
12857
|
+
};
|
|
12858
|
+
}
|
|
12859
|
+
/**
|
|
12860
|
+
* Delete an annotation via EventBus → Stower
|
|
12861
|
+
*/
|
|
12862
|
+
static async deleteAnnotation(id, resourceIdStr, userId3, eventBus, kb, logger) {
|
|
12863
|
+
const resId = resourceId(resourceIdStr);
|
|
12864
|
+
const projection = await AnnotationContext.getResourceAnnotations(resId, kb);
|
|
12865
|
+
const annotation = projection.annotations.find((a) => a.id === id);
|
|
12866
|
+
if (!annotation) {
|
|
12867
|
+
throw new Error("Annotation not found in resource");
|
|
12868
|
+
}
|
|
12869
|
+
logger?.debug("Removing annotation via EventBus", { annotationId: id });
|
|
12870
|
+
eventBus.get("mark:delete").next({
|
|
12871
|
+
annotationId: annotationId(id),
|
|
12872
|
+
_userId: userId3,
|
|
12873
|
+
resourceId: resId
|
|
12874
|
+
});
|
|
12875
|
+
logger?.debug("Annotation delete event emitted");
|
|
12876
|
+
}
|
|
12877
|
+
};
|
|
12878
|
+
|
|
12879
|
+
// src/handlers/annotation-assembly.ts
|
|
12880
|
+
function registerAnnotationAssemblyHandler(eventBus, kb, parentLogger) {
|
|
12594
12881
|
const logger = parentLogger.child({ component: "annotation-assembly" });
|
|
12595
12882
|
const inflight = /* @__PURE__ */ new Map();
|
|
12596
12883
|
eventBus.get("mark:create-request").subscribe((command) => {
|
|
12597
|
-
|
|
12598
|
-
|
|
12599
|
-
|
|
12600
|
-
|
|
12601
|
-
|
|
12602
|
-
|
|
12603
|
-
|
|
12604
|
-
|
|
12884
|
+
void (async () => {
|
|
12885
|
+
const { correlationId, resourceId: resId, request, _userId } = command;
|
|
12886
|
+
const cid = correlationId;
|
|
12887
|
+
try {
|
|
12888
|
+
if (!_userId || typeof _userId !== "string") {
|
|
12889
|
+
throw new Error("_userId is required (injected by bus gateway)");
|
|
12890
|
+
}
|
|
12891
|
+
if (!cid) {
|
|
12892
|
+
throw new Error("correlationId is required on mark:create-request");
|
|
12893
|
+
}
|
|
12894
|
+
await assertAnnotatableTarget(kb, resId);
|
|
12895
|
+
const agent = didToAgent(_userId);
|
|
12896
|
+
const { annotation } = assembleAnnotation(request, agent);
|
|
12897
|
+
inflight.set(cid, { annotationId: annotation.id });
|
|
12898
|
+
eventBus.get("mark:create").next({
|
|
12899
|
+
correlationId: cid,
|
|
12900
|
+
annotation,
|
|
12901
|
+
_userId,
|
|
12902
|
+
resourceId: resourceId(resId)
|
|
12903
|
+
});
|
|
12904
|
+
logger.info("Annotation assembled, awaiting persistence", {
|
|
12905
|
+
annotationId: annotation.id,
|
|
12906
|
+
correlationId: cid
|
|
12907
|
+
});
|
|
12908
|
+
} catch (error) {
|
|
12909
|
+
logger.warn("mark:create-request failed during assembly", {
|
|
12910
|
+
correlationId: cid,
|
|
12911
|
+
error: error.message
|
|
12912
|
+
});
|
|
12913
|
+
eventBus.get("mark:create-failed").next({
|
|
12914
|
+
correlationId: cid,
|
|
12915
|
+
message: error.message
|
|
12916
|
+
});
|
|
12605
12917
|
}
|
|
12606
|
-
|
|
12607
|
-
const { annotation } = assembleAnnotation(request, agent);
|
|
12608
|
-
inflight.set(cid, { annotationId: annotation.id });
|
|
12609
|
-
eventBus.get("mark:create").next({
|
|
12610
|
-
correlationId: cid,
|
|
12611
|
-
annotation,
|
|
12612
|
-
_userId,
|
|
12613
|
-
resourceId: resourceId(resId)
|
|
12614
|
-
});
|
|
12615
|
-
logger.info("Annotation assembled, awaiting persistence", {
|
|
12616
|
-
annotationId: annotation.id,
|
|
12617
|
-
correlationId: cid
|
|
12618
|
-
});
|
|
12619
|
-
} catch (error) {
|
|
12620
|
-
logger.warn("mark:create-request failed during assembly", {
|
|
12621
|
-
correlationId: cid,
|
|
12622
|
-
error: error.message
|
|
12623
|
-
});
|
|
12624
|
-
eventBus.get("mark:create-failed").next({
|
|
12625
|
-
correlationId: cid,
|
|
12626
|
-
message: error.message
|
|
12627
|
-
});
|
|
12628
|
-
}
|
|
12918
|
+
})();
|
|
12629
12919
|
});
|
|
12630
12920
|
eventBus.get("mark:added").subscribe((event) => {
|
|
12631
12921
|
const cid = event.metadata?.correlationId;
|
|
@@ -12645,7 +12935,7 @@ function registerAnnotationAssemblyHandler(eventBus, parentLogger) {
|
|
|
12645
12935
|
inflight.delete(cid);
|
|
12646
12936
|
});
|
|
12647
12937
|
}
|
|
12648
|
-
function
|
|
12938
|
+
function registerAnnotationContextHandler(eventBus, kb, parentLogger) {
|
|
12649
12939
|
const logger = parentLogger.child({ component: "annotation-lookups" });
|
|
12650
12940
|
eventBus.get("browse:annotation-context-requested").subscribe(async (command) => {
|
|
12651
12941
|
const { correlationId } = command;
|
|
@@ -12673,6 +12963,9 @@ function registerAnnotationLookupHandlers(eventBus, kb, gatherer, parentLogger)
|
|
|
12673
12963
|
});
|
|
12674
12964
|
}
|
|
12675
12965
|
});
|
|
12966
|
+
}
|
|
12967
|
+
function registerGatherSummaryHandler(eventBus, gatherer, parentLogger) {
|
|
12968
|
+
const logger = parentLogger.child({ component: "annotation-lookups" });
|
|
12676
12969
|
eventBus.get("gather:summary-requested").subscribe(async (command) => {
|
|
12677
12970
|
const { correlationId } = command;
|
|
12678
12971
|
const annId = command.annotationId;
|
|
@@ -12758,7 +13051,7 @@ function parseDidUser(did) {
|
|
|
12758
13051
|
const email = decodeURIComponent(parts.slice(usersIdx + 1).join(":"));
|
|
12759
13052
|
return { userId: did, email, domain };
|
|
12760
13053
|
}
|
|
12761
|
-
function registerJobCommandHandlers(eventBus, jobQueue,
|
|
13054
|
+
function registerJobCommandHandlers(eventBus, jobQueue, state, parentLogger) {
|
|
12762
13055
|
const logger = parentLogger.child({ component: "job-commands" });
|
|
12763
13056
|
eventBus.get("job:create").subscribe(async (command) => {
|
|
12764
13057
|
const { correlationId, jobType, resourceId: resId, params, _userId } = command;
|
|
@@ -12823,7 +13116,7 @@ function registerJobCommandHandlers(eventBus, jobQueue, project, parentLogger) {
|
|
|
12823
13116
|
};
|
|
12824
13117
|
const jobParams = job.params;
|
|
12825
13118
|
if ((jobType === "reference-annotation" || jobType === "generation") && Array.isArray(jobParams.entityTypes) && jobParams.entityTypes.length > 0) {
|
|
12826
|
-
const registered = await readEntityTypesProjection(
|
|
13119
|
+
const registered = await readEntityTypesProjection(state);
|
|
12827
13120
|
const result = validateEntityTypes(registered, jobParams.entityTypes);
|
|
12828
13121
|
if (!result.ok) {
|
|
12829
13122
|
throw new Error(entityTypesNotRegisteredMessage(result.unknown));
|
|
@@ -12833,7 +13126,7 @@ function registerJobCommandHandlers(eventBus, jobQueue, project, parentLogger) {
|
|
|
12833
13126
|
jobParams.entityTypes = jobParams.entityTypes.map((et) => entityType(et));
|
|
12834
13127
|
}
|
|
12835
13128
|
if (jobType === "tag-annotation") {
|
|
12836
|
-
const schemas = await readTagSchemasProjection(
|
|
13129
|
+
const schemas = await readTagSchemasProjection(state);
|
|
12837
13130
|
const result = resolveTagSchema(schemas, jobParams.schemaId);
|
|
12838
13131
|
if (result.error !== void 0) {
|
|
12839
13132
|
throw new Error(result.error);
|
|
@@ -12949,17 +13242,27 @@ function registerJobCommandHandlers(eventBus, jobQueue, project, parentLogger) {
|
|
|
12949
13242
|
}
|
|
12950
13243
|
|
|
12951
13244
|
// src/handlers/index.ts
|
|
12952
|
-
function registerBusHandlers(eventBus, knowledgeSystem, jobQueue,
|
|
12953
|
-
|
|
12954
|
-
|
|
13245
|
+
function registerBusHandlers(eventBus, knowledgeSystem, jobQueue, state, logger) {
|
|
13246
|
+
const { kb } = knowledgeSystem;
|
|
13247
|
+
registerAnnotationAssemblyHandler(eventBus, kb, logger);
|
|
13248
|
+
registerAnnotationContextHandler(
|
|
13249
|
+
eventBus,
|
|
13250
|
+
{ views: kb.views, content: workingTreeContentReads(kb.views, kb.content) },
|
|
13251
|
+
logger
|
|
13252
|
+
);
|
|
13253
|
+
registerGatherSummaryHandler(eventBus, knowledgeSystem.gatherer, logger);
|
|
12955
13254
|
registerBindUpdateBodyHandler(eventBus, logger);
|
|
12956
|
-
registerJobCommandHandlers(eventBus, jobQueue,
|
|
13255
|
+
registerJobCommandHandlers(eventBus, jobQueue, state, logger);
|
|
13256
|
+
}
|
|
13257
|
+
function registerGatewayBusHandlers(eventBus, jobQueue, state, logger) {
|
|
13258
|
+
registerBindUpdateBodyHandler(eventBus, logger);
|
|
13259
|
+
registerJobCommandHandlers(eventBus, jobQueue, state, logger);
|
|
12957
13260
|
}
|
|
12958
13261
|
|
|
12959
13262
|
// src/service.ts
|
|
12960
|
-
async function createJobQueue(
|
|
13263
|
+
async function createJobQueue(state, eventBus, logger) {
|
|
12961
13264
|
const jobQueueLogger = logger.child({ component: "job-queue" });
|
|
12962
|
-
const jobQueue = new FsJobQueue(
|
|
13265
|
+
const jobQueue = new FsJobQueue(state, jobQueueLogger, eventBus);
|
|
12963
13266
|
await jobQueue.initialize();
|
|
12964
13267
|
registerJobQueueProvider(() => jobQueue.getStats());
|
|
12965
13268
|
const jobStatusSubscription = eventBus.get("job:status-requested").pipe(
|
|
@@ -12999,12 +13302,12 @@ async function createJobQueue(project, eventBus, logger) {
|
|
|
12999
13302
|
}
|
|
13000
13303
|
var STARTUP_CONNECT_TIMEOUT_MS = 6e4;
|
|
13001
13304
|
async function withStartupTimeout(what, work) {
|
|
13002
|
-
let
|
|
13305
|
+
let timer;
|
|
13003
13306
|
try {
|
|
13004
13307
|
return await Promise.race([
|
|
13005
13308
|
work,
|
|
13006
13309
|
new Promise((_resolve, reject) => {
|
|
13007
|
-
|
|
13310
|
+
timer = setTimeout(
|
|
13008
13311
|
() => reject(
|
|
13009
13312
|
new Error(
|
|
13010
13313
|
`${what} did not become available within ${STARTUP_CONNECT_TIMEOUT_MS / 1e3}s. Exiting so the container restart policy can retry \u2014 it is normal for a dependency to be slow when every service restarts at once.`
|
|
@@ -13015,10 +13318,10 @@ async function withStartupTimeout(what, work) {
|
|
|
13015
13318
|
})
|
|
13016
13319
|
]);
|
|
13017
13320
|
} finally {
|
|
13018
|
-
if (
|
|
13321
|
+
if (timer !== void 0) clearTimeout(timer);
|
|
13019
13322
|
}
|
|
13020
13323
|
}
|
|
13021
|
-
async function
|
|
13324
|
+
async function connectStores(project, config, eventBus, logger, skipRebuild) {
|
|
13022
13325
|
const graphConfig = config.services.graph;
|
|
13023
13326
|
logger.info("Connecting to graph database", { type: graphConfig.type });
|
|
13024
13327
|
const graphDb = await withStartupTimeout("Graph database", getGraphDatabase(graphConfig));
|
|
@@ -13061,18 +13364,18 @@ async function createKnowledgeSystemFromConfig(project, config, eventBus, logger
|
|
|
13061
13364
|
vectorStore,
|
|
13062
13365
|
skipRebuild
|
|
13063
13366
|
});
|
|
13064
|
-
|
|
13065
|
-
|
|
13066
|
-
|
|
13067
|
-
|
|
13068
|
-
|
|
13069
|
-
return { ...event, annotation };
|
|
13070
|
-
});
|
|
13367
|
+
return { kb, eventStore, embeddingProvider };
|
|
13368
|
+
}
|
|
13369
|
+
async function createKnowledgeSystemFromConfig(project, config, eventBus, logger, skipRebuild) {
|
|
13370
|
+
const { kb, eventStore, embeddingProvider } = await connectStores(project, config, eventBus, logger, skipRebuild ?? false);
|
|
13371
|
+
wireEnrichment(eventStore, kb);
|
|
13071
13372
|
const stower = new Stower(kb, eventBus, project, logger.child({ component: "stower" }));
|
|
13072
13373
|
await stower.initialize();
|
|
13073
13374
|
await bootstrapEntityTypes(eventBus, eventStore, logger.child({ component: "entity-types-bootstrap" }));
|
|
13074
13375
|
const gatherer = new Gatherer(
|
|
13075
|
-
|
|
13376
|
+
// The content capability is ResourceId-keyed (D-CONTENT b); in-process
|
|
13377
|
+
// it wraps this root's own working tree behind the transport shape.
|
|
13378
|
+
{ ...kb, content: workingTreeContentReads(kb.views, kb.content) },
|
|
13076
13379
|
eventBus,
|
|
13077
13380
|
createInferenceClient(resolveActorInference(config, "gatherer"), logger.child({ component: "inference-client-gatherer" })),
|
|
13078
13381
|
config.gather.settleTimeoutMs,
|
|
@@ -13089,14 +13392,14 @@ async function createKnowledgeSystemFromConfig(project, config, eventBus, logger
|
|
|
13089
13392
|
);
|
|
13090
13393
|
await matcher.initialize();
|
|
13091
13394
|
const limitsDiscovery = createLimitsDiscovery(config, logger.child({ component: "limits-discovery" }));
|
|
13092
|
-
const browser = new Browser(kb
|
|
13395
|
+
const browser = new Browser(kb, eventBus, project, config, limitsDiscovery, embeddingProvider, logger.child({ component: "browser" }));
|
|
13093
13396
|
await browser.initialize();
|
|
13094
13397
|
const cloneTokenManager = new CloneTokenManager(kb, eventBus, logger.child({ component: "clone-token-manager" }));
|
|
13095
13398
|
await cloneTokenManager.initialize();
|
|
13096
13399
|
const ks = { kb, stower, gatherer, matcher, browser, cloneTokenManager, stop: () => stopKnowledgeSystem(ks) };
|
|
13097
13400
|
return ks;
|
|
13098
13401
|
}
|
|
13099
|
-
|
|
13402
|
+
function assertMakeMeaningConfig(config) {
|
|
13100
13403
|
if (!config.services?.graph) {
|
|
13101
13404
|
throw new Error("services.graph is required for make-meaning service");
|
|
13102
13405
|
}
|
|
@@ -13108,6 +13411,9 @@ async function startMakeMeaning(project, config, eventBus, logger, options) {
|
|
|
13108
13411
|
`gather.settleTimeoutMs (${config.gather.settleTimeoutMs}ms) plus the graph barrier budget (${GRAPH_BARRIER_BUDGET_MS}ms) must nest inside the job-worker stall watchdog (${STALL_THRESHOLD_MS}ms) \u2014 lower settleTimeoutMs (A4)`
|
|
13109
13412
|
);
|
|
13110
13413
|
}
|
|
13414
|
+
}
|
|
13415
|
+
async function startMakeMeaning(project, config, eventBus, logger, options) {
|
|
13416
|
+
assertMakeMeaningConfig(config);
|
|
13111
13417
|
const skipRebuild = options?.skipRebuild ?? process.env.SEMIONT_SKIP_REBUILD === "true";
|
|
13112
13418
|
const { jobQueue, jobStatusSubscription } = await createJobQueue(project, eventBus, logger);
|
|
13113
13419
|
const knowledgeSystem = await createKnowledgeSystemFromConfig(project, config, eventBus, logger, skipRebuild);
|
|
@@ -13124,6 +13430,106 @@ async function startMakeMeaning(project, config, eventBus, logger, options) {
|
|
|
13124
13430
|
}
|
|
13125
13431
|
};
|
|
13126
13432
|
}
|
|
13433
|
+
async function startMakeMeaningGateway(state, config, eventBus, logger) {
|
|
13434
|
+
assertMakeMeaningConfig(config);
|
|
13435
|
+
const { jobQueue, jobStatusSubscription } = await createJobQueue(state, eventBus, logger);
|
|
13436
|
+
registerGatewayBusHandlers(eventBus, jobQueue, state, logger);
|
|
13437
|
+
return {
|
|
13438
|
+
jobQueue,
|
|
13439
|
+
state,
|
|
13440
|
+
stop: async () => {
|
|
13441
|
+
logger.info("Stopping gateway make-meaning");
|
|
13442
|
+
jobStatusSubscription.unsubscribe();
|
|
13443
|
+
logger.info("Gateway make-meaning stopped");
|
|
13444
|
+
}
|
|
13445
|
+
};
|
|
13446
|
+
}
|
|
13447
|
+
var json = (res, status, body) => {
|
|
13448
|
+
res.writeHead(status, { "Content-Type": "application/json" });
|
|
13449
|
+
res.end(JSON.stringify(body));
|
|
13450
|
+
};
|
|
13451
|
+
function createArchivistServer(deps) {
|
|
13452
|
+
const { events, content, views, workerSecret, health, branch, logger } = deps;
|
|
13453
|
+
const authorized = (req, res) => {
|
|
13454
|
+
if (!workerSecret) {
|
|
13455
|
+
json(res, 503, { error: "disabled: no SEMIONT_WORKER_SECRET configured" });
|
|
13456
|
+
return false;
|
|
13457
|
+
}
|
|
13458
|
+
if (req.headers.authorization !== `Bearer ${workerSecret}`) {
|
|
13459
|
+
json(res, 401, { error: "unauthorized" });
|
|
13460
|
+
return false;
|
|
13461
|
+
}
|
|
13462
|
+
return true;
|
|
13463
|
+
};
|
|
13464
|
+
return createServer((req, res) => {
|
|
13465
|
+
const url = new URL(req.url ?? "/", "http://archivist");
|
|
13466
|
+
if (req.method === "GET" && url.pathname === "/health") {
|
|
13467
|
+
json(res, 200, health());
|
|
13468
|
+
return;
|
|
13469
|
+
}
|
|
13470
|
+
if (req.method === "GET" && url.pathname === "/kb/branch") {
|
|
13471
|
+
if (!authorized(req, res)) return;
|
|
13472
|
+
json(res, 200, { branch: branch() });
|
|
13473
|
+
return;
|
|
13474
|
+
}
|
|
13475
|
+
if (req.method === "GET" && url.pathname.startsWith("/events/")) {
|
|
13476
|
+
if (!authorized(req, res)) return;
|
|
13477
|
+
const rawId = decodeURIComponent(url.pathname.slice("/events/".length));
|
|
13478
|
+
const rawFrom = url.searchParams.get("fromSequence");
|
|
13479
|
+
const fromSequence = rawFrom === null ? NaN : Number(rawFrom);
|
|
13480
|
+
if (!rawId || !Number.isInteger(fromSequence) || fromSequence < 1) {
|
|
13481
|
+
json(res, 400, { error: "resourceId path segment and integer fromSequence >= 1 are required" });
|
|
13482
|
+
return;
|
|
13483
|
+
}
|
|
13484
|
+
events.queryEvents(resourceId(rawId), { fromSequence }).then((replay) => json(res, 200, { events: replay })).catch((error) => {
|
|
13485
|
+
logger.error("D1 read path failed", { resourceId: rawId, fromSequence, error: errField(error) });
|
|
13486
|
+
json(res, 500, { error: "event read failed" });
|
|
13487
|
+
});
|
|
13488
|
+
return;
|
|
13489
|
+
}
|
|
13490
|
+
const contentMatch = req.method === "GET" && /^\/resources\/(.+)\/content$/.exec(url.pathname);
|
|
13491
|
+
if (contentMatch) {
|
|
13492
|
+
if (!authorized(req, res)) return;
|
|
13493
|
+
const rid = decodeURIComponent(contentMatch[1]);
|
|
13494
|
+
resolveRepresentation({ views, content }, resourceId(rid)).then(async ({ stream, mediaType }) => {
|
|
13495
|
+
res.writeHead(200, { "Content-Type": mediaType });
|
|
13496
|
+
await pipeline(stream, res);
|
|
13497
|
+
}).catch((error) => {
|
|
13498
|
+
if (error instanceof RepresentationMissing) {
|
|
13499
|
+
json(res, 404, { error: error.message, reason: error.reason });
|
|
13500
|
+
return;
|
|
13501
|
+
}
|
|
13502
|
+
logger.error("Content read failed", { resourceId: rid, error: errField(error) });
|
|
13503
|
+
if (res.headersSent) res.destroy();
|
|
13504
|
+
else json(res, 500, { error: "content read failed" });
|
|
13505
|
+
});
|
|
13506
|
+
return;
|
|
13507
|
+
}
|
|
13508
|
+
if (req.method === "PUT" && url.pathname.startsWith("/content/")) {
|
|
13509
|
+
if (!authorized(req, res)) return;
|
|
13510
|
+
const storageUri = decodeURIComponent(url.pathname.slice("/content/".length));
|
|
13511
|
+
if (!storageUri) {
|
|
13512
|
+
json(res, 400, { error: "storageUri path segment is required" });
|
|
13513
|
+
return;
|
|
13514
|
+
}
|
|
13515
|
+
const expectedChecksum = url.searchParams.get("checksum");
|
|
13516
|
+
content.store(req, storageUri, {
|
|
13517
|
+
noGit: true,
|
|
13518
|
+
...expectedChecksum !== null ? { expectedChecksum } : {}
|
|
13519
|
+
}).then((stored) => json(res, 200, stored)).catch((error) => {
|
|
13520
|
+
if (error instanceof ChecksumMismatchError) {
|
|
13521
|
+
json(res, 409, { error: "checksum mismatch: body does not match the checksum the caller supplied" });
|
|
13522
|
+
return;
|
|
13523
|
+
}
|
|
13524
|
+
logger.error("Content write failed", { storageUri, error: errField(error) });
|
|
13525
|
+
json(res, 500, { error: "content write failed" });
|
|
13526
|
+
});
|
|
13527
|
+
return;
|
|
13528
|
+
}
|
|
13529
|
+
res.writeHead(404);
|
|
13530
|
+
res.end();
|
|
13531
|
+
});
|
|
13532
|
+
}
|
|
13127
13533
|
|
|
13128
13534
|
// src/local-transport.ts
|
|
13129
13535
|
var import_rxjs8 = __toESM(require_cjs());
|
|
@@ -13205,7 +13611,7 @@ var LocalTransport = class {
|
|
|
13205
13611
|
}
|
|
13206
13612
|
}
|
|
13207
13613
|
// LocalTransport implements `ITransport` only. It does not implement
|
|
13208
|
-
// `
|
|
13614
|
+
// `IGatewayOperations` — a SemiontClient built over LocalTransport has
|
|
13209
13615
|
// no `client.auth` / `client.admin` namespaces by design (no
|
|
13210
13616
|
// credentials, no admin routes, no exchange machinery in-process).
|
|
13211
13617
|
// ── Lifecycle ───────────────────────────────────────────────────────────
|
|
@@ -13221,10 +13627,10 @@ var LocalTransport = class {
|
|
|
13221
13627
|
}
|
|
13222
13628
|
};
|
|
13223
13629
|
var LocalContentTransport = class {
|
|
13224
|
-
constructor(
|
|
13225
|
-
this.
|
|
13630
|
+
constructor(kb) {
|
|
13631
|
+
this.kb = kb;
|
|
13226
13632
|
}
|
|
13227
|
-
|
|
13633
|
+
kb;
|
|
13228
13634
|
async putBinary(_request, _options) {
|
|
13229
13635
|
throw new Error(
|
|
13230
13636
|
"LocalContentTransport does not support putBinary() \u2014 create resources via bus emits (mark/yield namespaces) in local mode"
|
|
@@ -13239,12 +13645,12 @@ var LocalContentTransport = class {
|
|
|
13239
13645
|
*/
|
|
13240
13646
|
async putAnchoredText(checksum, outcome, _options) {
|
|
13241
13647
|
busLog("PUT", "anchored-text", { checksum });
|
|
13242
|
-
await this.
|
|
13648
|
+
await this.kb.anchoredText.write(checksum, outcome);
|
|
13243
13649
|
}
|
|
13244
13650
|
/** The stored outcome, or null when nothing has derived one — the common case. */
|
|
13245
13651
|
async getAnchoredText(resourceId10, _options) {
|
|
13246
13652
|
busLog("GET", "anchored-text", { resourceId: resourceId10 });
|
|
13247
|
-
return readAnchoredText(this.
|
|
13653
|
+
return readAnchoredText(this.kb, resourceId10);
|
|
13248
13654
|
}
|
|
13249
13655
|
/**
|
|
13250
13656
|
* The cache-consult read (PERSIST-ANCHORS P2c), straight from the store —
|
|
@@ -13253,7 +13659,7 @@ var LocalContentTransport = class {
|
|
|
13253
13659
|
*/
|
|
13254
13660
|
async getAnchoredTextByChecksum(checksum, _options) {
|
|
13255
13661
|
busLog("GET", "anchored-text-by-checksum", { checksum });
|
|
13256
|
-
return this.
|
|
13662
|
+
return this.kb.anchoredText.read(checksum);
|
|
13257
13663
|
}
|
|
13258
13664
|
/**
|
|
13259
13665
|
* The store's would-hit keys, straight from the store — planning data for
|
|
@@ -13262,7 +13668,7 @@ var LocalContentTransport = class {
|
|
|
13262
13668
|
*/
|
|
13263
13669
|
async listAnchoredTextKeys(_options) {
|
|
13264
13670
|
busLog("GET", "anchored-text-keys", {});
|
|
13265
|
-
return this.
|
|
13671
|
+
return this.kb.anchoredText.list();
|
|
13266
13672
|
}
|
|
13267
13673
|
async getBinary(resourceId10, _options) {
|
|
13268
13674
|
busLog("GET", "content", { resourceId: resourceId10 });
|
|
@@ -13293,16 +13699,14 @@ var LocalContentTransport = class {
|
|
|
13293
13699
|
}
|
|
13294
13700
|
);
|
|
13295
13701
|
}
|
|
13296
|
-
|
|
13297
|
-
|
|
13298
|
-
|
|
13299
|
-
|
|
13300
|
-
|
|
13301
|
-
|
|
13302
|
-
|
|
13303
|
-
|
|
13304
|
-
const data = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
|
|
13305
|
-
return { data, contentType: rep.mediaType };
|
|
13702
|
+
/**
|
|
13703
|
+
* The same resolution the wire path serves, buffered — local and hosted
|
|
13704
|
+
* modes must answer identically, which they did not before
|
|
13705
|
+
* SINGLE-KB-MOUNT P3: this resolved through `representations[].storageUri`,
|
|
13706
|
+
* a field `ViewMaterializer` never writes, so every binary read here threw.
|
|
13707
|
+
*/
|
|
13708
|
+
loadBinary(resourceId10) {
|
|
13709
|
+
return workingTreeContentReads(this.kb.views, this.kb.content).getBinary(resourceId10);
|
|
13306
13710
|
}
|
|
13307
13711
|
/**
|
|
13308
13712
|
* Assemble the resource's JSON-LD graph in-process from the KB — the local
|
|
@@ -13310,7 +13714,7 @@ var LocalContentTransport = class {
|
|
|
13310
13714
|
* getBinary; SIMPLER-JSON-LD.md decision 7). Local mode has no auth.
|
|
13311
13715
|
*/
|
|
13312
13716
|
async getResourceGraph(resourceId10, _options) {
|
|
13313
|
-
const graph = await assembleResourceGraph(this.
|
|
13717
|
+
const graph = await assembleResourceGraph(this.kb, resourceId10);
|
|
13314
13718
|
if (!graph) throw new Error(`Resource not found: ${resourceId10}`);
|
|
13315
13719
|
return graph;
|
|
13316
13720
|
}
|
|
@@ -13355,23 +13759,24 @@ function isWorkItem(input) {
|
|
|
13355
13759
|
return WORK_ITEM_TYPES.has(input.type);
|
|
13356
13760
|
}
|
|
13357
13761
|
var Smelter = class _Smelter {
|
|
13358
|
-
constructor(events$, rebuildAnchors$, vectorStore, embeddingProvider, content, bus, chunkingConfig, timing, logger) {
|
|
13762
|
+
constructor(events$, rebuildAnchors$, vectorStore, embeddingProvider, content, anchoredStore, bus, chunkingConfig, timing, logger) {
|
|
13359
13763
|
this.events$ = events$;
|
|
13360
13764
|
this.rebuildAnchors$ = rebuildAnchors$;
|
|
13361
13765
|
this.vectorStore = vectorStore;
|
|
13362
13766
|
this.embeddingProvider = embeddingProvider;
|
|
13363
13767
|
this.content = content;
|
|
13768
|
+
this.anchoredStore = anchoredStore;
|
|
13364
13769
|
this.bus = bus;
|
|
13365
13770
|
this.chunkingConfig = chunkingConfig;
|
|
13366
13771
|
this.timing = timing;
|
|
13367
13772
|
this.logger = logger;
|
|
13368
|
-
this.anchoredStore = anchoredTextStoreOverTransport(content, logger.child({ component: "anchored-text-cache" }));
|
|
13369
13773
|
}
|
|
13370
13774
|
events$;
|
|
13371
13775
|
rebuildAnchors$;
|
|
13372
13776
|
vectorStore;
|
|
13373
13777
|
embeddingProvider;
|
|
13374
13778
|
content;
|
|
13779
|
+
anchoredStore;
|
|
13375
13780
|
bus;
|
|
13376
13781
|
chunkingConfig;
|
|
13377
13782
|
timing;
|
|
@@ -13395,7 +13800,6 @@ var Smelter = class _Smelter {
|
|
|
13395
13800
|
* OCR pass.
|
|
13396
13801
|
*/
|
|
13397
13802
|
drainChain = Promise.resolve();
|
|
13398
|
-
anchoredStore;
|
|
13399
13803
|
get eventsProcessed() {
|
|
13400
13804
|
return this._eventsProcessed;
|
|
13401
13805
|
}
|
|
@@ -13617,15 +14021,15 @@ var Smelter = class _Smelter {
|
|
|
13617
14021
|
key: checksum,
|
|
13618
14022
|
store: this.anchoredStore
|
|
13619
14023
|
});
|
|
13620
|
-
if ("declined"
|
|
14024
|
+
if (extracted.kind === "declined" || !extracted.items?.length) {
|
|
13621
14025
|
this.logger.info("Re-anchor extraction yielded no geometry", {
|
|
13622
14026
|
resourceId: rid,
|
|
13623
14027
|
contentType,
|
|
13624
|
-
..."declined"
|
|
14028
|
+
...extracted.kind === "declined" ? { declined: extracted.declined } : {}
|
|
13625
14029
|
});
|
|
13626
14030
|
return;
|
|
13627
14031
|
}
|
|
13628
|
-
await this.
|
|
14032
|
+
await this.anchoredStore.write(checksum, { ...extracted, items: extracted.items });
|
|
13629
14033
|
this.logger.info("Re-anchored resource", { resourceId: rid, checksum, items: extracted.items.length });
|
|
13630
14034
|
}
|
|
13631
14035
|
async handleResourcePurge(event) {
|
|
@@ -13656,7 +14060,7 @@ var Smelter = class _Smelter {
|
|
|
13656
14060
|
key: checksum,
|
|
13657
14061
|
store: this.anchoredStore
|
|
13658
14062
|
});
|
|
13659
|
-
if ("declined"
|
|
14063
|
+
if (extracted.kind === "declined") {
|
|
13660
14064
|
this.logger.debug("Extractor declined", { resourceId: resourceId10, contentType, reason: extracted.declined });
|
|
13661
14065
|
return { kind: "skipped", checksum, reason: extracted.declined };
|
|
13662
14066
|
}
|
|
@@ -13912,7 +14316,7 @@ var Smelter = class _Smelter {
|
|
|
13912
14316
|
// per resource (PERSIST-ANCHORS P0). Keys are resource ids today;
|
|
13913
14317
|
// P1 rekeys the store by content checksum and this lookup moves
|
|
13914
14318
|
// with it.
|
|
13915
|
-
this.
|
|
14319
|
+
this.anchoredStore.list().then((keys) => new Set(keys))
|
|
13916
14320
|
]);
|
|
13917
14321
|
const resources = await this.listAllResources();
|
|
13918
14322
|
this.logger.info("Reconcile started", {
|
|
@@ -14149,864 +14553,7 @@ function createSmelterActorStateUnit(options) {
|
|
|
14149
14553
|
}
|
|
14150
14554
|
};
|
|
14151
14555
|
}
|
|
14152
|
-
var pipelineAsync = promisify(pipeline);
|
|
14153
|
-
var BLOCK_SIZE = 512;
|
|
14154
|
-
function createTarHeader(name, size) {
|
|
14155
|
-
const header = Buffer.alloc(BLOCK_SIZE, 0);
|
|
14156
|
-
header.write(name.slice(0, 100), 0, 100, "utf8");
|
|
14157
|
-
header.write("0000644\0", 100, 8, "utf8");
|
|
14158
|
-
header.write("0000000\0", 108, 8, "utf8");
|
|
14159
|
-
header.write("0000000\0", 116, 8, "utf8");
|
|
14160
|
-
header.write(size.toString(8).padStart(11, "0") + "\0", 124, 12, "utf8");
|
|
14161
|
-
const mtime = Math.floor(Date.now() / 1e3);
|
|
14162
|
-
header.write(mtime.toString(8).padStart(11, "0") + "\0", 136, 12, "utf8");
|
|
14163
|
-
header.write(" ", 148, 8, "utf8");
|
|
14164
|
-
header.write("0", 156, 1, "utf8");
|
|
14165
|
-
header.write("ustar\0", 257, 6, "utf8");
|
|
14166
|
-
header.write("00", 263, 2, "utf8");
|
|
14167
|
-
let checksum = 0;
|
|
14168
|
-
for (let i = 0; i < BLOCK_SIZE; i++) {
|
|
14169
|
-
checksum += header[i];
|
|
14170
|
-
}
|
|
14171
|
-
header.write(checksum.toString(8).padStart(6, "0") + "\0 ", 148, 8, "utf8");
|
|
14172
|
-
return header;
|
|
14173
|
-
}
|
|
14174
|
-
function paddingBytes(size) {
|
|
14175
|
-
const remainder = size % BLOCK_SIZE;
|
|
14176
|
-
return remainder === 0 ? 0 : BLOCK_SIZE - remainder;
|
|
14177
|
-
}
|
|
14178
|
-
async function writeTarGz(entries, output) {
|
|
14179
|
-
const gzip = createGzip();
|
|
14180
|
-
const tarStream = new Readable({ read() {
|
|
14181
|
-
} });
|
|
14182
|
-
const pipePromise = pipelineAsync(tarStream, gzip, output);
|
|
14183
|
-
for await (const entry of entries) {
|
|
14184
|
-
const header = createTarHeader(entry.name, entry.data.length);
|
|
14185
|
-
tarStream.push(header);
|
|
14186
|
-
tarStream.push(entry.data);
|
|
14187
|
-
const pad = paddingBytes(entry.data.length);
|
|
14188
|
-
if (pad > 0) {
|
|
14189
|
-
tarStream.push(Buffer.alloc(pad, 0));
|
|
14190
|
-
}
|
|
14191
|
-
}
|
|
14192
|
-
tarStream.push(Buffer.alloc(BLOCK_SIZE * 2, 0));
|
|
14193
|
-
tarStream.push(null);
|
|
14194
|
-
await pipePromise;
|
|
14195
|
-
}
|
|
14196
|
-
async function decompressStream(input) {
|
|
14197
|
-
const gunzip = createGunzip();
|
|
14198
|
-
const chunks = [];
|
|
14199
|
-
return new Promise((resolve2, reject) => {
|
|
14200
|
-
gunzip.on("data", (chunk) => chunks.push(chunk));
|
|
14201
|
-
gunzip.on("end", () => resolve2(Buffer.concat(chunks)));
|
|
14202
|
-
gunzip.on("error", reject);
|
|
14203
|
-
input.on("error", reject);
|
|
14204
|
-
input.pipe(gunzip);
|
|
14205
|
-
});
|
|
14206
|
-
}
|
|
14207
|
-
function* parseTarEntries(decompressed) {
|
|
14208
|
-
let offset = 0;
|
|
14209
|
-
while (offset + BLOCK_SIZE <= decompressed.length) {
|
|
14210
|
-
const header = decompressed.subarray(offset, offset + BLOCK_SIZE);
|
|
14211
|
-
if (header.every((b) => b === 0)) break;
|
|
14212
|
-
const nameEnd = header.indexOf(0, 0);
|
|
14213
|
-
const name = header.subarray(0, Math.min(nameEnd, 100)).toString("utf8");
|
|
14214
|
-
const sizeStr = header.subarray(124, 135).toString("utf8").trim();
|
|
14215
|
-
const size = parseInt(sizeStr, 8);
|
|
14216
|
-
offset += BLOCK_SIZE;
|
|
14217
|
-
const data = decompressed.subarray(offset, offset + size);
|
|
14218
|
-
offset += size;
|
|
14219
|
-
offset += paddingBytes(size);
|
|
14220
|
-
yield { name, size, data };
|
|
14221
|
-
}
|
|
14222
|
-
}
|
|
14223
|
-
async function* readTarGz(input) {
|
|
14224
|
-
const decompressed = await decompressStream(input);
|
|
14225
|
-
yield* parseTarEntries(decompressed);
|
|
14226
|
-
}
|
|
14227
|
-
|
|
14228
|
-
// src/exchange/manifest.ts
|
|
14229
|
-
var BACKUP_FORMAT = "semiont-backup";
|
|
14230
|
-
var FORMAT_VERSION = 1;
|
|
14231
|
-
var LINKED_DATA_FORMAT = "semiont-linked-data";
|
|
14232
|
-
function isLinkedDataManifest(obj) {
|
|
14233
|
-
return typeof obj === "object" && obj !== null && obj["semiont:format"] === LINKED_DATA_FORMAT;
|
|
14234
|
-
}
|
|
14235
|
-
function isBackupManifest(obj) {
|
|
14236
|
-
return typeof obj === "object" && obj !== null && obj.format === BACKUP_FORMAT;
|
|
14237
|
-
}
|
|
14238
|
-
function validateManifestVersion(version) {
|
|
14239
|
-
if (version > FORMAT_VERSION) {
|
|
14240
|
-
throw new Error(
|
|
14241
|
-
`Unsupported format version ${version}. This tool supports version ${FORMAT_VERSION}.`
|
|
14242
|
-
);
|
|
14243
|
-
}
|
|
14244
|
-
}
|
|
14245
|
-
|
|
14246
|
-
// src/exchange/backup-exporter.ts
|
|
14247
|
-
var SYSTEM_STREAM = "__system__";
|
|
14248
|
-
async function exportBackup(options, output) {
|
|
14249
|
-
const { eventStore, content, sourceUrl, logger } = options;
|
|
14250
|
-
const resourceIds = await eventStore.log.storage.getAllResourceIds();
|
|
14251
|
-
logger?.info("Backup export: enumerating streams", { resourceCount: resourceIds.length });
|
|
14252
|
-
const allStreamIds = [SYSTEM_STREAM, ...resourceIds];
|
|
14253
|
-
const streamData = /* @__PURE__ */ new Map();
|
|
14254
|
-
let totalEvents = 0;
|
|
14255
|
-
for (const id of allStreamIds) {
|
|
14256
|
-
const events = await eventStore.log.getEvents(id);
|
|
14257
|
-
if (events.length > 0) {
|
|
14258
|
-
streamData.set(id, events);
|
|
14259
|
-
totalEvents += events.length;
|
|
14260
|
-
}
|
|
14261
|
-
}
|
|
14262
|
-
const contentRefs = collectContentRefs(streamData);
|
|
14263
|
-
logger?.info("Backup export: collected content refs", {
|
|
14264
|
-
streams: streamData.size,
|
|
14265
|
-
events: totalEvents,
|
|
14266
|
-
blobs: contentRefs.size
|
|
14267
|
-
});
|
|
14268
|
-
const contentBlobs = /* @__PURE__ */ new Map();
|
|
14269
|
-
let totalContentBytes = 0;
|
|
14270
|
-
for (const [storageUri, mediaType] of contentRefs) {
|
|
14271
|
-
const data = await content.retrieve(storageUri);
|
|
14272
|
-
const ext = extensionForMediaType(mediaType);
|
|
14273
|
-
contentBlobs.set(storageUri, { data, ext });
|
|
14274
|
-
totalContentBytes += data.length;
|
|
14275
|
-
}
|
|
14276
|
-
const streamSummaries = [];
|
|
14277
|
-
for (const [streamId, events] of streamData) {
|
|
14278
|
-
streamSummaries.push({
|
|
14279
|
-
stream: streamId,
|
|
14280
|
-
eventCount: events.length
|
|
14281
|
-
});
|
|
14282
|
-
}
|
|
14283
|
-
const manifestHeader = {
|
|
14284
|
-
format: BACKUP_FORMAT,
|
|
14285
|
-
version: FORMAT_VERSION,
|
|
14286
|
-
exportedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
14287
|
-
sourceUrl,
|
|
14288
|
-
stats: {
|
|
14289
|
-
streams: streamData.size,
|
|
14290
|
-
events: totalEvents,
|
|
14291
|
-
blobs: contentBlobs.size,
|
|
14292
|
-
contentBytes: totalContentBytes
|
|
14293
|
-
}
|
|
14294
|
-
};
|
|
14295
|
-
async function* generateEntries() {
|
|
14296
|
-
const manifestLines = [
|
|
14297
|
-
JSON.stringify(manifestHeader),
|
|
14298
|
-
...streamSummaries.map((s) => JSON.stringify(s))
|
|
14299
|
-
].join("\n") + "\n";
|
|
14300
|
-
yield { name: ".semiont/manifest.jsonl", data: Buffer.from(manifestLines, "utf8") };
|
|
14301
|
-
for (const [streamId, events] of streamData) {
|
|
14302
|
-
const fileName = streamId === SYSTEM_STREAM ? ".semiont/events/__system__.jsonl" : `.semiont/events/${streamId}.jsonl`;
|
|
14303
|
-
const jsonl = events.map((e) => JSON.stringify(e)).join("\n") + "\n";
|
|
14304
|
-
yield { name: fileName, data: Buffer.from(jsonl, "utf8") };
|
|
14305
|
-
}
|
|
14306
|
-
for (const [storageUri, { data, ext }] of contentBlobs) {
|
|
14307
|
-
const blobPath = storageUri.startsWith("file://") ? storageUri.slice(7) : storageUri;
|
|
14308
|
-
yield { name: `${blobPath}${ext}`, data };
|
|
14309
|
-
}
|
|
14310
|
-
}
|
|
14311
|
-
await writeTarGz(generateEntries(), output);
|
|
14312
|
-
logger?.info("Backup export complete", {
|
|
14313
|
-
streams: streamData.size,
|
|
14314
|
-
events: totalEvents,
|
|
14315
|
-
blobs: contentBlobs.size,
|
|
14316
|
-
contentBytes: totalContentBytes
|
|
14317
|
-
});
|
|
14318
|
-
return manifestHeader;
|
|
14319
|
-
}
|
|
14320
|
-
function collectContentRefs(streamData) {
|
|
14321
|
-
const refs = /* @__PURE__ */ new Map();
|
|
14322
|
-
for (const [, events] of streamData) {
|
|
14323
|
-
for (const stored of events) {
|
|
14324
|
-
if (stored.type === "yield:created") {
|
|
14325
|
-
const payload = stored.payload;
|
|
14326
|
-
if (payload.storageUri && payload.format) {
|
|
14327
|
-
refs.set(payload.storageUri, payload.format);
|
|
14328
|
-
}
|
|
14329
|
-
}
|
|
14330
|
-
}
|
|
14331
|
-
}
|
|
14332
|
-
return refs;
|
|
14333
|
-
}
|
|
14334
|
-
|
|
14335
|
-
// src/exchange/await-reply.ts
|
|
14336
|
-
var import_rxjs11 = __toESM(require_cjs());
|
|
14337
|
-
var import_operators9 = __toESM(require_operators());
|
|
14338
|
-
function awaitReply(eventBus, command, payload, okChannel, failedChannel, timeoutMs) {
|
|
14339
|
-
const result$ = (0, import_rxjs11.race)(
|
|
14340
|
-
eventBus.get(okChannel).pipe((0, import_operators9.map)(() => void 0)),
|
|
14341
|
-
eventBus.get(failedChannel).pipe(
|
|
14342
|
-
(0, import_operators9.map)((e) => {
|
|
14343
|
-
throw new Error(e.message ?? `${command} failed`);
|
|
14344
|
-
})
|
|
14345
|
-
),
|
|
14346
|
-
(0, import_rxjs11.timer)(timeoutMs).pipe(
|
|
14347
|
-
(0, import_operators9.map)(() => {
|
|
14348
|
-
throw new Error(`Timeout waiting for ${okChannel}`);
|
|
14349
|
-
})
|
|
14350
|
-
)
|
|
14351
|
-
);
|
|
14352
|
-
eventBus.get(command).next(payload);
|
|
14353
|
-
return (0, import_rxjs11.firstValueFrom)(result$);
|
|
14354
|
-
}
|
|
14355
|
-
var REPLAY_TIMEOUT_MS = 3e4;
|
|
14356
|
-
async function replayEventStream(jsonl, eventBus, eventStore, resolveBlob, contentStore, logger) {
|
|
14357
|
-
const lines = jsonl.trim().split("\n").filter((l) => l.length > 0);
|
|
14358
|
-
const storedEvents = lines.map((line) => JSON.parse(line));
|
|
14359
|
-
const stats = {
|
|
14360
|
-
eventsReplayed: 0,
|
|
14361
|
-
resourcesCreated: 0,
|
|
14362
|
-
annotationsCreated: 0,
|
|
14363
|
-
entityTypesAdded: 0
|
|
14364
|
-
};
|
|
14365
|
-
for (const stored of storedEvents) {
|
|
14366
|
-
await replayEvent(stored, eventBus, eventStore, resolveBlob, contentStore, stats, logger);
|
|
14367
|
-
stats.eventsReplayed++;
|
|
14368
|
-
}
|
|
14369
|
-
return { stats };
|
|
14370
|
-
}
|
|
14371
|
-
async function replayEvent(event, eventBus, eventStore, resolveBlob, contentStore, stats, logger) {
|
|
14372
|
-
switch (event.type) {
|
|
14373
|
-
case "frame:entity-type-added":
|
|
14374
|
-
await replayEntityTypeAdded(event, eventBus, logger);
|
|
14375
|
-
stats.entityTypesAdded++;
|
|
14376
|
-
break;
|
|
14377
|
-
case "yield:created":
|
|
14378
|
-
await replayResourceCreated(event, eventBus, resolveBlob, contentStore, logger);
|
|
14379
|
-
stats.resourcesCreated++;
|
|
14380
|
-
break;
|
|
14381
|
-
case "mark:added":
|
|
14382
|
-
await replayAnnotationAdded(event, eventBus, logger);
|
|
14383
|
-
stats.annotationsCreated++;
|
|
14384
|
-
break;
|
|
14385
|
-
case "mark:body-updated":
|
|
14386
|
-
await replayAnnotationBodyUpdated(event, eventBus, logger);
|
|
14387
|
-
break;
|
|
14388
|
-
case "mark:removed":
|
|
14389
|
-
await replayAnnotationRemoved(event, eventBus, logger);
|
|
14390
|
-
break;
|
|
14391
|
-
case "mark:archived":
|
|
14392
|
-
await replayResourceArchived(event, eventBus, logger);
|
|
14393
|
-
break;
|
|
14394
|
-
case "mark:unarchived":
|
|
14395
|
-
await replayResourceUnarchived(event, eventBus, logger);
|
|
14396
|
-
break;
|
|
14397
|
-
case "mark:entity-tag-added":
|
|
14398
|
-
case "mark:entity-tag-removed":
|
|
14399
|
-
await replayEntityTagChange(event, eventStore, logger);
|
|
14400
|
-
break;
|
|
14401
|
-
// Job events are transient — skip during replay
|
|
14402
|
-
case "job:started":
|
|
14403
|
-
case "job:completed":
|
|
14404
|
-
case "job:failed":
|
|
14405
|
-
logger?.debug("Skipping job event during replay", { type: event.type });
|
|
14406
|
-
break;
|
|
14407
|
-
// Representation events — content is already stored via yield:created replay
|
|
14408
|
-
case "yield:representation-added":
|
|
14409
|
-
case "yield:representation-removed":
|
|
14410
|
-
logger?.debug("Skipping representation event during replay", { type: event.type });
|
|
14411
|
-
break;
|
|
14412
|
-
default:
|
|
14413
|
-
logger?.warn("Unknown event type during replay", { type: event.type });
|
|
14414
|
-
}
|
|
14415
|
-
}
|
|
14416
|
-
async function replayEntityTypeAdded(event, eventBus, logger) {
|
|
14417
|
-
await busRequest(
|
|
14418
|
-
asBusRequestPrimitive(eventBus),
|
|
14419
|
-
"frame:add-entity-type",
|
|
14420
|
-
{ tag: event.payload.entityType, _userId: event.userId },
|
|
14421
|
-
REPLAY_TIMEOUT_MS
|
|
14422
|
-
);
|
|
14423
|
-
logger?.debug("Replayed entitytype.added", { entityType: event.payload.entityType });
|
|
14424
|
-
}
|
|
14425
|
-
async function replayResourceCreated(event, eventBus, resolveBlob, contentStore, logger) {
|
|
14426
|
-
const { payload } = event;
|
|
14427
|
-
const blob = resolveBlob(payload.contentChecksum);
|
|
14428
|
-
if (!blob) {
|
|
14429
|
-
throw new Error(`Missing content blob for checksum ${payload.contentChecksum}`);
|
|
14430
|
-
}
|
|
14431
|
-
const base = baseMediaType(payload.format);
|
|
14432
|
-
const resolvedUri = payload.storageUri || deriveStorageUri(payload.name, isSupportedMediaType(base) ? base : "application/octet-stream");
|
|
14433
|
-
const stored = await contentStore.store(blob, resolvedUri);
|
|
14434
|
-
await busRequest(
|
|
14435
|
-
asBusRequestPrimitive(eventBus),
|
|
14436
|
-
"yield:create",
|
|
14437
|
-
{
|
|
14438
|
-
name: payload.name,
|
|
14439
|
-
storageUri: resolvedUri,
|
|
14440
|
-
contentChecksum: stored.checksum,
|
|
14441
|
-
byteSize: stored.byteSize,
|
|
14442
|
-
format: payload.format,
|
|
14443
|
-
_userId: event.userId,
|
|
14444
|
-
language: payload.language,
|
|
14445
|
-
entityTypes: payload.entityTypes,
|
|
14446
|
-
isDraft: payload.isDraft,
|
|
14447
|
-
generatedFrom: payload.generatedFrom,
|
|
14448
|
-
generationPrompt: payload.generationPrompt
|
|
14449
|
-
},
|
|
14450
|
-
REPLAY_TIMEOUT_MS
|
|
14451
|
-
);
|
|
14452
|
-
logger?.debug("Replayed resource.created", { name: payload.name });
|
|
14453
|
-
}
|
|
14454
|
-
async function replayAnnotationAdded(event, eventBus, logger) {
|
|
14455
|
-
await awaitReply(
|
|
14456
|
-
eventBus,
|
|
14457
|
-
"mark:create",
|
|
14458
|
-
{
|
|
14459
|
-
annotation: event.payload.annotation,
|
|
14460
|
-
_userId: event.userId,
|
|
14461
|
-
resourceId: event.resourceId
|
|
14462
|
-
},
|
|
14463
|
-
"mark:create-ok",
|
|
14464
|
-
"mark:create-failed",
|
|
14465
|
-
REPLAY_TIMEOUT_MS
|
|
14466
|
-
);
|
|
14467
|
-
logger?.debug("Replayed annotation.added", { annotationId: event.payload.annotation.id });
|
|
14468
|
-
}
|
|
14469
|
-
async function replayAnnotationBodyUpdated(event, eventBus, logger) {
|
|
14470
|
-
await awaitReply(
|
|
14471
|
-
eventBus,
|
|
14472
|
-
"mark:update-body",
|
|
14473
|
-
{
|
|
14474
|
-
annotationId: event.payload.annotationId,
|
|
14475
|
-
_userId: event.userId,
|
|
14476
|
-
resourceId: event.resourceId,
|
|
14477
|
-
operations: event.payload.operations
|
|
14478
|
-
},
|
|
14479
|
-
"mark:body-updated",
|
|
14480
|
-
"mark:body-update-failed",
|
|
14481
|
-
REPLAY_TIMEOUT_MS
|
|
14482
|
-
);
|
|
14483
|
-
logger?.debug("Replayed annotation.body.updated", { annotationId: event.payload.annotationId });
|
|
14484
|
-
}
|
|
14485
|
-
async function replayAnnotationRemoved(event, eventBus, logger) {
|
|
14486
|
-
await awaitReply(
|
|
14487
|
-
eventBus,
|
|
14488
|
-
"mark:delete",
|
|
14489
|
-
{
|
|
14490
|
-
annotationId: event.payload.annotationId,
|
|
14491
|
-
_userId: event.userId,
|
|
14492
|
-
resourceId: event.resourceId
|
|
14493
|
-
},
|
|
14494
|
-
"mark:delete-ok",
|
|
14495
|
-
"mark:delete-failed",
|
|
14496
|
-
REPLAY_TIMEOUT_MS
|
|
14497
|
-
);
|
|
14498
|
-
logger?.debug("Replayed annotation.removed", { annotationId: event.payload.annotationId });
|
|
14499
|
-
}
|
|
14500
|
-
async function replayResourceArchived(event, eventBus, logger) {
|
|
14501
|
-
eventBus.get("mark:archive").next({
|
|
14502
|
-
_userId: event.userId,
|
|
14503
|
-
resourceId: event.resourceId
|
|
14504
|
-
});
|
|
14505
|
-
logger?.debug("Replayed resource.archived", { resourceId: event.resourceId });
|
|
14506
|
-
}
|
|
14507
|
-
async function replayResourceUnarchived(event, eventBus, logger) {
|
|
14508
|
-
eventBus.get("mark:unarchive").next({
|
|
14509
|
-
_userId: event.userId,
|
|
14510
|
-
resourceId: event.resourceId
|
|
14511
|
-
});
|
|
14512
|
-
logger?.debug("Replayed resource.unarchived", { resourceId: event.resourceId });
|
|
14513
|
-
}
|
|
14514
|
-
async function replayEntityTagChange(event, eventStore, logger) {
|
|
14515
|
-
const base = {
|
|
14516
|
-
resourceId: event.resourceId,
|
|
14517
|
-
userId: event.userId,
|
|
14518
|
-
version: event.version
|
|
14519
|
-
};
|
|
14520
|
-
if (event.type === "mark:entity-tag-added") {
|
|
14521
|
-
await eventStore.appendEvent({ ...base, type: "mark:entity-tag-added", payload: event.payload });
|
|
14522
|
-
} else {
|
|
14523
|
-
await eventStore.appendEvent({ ...base, type: "mark:entity-tag-removed", payload: event.payload });
|
|
14524
|
-
}
|
|
14525
|
-
logger?.debug("Replayed entity tag change", { type: event.type, entityType: event.payload.entityType });
|
|
14526
|
-
}
|
|
14527
|
-
|
|
14528
|
-
// src/exchange/backup-importer.ts
|
|
14529
|
-
function buildBlobResolver(entries) {
|
|
14530
|
-
const checksumIndex = /* @__PURE__ */ new Map();
|
|
14531
|
-
for (const name of entries.keys()) {
|
|
14532
|
-
if (!name.startsWith(".semiont/")) {
|
|
14533
|
-
const dotIndex = name.lastIndexOf(".");
|
|
14534
|
-
const checksum = dotIndex >= 0 ? name.slice(0, dotIndex) : name;
|
|
14535
|
-
checksumIndex.set(checksum, name);
|
|
14536
|
-
}
|
|
14537
|
-
}
|
|
14538
|
-
return (checksum) => {
|
|
14539
|
-
const entryName = checksumIndex.get(checksum);
|
|
14540
|
-
return entryName ? entries.get(entryName) : void 0;
|
|
14541
|
-
};
|
|
14542
|
-
}
|
|
14543
|
-
async function importBackup(archive, options) {
|
|
14544
|
-
const { eventBus, eventStore, contentStore, logger } = options;
|
|
14545
|
-
const entries = /* @__PURE__ */ new Map();
|
|
14546
|
-
for await (const entry of readTarGz(archive)) {
|
|
14547
|
-
entries.set(entry.name, entry.data);
|
|
14548
|
-
}
|
|
14549
|
-
const manifestData = entries.get(".semiont/manifest.jsonl");
|
|
14550
|
-
if (!manifestData) {
|
|
14551
|
-
throw new Error("Invalid backup: missing .semiont/manifest.jsonl");
|
|
14552
|
-
}
|
|
14553
|
-
const manifestLines = manifestData.toString("utf8").trim().split("\n");
|
|
14554
|
-
const header = JSON.parse(manifestLines[0]);
|
|
14555
|
-
if (!isBackupManifest(header)) {
|
|
14556
|
-
throw new Error(`Invalid backup: expected format "${BACKUP_FORMAT}", got "${header.format}"`);
|
|
14557
|
-
}
|
|
14558
|
-
validateManifestVersion(header.version);
|
|
14559
|
-
const streamSummaries = manifestLines.slice(1).map((line) => JSON.parse(line));
|
|
14560
|
-
logger?.info("Backup import: parsed manifest", {
|
|
14561
|
-
streams: header.stats.streams,
|
|
14562
|
-
events: header.stats.events,
|
|
14563
|
-
blobs: header.stats.blobs
|
|
14564
|
-
});
|
|
14565
|
-
const resolveBlob = buildBlobResolver(entries);
|
|
14566
|
-
const systemData = entries.get(".semiont/events/__system__.jsonl");
|
|
14567
|
-
let stats = { eventsReplayed: 0, resourcesCreated: 0, annotationsCreated: 0, entityTypesAdded: 0 };
|
|
14568
|
-
if (systemData) {
|
|
14569
|
-
const result = await replayEventStream(
|
|
14570
|
-
systemData.toString("utf8"),
|
|
14571
|
-
eventBus,
|
|
14572
|
-
eventStore,
|
|
14573
|
-
resolveBlob,
|
|
14574
|
-
contentStore,
|
|
14575
|
-
logger
|
|
14576
|
-
);
|
|
14577
|
-
stats = mergeStats(stats, result.stats);
|
|
14578
|
-
}
|
|
14579
|
-
for (const summary of streamSummaries) {
|
|
14580
|
-
if (summary.stream === "__system__") continue;
|
|
14581
|
-
const eventData = entries.get(`.semiont/events/${summary.stream}.jsonl`);
|
|
14582
|
-
if (!eventData) {
|
|
14583
|
-
logger?.warn("Backup import: missing event stream", { stream: summary.stream });
|
|
14584
|
-
continue;
|
|
14585
|
-
}
|
|
14586
|
-
const result = await replayEventStream(
|
|
14587
|
-
eventData.toString("utf8"),
|
|
14588
|
-
eventBus,
|
|
14589
|
-
eventStore,
|
|
14590
|
-
resolveBlob,
|
|
14591
|
-
contentStore,
|
|
14592
|
-
logger
|
|
14593
|
-
);
|
|
14594
|
-
stats = mergeStats(stats, result.stats);
|
|
14595
|
-
}
|
|
14596
|
-
logger?.info("Backup import complete", { ...stats });
|
|
14597
|
-
return { manifest: header, stats };
|
|
14598
|
-
}
|
|
14599
|
-
function mergeStats(a, b) {
|
|
14600
|
-
return {
|
|
14601
|
-
eventsReplayed: a.eventsReplayed + b.eventsReplayed,
|
|
14602
|
-
resourcesCreated: a.resourcesCreated + b.resourcesCreated,
|
|
14603
|
-
annotationsCreated: a.annotationsCreated + b.annotationsCreated,
|
|
14604
|
-
entityTypesAdded: a.entityTypesAdded + b.entityTypesAdded
|
|
14605
|
-
};
|
|
14606
|
-
}
|
|
14607
|
-
var SEMIONT_CONTEXT = [
|
|
14608
|
-
"https://schema.org/",
|
|
14609
|
-
"http://www.w3.org/ns/anno.jsonld",
|
|
14610
|
-
{
|
|
14611
|
-
"semiont": "https://semiont.org/vocab/",
|
|
14612
|
-
"entityTypes": "semiont:entityTypes",
|
|
14613
|
-
"archived": "semiont:archived",
|
|
14614
|
-
"representations": { "@id": "semiont:representations", "@container": "@set" },
|
|
14615
|
-
"annotations": { "@id": "semiont:annotations", "@container": "@set" }
|
|
14616
|
-
}
|
|
14617
|
-
];
|
|
14618
|
-
var MANIFEST_CONTEXT = {
|
|
14619
|
-
"semiont": "https://semiont.org/vocab/",
|
|
14620
|
-
"schema": "https://schema.org/",
|
|
14621
|
-
"dct": "http://purl.org/dc/terms/",
|
|
14622
|
-
"prov": "http://www.w3.org/ns/prov#",
|
|
14623
|
-
"void": "http://rdfs.org/ns/void#"
|
|
14624
|
-
};
|
|
14625
|
-
function hydrateAnnotation(annotation, baseUrl) {
|
|
14626
|
-
const hydrated = { ...annotation };
|
|
14627
|
-
if (hydrated.id && !hydrated.id.startsWith("http")) {
|
|
14628
|
-
hydrated.id = `${baseUrl}/annotations/${hydrated.id}`;
|
|
14629
|
-
}
|
|
14630
|
-
if (typeof hydrated.target === "string") {
|
|
14631
|
-
if (!hydrated.target.startsWith("http")) {
|
|
14632
|
-
hydrated.target = `${baseUrl}/resources/${hydrated.target}`;
|
|
14633
|
-
}
|
|
14634
|
-
} else if (hydrated.target && typeof hydrated.target === "object") {
|
|
14635
|
-
const target = { ...hydrated.target };
|
|
14636
|
-
if (target.source && !target.source.startsWith("http")) {
|
|
14637
|
-
target.source = `${baseUrl}/resources/${target.source}`;
|
|
14638
|
-
}
|
|
14639
|
-
hydrated.target = target;
|
|
14640
|
-
}
|
|
14641
|
-
hydrated.body = hydrateBody(hydrated.body, baseUrl);
|
|
14642
|
-
return hydrated;
|
|
14643
|
-
}
|
|
14644
|
-
function hydrateBody(body, baseUrl) {
|
|
14645
|
-
if (Array.isArray(body)) {
|
|
14646
|
-
return body.map((b) => hydrateBodyItem(b, baseUrl));
|
|
14647
|
-
}
|
|
14648
|
-
return hydrateBodyItem(body, baseUrl);
|
|
14649
|
-
}
|
|
14650
|
-
function hydrateBodyItem(item, baseUrl) {
|
|
14651
|
-
if (item && typeof item === "object" && "source" in item) {
|
|
14652
|
-
const source = item.source;
|
|
14653
|
-
if (typeof source === "string" && !source.startsWith("http")) {
|
|
14654
|
-
return { ...item, source: `${baseUrl}/resources/${source}` };
|
|
14655
|
-
}
|
|
14656
|
-
}
|
|
14657
|
-
return item;
|
|
14658
|
-
}
|
|
14659
|
-
async function exportLinkedData(options, output) {
|
|
14660
|
-
const { views, content, sourceUrl, entityTypes, includeArchived, logger } = options;
|
|
14661
|
-
const allViews = await views.getAll();
|
|
14662
|
-
const resourceViews = includeArchived ? allViews : allViews.filter((v) => !v.resource.archived);
|
|
14663
|
-
logger?.info("Linked data export: enumerating resources", { count: resourceViews.length });
|
|
14664
|
-
const contentRefs = /* @__PURE__ */ new Map();
|
|
14665
|
-
for (const view of resourceViews) {
|
|
14666
|
-
collectContentRefsFromResource(view.resource, contentRefs);
|
|
14667
|
-
}
|
|
14668
|
-
const contentBlobs = /* @__PURE__ */ new Map();
|
|
14669
|
-
for (const [storageUri, mediaType] of contentRefs) {
|
|
14670
|
-
try {
|
|
14671
|
-
const data = await content.retrieve(storageUri);
|
|
14672
|
-
const ext = extensionForMediaType(mediaType);
|
|
14673
|
-
contentBlobs.set(storageUri, { data, ext });
|
|
14674
|
-
} catch (err) {
|
|
14675
|
-
logger?.warn("Failed to retrieve content blob", { storageUri, error: String(err) });
|
|
14676
|
-
}
|
|
14677
|
-
}
|
|
14678
|
-
const manifest = {
|
|
14679
|
-
"@context": MANIFEST_CONTEXT,
|
|
14680
|
-
"@type": "void:Dataset",
|
|
14681
|
-
"semiont:format": LINKED_DATA_FORMAT,
|
|
14682
|
-
"semiont:version": FORMAT_VERSION,
|
|
14683
|
-
"dct:created": (/* @__PURE__ */ new Date()).toISOString(),
|
|
14684
|
-
"prov:wasGeneratedBy": {
|
|
14685
|
-
"@type": "prov:Activity",
|
|
14686
|
-
"prov:used": sourceUrl
|
|
14687
|
-
},
|
|
14688
|
-
"semiont:entityTypes": entityTypes,
|
|
14689
|
-
"void:entities": resourceViews.length
|
|
14690
|
-
};
|
|
14691
|
-
async function* generateEntries() {
|
|
14692
|
-
yield {
|
|
14693
|
-
name: ".semiont/manifest.jsonld",
|
|
14694
|
-
data: Buffer.from(JSON.stringify(manifest, null, 2), "utf8")
|
|
14695
|
-
};
|
|
14696
|
-
for (const view of resourceViews) {
|
|
14697
|
-
const resourceId10 = view.resource["@id"];
|
|
14698
|
-
const jsonld = buildResourceJsonLd(view.resource, view.annotations.annotations, sourceUrl);
|
|
14699
|
-
yield {
|
|
14700
|
-
name: `.semiont/resources/${resourceId10}.jsonld`,
|
|
14701
|
-
data: Buffer.from(JSON.stringify(jsonld, null, 2), "utf8")
|
|
14702
|
-
};
|
|
14703
|
-
}
|
|
14704
|
-
for (const [storageUri, { data, ext }] of contentBlobs) {
|
|
14705
|
-
const blobPath = storageUri.startsWith("file://") ? storageUri.slice(7) : storageUri;
|
|
14706
|
-
yield { name: `${blobPath}${ext}`, data };
|
|
14707
|
-
}
|
|
14708
|
-
}
|
|
14709
|
-
await writeTarGz(generateEntries(), output);
|
|
14710
|
-
logger?.info("Linked data export complete", {
|
|
14711
|
-
resources: resourceViews.length,
|
|
14712
|
-
blobs: contentBlobs.size
|
|
14713
|
-
});
|
|
14714
|
-
return manifest;
|
|
14715
|
-
}
|
|
14716
|
-
function buildResourceJsonLd(resource, annotations, sourceUrl) {
|
|
14717
|
-
const resourceId10 = resource["@id"];
|
|
14718
|
-
const resourceUri = resourceId10.startsWith("http") ? resourceId10 : `${sourceUrl}/resources/${resourceId10}`;
|
|
14719
|
-
const doc = {
|
|
14720
|
-
"@context": SEMIONT_CONTEXT,
|
|
14721
|
-
"@id": resourceUri,
|
|
14722
|
-
"@type": resource["@type"] ?? "DigitalDocument",
|
|
14723
|
-
"name": resource.name
|
|
14724
|
-
};
|
|
14725
|
-
if (resource.dateCreated) doc["dateCreated"] = resource.dateCreated;
|
|
14726
|
-
if (resource.dateModified) doc["dateModified"] = resource.dateModified;
|
|
14727
|
-
if (resource.description) doc["description"] = resource.description;
|
|
14728
|
-
const reps = normalizeRepresentations(resource.representations);
|
|
14729
|
-
if (reps.length > 0) {
|
|
14730
|
-
const primary = reps[0];
|
|
14731
|
-
if (primary.language) doc["inLanguage"] = primary.language;
|
|
14732
|
-
if (primary.mediaType) doc["encodingFormat"] = primary.mediaType;
|
|
14733
|
-
}
|
|
14734
|
-
if (resource.entityTypes && resource.entityTypes.length > 0) doc["entityTypes"] = resource.entityTypes;
|
|
14735
|
-
if (resource.archived) doc["archived"] = resource.archived;
|
|
14736
|
-
if (resource.wasDerivedFrom) doc["wasDerivedFrom"] = resource.wasDerivedFrom;
|
|
14737
|
-
if (resource.wasAttributedTo) doc["wasAttributedTo"] = resource.wasAttributedTo;
|
|
14738
|
-
if (resource.sameAs && resource.sameAs.length > 0) doc["sameAs"] = resource.sameAs;
|
|
14739
|
-
if (resource.isPartOf && resource.isPartOf.length > 0) doc["isPartOf"] = resource.isPartOf;
|
|
14740
|
-
if (resource.hasPart && resource.hasPart.length > 0) doc["hasPart"] = resource.hasPart;
|
|
14741
|
-
if (reps.length > 0) {
|
|
14742
|
-
doc["representations"] = reps.map((rep) => {
|
|
14743
|
-
const mediaObj = {
|
|
14744
|
-
"@type": "schema:MediaObject",
|
|
14745
|
-
"encodingFormat": rep.mediaType
|
|
14746
|
-
};
|
|
14747
|
-
if (rep.byteSize !== void 0) mediaObj["contentSize"] = rep.byteSize;
|
|
14748
|
-
if (rep.checksum) {
|
|
14749
|
-
const rawChecksum = rep.checksum.startsWith("sha256:") ? rep.checksum.slice(7) : rep.checksum;
|
|
14750
|
-
mediaObj["sha256"] = rawChecksum;
|
|
14751
|
-
const ext = extensionForMediaType(rep.mediaType);
|
|
14752
|
-
mediaObj["name"] = `${rawChecksum}${ext}`;
|
|
14753
|
-
}
|
|
14754
|
-
if (rep.language) mediaObj["inLanguage"] = rep.language;
|
|
14755
|
-
return mediaObj;
|
|
14756
|
-
});
|
|
14757
|
-
}
|
|
14758
|
-
if (annotations.length > 0) {
|
|
14759
|
-
doc["annotations"] = annotations.map((ann) => hydrateAnnotation(ann, sourceUrl));
|
|
14760
|
-
}
|
|
14761
|
-
return doc;
|
|
14762
|
-
}
|
|
14763
|
-
function normalizeRepresentations(reps) {
|
|
14764
|
-
if (!reps) return [];
|
|
14765
|
-
if (Array.isArray(reps)) return reps;
|
|
14766
|
-
return [reps];
|
|
14767
|
-
}
|
|
14768
|
-
function collectContentRefsFromResource(resource, refs) {
|
|
14769
|
-
if (resource.storageUri) {
|
|
14770
|
-
const reps = normalizeRepresentations(resource.representations);
|
|
14771
|
-
const mediaType = reps[0]?.mediaType ?? "application/octet-stream";
|
|
14772
|
-
refs.set(resource.storageUri, mediaType);
|
|
14773
|
-
}
|
|
14774
|
-
}
|
|
14775
|
-
var IMPORT_TIMEOUT_MS = 3e4;
|
|
14776
|
-
function stripUriToId(uri) {
|
|
14777
|
-
if (!uri.includes("/")) return uri;
|
|
14778
|
-
const lastSlash = uri.lastIndexOf("/");
|
|
14779
|
-
return uri.slice(lastSlash + 1);
|
|
14780
|
-
}
|
|
14781
|
-
function dehydrateAnnotation(annotation) {
|
|
14782
|
-
const dehydrated = { ...annotation };
|
|
14783
|
-
if (dehydrated.id) {
|
|
14784
|
-
dehydrated.id = annotationId(stripUriToId(dehydrated.id));
|
|
14785
|
-
}
|
|
14786
|
-
if (typeof dehydrated.target === "string") {
|
|
14787
|
-
dehydrated.target = stripUriToId(dehydrated.target);
|
|
14788
|
-
} else if (dehydrated.target && typeof dehydrated.target === "object") {
|
|
14789
|
-
const target = { ...dehydrated.target };
|
|
14790
|
-
if (target.source) {
|
|
14791
|
-
target.source = stripUriToId(target.source);
|
|
14792
|
-
}
|
|
14793
|
-
dehydrated.target = target;
|
|
14794
|
-
}
|
|
14795
|
-
dehydrated.body = dehydrateBody(dehydrated.body);
|
|
14796
|
-
return dehydrated;
|
|
14797
|
-
}
|
|
14798
|
-
function dehydrateBody(body) {
|
|
14799
|
-
if (Array.isArray(body)) {
|
|
14800
|
-
return body.map((b) => dehydrateBodyItem(b));
|
|
14801
|
-
}
|
|
14802
|
-
return dehydrateBodyItem(body);
|
|
14803
|
-
}
|
|
14804
|
-
function dehydrateBodyItem(item) {
|
|
14805
|
-
if (item && typeof item === "object" && "source" in item) {
|
|
14806
|
-
const source = item.source;
|
|
14807
|
-
if (typeof source === "string" && source.includes("/")) {
|
|
14808
|
-
return { ...item, source: stripUriToId(source) };
|
|
14809
|
-
}
|
|
14810
|
-
}
|
|
14811
|
-
return item;
|
|
14812
|
-
}
|
|
14813
|
-
function buildBlobResolver2(entries) {
|
|
14814
|
-
const checksumIndex = /* @__PURE__ */ new Map();
|
|
14815
|
-
for (const name of entries.keys()) {
|
|
14816
|
-
if (!name.startsWith(".semiont/")) {
|
|
14817
|
-
const dotIndex = name.lastIndexOf(".");
|
|
14818
|
-
const checksum = dotIndex >= 0 ? name.slice(0, dotIndex) : name;
|
|
14819
|
-
checksumIndex.set(checksum, name);
|
|
14820
|
-
}
|
|
14821
|
-
}
|
|
14822
|
-
return (checksum) => {
|
|
14823
|
-
const entryName = checksumIndex.get(checksum);
|
|
14824
|
-
return entryName ? entries.get(entryName) : void 0;
|
|
14825
|
-
};
|
|
14826
|
-
}
|
|
14827
|
-
async function importLinkedData(archive, options) {
|
|
14828
|
-
const { eventBus, contentStore, userId: userId3, logger } = options;
|
|
14829
|
-
const entries = /* @__PURE__ */ new Map();
|
|
14830
|
-
for await (const entry of readTarGz(archive)) {
|
|
14831
|
-
entries.set(entry.name, entry.data);
|
|
14832
|
-
}
|
|
14833
|
-
const manifestData = entries.get(".semiont/manifest.jsonld");
|
|
14834
|
-
if (!manifestData) {
|
|
14835
|
-
throw new Error("Invalid linked data archive: missing .semiont/manifest.jsonld");
|
|
14836
|
-
}
|
|
14837
|
-
const manifest = JSON.parse(manifestData.toString("utf8"));
|
|
14838
|
-
if (!isLinkedDataManifest(manifest)) {
|
|
14839
|
-
throw new Error(
|
|
14840
|
-
`Invalid linked data archive: expected format "${LINKED_DATA_FORMAT}", got "${manifest["semiont:format"]}"`
|
|
14841
|
-
);
|
|
14842
|
-
}
|
|
14843
|
-
validateManifestVersion(manifest["semiont:version"]);
|
|
14844
|
-
logger?.info("Linked data import: parsed manifest", {
|
|
14845
|
-
entityTypes: manifest["semiont:entityTypes"].length,
|
|
14846
|
-
resources: manifest["void:entities"]
|
|
14847
|
-
});
|
|
14848
|
-
const resolveBlob = buildBlobResolver2(entries);
|
|
14849
|
-
let entityTypesAdded = 0;
|
|
14850
|
-
for (const entityType2 of manifest["semiont:entityTypes"]) {
|
|
14851
|
-
await addEntityType(entityType2, userId3, eventBus, logger);
|
|
14852
|
-
entityTypesAdded++;
|
|
14853
|
-
}
|
|
14854
|
-
const resourceEntries = [...entries.keys()].filter((name) => name.startsWith(".semiont/resources/") && name.endsWith(".jsonld")).sort();
|
|
14855
|
-
let resourcesCreated = 0;
|
|
14856
|
-
let annotationsCreated = 0;
|
|
14857
|
-
for (const entryName of resourceEntries) {
|
|
14858
|
-
const resourceDoc = JSON.parse(entries.get(entryName).toString("utf8"));
|
|
14859
|
-
const result = await importResource(resourceDoc, userId3, eventBus, contentStore, resolveBlob, logger);
|
|
14860
|
-
resourcesCreated++;
|
|
14861
|
-
annotationsCreated += result.annotationsCreated;
|
|
14862
|
-
}
|
|
14863
|
-
logger?.info("Linked data import complete", {
|
|
14864
|
-
resourcesCreated,
|
|
14865
|
-
annotationsCreated,
|
|
14866
|
-
entityTypesAdded
|
|
14867
|
-
});
|
|
14868
|
-
return {
|
|
14869
|
-
manifest,
|
|
14870
|
-
resourcesCreated,
|
|
14871
|
-
annotationsCreated,
|
|
14872
|
-
entityTypesAdded
|
|
14873
|
-
};
|
|
14874
|
-
}
|
|
14875
|
-
async function addEntityType(entityType2, userId3, eventBus, logger) {
|
|
14876
|
-
await busRequest(
|
|
14877
|
-
asBusRequestPrimitive(eventBus),
|
|
14878
|
-
"frame:add-entity-type",
|
|
14879
|
-
{ tag: entityType2, _userId: userId3 },
|
|
14880
|
-
IMPORT_TIMEOUT_MS
|
|
14881
|
-
);
|
|
14882
|
-
logger?.debug("Added entity type", { entityType: entityType2 });
|
|
14883
|
-
}
|
|
14884
|
-
async function importResource(doc, userId3, eventBus, contentStore, resolveBlob, logger) {
|
|
14885
|
-
const name = doc["name"];
|
|
14886
|
-
const representations = doc["representations"];
|
|
14887
|
-
const annotations = doc["annotations"];
|
|
14888
|
-
const entityTypes = doc["entityTypes"];
|
|
14889
|
-
let format = "text/markdown";
|
|
14890
|
-
let language;
|
|
14891
|
-
let contentChecksum;
|
|
14892
|
-
if (representations && representations.length > 0) {
|
|
14893
|
-
const primary = representations[0];
|
|
14894
|
-
if (primary["encodingFormat"]) format = primary["encodingFormat"];
|
|
14895
|
-
if (primary["inLanguage"]) language = primary["inLanguage"];
|
|
14896
|
-
if (primary["sha256"]) contentChecksum = primary["sha256"];
|
|
14897
|
-
}
|
|
14898
|
-
if (!contentChecksum) {
|
|
14899
|
-
throw new Error(`Resource "${name}" has no content checksum in representations`);
|
|
14900
|
-
}
|
|
14901
|
-
const blob = resolveBlob(contentChecksum);
|
|
14902
|
-
if (!blob) {
|
|
14903
|
-
throw new Error(`Missing content blob for checksum ${contentChecksum} (resource "${name}")`);
|
|
14904
|
-
}
|
|
14905
|
-
const base = baseMediaType(format);
|
|
14906
|
-
const resolvedUri = deriveStorageUri(name, isSupportedMediaType(base) ? base : "application/octet-stream");
|
|
14907
|
-
const stored = await contentStore.store(blob, resolvedUri);
|
|
14908
|
-
const { resourceId: createdId } = await busRequest(
|
|
14909
|
-
asBusRequestPrimitive(eventBus),
|
|
14910
|
-
"yield:create",
|
|
14911
|
-
{
|
|
14912
|
-
name,
|
|
14913
|
-
storageUri: resolvedUri,
|
|
14914
|
-
contentChecksum: stored.checksum,
|
|
14915
|
-
byteSize: stored.byteSize,
|
|
14916
|
-
format,
|
|
14917
|
-
_userId: userId3,
|
|
14918
|
-
language,
|
|
14919
|
-
entityTypes: entityTypes ?? []
|
|
14920
|
-
},
|
|
14921
|
-
IMPORT_TIMEOUT_MS
|
|
14922
|
-
);
|
|
14923
|
-
const resourceId10 = resourceId(createdId);
|
|
14924
|
-
logger?.debug("Created resource from JSON-LD", { name, resourceId: resourceId10 });
|
|
14925
|
-
let annotationsCreated = 0;
|
|
14926
|
-
if (annotations && annotations.length > 0) {
|
|
14927
|
-
for (const annotation of annotations) {
|
|
14928
|
-
await createAnnotation(annotation, resourceId10, userId3, eventBus, logger);
|
|
14929
|
-
annotationsCreated++;
|
|
14930
|
-
}
|
|
14931
|
-
}
|
|
14932
|
-
return { annotationsCreated };
|
|
14933
|
-
}
|
|
14934
|
-
async function createAnnotation(annotation, resourceId10, userId3, eventBus, logger) {
|
|
14935
|
-
await awaitReply(
|
|
14936
|
-
eventBus,
|
|
14937
|
-
"mark:create",
|
|
14938
|
-
{
|
|
14939
|
-
annotation: dehydrateAnnotation(annotation),
|
|
14940
|
-
_userId: userId3,
|
|
14941
|
-
resourceId: resourceId10
|
|
14942
|
-
},
|
|
14943
|
-
"mark:create-ok",
|
|
14944
|
-
"mark:create-failed",
|
|
14945
|
-
IMPORT_TIMEOUT_MS
|
|
14946
|
-
);
|
|
14947
|
-
logger?.debug("Created annotation", { annotationId: annotation.id });
|
|
14948
|
-
}
|
|
14949
|
-
var AnnotationOperations = class {
|
|
14950
|
-
/**
|
|
14951
|
-
* Create a new annotation via EventBus → Stower
|
|
14952
|
-
*/
|
|
14953
|
-
static async createAnnotation(request, userId3, creator, eventBus) {
|
|
14954
|
-
const { annotation } = assembleAnnotation(request, creator);
|
|
14955
|
-
const resId = resourceId(request.target.source);
|
|
14956
|
-
eventBus.get("mark:create").next({
|
|
14957
|
-
annotation,
|
|
14958
|
-
_userId: userId3,
|
|
14959
|
-
resourceId: resId
|
|
14960
|
-
});
|
|
14961
|
-
return { annotation };
|
|
14962
|
-
}
|
|
14963
|
-
/**
|
|
14964
|
-
* Update annotation body via EventBus → Stower
|
|
14965
|
-
*/
|
|
14966
|
-
static async updateAnnotationBody(id, request, userId3, eventBus, kb) {
|
|
14967
|
-
const resId = resourceId(request.resourceId);
|
|
14968
|
-
const annotation = await AnnotationContext.getAnnotation(
|
|
14969
|
-
annotationId(id),
|
|
14970
|
-
resId,
|
|
14971
|
-
kb
|
|
14972
|
-
);
|
|
14973
|
-
if (!annotation) {
|
|
14974
|
-
throw new Error("Annotation not found");
|
|
14975
|
-
}
|
|
14976
|
-
eventBus.get("mark:update-body").next({
|
|
14977
|
-
annotationId: annotationId(id),
|
|
14978
|
-
_userId: userId3,
|
|
14979
|
-
resourceId: resId,
|
|
14980
|
-
operations: request.operations
|
|
14981
|
-
});
|
|
14982
|
-
const updatedBody = applyBodyOperations(annotation.body, request.operations);
|
|
14983
|
-
return {
|
|
14984
|
-
annotation: {
|
|
14985
|
-
...annotation,
|
|
14986
|
-
body: updatedBody
|
|
14987
|
-
}
|
|
14988
|
-
};
|
|
14989
|
-
}
|
|
14990
|
-
/**
|
|
14991
|
-
* Delete an annotation via EventBus → Stower
|
|
14992
|
-
*/
|
|
14993
|
-
static async deleteAnnotation(id, resourceIdStr, userId3, eventBus, kb, logger) {
|
|
14994
|
-
const resId = resourceId(resourceIdStr);
|
|
14995
|
-
const projection = await AnnotationContext.getResourceAnnotations(resId, kb);
|
|
14996
|
-
const annotation = projection.annotations.find((a) => a.id === id);
|
|
14997
|
-
if (!annotation) {
|
|
14998
|
-
throw new Error("Annotation not found in resource");
|
|
14999
|
-
}
|
|
15000
|
-
logger?.debug("Removing annotation via EventBus", { annotationId: id });
|
|
15001
|
-
eventBus.get("mark:delete").next({
|
|
15002
|
-
annotationId: annotationId(id),
|
|
15003
|
-
_userId: userId3,
|
|
15004
|
-
resourceId: resId
|
|
15005
|
-
});
|
|
15006
|
-
logger?.debug("Annotation delete event emitted");
|
|
15007
|
-
}
|
|
15008
|
-
};
|
|
15009
14556
|
|
|
15010
|
-
export { AnnotationContext, AnnotationOperations,
|
|
14557
|
+
export { AnnotationContext, AnnotationOperations, Browser, CloneTokenManager, Gatherer, GraphContext, LLMContext, LocalContentTransport, LocalTransport, Matcher, ResourceContext, ResourceOperations, Smelter, Stower, asBusRequestPrimitive, bootstrapEntityTypes, createArchivistServer, createKnowledgeBase, createSmelterActorStateUnit, generateReferenceSuggestions, generateResourceSummary, makeMeaningConfigFrom, readEntityTypesProjection, registerAnnotationAssemblyHandler, registerAnnotationContextHandler, registerBindUpdateBodyHandler, registerBusHandlers, registerGatherSummaryHandler, registerJobCommandHandlers, requireKBName, startMakeMeaning, startMakeMeaningGateway, stopKnowledgeSystem };
|
|
15011
14558
|
//# sourceMappingURL=index.js.map
|
|
15012
14559
|
//# sourceMappingURL=index.js.map
|