@semiont/make-meaning 0.5.28 → 0.5.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -15
- package/dist/archivist-main.d.ts +2 -0
- package/dist/archivist-main.js +12673 -0
- package/dist/archivist-main.js.map +1 -0
- package/dist/index.d.ts +668 -460
- package/dist/index.js +684 -1068
- package/dist/index.js.map +1 -1
- package/dist/librarian-main.d.ts +2 -0
- package/dist/librarian-main.js +11518 -0
- package/dist/librarian-main.js.map +1 -0
- package/dist/smelter-main.js +145 -39
- package/dist/smelter-main.js.map +1 -1
- package/dist/weaver-main.d.ts +2 -0
- package/dist/weaver-main.js +143 -36
- package/dist/weaver-main.js.map +1 -1
- package/package.json +27 -16
package/dist/index.js
CHANGED
|
@@ -1,17 +1,16 @@
|
|
|
1
1
|
import { STALL_THRESHOLD_MS, FsJobQueue } from '@semiont/jobs';
|
|
2
2
|
import { FilesystemViewStorage, resolveStorageUri, EventQuery, createEventStore } from '@semiont/event-sourcing';
|
|
3
|
-
import { getResourceEntityTypes, getResourceId, getTargetSource, resourceId,
|
|
3
|
+
import { getResourceEntityTypes, getResourceId, getTargetSource, resourceId, decodeRepresentation, getBodySource, getStorageUri, getPrimaryRepresentation, getTargetSelector, deriveViews, getTextPositionSelector, annotationId, errField, userId, generateUuid, getExactText, busRequest, cloneToken, assembleAnnotation, applyBodyOperations, didToAgent, isGenerationJobParams, jobId, entityType, baseUrl, busLog, BRIDGED_CHANNELS, burstBuffer, textExtractionOf, chunkText, getPrimaryMediaType, isAnnotatable, softwareToAgent } from '@semiont/core';
|
|
4
4
|
import { recordGatherDegrade, withActorSpan, recordBusEmit, withSpan, SpanKind, registerJobQueueProvider, registerVectorIndexSizeProvider } from '@semiont/observability';
|
|
5
5
|
import { createInferenceClient } from '@semiont/inference';
|
|
6
6
|
import { compareByRecencyThenId, getGraphDatabase } from '@semiont/graph';
|
|
7
|
-
import { WorkingTreeStore, createAnchoredTextStore,
|
|
7
|
+
import { WorkingTreeStore, createAnchoredTextStore, RepresentationMissing, ChecksumMismatchError, EXTRACTORS, calculateChecksum } from '@semiont/content';
|
|
8
8
|
import { getEntityTypes, DEFAULT_ENTITY_TYPES } from '@semiont/ontology';
|
|
9
9
|
import { mergeByResource } from '@semiont/vectors';
|
|
10
10
|
import { promises } from 'fs';
|
|
11
11
|
import * as path3 from 'path';
|
|
12
|
-
import {
|
|
13
|
-
import { pipeline
|
|
14
|
-
import { promisify } from 'util';
|
|
12
|
+
import { createServer } from 'http';
|
|
13
|
+
import { pipeline } from 'stream/promises';
|
|
15
14
|
|
|
16
15
|
var __create = Object.create;
|
|
17
16
|
var __defProp = Object.defineProperty;
|
|
@@ -3568,7 +3567,7 @@ var require_firstValueFrom = __commonJS({
|
|
|
3568
3567
|
exports.firstValueFrom = void 0;
|
|
3569
3568
|
var EmptyError_1 = require_EmptyError();
|
|
3570
3569
|
var Subscriber_1 = require_Subscriber();
|
|
3571
|
-
function
|
|
3570
|
+
function firstValueFrom(source, config) {
|
|
3572
3571
|
var hasConfig = typeof config === "object";
|
|
3573
3572
|
return new Promise(function(resolve2, reject) {
|
|
3574
3573
|
var subscriber = new Subscriber_1.SafeSubscriber({
|
|
@@ -3588,7 +3587,7 @@ var require_firstValueFrom = __commonJS({
|
|
|
3588
3587
|
source.subscribe(subscriber);
|
|
3589
3588
|
});
|
|
3590
3589
|
}
|
|
3591
|
-
exports.firstValueFrom =
|
|
3590
|
+
exports.firstValueFrom = firstValueFrom;
|
|
3592
3591
|
}
|
|
3593
3592
|
});
|
|
3594
3593
|
|
|
@@ -3727,7 +3726,7 @@ var require_map = __commonJS({
|
|
|
3727
3726
|
exports.map = void 0;
|
|
3728
3727
|
var lift_1 = require_lift();
|
|
3729
3728
|
var OperatorSubscriber_1 = require_OperatorSubscriber();
|
|
3730
|
-
function
|
|
3729
|
+
function map2(project, thisArg) {
|
|
3731
3730
|
return lift_1.operate(function(source, subscriber) {
|
|
3732
3731
|
var index = 0;
|
|
3733
3732
|
source.subscribe(OperatorSubscriber_1.createOperatorSubscriber(subscriber, function(value) {
|
|
@@ -3735,7 +3734,7 @@ var require_map = __commonJS({
|
|
|
3735
3734
|
}));
|
|
3736
3735
|
});
|
|
3737
3736
|
}
|
|
3738
|
-
exports.map =
|
|
3737
|
+
exports.map = map2;
|
|
3739
3738
|
}
|
|
3740
3739
|
});
|
|
3741
3740
|
|
|
@@ -4565,7 +4564,7 @@ var require_timer = __commonJS({
|
|
|
4565
4564
|
var async_1 = require_async();
|
|
4566
4565
|
var isScheduler_1 = require_isScheduler();
|
|
4567
4566
|
var isDate_1 = require_isDate();
|
|
4568
|
-
function
|
|
4567
|
+
function timer(dueTime, intervalOrScheduler, scheduler) {
|
|
4569
4568
|
if (dueTime === void 0) {
|
|
4570
4569
|
dueTime = 0;
|
|
4571
4570
|
}
|
|
@@ -4598,7 +4597,7 @@ var require_timer = __commonJS({
|
|
|
4598
4597
|
}, due);
|
|
4599
4598
|
});
|
|
4600
4599
|
}
|
|
4601
|
-
exports.timer =
|
|
4600
|
+
exports.timer = timer;
|
|
4602
4601
|
}
|
|
4603
4602
|
});
|
|
4604
4603
|
|
|
@@ -4788,7 +4787,7 @@ var require_race = __commonJS({
|
|
|
4788
4787
|
var innerFrom_1 = require_innerFrom();
|
|
4789
4788
|
var argsOrArgArray_1 = require_argsOrArgArray();
|
|
4790
4789
|
var OperatorSubscriber_1 = require_OperatorSubscriber();
|
|
4791
|
-
function
|
|
4790
|
+
function race() {
|
|
4792
4791
|
var sources = [];
|
|
4793
4792
|
for (var _i = 0; _i < arguments.length; _i++) {
|
|
4794
4793
|
sources[_i] = arguments[_i];
|
|
@@ -4796,7 +4795,7 @@ var require_race = __commonJS({
|
|
|
4796
4795
|
sources = argsOrArgArray_1.argsOrArgArray(sources);
|
|
4797
4796
|
return sources.length === 1 ? innerFrom_1.innerFrom(sources[0]) : new Observable_1.Observable(raceInit(sources));
|
|
4798
4797
|
}
|
|
4799
|
-
exports.race =
|
|
4798
|
+
exports.race = race;
|
|
4800
4799
|
function raceInit(sources) {
|
|
4801
4800
|
return function(subscriber) {
|
|
4802
4801
|
var subscriptions = [];
|
|
@@ -9252,14 +9251,14 @@ var require_race2 = __commonJS({
|
|
|
9252
9251
|
exports.race = void 0;
|
|
9253
9252
|
var argsOrArgArray_1 = require_argsOrArgArray();
|
|
9254
9253
|
var raceWith_1 = require_raceWith();
|
|
9255
|
-
function
|
|
9254
|
+
function race() {
|
|
9256
9255
|
var args = [];
|
|
9257
9256
|
for (var _i = 0; _i < arguments.length; _i++) {
|
|
9258
9257
|
args[_i] = arguments[_i];
|
|
9259
9258
|
}
|
|
9260
9259
|
return raceWith_1.raceWith.apply(void 0, __spreadArray([], __read(argsOrArgArray_1.argsOrArgArray(args))));
|
|
9261
9260
|
}
|
|
9262
|
-
exports.race =
|
|
9261
|
+
exports.race = race;
|
|
9263
9262
|
}
|
|
9264
9263
|
});
|
|
9265
9264
|
|
|
@@ -9726,6 +9725,45 @@ var require_operators = __commonJS({
|
|
|
9726
9725
|
});
|
|
9727
9726
|
|
|
9728
9727
|
// src/config.ts
|
|
9728
|
+
function requireKBName(config) {
|
|
9729
|
+
const name = config.kb?.name;
|
|
9730
|
+
if (!name) {
|
|
9731
|
+
throw new Error(
|
|
9732
|
+
"[kb] name is missing from the environment config. The launcher stages the KB's committed identity into each service's config (SINGLE-KB-MOUNT D4); without it this service cannot locate the state tree."
|
|
9733
|
+
);
|
|
9734
|
+
}
|
|
9735
|
+
return name;
|
|
9736
|
+
}
|
|
9737
|
+
function makeMeaningConfigFrom(config) {
|
|
9738
|
+
const meta = config._metadata;
|
|
9739
|
+
const gather = meta?.gather;
|
|
9740
|
+
if (!gather) {
|
|
9741
|
+
throw new Error("make-meaning gather config missing \u2014 load config via loadEnvironmentConfig (the TOML loader owns the settleTimeoutMs default)");
|
|
9742
|
+
}
|
|
9743
|
+
const search = meta?.search;
|
|
9744
|
+
if (!search) {
|
|
9745
|
+
throw new Error("make-meaning search config missing \u2014 load config via loadEnvironmentConfig (the TOML loader owns the semanticFloor default)");
|
|
9746
|
+
}
|
|
9747
|
+
return {
|
|
9748
|
+
gather,
|
|
9749
|
+
search,
|
|
9750
|
+
services: {
|
|
9751
|
+
// vectors/embedding are required on both sides (MANDATORY-EMBEDDING P3):
|
|
9752
|
+
// core's ServicesConfig requires the pair, so a config missing either
|
|
9753
|
+
// already refused at the TOML loader — nothing to re-check here.
|
|
9754
|
+
graph: config.services.graph,
|
|
9755
|
+
vectors: config.services.vectors,
|
|
9756
|
+
embedding: config.services.embedding,
|
|
9757
|
+
archivist: config.services.archivist
|
|
9758
|
+
},
|
|
9759
|
+
// The KB's canonical identity — the agent roster mints DIDs from this,
|
|
9760
|
+
// the SAME value /api/tokens/agent uses (agent-did-host-skew fix). The
|
|
9761
|
+
// value, not JWTService, so make-meaning stays gateway-agnostic.
|
|
9762
|
+
...config.site?.domain ? { site: { domain: config.site.domain } } : {},
|
|
9763
|
+
actors: meta?.actors,
|
|
9764
|
+
workers: meta?.workers
|
|
9765
|
+
};
|
|
9766
|
+
}
|
|
9729
9767
|
function resolveActorInference(config, actor) {
|
|
9730
9768
|
const specific = config.actors?.[actor];
|
|
9731
9769
|
if (specific) return specific;
|
|
@@ -9746,6 +9784,21 @@ function resolveWorkerInference(config, workerType) {
|
|
|
9746
9784
|
// src/service.ts
|
|
9747
9785
|
var import_rxjs7 = __toESM(require_cjs());
|
|
9748
9786
|
var import_operators6 = __toESM(require_operators());
|
|
9787
|
+
function representationSource(resource) {
|
|
9788
|
+
const primary = getPrimaryRepresentation(resource);
|
|
9789
|
+
if (!primary?.storageUri) return null;
|
|
9790
|
+
return {
|
|
9791
|
+
storageUri: primary.storageUri,
|
|
9792
|
+
mediaType: primary.mediaType
|
|
9793
|
+
};
|
|
9794
|
+
}
|
|
9795
|
+
async function resolveRepresentation(deps, resourceId10) {
|
|
9796
|
+
const view = await deps.views.get(resourceId10);
|
|
9797
|
+
if (!view?.resource) throw new RepresentationMissing(String(resourceId10), "resource");
|
|
9798
|
+
const source = representationSource(view.resource);
|
|
9799
|
+
if (!source) throw new RepresentationMissing(String(resourceId10), "representation");
|
|
9800
|
+
return { stream: deps.content.retrieveStream(source.storageUri), mediaType: source.mediaType };
|
|
9801
|
+
}
|
|
9749
9802
|
|
|
9750
9803
|
// src/weave-progress.ts
|
|
9751
9804
|
var WeaveProgressTimeout = class extends Error {
|
|
@@ -9883,6 +9936,19 @@ function createSmeltProgress(eventBus) {
|
|
|
9883
9936
|
}
|
|
9884
9937
|
|
|
9885
9938
|
// src/knowledge-base.ts
|
|
9939
|
+
function workingTreeContentReads(views, content) {
|
|
9940
|
+
return {
|
|
9941
|
+
getBinary: async (resourceId10) => {
|
|
9942
|
+
const { stream, mediaType } = await resolveRepresentation({ views, content }, resourceId10);
|
|
9943
|
+
const chunks = [];
|
|
9944
|
+
for await (const chunk of stream) chunks.push(chunk);
|
|
9945
|
+
const buf = Buffer.concat(chunks);
|
|
9946
|
+
const data = new ArrayBuffer(buf.byteLength);
|
|
9947
|
+
new Uint8Array(data).set(buf);
|
|
9948
|
+
return { data, contentType: mediaType };
|
|
9949
|
+
}
|
|
9950
|
+
};
|
|
9951
|
+
}
|
|
9886
9952
|
async function createKnowledgeBase(eventStore, project, graphDb, eventBus, logger, options) {
|
|
9887
9953
|
const views = new FilesystemViewStorage(project, logger.child({ component: "view-storage" }));
|
|
9888
9954
|
const content = new WorkingTreeStore(
|
|
@@ -9915,27 +9981,6 @@ var PROJECTION_LAG_BACKOFF_MS = [25, 50, 100, 200];
|
|
|
9915
9981
|
var PROJECTION_BARRIER_TIMEOUT_MS = 500;
|
|
9916
9982
|
var GRAPH_BARRIER_BUDGET_MS = PROJECTION_BARRIER_TIMEOUT_MS + PROJECTION_LAG_BACKOFF_MS.reduce((a, b) => a + b, 0);
|
|
9917
9983
|
var GraphContext = class {
|
|
9918
|
-
/**
|
|
9919
|
-
* Get all resources referencing this resource (backlinks)
|
|
9920
|
-
* Requires graph traversal - must use graph database
|
|
9921
|
-
*/
|
|
9922
|
-
static async getBacklinks(resourceId10, kb) {
|
|
9923
|
-
return kb.graph.getResourceReferencedBy(resourceId10);
|
|
9924
|
-
}
|
|
9925
|
-
/**
|
|
9926
|
-
* Find shortest path between two resources
|
|
9927
|
-
* Requires graph traversal - must use graph database
|
|
9928
|
-
*/
|
|
9929
|
-
static async findPath(fromResourceId, toResourceId, kb, maxDepth) {
|
|
9930
|
-
return kb.graph.findPath(fromResourceId, toResourceId, maxDepth);
|
|
9931
|
-
}
|
|
9932
|
-
/**
|
|
9933
|
-
* Get resource connections (graph edges)
|
|
9934
|
-
* Requires graph traversal - must use graph database
|
|
9935
|
-
*/
|
|
9936
|
-
static async getResourceConnections(resourceId10, kb) {
|
|
9937
|
-
return kb.graph.getResourceConnections(resourceId10);
|
|
9938
|
-
}
|
|
9939
9984
|
/**
|
|
9940
9985
|
* Build the unified knowledge graph for a resource's neighborhood:
|
|
9941
9986
|
* resources AND annotations as typed nodes, typed/directional edges.
|
|
@@ -10168,10 +10213,10 @@ var ResourceContext = class _ResourceContext {
|
|
|
10168
10213
|
return Promise.all(
|
|
10169
10214
|
resources.map(async (doc) => {
|
|
10170
10215
|
try {
|
|
10171
|
-
|
|
10172
|
-
|
|
10173
|
-
const
|
|
10174
|
-
const contentPreview = decodeRepresentation(contentBuffer,
|
|
10216
|
+
const source = representationSource(doc);
|
|
10217
|
+
if (source) {
|
|
10218
|
+
const contentBuffer = await kb.content.retrieve(source.storageUri);
|
|
10219
|
+
const contentPreview = decodeRepresentation(contentBuffer, source.mediaType).slice(0, 200);
|
|
10175
10220
|
return { ...doc, content: contentPreview };
|
|
10176
10221
|
}
|
|
10177
10222
|
return { ...doc, content: "" };
|
|
@@ -10183,13 +10228,17 @@ var ResourceContext = class _ResourceContext {
|
|
|
10183
10228
|
}
|
|
10184
10229
|
/**
|
|
10185
10230
|
* Get full content for a resource
|
|
10186
|
-
* Retrieves and decodes the primary representation
|
|
10231
|
+
* Retrieves and decodes the primary representation. ResourceId-keyed
|
|
10232
|
+
* (EXTRACT-LIBRARIAN P3, D-CONTENT b): the descriptor's `storageUri` is the
|
|
10233
|
+
* has-content signal; the fetch itself goes by id, so the standalone
|
|
10234
|
+
* Librarian serves it over the transport.
|
|
10187
10235
|
*/
|
|
10188
10236
|
static async getResourceContent(resource, kb) {
|
|
10189
|
-
|
|
10190
|
-
|
|
10191
|
-
|
|
10192
|
-
|
|
10237
|
+
const id = getResourceId(resource);
|
|
10238
|
+
const source = representationSource(resource);
|
|
10239
|
+
if (source && id) {
|
|
10240
|
+
const { data, contentType } = await kb.content.getBinary(resourceId(id));
|
|
10241
|
+
return decodeRepresentation(Buffer.from(data), contentType);
|
|
10193
10242
|
}
|
|
10194
10243
|
return void 0;
|
|
10195
10244
|
}
|
|
@@ -10256,12 +10305,12 @@ var AnnotationContext = class {
|
|
|
10256
10305
|
}
|
|
10257
10306
|
let sourceContext;
|
|
10258
10307
|
if (includeSourceContext) {
|
|
10259
|
-
if (!sourceDoc
|
|
10308
|
+
if (!getStorageUri(sourceDoc)) {
|
|
10260
10309
|
throw new Error("Source content not found: no storageUri");
|
|
10261
10310
|
}
|
|
10262
10311
|
const primaryRep = getPrimaryRepresentation(sourceDoc);
|
|
10263
|
-
const sourceContent = await kb.content.
|
|
10264
|
-
const contentStr = decodeRepresentation(sourceContent, primaryRep?.mediaType ?? "text/plain");
|
|
10312
|
+
const { data: sourceContent } = await kb.content.getBinary(resourceId10);
|
|
10313
|
+
const contentStr = decodeRepresentation(Buffer.from(sourceContent), primaryRep?.mediaType ?? "text/plain");
|
|
10265
10314
|
const targetSelectorRaw = getTargetSelector(annotation.target);
|
|
10266
10315
|
const targetSelector = Array.isArray(targetSelectorRaw) ? targetSelectorRaw[0] : targetSelectorRaw;
|
|
10267
10316
|
logger?.debug("Target selector", { type: targetSelector?.type });
|
|
@@ -10297,10 +10346,10 @@ var AnnotationContext = class {
|
|
|
10297
10346
|
}
|
|
10298
10347
|
let targetContext;
|
|
10299
10348
|
if (includeTargetContext && targetDoc) {
|
|
10300
|
-
if (targetDoc
|
|
10349
|
+
if (getStorageUri(targetDoc) && bodySource) {
|
|
10301
10350
|
const targetRep = getPrimaryRepresentation(targetDoc);
|
|
10302
|
-
const targetContent = await kb.content.
|
|
10303
|
-
const contentStr = decodeRepresentation(targetContent, targetRep?.mediaType ?? "text/plain");
|
|
10351
|
+
const { data: targetContent } = await kb.content.getBinary(resourceId(bodySource));
|
|
10352
|
+
const contentStr = decodeRepresentation(Buffer.from(targetContent), targetRep?.mediaType ?? "text/plain");
|
|
10304
10353
|
targetContext = {
|
|
10305
10354
|
content: contentStr.slice(0, contextWindow * 2),
|
|
10306
10355
|
summary: inferenceClient ? await generateResourceSummary(targetDoc.name, contentStr, getResourceEntityTypes(targetDoc), inferenceClient) : void 0
|
|
@@ -10582,15 +10631,18 @@ Summary:`;
|
|
|
10582
10631
|
};
|
|
10583
10632
|
}
|
|
10584
10633
|
/**
|
|
10585
|
-
* Get resource content as string
|
|
10634
|
+
* Get resource content as string. ResourceId-keyed (D-CONTENT b): the
|
|
10635
|
+
* primary representation's `storageUri` stays the has-content signal, the
|
|
10636
|
+
* fetch goes by id.
|
|
10586
10637
|
*/
|
|
10587
10638
|
static async getResourceContent(resource, content) {
|
|
10588
|
-
|
|
10639
|
+
const id = getResourceId(resource);
|
|
10640
|
+
if (!getStorageUri(resource) || !id) {
|
|
10589
10641
|
throw new Error("Resource content not found: no storageUri");
|
|
10590
10642
|
}
|
|
10591
10643
|
const primaryRep = getPrimaryRepresentation(resource);
|
|
10592
|
-
const
|
|
10593
|
-
return decodeRepresentation(
|
|
10644
|
+
const { data } = await content.getBinary(resourceId(id));
|
|
10645
|
+
return decodeRepresentation(Buffer.from(data), primaryRep?.mediaType ?? "text/plain");
|
|
10594
10646
|
}
|
|
10595
10647
|
/**
|
|
10596
10648
|
* Extract annotation context from resource content
|
|
@@ -10745,15 +10797,15 @@ var LLMContext = class {
|
|
|
10745
10797
|
|
|
10746
10798
|
// src/gatherer.ts
|
|
10747
10799
|
var Gatherer = class {
|
|
10748
|
-
constructor(
|
|
10749
|
-
this.
|
|
10800
|
+
constructor(stores, eventBus, inferenceClient, settleTimeoutMs, logger, embeddingProvider) {
|
|
10801
|
+
this.stores = stores;
|
|
10750
10802
|
this.eventBus = eventBus;
|
|
10751
10803
|
this.inferenceClient = inferenceClient;
|
|
10752
10804
|
this.settleTimeoutMs = settleTimeoutMs;
|
|
10753
10805
|
this.embeddingProvider = embeddingProvider;
|
|
10754
10806
|
this.logger = logger;
|
|
10755
10807
|
}
|
|
10756
|
-
|
|
10808
|
+
stores;
|
|
10757
10809
|
eventBus;
|
|
10758
10810
|
inferenceClient;
|
|
10759
10811
|
settleTimeoutMs;
|
|
@@ -10800,7 +10852,7 @@ var Gatherer = class {
|
|
|
10800
10852
|
const response = await AnnotationContext.buildLLMContext(
|
|
10801
10853
|
annotationId(event.annotationId),
|
|
10802
10854
|
resourceId(event.resourceId),
|
|
10803
|
-
this.
|
|
10855
|
+
this.stores,
|
|
10804
10856
|
this.embeddingProvider,
|
|
10805
10857
|
event.options ?? {},
|
|
10806
10858
|
this.inferenceClient,
|
|
@@ -10831,7 +10883,7 @@ var Gatherer = class {
|
|
|
10831
10883
|
const result = await LLMContext.getResourceContext(
|
|
10832
10884
|
resourceId(event.resourceId),
|
|
10833
10885
|
event.options,
|
|
10834
|
-
this.
|
|
10886
|
+
this.stores,
|
|
10835
10887
|
this.inferenceClient,
|
|
10836
10888
|
this.settleTimeoutMs,
|
|
10837
10889
|
this.logger
|
|
@@ -10857,7 +10909,7 @@ var Gatherer = class {
|
|
|
10857
10909
|
return AnnotationContext.generateAnnotationSummary(
|
|
10858
10910
|
annotationId5,
|
|
10859
10911
|
resourceId10,
|
|
10860
|
-
this.
|
|
10912
|
+
this.stores,
|
|
10861
10913
|
this.inferenceClient
|
|
10862
10914
|
);
|
|
10863
10915
|
}
|
|
@@ -10874,14 +10926,14 @@ var Gatherer = class {
|
|
|
10874
10926
|
var import_rxjs2 = __toESM(require_cjs());
|
|
10875
10927
|
var import_operators2 = __toESM(require_operators());
|
|
10876
10928
|
var Matcher = class {
|
|
10877
|
-
constructor(
|
|
10878
|
-
this.
|
|
10929
|
+
constructor(stores, eventBus, logger, inferenceClient, embeddingProvider) {
|
|
10930
|
+
this.stores = stores;
|
|
10879
10931
|
this.eventBus = eventBus;
|
|
10880
10932
|
this.inferenceClient = inferenceClient;
|
|
10881
10933
|
this.embeddingProvider = embeddingProvider;
|
|
10882
10934
|
this.logger = logger;
|
|
10883
10935
|
}
|
|
10884
|
-
|
|
10936
|
+
stores;
|
|
10885
10937
|
eventBus;
|
|
10886
10938
|
inferenceClient;
|
|
10887
10939
|
embeddingProvider;
|
|
@@ -10961,13 +11013,13 @@ var Matcher = class {
|
|
|
10961
11013
|
const views = deriveViews(context.graph, mainResourceId);
|
|
10962
11014
|
const connections = views.connections;
|
|
10963
11015
|
const [nameMatches, entityTypeMatches, semanticMatches] = await Promise.all([
|
|
10964
|
-
this.
|
|
10965
|
-
annotationEntityTypes.length > 0 ? this.
|
|
11016
|
+
this.stores.graph.listResources({ search: searchTerm, limit: 20 }).then((r) => r.resources),
|
|
11017
|
+
annotationEntityTypes.length > 0 ? this.stores.graph.listResources({ entityTypes: annotationEntityTypes, limit: 50 }).then((r) => r.resources) : Promise.resolve([]),
|
|
10966
11018
|
// 4. Semantic match — vector similarity search (if vectors configured)
|
|
10967
11019
|
this.searchVectors(searchTerm)
|
|
10968
11020
|
]);
|
|
10969
11021
|
const neighborResolved = await Promise.all(
|
|
10970
|
-
connections.map((conn) => resourceWithViewGrace(this.
|
|
11022
|
+
connections.map((conn) => resourceWithViewGrace(this.stores, resourceId(conn.resourceId)))
|
|
10971
11023
|
);
|
|
10972
11024
|
const laggedNeighbors = neighborResolved.filter((r) => r.laggedBehindView).length;
|
|
10973
11025
|
if (laggedNeighbors > 0) {
|
|
@@ -10994,7 +11046,7 @@ var Matcher = class {
|
|
|
10994
11046
|
let laggedSemantic = 0;
|
|
10995
11047
|
for (const sm of semanticMatches) {
|
|
10996
11048
|
semanticScores.set(sm.resourceId, sm.score);
|
|
10997
|
-
const { resource, laggedBehindView } = await resourceWithViewGrace(this.
|
|
11049
|
+
const { resource, laggedBehindView } = await resourceWithViewGrace(this.stores, resourceId(sm.resourceId));
|
|
10998
11050
|
if (laggedBehindView) laggedSemantic++;
|
|
10999
11051
|
if (resource) addCandidate(resource, "semantic");
|
|
11000
11052
|
}
|
|
@@ -11190,7 +11242,7 @@ For each candidate, output a line with the number and score, like:
|
|
|
11190
11242
|
if (!searchTerm.trim()) return [];
|
|
11191
11243
|
try {
|
|
11192
11244
|
const embedding = await this.embeddingProvider.embed(searchTerm);
|
|
11193
|
-
const results = await this.
|
|
11245
|
+
const results = await this.stores.vectors.searchResources(embedding, {
|
|
11194
11246
|
limit: 40,
|
|
11195
11247
|
scoreThreshold: 0.4
|
|
11196
11248
|
});
|
|
@@ -11215,9 +11267,9 @@ For each candidate, output a line with the number and score, like:
|
|
|
11215
11267
|
// src/stower.ts
|
|
11216
11268
|
var import_rxjs3 = __toESM(require_cjs());
|
|
11217
11269
|
var import_operators3 = __toESM(require_operators());
|
|
11218
|
-
async function readEntityTypesProjection(
|
|
11270
|
+
async function readEntityTypesProjection(state) {
|
|
11219
11271
|
const entityTypesPath = path3.join(
|
|
11220
|
-
|
|
11272
|
+
state.stateDir,
|
|
11221
11273
|
"projections",
|
|
11222
11274
|
"__system__",
|
|
11223
11275
|
"entitytypes.json"
|
|
@@ -11259,13 +11311,13 @@ function entityTypesNotRegisteredMessage(unknown) {
|
|
|
11259
11311
|
|
|
11260
11312
|
// src/stower.ts
|
|
11261
11313
|
var Stower = class {
|
|
11262
|
-
constructor(
|
|
11263
|
-
this.
|
|
11314
|
+
constructor(stores, eventBus, project, logger) {
|
|
11315
|
+
this.stores = stores;
|
|
11264
11316
|
this.eventBus = eventBus;
|
|
11265
11317
|
this.project = project;
|
|
11266
11318
|
this.logger = logger;
|
|
11267
11319
|
}
|
|
11268
|
-
|
|
11320
|
+
stores;
|
|
11269
11321
|
eventBus;
|
|
11270
11322
|
project;
|
|
11271
11323
|
subscription = null;
|
|
@@ -11279,9 +11331,11 @@ var Stower = class {
|
|
|
11279
11331
|
);
|
|
11280
11332
|
this.subscription = (0, import_rxjs3.merge)(
|
|
11281
11333
|
pipe("yield:create", (e) => this.handleYieldCreate(e)),
|
|
11334
|
+
pipe("yield:clone-persist", (e) => this.handleYieldClonePersist(e)),
|
|
11282
11335
|
pipe("yield:update", (e) => this.handleYieldUpdate(e)),
|
|
11283
11336
|
pipe("yield:mv", (e) => this.handleYieldMv(e)),
|
|
11284
11337
|
pipe("mark:create", (e) => this.handleMarkCreate(e)),
|
|
11338
|
+
pipe("mark:commit", (e) => this.handleMarkCommit(e)),
|
|
11285
11339
|
pipe("mark:delete", (e) => this.handleMarkDelete(e)),
|
|
11286
11340
|
pipe("mark:update-body", (e) => this.handleMarkUpdateBody(e)),
|
|
11287
11341
|
pipe("frame:add-entity-type", (e) => this.handleAddEntityType(e)),
|
|
@@ -11306,14 +11360,14 @@ var Stower = class {
|
|
|
11306
11360
|
const uid = userId(event._userId);
|
|
11307
11361
|
try {
|
|
11308
11362
|
const rId = resourceId(generateUuid());
|
|
11309
|
-
const stored = await this.
|
|
11363
|
+
const stored = await this.stores.content.register(event.storageUri, event.contentChecksum, { noGit: event.noGit });
|
|
11310
11364
|
const checksum = stored.checksum;
|
|
11311
11365
|
const byteSize = event.byteSize;
|
|
11312
11366
|
const generatedFrom = event.generatedFrom?.resourceId && event.generatedFrom?.annotationId ? {
|
|
11313
11367
|
resourceId: resourceId(event.generatedFrom.resourceId),
|
|
11314
11368
|
annotationId: annotationId(event.generatedFrom.annotationId)
|
|
11315
11369
|
} : void 0;
|
|
11316
|
-
await this.
|
|
11370
|
+
await this.stores.eventStore.appendEvent({
|
|
11317
11371
|
type: "yield:created",
|
|
11318
11372
|
resourceId: rId,
|
|
11319
11373
|
userId: uid,
|
|
@@ -11366,13 +11420,66 @@ var Stower = class {
|
|
|
11366
11420
|
});
|
|
11367
11421
|
}
|
|
11368
11422
|
}
|
|
11423
|
+
/**
|
|
11424
|
+
* Persist a clone (the CloneTokenManager's second half).
|
|
11425
|
+
*
|
|
11426
|
+
* A clone is its own domain fact, not a creation with an extra field: it
|
|
11427
|
+
* NAMES the resource it came from, and `ResourceClonedPayload` requires
|
|
11428
|
+
* that name. So it gets its own command and appends its own event, rather
|
|
11429
|
+
* than an optional parent turning `yield:created` into something else at
|
|
11430
|
+
* runtime.
|
|
11431
|
+
*
|
|
11432
|
+
* The token was already validated and the source's entity types already
|
|
11433
|
+
* read by the CloneTokenManager — the one party that can do either. What
|
|
11434
|
+
* happens here is what only the Stower may do: register the bytes and
|
|
11435
|
+
* append the event (single-writer, GATEWAY.md D4b).
|
|
11436
|
+
*
|
|
11437
|
+
* Generated resources do NOT come through here. Their provenance is
|
|
11438
|
+
* `generatedFrom` on `yield:created`, which is a different relation: a
|
|
11439
|
+
* generated resource is derived from a source, a clone IS a copy of one.
|
|
11440
|
+
*/
|
|
11441
|
+
async handleYieldClonePersist(event) {
|
|
11442
|
+
if (!event._userId) {
|
|
11443
|
+
throw new Error("yield:clone-persist missing _userId (gateway injection)");
|
|
11444
|
+
}
|
|
11445
|
+
try {
|
|
11446
|
+
const rId = resourceId(generateUuid());
|
|
11447
|
+
const stored = await this.stores.content.register(event.storageUri, event.contentChecksum, { noGit: event.noGit });
|
|
11448
|
+
await this.stores.eventStore.appendEvent({
|
|
11449
|
+
type: "yield:cloned",
|
|
11450
|
+
resourceId: rId,
|
|
11451
|
+
userId: userId(event._userId),
|
|
11452
|
+
version: 1,
|
|
11453
|
+
payload: {
|
|
11454
|
+
name: event.name,
|
|
11455
|
+
format: event.format,
|
|
11456
|
+
contentChecksum: stored.checksum,
|
|
11457
|
+
contentByteSize: event.byteSize,
|
|
11458
|
+
storageUri: event.storageUri,
|
|
11459
|
+
parentResourceId: event.parentResourceId,
|
|
11460
|
+
entityTypes: event.entityTypes || [],
|
|
11461
|
+
language: event.language || void 0
|
|
11462
|
+
}
|
|
11463
|
+
});
|
|
11464
|
+
this.eventBus.get("yield:clone-persist-ok").next({
|
|
11465
|
+
correlationId: event.correlationId,
|
|
11466
|
+
response: { resourceId: rId }
|
|
11467
|
+
});
|
|
11468
|
+
} catch (error) {
|
|
11469
|
+
this.logger.error("Failed to persist clone", { error: errField(error) });
|
|
11470
|
+
this.eventBus.get("yield:clone-persist-failed").next({
|
|
11471
|
+
correlationId: event.correlationId,
|
|
11472
|
+
message: error instanceof Error ? error.message : String(error)
|
|
11473
|
+
});
|
|
11474
|
+
}
|
|
11475
|
+
}
|
|
11369
11476
|
async handleYieldUpdate(event) {
|
|
11370
11477
|
if (!event._userId) {
|
|
11371
11478
|
throw new Error("yield:update missing _userId (gateway injection)");
|
|
11372
11479
|
}
|
|
11373
11480
|
try {
|
|
11374
|
-
await this.
|
|
11375
|
-
await this.
|
|
11481
|
+
await this.stores.content.register(event.storageUri, event.contentChecksum, { noGit: event.noGit });
|
|
11482
|
+
await this.stores.eventStore.appendEvent({
|
|
11376
11483
|
type: "yield:updated",
|
|
11377
11484
|
resourceId: resourceId(event.resourceId),
|
|
11378
11485
|
userId: userId(event._userId),
|
|
@@ -11397,7 +11504,7 @@ var Stower = class {
|
|
|
11397
11504
|
async handleYieldMv(event) {
|
|
11398
11505
|
let rId;
|
|
11399
11506
|
try {
|
|
11400
|
-
const resolved = await resolveStorageUri(this.
|
|
11507
|
+
const resolved = await resolveStorageUri(this.project.projectionsDir, event.fromUri);
|
|
11401
11508
|
rId = resolved;
|
|
11402
11509
|
} catch (error) {
|
|
11403
11510
|
this.logger.error("Failed to resolve resource for move", { fromUri: event.fromUri, error });
|
|
@@ -11411,8 +11518,8 @@ var Stower = class {
|
|
|
11411
11518
|
throw new Error("yield:mv missing _userId (gateway injection)");
|
|
11412
11519
|
}
|
|
11413
11520
|
try {
|
|
11414
|
-
await this.
|
|
11415
|
-
await this.
|
|
11521
|
+
await this.stores.content.move(event.fromUri, event.toUri, { noGit: event.noGit });
|
|
11522
|
+
await this.stores.eventStore.appendEvent({
|
|
11416
11523
|
type: "yield:moved",
|
|
11417
11524
|
resourceId: rId,
|
|
11418
11525
|
userId: userId(event._userId),
|
|
@@ -11436,7 +11543,7 @@ var Stower = class {
|
|
|
11436
11543
|
}
|
|
11437
11544
|
try {
|
|
11438
11545
|
this.logger.debug("Stowing annotation", { annotationId: event.annotation.id });
|
|
11439
|
-
await this.
|
|
11546
|
+
await this.stores.eventStore.appendEvent(
|
|
11440
11547
|
{
|
|
11441
11548
|
type: "mark:added",
|
|
11442
11549
|
resourceId: resourceId(event.resourceId),
|
|
@@ -11454,6 +11561,61 @@ var Stower = class {
|
|
|
11454
11561
|
});
|
|
11455
11562
|
}
|
|
11456
11563
|
}
|
|
11564
|
+
/**
|
|
11565
|
+
* Persist a detection unit's annotations as ONE acknowledged batch, then
|
|
11566
|
+
* answer (JOB-RESTART-SAFETY P6).
|
|
11567
|
+
*
|
|
11568
|
+
* The difference from `mark:create` is the reply, and it is the whole point.
|
|
11569
|
+
* `mark:create` is fire-and-forget: the worker's emit resolves when the bus
|
|
11570
|
+
* accepts it, which says nothing about the event log, so a down Stower loses
|
|
11571
|
+
* a unit silently and a flapping one hangs the worker forever. This answers
|
|
11572
|
+
* only after every append has returned, so the worker can gate unit
|
|
11573
|
+
* completion — and its checkpoint — on durability.
|
|
11574
|
+
*
|
|
11575
|
+
* Appends are sequential, not concurrent: the event log is the system of
|
|
11576
|
+
* record and a batch that half-lands under concurrency is harder to reason
|
|
11577
|
+
* about than one that stops at the first failure. A partial batch is
|
|
11578
|
+
* reported as a failure and the worker retries the WHOLE unit, which is safe
|
|
11579
|
+
* because ids are deterministic (P3) and the annotation fold is idempotent
|
|
11580
|
+
* by id — re-appending what already landed changes nothing.
|
|
11581
|
+
*/
|
|
11582
|
+
async handleMarkCommit(event) {
|
|
11583
|
+
if (!event._userId) {
|
|
11584
|
+
throw new Error("mark:commit missing _userId (gateway injection)");
|
|
11585
|
+
}
|
|
11586
|
+
const annotations = event.annotations ?? [];
|
|
11587
|
+
try {
|
|
11588
|
+
let persisted = 0;
|
|
11589
|
+
for (const annotation of annotations) {
|
|
11590
|
+
await this.stores.eventStore.appendEvent({
|
|
11591
|
+
type: "mark:added",
|
|
11592
|
+
resourceId: resourceId(event.resourceId),
|
|
11593
|
+
userId: userId(event._userId),
|
|
11594
|
+
version: 1,
|
|
11595
|
+
payload: { annotation }
|
|
11596
|
+
});
|
|
11597
|
+
persisted++;
|
|
11598
|
+
}
|
|
11599
|
+
this.logger.debug("Committed annotation batch", {
|
|
11600
|
+
correlationId: event.correlationId,
|
|
11601
|
+
resourceId: event.resourceId,
|
|
11602
|
+
persisted
|
|
11603
|
+
});
|
|
11604
|
+
this.eventBus.get("mark:commit-ok").next({
|
|
11605
|
+
correlationId: event.correlationId,
|
|
11606
|
+
response: { persisted, annotationIds: annotations.map((a) => String(a.id)) }
|
|
11607
|
+
});
|
|
11608
|
+
} catch (error) {
|
|
11609
|
+
this.logger.error("Failed to commit annotation batch", {
|
|
11610
|
+
correlationId: event.correlationId,
|
|
11611
|
+
error: errField(error)
|
|
11612
|
+
});
|
|
11613
|
+
this.eventBus.get("mark:commit-failed").next({
|
|
11614
|
+
correlationId: event.correlationId,
|
|
11615
|
+
message: error instanceof Error ? error.message : String(error)
|
|
11616
|
+
});
|
|
11617
|
+
}
|
|
11618
|
+
}
|
|
11457
11619
|
async handleMarkDelete(event) {
|
|
11458
11620
|
if (!event._userId) {
|
|
11459
11621
|
throw new Error("mark:delete missing _userId (gateway injection)");
|
|
@@ -11462,7 +11624,7 @@ var Stower = class {
|
|
|
11462
11624
|
throw new Error("mark:delete missing resourceId");
|
|
11463
11625
|
}
|
|
11464
11626
|
try {
|
|
11465
|
-
await this.
|
|
11627
|
+
await this.stores.eventStore.appendEvent({
|
|
11466
11628
|
type: "mark:removed",
|
|
11467
11629
|
resourceId: resourceId(event.resourceId),
|
|
11468
11630
|
userId: userId(event._userId),
|
|
@@ -11483,7 +11645,7 @@ var Stower = class {
|
|
|
11483
11645
|
throw new Error("mark:update-body missing _userId (gateway injection)");
|
|
11484
11646
|
}
|
|
11485
11647
|
try {
|
|
11486
|
-
await this.
|
|
11648
|
+
await this.stores.eventStore.appendEvent(
|
|
11487
11649
|
{
|
|
11488
11650
|
type: "mark:body-updated",
|
|
11489
11651
|
resourceId: resourceId(event.resourceId),
|
|
@@ -11509,9 +11671,9 @@ var Stower = class {
|
|
|
11509
11671
|
}
|
|
11510
11672
|
try {
|
|
11511
11673
|
if (event.storageUri) {
|
|
11512
|
-
await this.
|
|
11674
|
+
await this.stores.content.remove(event.storageUri, { keepFile: event.keepFile, noGit: event.noGit });
|
|
11513
11675
|
}
|
|
11514
|
-
await this.
|
|
11676
|
+
await this.stores.eventStore.appendEvent({
|
|
11515
11677
|
type: "mark:archived",
|
|
11516
11678
|
resourceId: resourceId(event.resourceId),
|
|
11517
11679
|
userId: userId(event._userId),
|
|
@@ -11533,14 +11695,14 @@ var Stower = class {
|
|
|
11533
11695
|
}
|
|
11534
11696
|
try {
|
|
11535
11697
|
if (event.storageUri) {
|
|
11536
|
-
const absPath = this.
|
|
11698
|
+
const absPath = this.stores.content.resolveUri(event.storageUri);
|
|
11537
11699
|
try {
|
|
11538
11700
|
await promises.access(absPath);
|
|
11539
11701
|
} catch {
|
|
11540
11702
|
throw new Error(`Cannot unarchive: file not found at ${event.storageUri}`);
|
|
11541
11703
|
}
|
|
11542
11704
|
}
|
|
11543
|
-
await this.
|
|
11705
|
+
await this.stores.eventStore.appendEvent({
|
|
11544
11706
|
type: "mark:unarchived",
|
|
11545
11707
|
resourceId: resourceId(event.resourceId),
|
|
11546
11708
|
userId: userId(event._userId),
|
|
@@ -11561,7 +11723,7 @@ var Stower = class {
|
|
|
11561
11723
|
throw new Error("frame:add-entity-type missing _userId (gateway injection)");
|
|
11562
11724
|
}
|
|
11563
11725
|
try {
|
|
11564
|
-
await this.
|
|
11726
|
+
await this.stores.eventStore.appendEvent({
|
|
11565
11727
|
type: "frame:entity-type-added",
|
|
11566
11728
|
userId: userId(event._userId),
|
|
11567
11729
|
version: 1,
|
|
@@ -11581,7 +11743,7 @@ var Stower = class {
|
|
|
11581
11743
|
throw new Error("frame:add-tag-schema missing _userId (gateway injection)");
|
|
11582
11744
|
}
|
|
11583
11745
|
try {
|
|
11584
|
-
await this.
|
|
11746
|
+
await this.stores.eventStore.appendEvent({
|
|
11585
11747
|
type: "frame:tag-schema-added",
|
|
11586
11748
|
userId: userId(event._userId),
|
|
11587
11749
|
version: 1,
|
|
@@ -11612,7 +11774,7 @@ var Stower = class {
|
|
|
11612
11774
|
}
|
|
11613
11775
|
}
|
|
11614
11776
|
for (const entityType2 of added) {
|
|
11615
|
-
await this.
|
|
11777
|
+
await this.stores.eventStore.appendEvent({
|
|
11616
11778
|
type: "mark:entity-tag-added",
|
|
11617
11779
|
resourceId: resourceId(event.resourceId),
|
|
11618
11780
|
userId: uid,
|
|
@@ -11621,7 +11783,7 @@ var Stower = class {
|
|
|
11621
11783
|
});
|
|
11622
11784
|
}
|
|
11623
11785
|
for (const entityType2 of removed) {
|
|
11624
|
-
await this.
|
|
11786
|
+
await this.stores.eventStore.appendEvent({
|
|
11625
11787
|
type: "mark:entity-tag-removed",
|
|
11626
11788
|
resourceId: resourceId(event.resourceId),
|
|
11627
11789
|
userId: uid,
|
|
@@ -11642,7 +11804,7 @@ var Stower = class {
|
|
|
11642
11804
|
if (!event._userId) {
|
|
11643
11805
|
throw new Error("job:start missing _userId (gateway injection)");
|
|
11644
11806
|
}
|
|
11645
|
-
await this.
|
|
11807
|
+
await this.stores.eventStore.appendEvent({
|
|
11646
11808
|
type: "job:started",
|
|
11647
11809
|
resourceId: resourceId(event.resourceId),
|
|
11648
11810
|
userId: userId(event._userId),
|
|
@@ -11658,7 +11820,7 @@ var Stower = class {
|
|
|
11658
11820
|
if (!event._userId) {
|
|
11659
11821
|
throw new Error("job:complete missing _userId (gateway injection)");
|
|
11660
11822
|
}
|
|
11661
|
-
await this.
|
|
11823
|
+
await this.stores.eventStore.appendEvent({
|
|
11662
11824
|
type: "job:completed",
|
|
11663
11825
|
resourceId: resourceId(event.resourceId),
|
|
11664
11826
|
userId: userId(event._userId),
|
|
@@ -11675,7 +11837,7 @@ var Stower = class {
|
|
|
11675
11837
|
if (!event._userId) {
|
|
11676
11838
|
throw new Error("job:fail missing _userId (gateway injection)");
|
|
11677
11839
|
}
|
|
11678
|
-
await this.
|
|
11840
|
+
await this.stores.eventStore.appendEvent({
|
|
11679
11841
|
type: "job:failed",
|
|
11680
11842
|
resourceId: resourceId(event.resourceId),
|
|
11681
11843
|
userId: userId(event._userId),
|
|
@@ -11713,9 +11875,9 @@ async function readAnchoredText(kb, resourceId10) {
|
|
|
11713
11875
|
throw error;
|
|
11714
11876
|
}
|
|
11715
11877
|
}
|
|
11716
|
-
async function readTagSchemasProjection(
|
|
11878
|
+
async function readTagSchemasProjection(state) {
|
|
11717
11879
|
const tagSchemasPath = path3.join(
|
|
11718
|
-
|
|
11880
|
+
state.stateDir,
|
|
11719
11881
|
"projections",
|
|
11720
11882
|
"__system__",
|
|
11721
11883
|
"tagschemas.json"
|
|
@@ -11801,8 +11963,7 @@ function deriveAgentRoster(config) {
|
|
|
11801
11963
|
|
|
11802
11964
|
// src/browser.ts
|
|
11803
11965
|
var Browser = class {
|
|
11804
|
-
constructor(
|
|
11805
|
-
this.views = views;
|
|
11966
|
+
constructor(kb, eventBus, project, config, limitsDiscovery, embeddingProvider, logger) {
|
|
11806
11967
|
this.kb = kb;
|
|
11807
11968
|
this.eventBus = eventBus;
|
|
11808
11969
|
this.project = project;
|
|
@@ -11811,7 +11972,6 @@ var Browser = class {
|
|
|
11811
11972
|
this.embeddingProvider = embeddingProvider;
|
|
11812
11973
|
this.logger = logger;
|
|
11813
11974
|
}
|
|
11814
|
-
views;
|
|
11815
11975
|
kb;
|
|
11816
11976
|
eventBus;
|
|
11817
11977
|
project;
|
|
@@ -11843,6 +12003,7 @@ var Browser = class {
|
|
|
11843
12003
|
this.subscriptions.push(
|
|
11844
12004
|
pipe("browse:resource-requested", (e) => this.handleBrowseResource(e)).subscribe({ error: errorHandler }),
|
|
11845
12005
|
pipe("browse:anchored-text-requested", (e) => this.handleAnchoredText(e)).subscribe({ error: errorHandler }),
|
|
12006
|
+
pipe("browse:anchored-text-by-checksum-requested", (e) => this.handleAnchoredTextByChecksum(e)).subscribe({ error: errorHandler }),
|
|
11846
12007
|
pipe("browse:resources-requested", (e) => this.handleBrowseResources(e)).subscribe({ error: errorHandler }),
|
|
11847
12008
|
pipe("browse:annotations-requested", (e) => this.handleBrowseAnnotations(e)).subscribe({ error: errorHandler }),
|
|
11848
12009
|
pipe("browse:annotation-requested", (e) => this.handleBrowseAnnotation(e)).subscribe({ error: errorHandler }),
|
|
@@ -11886,6 +12047,30 @@ var Browser = class {
|
|
|
11886
12047
|
});
|
|
11887
12048
|
}
|
|
11888
12049
|
}
|
|
12050
|
+
/**
|
|
12051
|
+
* Serve the checksum-addressed consult (ANCHORED-TEXT-TO-SMELTER P2, D2) —
|
|
12052
|
+
* the detection workers' read-through cache. The caller computed the key
|
|
12053
|
+
* from bytes it already holds, so unlike the resource-addressed read there
|
|
12054
|
+
* is nothing to resolve and no settle barrier: with no resourceId there is
|
|
12055
|
+
* no content generation to wait on, and a miss means "extract it yourself".
|
|
12056
|
+
* A hit — success or decline — is served whole, so the caller runs neither
|
|
12057
|
+
* parser nor engine. Read-only over the wire: the Smelter is the sole
|
|
12058
|
+
* writer and never answers here.
|
|
12059
|
+
*/
|
|
12060
|
+
async handleAnchoredTextByChecksum(event) {
|
|
12061
|
+
try {
|
|
12062
|
+
this.eventBus.get("browse:anchored-text-by-checksum-result").next({
|
|
12063
|
+
correlationId: event.correlationId,
|
|
12064
|
+
response: await this.kb.anchoredText.read(event.checksum)
|
|
12065
|
+
});
|
|
12066
|
+
} catch (error) {
|
|
12067
|
+
this.logger.error("Browse anchored text by checksum failed", { checksum: event.checksum, error: errField(error) });
|
|
12068
|
+
this.eventBus.get("browse:anchored-text-by-checksum-failed").next({
|
|
12069
|
+
correlationId: event.correlationId,
|
|
12070
|
+
message: error instanceof Error ? error.message : String(error)
|
|
12071
|
+
});
|
|
12072
|
+
}
|
|
12073
|
+
}
|
|
11889
12074
|
async handleBrowseResource(event) {
|
|
11890
12075
|
try {
|
|
11891
12076
|
const response = await assembleResourceGraph(this.kb, resourceId(event.resourceId));
|
|
@@ -12183,10 +12368,13 @@ var Browser = class {
|
|
|
12183
12368
|
return;
|
|
12184
12369
|
}
|
|
12185
12370
|
const visible = dirents.filter((d) => d.name !== ".semiont" && !d.name.startsWith("."));
|
|
12186
|
-
const allViews = await this.views.getAll();
|
|
12371
|
+
const allViews = await this.kb.views.getAll();
|
|
12187
12372
|
const prefix = `file://${resolved}`;
|
|
12188
12373
|
const viewsByUri = new Map(
|
|
12189
|
-
allViews.filter((v) =>
|
|
12374
|
+
allViews.filter((v) => {
|
|
12375
|
+
const uri = getStorageUri(v.resource);
|
|
12376
|
+
return uri?.startsWith(prefix + "/") || uri?.startsWith(prefix + path3.sep);
|
|
12377
|
+
}).map((v) => [getStorageUri(v.resource), v])
|
|
12190
12378
|
);
|
|
12191
12379
|
const entries = [];
|
|
12192
12380
|
for (const dirent of visible) {
|
|
@@ -12275,13 +12463,13 @@ function createLimitsDiscovery(config, logger, options) {
|
|
|
12275
12463
|
}
|
|
12276
12464
|
}
|
|
12277
12465
|
const consult = async (pair, client) => {
|
|
12278
|
-
let
|
|
12466
|
+
let timer;
|
|
12279
12467
|
try {
|
|
12280
12468
|
const discovery = client.limits();
|
|
12281
12469
|
const raced = await Promise.race([
|
|
12282
12470
|
discovery,
|
|
12283
12471
|
new Promise((resolve2) => {
|
|
12284
|
-
|
|
12472
|
+
timer = setTimeout(() => resolve2(void 0), budgetMs);
|
|
12285
12473
|
})
|
|
12286
12474
|
]);
|
|
12287
12475
|
if (raced === void 0) {
|
|
@@ -12298,7 +12486,7 @@ function createLimitsDiscovery(config, logger, options) {
|
|
|
12298
12486
|
});
|
|
12299
12487
|
return void 0;
|
|
12300
12488
|
} finally {
|
|
12301
|
-
clearTimeout(
|
|
12489
|
+
clearTimeout(timer);
|
|
12302
12490
|
}
|
|
12303
12491
|
};
|
|
12304
12492
|
return {
|
|
@@ -12334,6 +12522,15 @@ async function readAnnotationFromView(kb, resourceId10, annotationId5) {
|
|
|
12334
12522
|
const allAnnotations = await AnnotationContext.getAllAnnotations(resourceId10, kb);
|
|
12335
12523
|
return allAnnotations.find((a) => a.id === annotationId5) ?? null;
|
|
12336
12524
|
}
|
|
12525
|
+
function wireEnrichment(eventStore, kb) {
|
|
12526
|
+
eventStore.setEnrichEvent(async (event, resourceId10) => {
|
|
12527
|
+
const annId = eventAnnotationId(event);
|
|
12528
|
+
if (annId === null) return event;
|
|
12529
|
+
const annotation = await readAnnotationFromView(kb, resourceId10, annId);
|
|
12530
|
+
if (annotation === null) return event;
|
|
12531
|
+
return { ...event, annotation };
|
|
12532
|
+
});
|
|
12533
|
+
}
|
|
12337
12534
|
|
|
12338
12535
|
// src/clone-token-manager.ts
|
|
12339
12536
|
var import_rxjs6 = __toESM(require_cjs());
|
|
@@ -12385,16 +12582,67 @@ var ResourceOperations = class {
|
|
|
12385
12582
|
);
|
|
12386
12583
|
return resourceId(rId);
|
|
12387
12584
|
}
|
|
12585
|
+
/**
|
|
12586
|
+
* Persist a CLONE via EventBus → Stower.
|
|
12587
|
+
*
|
|
12588
|
+
* Separate from `createResource` because a clone is a separate fact: it
|
|
12589
|
+
* names its parent, and `yield:cloned` requires that name. Callers reach
|
|
12590
|
+
* this only after the CloneTokenManager has validated the token — it is the
|
|
12591
|
+
* inner half of the flow, not a public entry point.
|
|
12592
|
+
*/
|
|
12593
|
+
static async persistClone(input, userId3, eventBus) {
|
|
12594
|
+
const { resourceId: rId } = await busRequest(
|
|
12595
|
+
asBusRequestPrimitive(eventBus),
|
|
12596
|
+
"yield:clone-persist",
|
|
12597
|
+
{
|
|
12598
|
+
name: input.name,
|
|
12599
|
+
storageUri: input.storageUri,
|
|
12600
|
+
contentChecksum: input.contentChecksum,
|
|
12601
|
+
byteSize: input.byteSize,
|
|
12602
|
+
format: input.format,
|
|
12603
|
+
parentResourceId: input.parentResourceId,
|
|
12604
|
+
entityTypes: input.entityTypes,
|
|
12605
|
+
language: input.language,
|
|
12606
|
+
_userId: userId3
|
|
12607
|
+
},
|
|
12608
|
+
3e4
|
|
12609
|
+
);
|
|
12610
|
+
return resourceId(rId);
|
|
12611
|
+
}
|
|
12612
|
+
/**
|
|
12613
|
+
* Create a resource from a clone token via EventBus → CloneTokenManager.
|
|
12614
|
+
* The bytes are already stored (the gateway's upload path, `noGit` — the
|
|
12615
|
+
* Archivist's register does the one `git add`, D4b); the command carries
|
|
12616
|
+
* storage coordinates only (EXTRACT-ARCHIVIST P3, D4a).
|
|
12617
|
+
*/
|
|
12618
|
+
static async createFromCloneToken(input, userId3, eventBus) {
|
|
12619
|
+
const { resourceId: rId } = await busRequest(
|
|
12620
|
+
asBusRequestPrimitive(eventBus),
|
|
12621
|
+
"yield:clone-create",
|
|
12622
|
+
{
|
|
12623
|
+
token: input.token,
|
|
12624
|
+
name: input.name,
|
|
12625
|
+
storageUri: input.storageUri,
|
|
12626
|
+
contentChecksum: input.contentChecksum,
|
|
12627
|
+
byteSize: input.byteSize,
|
|
12628
|
+
format: input.format,
|
|
12629
|
+
archiveOriginal: input.archiveOriginal,
|
|
12630
|
+
_userId: userId3
|
|
12631
|
+
},
|
|
12632
|
+
3e4
|
|
12633
|
+
);
|
|
12634
|
+
return resourceId(rId);
|
|
12635
|
+
}
|
|
12388
12636
|
};
|
|
12389
12637
|
|
|
12390
12638
|
// src/clone-token-manager.ts
|
|
12391
12639
|
var CloneTokenManager = class {
|
|
12392
|
-
constructor(
|
|
12393
|
-
this.
|
|
12640
|
+
constructor(stores, eventBus, logger) {
|
|
12641
|
+
this.stores = stores;
|
|
12394
12642
|
this.eventBus = eventBus;
|
|
12395
12643
|
this.logger = logger;
|
|
12396
12644
|
}
|
|
12397
|
-
|
|
12645
|
+
stores;
|
|
12398
12646
|
eventBus;
|
|
12399
12647
|
subscriptions = [];
|
|
12400
12648
|
logger;
|
|
@@ -12419,7 +12667,7 @@ var CloneTokenManager = class {
|
|
|
12419
12667
|
}
|
|
12420
12668
|
async handleGenerateToken(event) {
|
|
12421
12669
|
try {
|
|
12422
|
-
const resource = await ResourceContext.getResourceMetadata(resourceId(event.resourceId), this.
|
|
12670
|
+
const resource = await ResourceContext.getResourceMetadata(resourceId(event.resourceId), this.stores);
|
|
12423
12671
|
if (!resource) {
|
|
12424
12672
|
this.eventBus.get("yield:clone-token-failed").next({
|
|
12425
12673
|
correlationId: event.correlationId,
|
|
@@ -12427,7 +12675,8 @@ var CloneTokenManager = class {
|
|
|
12427
12675
|
});
|
|
12428
12676
|
return;
|
|
12429
12677
|
}
|
|
12430
|
-
|
|
12678
|
+
const storageUri = getStorageUri(resource);
|
|
12679
|
+
if (!storageUri) {
|
|
12431
12680
|
this.eventBus.get("yield:clone-token-failed").next({
|
|
12432
12681
|
correlationId: event.correlationId,
|
|
12433
12682
|
message: "Resource content not found"
|
|
@@ -12435,7 +12684,7 @@ var CloneTokenManager = class {
|
|
|
12435
12684
|
return;
|
|
12436
12685
|
}
|
|
12437
12686
|
try {
|
|
12438
|
-
await this.
|
|
12687
|
+
await promises.access(this.stores.content.resolveUri(storageUri));
|
|
12439
12688
|
} catch {
|
|
12440
12689
|
this.eventBus.get("yield:clone-token-failed").next({
|
|
12441
12690
|
correlationId: event.correlationId,
|
|
@@ -12482,7 +12731,7 @@ var CloneTokenManager = class {
|
|
|
12482
12731
|
});
|
|
12483
12732
|
return;
|
|
12484
12733
|
}
|
|
12485
|
-
const sourceResource = await ResourceContext.getResourceMetadata(tokenData.resourceId, this.
|
|
12734
|
+
const sourceResource = await ResourceContext.getResourceMetadata(tokenData.resourceId, this.stores);
|
|
12486
12735
|
if (!sourceResource) {
|
|
12487
12736
|
this.eventBus.get("yield:clone-resource-failed").next({
|
|
12488
12737
|
correlationId: event.correlationId,
|
|
@@ -12531,7 +12780,7 @@ var CloneTokenManager = class {
|
|
|
12531
12780
|
});
|
|
12532
12781
|
return;
|
|
12533
12782
|
}
|
|
12534
|
-
const sourceDoc = await ResourceContext.getResourceMetadata(tokenData.resourceId, this.
|
|
12783
|
+
const sourceDoc = await ResourceContext.getResourceMetadata(tokenData.resourceId, this.stores);
|
|
12535
12784
|
if (!sourceDoc) {
|
|
12536
12785
|
this.eventBus.get("yield:clone-create-failed").next({
|
|
12537
12786
|
correlationId: event.correlationId,
|
|
@@ -12539,17 +12788,14 @@ var CloneTokenManager = class {
|
|
|
12539
12788
|
});
|
|
12540
12789
|
return;
|
|
12541
12790
|
}
|
|
12542
|
-
const
|
|
12543
|
-
const format = isSupportedMediaType(base) && capabilitiesOf(base)?.authorable ? base : "text/plain";
|
|
12544
|
-
const resolvedUri = deriveStorageUri(event.name, format);
|
|
12545
|
-
const stored = await this.kb.content.store(Buffer.from(event.content), resolvedUri);
|
|
12546
|
-
const newResourceId = await ResourceOperations.createResource(
|
|
12791
|
+
const newResourceId = await ResourceOperations.persistClone(
|
|
12547
12792
|
{
|
|
12548
12793
|
name: event.name,
|
|
12549
|
-
storageUri:
|
|
12550
|
-
contentChecksum:
|
|
12551
|
-
byteSize:
|
|
12552
|
-
format,
|
|
12794
|
+
storageUri: event.storageUri,
|
|
12795
|
+
contentChecksum: event.contentChecksum,
|
|
12796
|
+
byteSize: event.byteSize,
|
|
12797
|
+
format: event.format,
|
|
12798
|
+
parentResourceId: String(tokenData.resourceId),
|
|
12553
12799
|
entityTypes: getResourceEntityTypes(sourceDoc)
|
|
12554
12800
|
},
|
|
12555
12801
|
userId(event._userId),
|
|
@@ -12617,42 +12863,115 @@ async function stopKnowledgeSystem(ks) {
|
|
|
12617
12863
|
ks.kb.weaveProgress.dispose();
|
|
12618
12864
|
await ks.kb.graph.disconnect();
|
|
12619
12865
|
}
|
|
12620
|
-
function
|
|
12866
|
+
async function assertAnnotatableTarget(kb, target) {
|
|
12867
|
+
const view = await kb.views.get(resourceId(target));
|
|
12868
|
+
const mediaType = getPrimaryRepresentation(view?.resource)?.mediaType;
|
|
12869
|
+
if (!mediaType || !isAnnotatable(mediaType)) {
|
|
12870
|
+
throw new Error(`"${mediaType ?? "unknown"}" cannot be annotated`);
|
|
12871
|
+
}
|
|
12872
|
+
}
|
|
12873
|
+
var AnnotationOperations = class {
|
|
12874
|
+
/**
|
|
12875
|
+
* Create a new annotation via EventBus → Stower
|
|
12876
|
+
*/
|
|
12877
|
+
static async createAnnotation(request, userId3, creator, eventBus, kb) {
|
|
12878
|
+
await assertAnnotatableTarget(kb, request.target.source);
|
|
12879
|
+
const { annotation } = assembleAnnotation(request, creator);
|
|
12880
|
+
const resId = resourceId(request.target.source);
|
|
12881
|
+
eventBus.get("mark:create").next({
|
|
12882
|
+
annotation,
|
|
12883
|
+
_userId: userId3,
|
|
12884
|
+
resourceId: resId
|
|
12885
|
+
});
|
|
12886
|
+
return { annotation };
|
|
12887
|
+
}
|
|
12888
|
+
/**
|
|
12889
|
+
* Update annotation body via EventBus → Stower
|
|
12890
|
+
*/
|
|
12891
|
+
static async updateAnnotationBody(id, request, userId3, eventBus, kb) {
|
|
12892
|
+
const resId = resourceId(request.resourceId);
|
|
12893
|
+
const annotation = await AnnotationContext.getAnnotation(
|
|
12894
|
+
annotationId(id),
|
|
12895
|
+
resId,
|
|
12896
|
+
kb
|
|
12897
|
+
);
|
|
12898
|
+
if (!annotation) {
|
|
12899
|
+
throw new Error("Annotation not found");
|
|
12900
|
+
}
|
|
12901
|
+
eventBus.get("mark:update-body").next({
|
|
12902
|
+
annotationId: annotationId(id),
|
|
12903
|
+
_userId: userId3,
|
|
12904
|
+
resourceId: resId,
|
|
12905
|
+
operations: request.operations
|
|
12906
|
+
});
|
|
12907
|
+
const updatedBody = applyBodyOperations(annotation.body, request.operations);
|
|
12908
|
+
return {
|
|
12909
|
+
annotation: {
|
|
12910
|
+
...annotation,
|
|
12911
|
+
body: updatedBody
|
|
12912
|
+
}
|
|
12913
|
+
};
|
|
12914
|
+
}
|
|
12915
|
+
/**
|
|
12916
|
+
* Delete an annotation via EventBus → Stower
|
|
12917
|
+
*/
|
|
12918
|
+
static async deleteAnnotation(id, resourceIdStr, userId3, eventBus, kb, logger) {
|
|
12919
|
+
const resId = resourceId(resourceIdStr);
|
|
12920
|
+
const projection = await AnnotationContext.getResourceAnnotations(resId, kb);
|
|
12921
|
+
const annotation = projection.annotations.find((a) => a.id === id);
|
|
12922
|
+
if (!annotation) {
|
|
12923
|
+
throw new Error("Annotation not found in resource");
|
|
12924
|
+
}
|
|
12925
|
+
logger?.debug("Removing annotation via EventBus", { annotationId: id });
|
|
12926
|
+
eventBus.get("mark:delete").next({
|
|
12927
|
+
annotationId: annotationId(id),
|
|
12928
|
+
_userId: userId3,
|
|
12929
|
+
resourceId: resId
|
|
12930
|
+
});
|
|
12931
|
+
logger?.debug("Annotation delete event emitted");
|
|
12932
|
+
}
|
|
12933
|
+
};
|
|
12934
|
+
|
|
12935
|
+
// src/handlers/annotation-assembly.ts
|
|
12936
|
+
function registerAnnotationAssemblyHandler(eventBus, kb, parentLogger) {
|
|
12621
12937
|
const logger = parentLogger.child({ component: "annotation-assembly" });
|
|
12622
12938
|
const inflight = /* @__PURE__ */ new Map();
|
|
12623
12939
|
eventBus.get("mark:create-request").subscribe((command) => {
|
|
12624
|
-
|
|
12625
|
-
|
|
12626
|
-
|
|
12627
|
-
|
|
12628
|
-
|
|
12629
|
-
|
|
12630
|
-
|
|
12631
|
-
|
|
12940
|
+
void (async () => {
|
|
12941
|
+
const { correlationId, resourceId: resId, request, _userId } = command;
|
|
12942
|
+
const cid = correlationId;
|
|
12943
|
+
try {
|
|
12944
|
+
if (!_userId || typeof _userId !== "string") {
|
|
12945
|
+
throw new Error("_userId is required (injected by bus gateway)");
|
|
12946
|
+
}
|
|
12947
|
+
if (!cid) {
|
|
12948
|
+
throw new Error("correlationId is required on mark:create-request");
|
|
12949
|
+
}
|
|
12950
|
+
await assertAnnotatableTarget(kb, resId);
|
|
12951
|
+
const agent = didToAgent(_userId);
|
|
12952
|
+
const { annotation } = assembleAnnotation(request, agent);
|
|
12953
|
+
inflight.set(cid, { annotationId: annotation.id });
|
|
12954
|
+
eventBus.get("mark:create").next({
|
|
12955
|
+
correlationId: cid,
|
|
12956
|
+
annotation,
|
|
12957
|
+
_userId,
|
|
12958
|
+
resourceId: resourceId(resId)
|
|
12959
|
+
});
|
|
12960
|
+
logger.info("Annotation assembled, awaiting persistence", {
|
|
12961
|
+
annotationId: annotation.id,
|
|
12962
|
+
correlationId: cid
|
|
12963
|
+
});
|
|
12964
|
+
} catch (error) {
|
|
12965
|
+
logger.warn("mark:create-request failed during assembly", {
|
|
12966
|
+
correlationId: cid,
|
|
12967
|
+
error: error.message
|
|
12968
|
+
});
|
|
12969
|
+
eventBus.get("mark:create-failed").next({
|
|
12970
|
+
correlationId: cid,
|
|
12971
|
+
message: error.message
|
|
12972
|
+
});
|
|
12632
12973
|
}
|
|
12633
|
-
|
|
12634
|
-
const { annotation } = assembleAnnotation(request, agent);
|
|
12635
|
-
inflight.set(cid, { annotationId: annotation.id });
|
|
12636
|
-
eventBus.get("mark:create").next({
|
|
12637
|
-
correlationId: cid,
|
|
12638
|
-
annotation,
|
|
12639
|
-
_userId,
|
|
12640
|
-
resourceId: resourceId(resId)
|
|
12641
|
-
});
|
|
12642
|
-
logger.info("Annotation assembled, awaiting persistence", {
|
|
12643
|
-
annotationId: annotation.id,
|
|
12644
|
-
correlationId: cid
|
|
12645
|
-
});
|
|
12646
|
-
} catch (error) {
|
|
12647
|
-
logger.warn("mark:create-request failed during assembly", {
|
|
12648
|
-
correlationId: cid,
|
|
12649
|
-
error: error.message
|
|
12650
|
-
});
|
|
12651
|
-
eventBus.get("mark:create-failed").next({
|
|
12652
|
-
correlationId: cid,
|
|
12653
|
-
message: error.message
|
|
12654
|
-
});
|
|
12655
|
-
}
|
|
12974
|
+
})();
|
|
12656
12975
|
});
|
|
12657
12976
|
eventBus.get("mark:added").subscribe((event) => {
|
|
12658
12977
|
const cid = event.metadata?.correlationId;
|
|
@@ -12672,7 +12991,7 @@ function registerAnnotationAssemblyHandler(eventBus, parentLogger) {
|
|
|
12672
12991
|
inflight.delete(cid);
|
|
12673
12992
|
});
|
|
12674
12993
|
}
|
|
12675
|
-
function
|
|
12994
|
+
function registerAnnotationContextHandler(eventBus, kb, parentLogger) {
|
|
12676
12995
|
const logger = parentLogger.child({ component: "annotation-lookups" });
|
|
12677
12996
|
eventBus.get("browse:annotation-context-requested").subscribe(async (command) => {
|
|
12678
12997
|
const { correlationId } = command;
|
|
@@ -12700,6 +13019,9 @@ function registerAnnotationLookupHandlers(eventBus, kb, gatherer, parentLogger)
|
|
|
12700
13019
|
});
|
|
12701
13020
|
}
|
|
12702
13021
|
});
|
|
13022
|
+
}
|
|
13023
|
+
function registerGatherSummaryHandler(eventBus, gatherer, parentLogger) {
|
|
13024
|
+
const logger = parentLogger.child({ component: "annotation-lookups" });
|
|
12703
13025
|
eventBus.get("gather:summary-requested").subscribe(async (command) => {
|
|
12704
13026
|
const { correlationId } = command;
|
|
12705
13027
|
const annId = command.annotationId;
|
|
@@ -12785,7 +13107,7 @@ function parseDidUser(did) {
|
|
|
12785
13107
|
const email = decodeURIComponent(parts.slice(usersIdx + 1).join(":"));
|
|
12786
13108
|
return { userId: did, email, domain };
|
|
12787
13109
|
}
|
|
12788
|
-
function registerJobCommandHandlers(eventBus, jobQueue,
|
|
13110
|
+
function registerJobCommandHandlers(eventBus, jobQueue, state, parentLogger) {
|
|
12789
13111
|
const logger = parentLogger.child({ component: "job-commands" });
|
|
12790
13112
|
eventBus.get("job:create").subscribe(async (command) => {
|
|
12791
13113
|
const { correlationId, jobType, resourceId: resId, params, _userId } = command;
|
|
@@ -12850,7 +13172,7 @@ function registerJobCommandHandlers(eventBus, jobQueue, project, parentLogger) {
|
|
|
12850
13172
|
};
|
|
12851
13173
|
const jobParams = job.params;
|
|
12852
13174
|
if ((jobType === "reference-annotation" || jobType === "generation") && Array.isArray(jobParams.entityTypes) && jobParams.entityTypes.length > 0) {
|
|
12853
|
-
const registered = await readEntityTypesProjection(
|
|
13175
|
+
const registered = await readEntityTypesProjection(state);
|
|
12854
13176
|
const result = validateEntityTypes(registered, jobParams.entityTypes);
|
|
12855
13177
|
if (!result.ok) {
|
|
12856
13178
|
throw new Error(entityTypesNotRegisteredMessage(result.unknown));
|
|
@@ -12860,7 +13182,7 @@ function registerJobCommandHandlers(eventBus, jobQueue, project, parentLogger) {
|
|
|
12860
13182
|
jobParams.entityTypes = jobParams.entityTypes.map((et) => entityType(et));
|
|
12861
13183
|
}
|
|
12862
13184
|
if (jobType === "tag-annotation") {
|
|
12863
|
-
const schemas = await readTagSchemasProjection(
|
|
13185
|
+
const schemas = await readTagSchemasProjection(state);
|
|
12864
13186
|
const result = resolveTagSchema(schemas, jobParams.schemaId);
|
|
12865
13187
|
if (result.error !== void 0) {
|
|
12866
13188
|
throw new Error(result.error);
|
|
@@ -12928,7 +13250,7 @@ function registerJobCommandHandlers(eventBus, jobQueue, project, parentLogger) {
|
|
|
12928
13250
|
});
|
|
12929
13251
|
eventBus.get("job:fail").subscribe(async (event) => {
|
|
12930
13252
|
try {
|
|
12931
|
-
const outcome = await jobQueue.failJob(jobId(event.jobId), event.error);
|
|
13253
|
+
const outcome = await jobQueue.failJob(jobId(event.jobId), event.error, event.completedUnits, event.failureClass);
|
|
12932
13254
|
if (outcome === "retried") {
|
|
12933
13255
|
logger.info("Job re-queued for retry", { jobId: event.jobId });
|
|
12934
13256
|
} else if (outcome === null) {
|
|
@@ -12954,16 +13276,45 @@ function registerJobCommandHandlers(eventBus, jobQueue, project, parentLogger) {
|
|
|
12954
13276
|
});
|
|
12955
13277
|
}
|
|
12956
13278
|
});
|
|
13279
|
+
eventBus.get("job:checkpoint").subscribe(async (event) => {
|
|
13280
|
+
try {
|
|
13281
|
+
await jobQueue.checkpointUnits(jobId(event.jobId), event.completedUnits);
|
|
13282
|
+
} catch (error) {
|
|
13283
|
+
logger.error("Failed to checkpoint job units", {
|
|
13284
|
+
jobId: event.jobId,
|
|
13285
|
+
error: error.message
|
|
13286
|
+
});
|
|
13287
|
+
}
|
|
13288
|
+
});
|
|
12957
13289
|
eventBus.get("job:cancel-requested").subscribe(async (event) => {
|
|
12958
13290
|
try {
|
|
12959
|
-
|
|
12960
|
-
|
|
13291
|
+
let cancelled;
|
|
13292
|
+
if (event.jobId) {
|
|
13293
|
+
const target = await jobQueue.getJob(jobId(event.jobId));
|
|
13294
|
+
if (!target) {
|
|
13295
|
+
cancelled = 0;
|
|
13296
|
+
} else if (target.status === "pending") {
|
|
13297
|
+
cancelled = await jobQueue.cancelJob(jobId(event.jobId)) ? 1 : 0;
|
|
13298
|
+
} else if (target.status === "running") {
|
|
13299
|
+
logger.info("Cancel of running job delegated to its worker", { jobId: event.jobId });
|
|
13300
|
+
cancelled = 1;
|
|
13301
|
+
} else {
|
|
13302
|
+
cancelled = 0;
|
|
13303
|
+
}
|
|
13304
|
+
logger.info("Cancel requested", { jobId: event.jobId, cancelled });
|
|
13305
|
+
} else if (event.jobType) {
|
|
13306
|
+
cancelled = await jobQueue.cancelPendingJobs(event.jobType);
|
|
13307
|
+
logger.info("Cancel requested", { jobType: event.jobType, cancelled });
|
|
13308
|
+
} else {
|
|
13309
|
+
cancelled = 0;
|
|
13310
|
+
}
|
|
12961
13311
|
eventBus.get("job:cancel-ok").next({
|
|
12962
13312
|
correlationId: event.correlationId,
|
|
12963
13313
|
response: { cancelled }
|
|
12964
13314
|
});
|
|
12965
13315
|
} catch (error) {
|
|
12966
|
-
logger.error("Failed to cancel
|
|
13316
|
+
logger.error("Failed to cancel jobs", {
|
|
13317
|
+
jobId: event.jobId,
|
|
12967
13318
|
jobType: event.jobType,
|
|
12968
13319
|
error: error.message
|
|
12969
13320
|
});
|
|
@@ -12973,20 +13324,41 @@ function registerJobCommandHandlers(eventBus, jobQueue, project, parentLogger) {
|
|
|
12973
13324
|
});
|
|
12974
13325
|
}
|
|
12975
13326
|
});
|
|
13327
|
+
eventBus.get("job:cancel").subscribe(async (event) => {
|
|
13328
|
+
try {
|
|
13329
|
+
await jobQueue.cancelJob(jobId(event.jobId));
|
|
13330
|
+
logger.info("Job cancelled by its worker", { jobId: event.jobId });
|
|
13331
|
+
} catch (error) {
|
|
13332
|
+
logger.error("Failed to cancel job", {
|
|
13333
|
+
jobId: event.jobId,
|
|
13334
|
+
error: error.message
|
|
13335
|
+
});
|
|
13336
|
+
}
|
|
13337
|
+
});
|
|
13338
|
+
}
|
|
13339
|
+
|
|
13340
|
+
// src/handlers/index.ts
|
|
13341
|
+
function registerBusHandlers(eventBus, knowledgeSystem, jobQueue, state, logger) {
|
|
13342
|
+
const { kb } = knowledgeSystem;
|
|
13343
|
+
registerAnnotationAssemblyHandler(eventBus, kb, logger);
|
|
13344
|
+
registerAnnotationContextHandler(
|
|
13345
|
+
eventBus,
|
|
13346
|
+
{ views: kb.views, content: workingTreeContentReads(kb.views, kb.content) },
|
|
13347
|
+
logger
|
|
13348
|
+
);
|
|
13349
|
+
registerGatherSummaryHandler(eventBus, knowledgeSystem.gatherer, logger);
|
|
13350
|
+
registerBindUpdateBodyHandler(eventBus, logger);
|
|
13351
|
+
registerJobCommandHandlers(eventBus, jobQueue, state, logger);
|
|
13352
|
+
}
|
|
13353
|
+
function registerGatewayBusHandlers(eventBus, jobQueue, state, logger) {
|
|
13354
|
+
registerBindUpdateBodyHandler(eventBus, logger);
|
|
13355
|
+
registerJobCommandHandlers(eventBus, jobQueue, state, logger);
|
|
12976
13356
|
}
|
|
12977
|
-
|
|
12978
|
-
// src/handlers/index.ts
|
|
12979
|
-
function registerBusHandlers(eventBus, knowledgeSystem, jobQueue, project, logger) {
|
|
12980
|
-
registerAnnotationAssemblyHandler(eventBus, logger);
|
|
12981
|
-
registerAnnotationLookupHandlers(eventBus, knowledgeSystem.kb, knowledgeSystem.gatherer, logger);
|
|
12982
|
-
registerBindUpdateBodyHandler(eventBus, logger);
|
|
12983
|
-
registerJobCommandHandlers(eventBus, jobQueue, project, logger);
|
|
12984
|
-
}
|
|
12985
13357
|
|
|
12986
13358
|
// src/service.ts
|
|
12987
|
-
async function createJobQueue(
|
|
13359
|
+
async function createJobQueue(state, eventBus, logger) {
|
|
12988
13360
|
const jobQueueLogger = logger.child({ component: "job-queue" });
|
|
12989
|
-
const jobQueue = new FsJobQueue(
|
|
13361
|
+
const jobQueue = new FsJobQueue(state, jobQueueLogger, eventBus);
|
|
12990
13362
|
await jobQueue.initialize();
|
|
12991
13363
|
registerJobQueueProvider(() => jobQueue.getStats());
|
|
12992
13364
|
const jobStatusSubscription = eventBus.get("job:status-requested").pipe(
|
|
@@ -13026,12 +13398,12 @@ async function createJobQueue(project, eventBus, logger) {
|
|
|
13026
13398
|
}
|
|
13027
13399
|
var STARTUP_CONNECT_TIMEOUT_MS = 6e4;
|
|
13028
13400
|
async function withStartupTimeout(what, work) {
|
|
13029
|
-
let
|
|
13401
|
+
let timer;
|
|
13030
13402
|
try {
|
|
13031
13403
|
return await Promise.race([
|
|
13032
13404
|
work,
|
|
13033
13405
|
new Promise((_resolve, reject) => {
|
|
13034
|
-
|
|
13406
|
+
timer = setTimeout(
|
|
13035
13407
|
() => reject(
|
|
13036
13408
|
new Error(
|
|
13037
13409
|
`${what} did not become available within ${STARTUP_CONNECT_TIMEOUT_MS / 1e3}s. Exiting so the container restart policy can retry \u2014 it is normal for a dependency to be slow when every service restarts at once.`
|
|
@@ -13042,10 +13414,10 @@ async function withStartupTimeout(what, work) {
|
|
|
13042
13414
|
})
|
|
13043
13415
|
]);
|
|
13044
13416
|
} finally {
|
|
13045
|
-
if (
|
|
13417
|
+
if (timer !== void 0) clearTimeout(timer);
|
|
13046
13418
|
}
|
|
13047
13419
|
}
|
|
13048
|
-
async function
|
|
13420
|
+
async function connectStores(project, config, eventBus, logger, skipRebuild) {
|
|
13049
13421
|
const graphConfig = config.services.graph;
|
|
13050
13422
|
logger.info("Connecting to graph database", { type: graphConfig.type });
|
|
13051
13423
|
const graphDb = await withStartupTimeout("Graph database", getGraphDatabase(graphConfig));
|
|
@@ -13088,18 +13460,18 @@ async function createKnowledgeSystemFromConfig(project, config, eventBus, logger
|
|
|
13088
13460
|
vectorStore,
|
|
13089
13461
|
skipRebuild
|
|
13090
13462
|
});
|
|
13091
|
-
|
|
13092
|
-
|
|
13093
|
-
|
|
13094
|
-
|
|
13095
|
-
|
|
13096
|
-
return { ...event, annotation };
|
|
13097
|
-
});
|
|
13463
|
+
return { kb, eventStore, embeddingProvider };
|
|
13464
|
+
}
|
|
13465
|
+
async function createKnowledgeSystemFromConfig(project, config, eventBus, logger, skipRebuild) {
|
|
13466
|
+
const { kb, eventStore, embeddingProvider } = await connectStores(project, config, eventBus, logger, skipRebuild ?? false);
|
|
13467
|
+
wireEnrichment(eventStore, kb);
|
|
13098
13468
|
const stower = new Stower(kb, eventBus, project, logger.child({ component: "stower" }));
|
|
13099
13469
|
await stower.initialize();
|
|
13100
13470
|
await bootstrapEntityTypes(eventBus, eventStore, logger.child({ component: "entity-types-bootstrap" }));
|
|
13101
13471
|
const gatherer = new Gatherer(
|
|
13102
|
-
|
|
13472
|
+
// The content capability is ResourceId-keyed (D-CONTENT b); in-process
|
|
13473
|
+
// it wraps this root's own working tree behind the transport shape.
|
|
13474
|
+
{ ...kb, content: workingTreeContentReads(kb.views, kb.content) },
|
|
13103
13475
|
eventBus,
|
|
13104
13476
|
createInferenceClient(resolveActorInference(config, "gatherer"), logger.child({ component: "inference-client-gatherer" })),
|
|
13105
13477
|
config.gather.settleTimeoutMs,
|
|
@@ -13116,14 +13488,14 @@ async function createKnowledgeSystemFromConfig(project, config, eventBus, logger
|
|
|
13116
13488
|
);
|
|
13117
13489
|
await matcher.initialize();
|
|
13118
13490
|
const limitsDiscovery = createLimitsDiscovery(config, logger.child({ component: "limits-discovery" }));
|
|
13119
|
-
const browser = new Browser(kb
|
|
13491
|
+
const browser = new Browser(kb, eventBus, project, config, limitsDiscovery, embeddingProvider, logger.child({ component: "browser" }));
|
|
13120
13492
|
await browser.initialize();
|
|
13121
13493
|
const cloneTokenManager = new CloneTokenManager(kb, eventBus, logger.child({ component: "clone-token-manager" }));
|
|
13122
13494
|
await cloneTokenManager.initialize();
|
|
13123
13495
|
const ks = { kb, stower, gatherer, matcher, browser, cloneTokenManager, stop: () => stopKnowledgeSystem(ks) };
|
|
13124
13496
|
return ks;
|
|
13125
13497
|
}
|
|
13126
|
-
|
|
13498
|
+
function assertMakeMeaningConfig(config) {
|
|
13127
13499
|
if (!config.services?.graph) {
|
|
13128
13500
|
throw new Error("services.graph is required for make-meaning service");
|
|
13129
13501
|
}
|
|
@@ -13135,6 +13507,9 @@ async function startMakeMeaning(project, config, eventBus, logger, options) {
|
|
|
13135
13507
|
`gather.settleTimeoutMs (${config.gather.settleTimeoutMs}ms) plus the graph barrier budget (${GRAPH_BARRIER_BUDGET_MS}ms) must nest inside the job-worker stall watchdog (${STALL_THRESHOLD_MS}ms) \u2014 lower settleTimeoutMs (A4)`
|
|
13136
13508
|
);
|
|
13137
13509
|
}
|
|
13510
|
+
}
|
|
13511
|
+
async function startMakeMeaning(project, config, eventBus, logger, options) {
|
|
13512
|
+
assertMakeMeaningConfig(config);
|
|
13138
13513
|
const skipRebuild = options?.skipRebuild ?? process.env.SEMIONT_SKIP_REBUILD === "true";
|
|
13139
13514
|
const { jobQueue, jobStatusSubscription } = await createJobQueue(project, eventBus, logger);
|
|
13140
13515
|
const knowledgeSystem = await createKnowledgeSystemFromConfig(project, config, eventBus, logger, skipRebuild);
|
|
@@ -13151,6 +13526,106 @@ async function startMakeMeaning(project, config, eventBus, logger, options) {
|
|
|
13151
13526
|
}
|
|
13152
13527
|
};
|
|
13153
13528
|
}
|
|
13529
|
+
async function startMakeMeaningGateway(state, config, eventBus, logger) {
|
|
13530
|
+
assertMakeMeaningConfig(config);
|
|
13531
|
+
const { jobQueue, jobStatusSubscription } = await createJobQueue(state, eventBus, logger);
|
|
13532
|
+
registerGatewayBusHandlers(eventBus, jobQueue, state, logger);
|
|
13533
|
+
return {
|
|
13534
|
+
jobQueue,
|
|
13535
|
+
state,
|
|
13536
|
+
stop: async () => {
|
|
13537
|
+
logger.info("Stopping gateway make-meaning");
|
|
13538
|
+
jobStatusSubscription.unsubscribe();
|
|
13539
|
+
logger.info("Gateway make-meaning stopped");
|
|
13540
|
+
}
|
|
13541
|
+
};
|
|
13542
|
+
}
|
|
13543
|
+
var json = (res, status, body) => {
|
|
13544
|
+
res.writeHead(status, { "Content-Type": "application/json" });
|
|
13545
|
+
res.end(JSON.stringify(body));
|
|
13546
|
+
};
|
|
13547
|
+
function createArchivistServer(deps) {
|
|
13548
|
+
const { events, content, views, workerSecret, health, branch, logger } = deps;
|
|
13549
|
+
const authorized = (req, res) => {
|
|
13550
|
+
if (!workerSecret) {
|
|
13551
|
+
json(res, 503, { error: "disabled: no SEMIONT_WORKER_SECRET configured" });
|
|
13552
|
+
return false;
|
|
13553
|
+
}
|
|
13554
|
+
if (req.headers.authorization !== `Bearer ${workerSecret}`) {
|
|
13555
|
+
json(res, 401, { error: "unauthorized" });
|
|
13556
|
+
return false;
|
|
13557
|
+
}
|
|
13558
|
+
return true;
|
|
13559
|
+
};
|
|
13560
|
+
return createServer((req, res) => {
|
|
13561
|
+
const url = new URL(req.url ?? "/", "http://archivist");
|
|
13562
|
+
if (req.method === "GET" && url.pathname === "/health") {
|
|
13563
|
+
json(res, 200, health());
|
|
13564
|
+
return;
|
|
13565
|
+
}
|
|
13566
|
+
if (req.method === "GET" && url.pathname === "/kb/branch") {
|
|
13567
|
+
if (!authorized(req, res)) return;
|
|
13568
|
+
json(res, 200, { branch: branch() });
|
|
13569
|
+
return;
|
|
13570
|
+
}
|
|
13571
|
+
if (req.method === "GET" && url.pathname.startsWith("/events/")) {
|
|
13572
|
+
if (!authorized(req, res)) return;
|
|
13573
|
+
const rawId = decodeURIComponent(url.pathname.slice("/events/".length));
|
|
13574
|
+
const rawFrom = url.searchParams.get("fromSequence");
|
|
13575
|
+
const fromSequence = rawFrom === null ? NaN : Number(rawFrom);
|
|
13576
|
+
if (!rawId || !Number.isInteger(fromSequence) || fromSequence < 1) {
|
|
13577
|
+
json(res, 400, { error: "resourceId path segment and integer fromSequence >= 1 are required" });
|
|
13578
|
+
return;
|
|
13579
|
+
}
|
|
13580
|
+
events.queryEvents(resourceId(rawId), { fromSequence }).then((replay) => json(res, 200, { events: replay })).catch((error) => {
|
|
13581
|
+
logger.error("D1 read path failed", { resourceId: rawId, fromSequence, error: errField(error) });
|
|
13582
|
+
json(res, 500, { error: "event read failed" });
|
|
13583
|
+
});
|
|
13584
|
+
return;
|
|
13585
|
+
}
|
|
13586
|
+
const contentMatch = req.method === "GET" && /^\/resources\/(.+)\/content$/.exec(url.pathname);
|
|
13587
|
+
if (contentMatch) {
|
|
13588
|
+
if (!authorized(req, res)) return;
|
|
13589
|
+
const rid = decodeURIComponent(contentMatch[1]);
|
|
13590
|
+
resolveRepresentation({ views, content }, resourceId(rid)).then(async ({ stream, mediaType }) => {
|
|
13591
|
+
res.writeHead(200, { "Content-Type": mediaType });
|
|
13592
|
+
await pipeline(stream, res);
|
|
13593
|
+
}).catch((error) => {
|
|
13594
|
+
if (error instanceof RepresentationMissing) {
|
|
13595
|
+
json(res, 404, { error: error.message, reason: error.reason });
|
|
13596
|
+
return;
|
|
13597
|
+
}
|
|
13598
|
+
logger.error("Content read failed", { resourceId: rid, error: errField(error) });
|
|
13599
|
+
if (res.headersSent) res.destroy();
|
|
13600
|
+
else json(res, 500, { error: "content read failed" });
|
|
13601
|
+
});
|
|
13602
|
+
return;
|
|
13603
|
+
}
|
|
13604
|
+
if (req.method === "PUT" && url.pathname.startsWith("/content/")) {
|
|
13605
|
+
if (!authorized(req, res)) return;
|
|
13606
|
+
const storageUri = decodeURIComponent(url.pathname.slice("/content/".length));
|
|
13607
|
+
if (!storageUri) {
|
|
13608
|
+
json(res, 400, { error: "storageUri path segment is required" });
|
|
13609
|
+
return;
|
|
13610
|
+
}
|
|
13611
|
+
const expectedChecksum = url.searchParams.get("checksum");
|
|
13612
|
+
content.store(req, storageUri, {
|
|
13613
|
+
noGit: true,
|
|
13614
|
+
...expectedChecksum !== null ? { expectedChecksum } : {}
|
|
13615
|
+
}).then((stored) => json(res, 200, stored)).catch((error) => {
|
|
13616
|
+
if (error instanceof ChecksumMismatchError) {
|
|
13617
|
+
json(res, 409, { error: "checksum mismatch: body does not match the checksum the caller supplied" });
|
|
13618
|
+
return;
|
|
13619
|
+
}
|
|
13620
|
+
logger.error("Content write failed", { storageUri, error: errField(error) });
|
|
13621
|
+
json(res, 500, { error: "content write failed" });
|
|
13622
|
+
});
|
|
13623
|
+
return;
|
|
13624
|
+
}
|
|
13625
|
+
res.writeHead(404);
|
|
13626
|
+
res.end();
|
|
13627
|
+
});
|
|
13628
|
+
}
|
|
13154
13629
|
|
|
13155
13630
|
// src/local-transport.ts
|
|
13156
13631
|
var import_rxjs8 = __toESM(require_cjs());
|
|
@@ -13232,7 +13707,7 @@ var LocalTransport = class {
|
|
|
13232
13707
|
}
|
|
13233
13708
|
}
|
|
13234
13709
|
// LocalTransport implements `ITransport` only. It does not implement
|
|
13235
|
-
// `
|
|
13710
|
+
// `IGatewayOperations` — a SemiontClient built over LocalTransport has
|
|
13236
13711
|
// no `client.auth` / `client.admin` namespaces by design (no
|
|
13237
13712
|
// credentials, no admin routes, no exchange machinery in-process).
|
|
13238
13713
|
// ── Lifecycle ───────────────────────────────────────────────────────────
|
|
@@ -13248,10 +13723,10 @@ var LocalTransport = class {
|
|
|
13248
13723
|
}
|
|
13249
13724
|
};
|
|
13250
13725
|
var LocalContentTransport = class {
|
|
13251
|
-
constructor(
|
|
13252
|
-
this.
|
|
13726
|
+
constructor(kb) {
|
|
13727
|
+
this.kb = kb;
|
|
13253
13728
|
}
|
|
13254
|
-
|
|
13729
|
+
kb;
|
|
13255
13730
|
async putBinary(_request, _options) {
|
|
13256
13731
|
throw new Error(
|
|
13257
13732
|
"LocalContentTransport does not support putBinary() \u2014 create resources via bus emits (mark/yield namespaces) in local mode"
|
|
@@ -13266,12 +13741,12 @@ var LocalContentTransport = class {
|
|
|
13266
13741
|
*/
|
|
13267
13742
|
async putAnchoredText(checksum, outcome, _options) {
|
|
13268
13743
|
busLog("PUT", "anchored-text", { checksum });
|
|
13269
|
-
await this.
|
|
13744
|
+
await this.kb.anchoredText.write(checksum, outcome);
|
|
13270
13745
|
}
|
|
13271
13746
|
/** The stored outcome, or null when nothing has derived one — the common case. */
|
|
13272
13747
|
async getAnchoredText(resourceId10, _options) {
|
|
13273
13748
|
busLog("GET", "anchored-text", { resourceId: resourceId10 });
|
|
13274
|
-
return readAnchoredText(this.
|
|
13749
|
+
return readAnchoredText(this.kb, resourceId10);
|
|
13275
13750
|
}
|
|
13276
13751
|
/**
|
|
13277
13752
|
* The cache-consult read (PERSIST-ANCHORS P2c), straight from the store —
|
|
@@ -13280,7 +13755,7 @@ var LocalContentTransport = class {
|
|
|
13280
13755
|
*/
|
|
13281
13756
|
async getAnchoredTextByChecksum(checksum, _options) {
|
|
13282
13757
|
busLog("GET", "anchored-text-by-checksum", { checksum });
|
|
13283
|
-
return this.
|
|
13758
|
+
return this.kb.anchoredText.read(checksum);
|
|
13284
13759
|
}
|
|
13285
13760
|
/**
|
|
13286
13761
|
* The store's would-hit keys, straight from the store — planning data for
|
|
@@ -13289,7 +13764,7 @@ var LocalContentTransport = class {
|
|
|
13289
13764
|
*/
|
|
13290
13765
|
async listAnchoredTextKeys(_options) {
|
|
13291
13766
|
busLog("GET", "anchored-text-keys", {});
|
|
13292
|
-
return this.
|
|
13767
|
+
return this.kb.anchoredText.list();
|
|
13293
13768
|
}
|
|
13294
13769
|
async getBinary(resourceId10, _options) {
|
|
13295
13770
|
busLog("GET", "content", { resourceId: resourceId10 });
|
|
@@ -13320,16 +13795,14 @@ var LocalContentTransport = class {
|
|
|
13320
13795
|
}
|
|
13321
13796
|
);
|
|
13322
13797
|
}
|
|
13323
|
-
|
|
13324
|
-
|
|
13325
|
-
|
|
13326
|
-
|
|
13327
|
-
|
|
13328
|
-
|
|
13329
|
-
|
|
13330
|
-
|
|
13331
|
-
const data = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
|
|
13332
|
-
return { data, contentType: rep.mediaType };
|
|
13798
|
+
/**
|
|
13799
|
+
* The same resolution the wire path serves, buffered — local and hosted
|
|
13800
|
+
* modes must answer identically, which they did not before
|
|
13801
|
+
* SINGLE-KB-MOUNT P3: this resolved through `representations[].storageUri`,
|
|
13802
|
+
* a field `ViewMaterializer` never writes, so every binary read here threw.
|
|
13803
|
+
*/
|
|
13804
|
+
loadBinary(resourceId10) {
|
|
13805
|
+
return workingTreeContentReads(this.kb.views, this.kb.content).getBinary(resourceId10);
|
|
13333
13806
|
}
|
|
13334
13807
|
/**
|
|
13335
13808
|
* Assemble the resource's JSON-LD graph in-process from the KB — the local
|
|
@@ -13337,7 +13810,7 @@ var LocalContentTransport = class {
|
|
|
13337
13810
|
* getBinary; SIMPLER-JSON-LD.md decision 7). Local mode has no auth.
|
|
13338
13811
|
*/
|
|
13339
13812
|
async getResourceGraph(resourceId10, _options) {
|
|
13340
|
-
const graph = await assembleResourceGraph(this.
|
|
13813
|
+
const graph = await assembleResourceGraph(this.kb, resourceId10);
|
|
13341
13814
|
if (!graph) throw new Error(`Resource not found: ${resourceId10}`);
|
|
13342
13815
|
return graph;
|
|
13343
13816
|
}
|
|
@@ -13382,23 +13855,24 @@ function isWorkItem(input) {
|
|
|
13382
13855
|
return WORK_ITEM_TYPES.has(input.type);
|
|
13383
13856
|
}
|
|
13384
13857
|
var Smelter = class _Smelter {
|
|
13385
|
-
constructor(events$, rebuildAnchors$, vectorStore, embeddingProvider, content, bus, chunkingConfig, timing, logger) {
|
|
13858
|
+
constructor(events$, rebuildAnchors$, vectorStore, embeddingProvider, content, anchoredStore, bus, chunkingConfig, timing, logger) {
|
|
13386
13859
|
this.events$ = events$;
|
|
13387
13860
|
this.rebuildAnchors$ = rebuildAnchors$;
|
|
13388
13861
|
this.vectorStore = vectorStore;
|
|
13389
13862
|
this.embeddingProvider = embeddingProvider;
|
|
13390
13863
|
this.content = content;
|
|
13864
|
+
this.anchoredStore = anchoredStore;
|
|
13391
13865
|
this.bus = bus;
|
|
13392
13866
|
this.chunkingConfig = chunkingConfig;
|
|
13393
13867
|
this.timing = timing;
|
|
13394
13868
|
this.logger = logger;
|
|
13395
|
-
this.anchoredStore = anchoredTextStoreOverTransport(content, logger.child({ component: "anchored-text-cache" }));
|
|
13396
13869
|
}
|
|
13397
13870
|
events$;
|
|
13398
13871
|
rebuildAnchors$;
|
|
13399
13872
|
vectorStore;
|
|
13400
13873
|
embeddingProvider;
|
|
13401
13874
|
content;
|
|
13875
|
+
anchoredStore;
|
|
13402
13876
|
bus;
|
|
13403
13877
|
chunkingConfig;
|
|
13404
13878
|
timing;
|
|
@@ -13422,7 +13896,6 @@ var Smelter = class _Smelter {
|
|
|
13422
13896
|
* OCR pass.
|
|
13423
13897
|
*/
|
|
13424
13898
|
drainChain = Promise.resolve();
|
|
13425
|
-
anchoredStore;
|
|
13426
13899
|
get eventsProcessed() {
|
|
13427
13900
|
return this._eventsProcessed;
|
|
13428
13901
|
}
|
|
@@ -13652,7 +14125,7 @@ var Smelter = class _Smelter {
|
|
|
13652
14125
|
});
|
|
13653
14126
|
return;
|
|
13654
14127
|
}
|
|
13655
|
-
await this.
|
|
14128
|
+
await this.anchoredStore.write(checksum, { ...extracted, items: extracted.items });
|
|
13656
14129
|
this.logger.info("Re-anchored resource", { resourceId: rid, checksum, items: extracted.items.length });
|
|
13657
14130
|
}
|
|
13658
14131
|
async handleResourcePurge(event) {
|
|
@@ -13939,7 +14412,7 @@ var Smelter = class _Smelter {
|
|
|
13939
14412
|
// per resource (PERSIST-ANCHORS P0). Keys are resource ids today;
|
|
13940
14413
|
// P1 rekeys the store by content checksum and this lookup moves
|
|
13941
14414
|
// with it.
|
|
13942
|
-
this.
|
|
14415
|
+
this.anchoredStore.list().then((keys) => new Set(keys))
|
|
13943
14416
|
]);
|
|
13944
14417
|
const resources = await this.listAllResources();
|
|
13945
14418
|
this.logger.info("Reconcile started", {
|
|
@@ -14176,864 +14649,7 @@ function createSmelterActorStateUnit(options) {
|
|
|
14176
14649
|
}
|
|
14177
14650
|
};
|
|
14178
14651
|
}
|
|
14179
|
-
var pipelineAsync = promisify(pipeline);
|
|
14180
|
-
var BLOCK_SIZE = 512;
|
|
14181
|
-
function createTarHeader(name, size) {
|
|
14182
|
-
const header = Buffer.alloc(BLOCK_SIZE, 0);
|
|
14183
|
-
header.write(name.slice(0, 100), 0, 100, "utf8");
|
|
14184
|
-
header.write("0000644\0", 100, 8, "utf8");
|
|
14185
|
-
header.write("0000000\0", 108, 8, "utf8");
|
|
14186
|
-
header.write("0000000\0", 116, 8, "utf8");
|
|
14187
|
-
header.write(size.toString(8).padStart(11, "0") + "\0", 124, 12, "utf8");
|
|
14188
|
-
const mtime = Math.floor(Date.now() / 1e3);
|
|
14189
|
-
header.write(mtime.toString(8).padStart(11, "0") + "\0", 136, 12, "utf8");
|
|
14190
|
-
header.write(" ", 148, 8, "utf8");
|
|
14191
|
-
header.write("0", 156, 1, "utf8");
|
|
14192
|
-
header.write("ustar\0", 257, 6, "utf8");
|
|
14193
|
-
header.write("00", 263, 2, "utf8");
|
|
14194
|
-
let checksum = 0;
|
|
14195
|
-
for (let i = 0; i < BLOCK_SIZE; i++) {
|
|
14196
|
-
checksum += header[i];
|
|
14197
|
-
}
|
|
14198
|
-
header.write(checksum.toString(8).padStart(6, "0") + "\0 ", 148, 8, "utf8");
|
|
14199
|
-
return header;
|
|
14200
|
-
}
|
|
14201
|
-
function paddingBytes(size) {
|
|
14202
|
-
const remainder = size % BLOCK_SIZE;
|
|
14203
|
-
return remainder === 0 ? 0 : BLOCK_SIZE - remainder;
|
|
14204
|
-
}
|
|
14205
|
-
async function writeTarGz(entries, output) {
|
|
14206
|
-
const gzip = createGzip();
|
|
14207
|
-
const tarStream = new Readable({ read() {
|
|
14208
|
-
} });
|
|
14209
|
-
const pipePromise = pipelineAsync(tarStream, gzip, output);
|
|
14210
|
-
for await (const entry of entries) {
|
|
14211
|
-
const header = createTarHeader(entry.name, entry.data.length);
|
|
14212
|
-
tarStream.push(header);
|
|
14213
|
-
tarStream.push(entry.data);
|
|
14214
|
-
const pad = paddingBytes(entry.data.length);
|
|
14215
|
-
if (pad > 0) {
|
|
14216
|
-
tarStream.push(Buffer.alloc(pad, 0));
|
|
14217
|
-
}
|
|
14218
|
-
}
|
|
14219
|
-
tarStream.push(Buffer.alloc(BLOCK_SIZE * 2, 0));
|
|
14220
|
-
tarStream.push(null);
|
|
14221
|
-
await pipePromise;
|
|
14222
|
-
}
|
|
14223
|
-
async function decompressStream(input) {
|
|
14224
|
-
const gunzip = createGunzip();
|
|
14225
|
-
const chunks = [];
|
|
14226
|
-
return new Promise((resolve2, reject) => {
|
|
14227
|
-
gunzip.on("data", (chunk) => chunks.push(chunk));
|
|
14228
|
-
gunzip.on("end", () => resolve2(Buffer.concat(chunks)));
|
|
14229
|
-
gunzip.on("error", reject);
|
|
14230
|
-
input.on("error", reject);
|
|
14231
|
-
input.pipe(gunzip);
|
|
14232
|
-
});
|
|
14233
|
-
}
|
|
14234
|
-
function* parseTarEntries(decompressed) {
|
|
14235
|
-
let offset = 0;
|
|
14236
|
-
while (offset + BLOCK_SIZE <= decompressed.length) {
|
|
14237
|
-
const header = decompressed.subarray(offset, offset + BLOCK_SIZE);
|
|
14238
|
-
if (header.every((b) => b === 0)) break;
|
|
14239
|
-
const nameEnd = header.indexOf(0, 0);
|
|
14240
|
-
const name = header.subarray(0, Math.min(nameEnd, 100)).toString("utf8");
|
|
14241
|
-
const sizeStr = header.subarray(124, 135).toString("utf8").trim();
|
|
14242
|
-
const size = parseInt(sizeStr, 8);
|
|
14243
|
-
offset += BLOCK_SIZE;
|
|
14244
|
-
const data = decompressed.subarray(offset, offset + size);
|
|
14245
|
-
offset += size;
|
|
14246
|
-
offset += paddingBytes(size);
|
|
14247
|
-
yield { name, size, data };
|
|
14248
|
-
}
|
|
14249
|
-
}
|
|
14250
|
-
async function* readTarGz(input) {
|
|
14251
|
-
const decompressed = await decompressStream(input);
|
|
14252
|
-
yield* parseTarEntries(decompressed);
|
|
14253
|
-
}
|
|
14254
|
-
|
|
14255
|
-
// src/exchange/manifest.ts
|
|
14256
|
-
var BACKUP_FORMAT = "semiont-backup";
|
|
14257
|
-
var FORMAT_VERSION = 1;
|
|
14258
|
-
var LINKED_DATA_FORMAT = "semiont-linked-data";
|
|
14259
|
-
function isLinkedDataManifest(obj) {
|
|
14260
|
-
return typeof obj === "object" && obj !== null && obj["semiont:format"] === LINKED_DATA_FORMAT;
|
|
14261
|
-
}
|
|
14262
|
-
function isBackupManifest(obj) {
|
|
14263
|
-
return typeof obj === "object" && obj !== null && obj.format === BACKUP_FORMAT;
|
|
14264
|
-
}
|
|
14265
|
-
function validateManifestVersion(version) {
|
|
14266
|
-
if (version > FORMAT_VERSION) {
|
|
14267
|
-
throw new Error(
|
|
14268
|
-
`Unsupported format version ${version}. This tool supports version ${FORMAT_VERSION}.`
|
|
14269
|
-
);
|
|
14270
|
-
}
|
|
14271
|
-
}
|
|
14272
|
-
|
|
14273
|
-
// src/exchange/backup-exporter.ts
|
|
14274
|
-
var SYSTEM_STREAM = "__system__";
|
|
14275
|
-
async function exportBackup(options, output) {
|
|
14276
|
-
const { eventStore, content, sourceUrl, logger } = options;
|
|
14277
|
-
const resourceIds = await eventStore.log.storage.getAllResourceIds();
|
|
14278
|
-
logger?.info("Backup export: enumerating streams", { resourceCount: resourceIds.length });
|
|
14279
|
-
const allStreamIds = [SYSTEM_STREAM, ...resourceIds];
|
|
14280
|
-
const streamData = /* @__PURE__ */ new Map();
|
|
14281
|
-
let totalEvents = 0;
|
|
14282
|
-
for (const id of allStreamIds) {
|
|
14283
|
-
const events = await eventStore.log.getEvents(id);
|
|
14284
|
-
if (events.length > 0) {
|
|
14285
|
-
streamData.set(id, events);
|
|
14286
|
-
totalEvents += events.length;
|
|
14287
|
-
}
|
|
14288
|
-
}
|
|
14289
|
-
const contentRefs = collectContentRefs(streamData);
|
|
14290
|
-
logger?.info("Backup export: collected content refs", {
|
|
14291
|
-
streams: streamData.size,
|
|
14292
|
-
events: totalEvents,
|
|
14293
|
-
blobs: contentRefs.size
|
|
14294
|
-
});
|
|
14295
|
-
const contentBlobs = /* @__PURE__ */ new Map();
|
|
14296
|
-
let totalContentBytes = 0;
|
|
14297
|
-
for (const [storageUri, mediaType] of contentRefs) {
|
|
14298
|
-
const data = await content.retrieve(storageUri);
|
|
14299
|
-
const ext = extensionForMediaType(mediaType);
|
|
14300
|
-
contentBlobs.set(storageUri, { data, ext });
|
|
14301
|
-
totalContentBytes += data.length;
|
|
14302
|
-
}
|
|
14303
|
-
const streamSummaries = [];
|
|
14304
|
-
for (const [streamId, events] of streamData) {
|
|
14305
|
-
streamSummaries.push({
|
|
14306
|
-
stream: streamId,
|
|
14307
|
-
eventCount: events.length
|
|
14308
|
-
});
|
|
14309
|
-
}
|
|
14310
|
-
const manifestHeader = {
|
|
14311
|
-
format: BACKUP_FORMAT,
|
|
14312
|
-
version: FORMAT_VERSION,
|
|
14313
|
-
exportedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
14314
|
-
sourceUrl,
|
|
14315
|
-
stats: {
|
|
14316
|
-
streams: streamData.size,
|
|
14317
|
-
events: totalEvents,
|
|
14318
|
-
blobs: contentBlobs.size,
|
|
14319
|
-
contentBytes: totalContentBytes
|
|
14320
|
-
}
|
|
14321
|
-
};
|
|
14322
|
-
async function* generateEntries() {
|
|
14323
|
-
const manifestLines = [
|
|
14324
|
-
JSON.stringify(manifestHeader),
|
|
14325
|
-
...streamSummaries.map((s) => JSON.stringify(s))
|
|
14326
|
-
].join("\n") + "\n";
|
|
14327
|
-
yield { name: ".semiont/manifest.jsonl", data: Buffer.from(manifestLines, "utf8") };
|
|
14328
|
-
for (const [streamId, events] of streamData) {
|
|
14329
|
-
const fileName = streamId === SYSTEM_STREAM ? ".semiont/events/__system__.jsonl" : `.semiont/events/${streamId}.jsonl`;
|
|
14330
|
-
const jsonl = events.map((e) => JSON.stringify(e)).join("\n") + "\n";
|
|
14331
|
-
yield { name: fileName, data: Buffer.from(jsonl, "utf8") };
|
|
14332
|
-
}
|
|
14333
|
-
for (const [storageUri, { data, ext }] of contentBlobs) {
|
|
14334
|
-
const blobPath = storageUri.startsWith("file://") ? storageUri.slice(7) : storageUri;
|
|
14335
|
-
yield { name: `${blobPath}${ext}`, data };
|
|
14336
|
-
}
|
|
14337
|
-
}
|
|
14338
|
-
await writeTarGz(generateEntries(), output);
|
|
14339
|
-
logger?.info("Backup export complete", {
|
|
14340
|
-
streams: streamData.size,
|
|
14341
|
-
events: totalEvents,
|
|
14342
|
-
blobs: contentBlobs.size,
|
|
14343
|
-
contentBytes: totalContentBytes
|
|
14344
|
-
});
|
|
14345
|
-
return manifestHeader;
|
|
14346
|
-
}
|
|
14347
|
-
function collectContentRefs(streamData) {
|
|
14348
|
-
const refs = /* @__PURE__ */ new Map();
|
|
14349
|
-
for (const [, events] of streamData) {
|
|
14350
|
-
for (const stored of events) {
|
|
14351
|
-
if (stored.type === "yield:created") {
|
|
14352
|
-
const payload = stored.payload;
|
|
14353
|
-
if (payload.storageUri && payload.format) {
|
|
14354
|
-
refs.set(payload.storageUri, payload.format);
|
|
14355
|
-
}
|
|
14356
|
-
}
|
|
14357
|
-
}
|
|
14358
|
-
}
|
|
14359
|
-
return refs;
|
|
14360
|
-
}
|
|
14361
|
-
|
|
14362
|
-
// src/exchange/await-reply.ts
|
|
14363
|
-
var import_rxjs11 = __toESM(require_cjs());
|
|
14364
|
-
var import_operators9 = __toESM(require_operators());
|
|
14365
|
-
function awaitReply(eventBus, command, payload, okChannel, failedChannel, timeoutMs) {
|
|
14366
|
-
const result$ = (0, import_rxjs11.race)(
|
|
14367
|
-
eventBus.get(okChannel).pipe((0, import_operators9.map)(() => void 0)),
|
|
14368
|
-
eventBus.get(failedChannel).pipe(
|
|
14369
|
-
(0, import_operators9.map)((e) => {
|
|
14370
|
-
throw new Error(e.message ?? `${command} failed`);
|
|
14371
|
-
})
|
|
14372
|
-
),
|
|
14373
|
-
(0, import_rxjs11.timer)(timeoutMs).pipe(
|
|
14374
|
-
(0, import_operators9.map)(() => {
|
|
14375
|
-
throw new Error(`Timeout waiting for ${okChannel}`);
|
|
14376
|
-
})
|
|
14377
|
-
)
|
|
14378
|
-
);
|
|
14379
|
-
eventBus.get(command).next(payload);
|
|
14380
|
-
return (0, import_rxjs11.firstValueFrom)(result$);
|
|
14381
|
-
}
|
|
14382
|
-
var REPLAY_TIMEOUT_MS = 3e4;
|
|
14383
|
-
async function replayEventStream(jsonl, eventBus, eventStore, resolveBlob, contentStore, logger) {
|
|
14384
|
-
const lines = jsonl.trim().split("\n").filter((l) => l.length > 0);
|
|
14385
|
-
const storedEvents = lines.map((line) => JSON.parse(line));
|
|
14386
|
-
const stats = {
|
|
14387
|
-
eventsReplayed: 0,
|
|
14388
|
-
resourcesCreated: 0,
|
|
14389
|
-
annotationsCreated: 0,
|
|
14390
|
-
entityTypesAdded: 0
|
|
14391
|
-
};
|
|
14392
|
-
for (const stored of storedEvents) {
|
|
14393
|
-
await replayEvent(stored, eventBus, eventStore, resolveBlob, contentStore, stats, logger);
|
|
14394
|
-
stats.eventsReplayed++;
|
|
14395
|
-
}
|
|
14396
|
-
return { stats };
|
|
14397
|
-
}
|
|
14398
|
-
async function replayEvent(event, eventBus, eventStore, resolveBlob, contentStore, stats, logger) {
|
|
14399
|
-
switch (event.type) {
|
|
14400
|
-
case "frame:entity-type-added":
|
|
14401
|
-
await replayEntityTypeAdded(event, eventBus, logger);
|
|
14402
|
-
stats.entityTypesAdded++;
|
|
14403
|
-
break;
|
|
14404
|
-
case "yield:created":
|
|
14405
|
-
await replayResourceCreated(event, eventBus, resolveBlob, contentStore, logger);
|
|
14406
|
-
stats.resourcesCreated++;
|
|
14407
|
-
break;
|
|
14408
|
-
case "mark:added":
|
|
14409
|
-
await replayAnnotationAdded(event, eventBus, logger);
|
|
14410
|
-
stats.annotationsCreated++;
|
|
14411
|
-
break;
|
|
14412
|
-
case "mark:body-updated":
|
|
14413
|
-
await replayAnnotationBodyUpdated(event, eventBus, logger);
|
|
14414
|
-
break;
|
|
14415
|
-
case "mark:removed":
|
|
14416
|
-
await replayAnnotationRemoved(event, eventBus, logger);
|
|
14417
|
-
break;
|
|
14418
|
-
case "mark:archived":
|
|
14419
|
-
await replayResourceArchived(event, eventBus, logger);
|
|
14420
|
-
break;
|
|
14421
|
-
case "mark:unarchived":
|
|
14422
|
-
await replayResourceUnarchived(event, eventBus, logger);
|
|
14423
|
-
break;
|
|
14424
|
-
case "mark:entity-tag-added":
|
|
14425
|
-
case "mark:entity-tag-removed":
|
|
14426
|
-
await replayEntityTagChange(event, eventStore, logger);
|
|
14427
|
-
break;
|
|
14428
|
-
// Job events are transient — skip during replay
|
|
14429
|
-
case "job:started":
|
|
14430
|
-
case "job:completed":
|
|
14431
|
-
case "job:failed":
|
|
14432
|
-
logger?.debug("Skipping job event during replay", { type: event.type });
|
|
14433
|
-
break;
|
|
14434
|
-
// Representation events — content is already stored via yield:created replay
|
|
14435
|
-
case "yield:representation-added":
|
|
14436
|
-
case "yield:representation-removed":
|
|
14437
|
-
logger?.debug("Skipping representation event during replay", { type: event.type });
|
|
14438
|
-
break;
|
|
14439
|
-
default:
|
|
14440
|
-
logger?.warn("Unknown event type during replay", { type: event.type });
|
|
14441
|
-
}
|
|
14442
|
-
}
|
|
14443
|
-
async function replayEntityTypeAdded(event, eventBus, logger) {
|
|
14444
|
-
await busRequest(
|
|
14445
|
-
asBusRequestPrimitive(eventBus),
|
|
14446
|
-
"frame:add-entity-type",
|
|
14447
|
-
{ tag: event.payload.entityType, _userId: event.userId },
|
|
14448
|
-
REPLAY_TIMEOUT_MS
|
|
14449
|
-
);
|
|
14450
|
-
logger?.debug("Replayed entitytype.added", { entityType: event.payload.entityType });
|
|
14451
|
-
}
|
|
14452
|
-
async function replayResourceCreated(event, eventBus, resolveBlob, contentStore, logger) {
|
|
14453
|
-
const { payload } = event;
|
|
14454
|
-
const blob = resolveBlob(payload.contentChecksum);
|
|
14455
|
-
if (!blob) {
|
|
14456
|
-
throw new Error(`Missing content blob for checksum ${payload.contentChecksum}`);
|
|
14457
|
-
}
|
|
14458
|
-
const base = baseMediaType(payload.format);
|
|
14459
|
-
const resolvedUri = payload.storageUri || deriveStorageUri(payload.name, isSupportedMediaType(base) ? base : "application/octet-stream");
|
|
14460
|
-
const stored = await contentStore.store(blob, resolvedUri);
|
|
14461
|
-
await busRequest(
|
|
14462
|
-
asBusRequestPrimitive(eventBus),
|
|
14463
|
-
"yield:create",
|
|
14464
|
-
{
|
|
14465
|
-
name: payload.name,
|
|
14466
|
-
storageUri: resolvedUri,
|
|
14467
|
-
contentChecksum: stored.checksum,
|
|
14468
|
-
byteSize: stored.byteSize,
|
|
14469
|
-
format: payload.format,
|
|
14470
|
-
_userId: event.userId,
|
|
14471
|
-
language: payload.language,
|
|
14472
|
-
entityTypes: payload.entityTypes,
|
|
14473
|
-
isDraft: payload.isDraft,
|
|
14474
|
-
generatedFrom: payload.generatedFrom,
|
|
14475
|
-
generationPrompt: payload.generationPrompt
|
|
14476
|
-
},
|
|
14477
|
-
REPLAY_TIMEOUT_MS
|
|
14478
|
-
);
|
|
14479
|
-
logger?.debug("Replayed resource.created", { name: payload.name });
|
|
14480
|
-
}
|
|
14481
|
-
async function replayAnnotationAdded(event, eventBus, logger) {
|
|
14482
|
-
await awaitReply(
|
|
14483
|
-
eventBus,
|
|
14484
|
-
"mark:create",
|
|
14485
|
-
{
|
|
14486
|
-
annotation: event.payload.annotation,
|
|
14487
|
-
_userId: event.userId,
|
|
14488
|
-
resourceId: event.resourceId
|
|
14489
|
-
},
|
|
14490
|
-
"mark:create-ok",
|
|
14491
|
-
"mark:create-failed",
|
|
14492
|
-
REPLAY_TIMEOUT_MS
|
|
14493
|
-
);
|
|
14494
|
-
logger?.debug("Replayed annotation.added", { annotationId: event.payload.annotation.id });
|
|
14495
|
-
}
|
|
14496
|
-
async function replayAnnotationBodyUpdated(event, eventBus, logger) {
|
|
14497
|
-
await awaitReply(
|
|
14498
|
-
eventBus,
|
|
14499
|
-
"mark:update-body",
|
|
14500
|
-
{
|
|
14501
|
-
annotationId: event.payload.annotationId,
|
|
14502
|
-
_userId: event.userId,
|
|
14503
|
-
resourceId: event.resourceId,
|
|
14504
|
-
operations: event.payload.operations
|
|
14505
|
-
},
|
|
14506
|
-
"mark:body-updated",
|
|
14507
|
-
"mark:body-update-failed",
|
|
14508
|
-
REPLAY_TIMEOUT_MS
|
|
14509
|
-
);
|
|
14510
|
-
logger?.debug("Replayed annotation.body.updated", { annotationId: event.payload.annotationId });
|
|
14511
|
-
}
|
|
14512
|
-
async function replayAnnotationRemoved(event, eventBus, logger) {
|
|
14513
|
-
await awaitReply(
|
|
14514
|
-
eventBus,
|
|
14515
|
-
"mark:delete",
|
|
14516
|
-
{
|
|
14517
|
-
annotationId: event.payload.annotationId,
|
|
14518
|
-
_userId: event.userId,
|
|
14519
|
-
resourceId: event.resourceId
|
|
14520
|
-
},
|
|
14521
|
-
"mark:delete-ok",
|
|
14522
|
-
"mark:delete-failed",
|
|
14523
|
-
REPLAY_TIMEOUT_MS
|
|
14524
|
-
);
|
|
14525
|
-
logger?.debug("Replayed annotation.removed", { annotationId: event.payload.annotationId });
|
|
14526
|
-
}
|
|
14527
|
-
async function replayResourceArchived(event, eventBus, logger) {
|
|
14528
|
-
eventBus.get("mark:archive").next({
|
|
14529
|
-
_userId: event.userId,
|
|
14530
|
-
resourceId: event.resourceId
|
|
14531
|
-
});
|
|
14532
|
-
logger?.debug("Replayed resource.archived", { resourceId: event.resourceId });
|
|
14533
|
-
}
|
|
14534
|
-
async function replayResourceUnarchived(event, eventBus, logger) {
|
|
14535
|
-
eventBus.get("mark:unarchive").next({
|
|
14536
|
-
_userId: event.userId,
|
|
14537
|
-
resourceId: event.resourceId
|
|
14538
|
-
});
|
|
14539
|
-
logger?.debug("Replayed resource.unarchived", { resourceId: event.resourceId });
|
|
14540
|
-
}
|
|
14541
|
-
async function replayEntityTagChange(event, eventStore, logger) {
|
|
14542
|
-
const base = {
|
|
14543
|
-
resourceId: event.resourceId,
|
|
14544
|
-
userId: event.userId,
|
|
14545
|
-
version: event.version
|
|
14546
|
-
};
|
|
14547
|
-
if (event.type === "mark:entity-tag-added") {
|
|
14548
|
-
await eventStore.appendEvent({ ...base, type: "mark:entity-tag-added", payload: event.payload });
|
|
14549
|
-
} else {
|
|
14550
|
-
await eventStore.appendEvent({ ...base, type: "mark:entity-tag-removed", payload: event.payload });
|
|
14551
|
-
}
|
|
14552
|
-
logger?.debug("Replayed entity tag change", { type: event.type, entityType: event.payload.entityType });
|
|
14553
|
-
}
|
|
14554
|
-
|
|
14555
|
-
// src/exchange/backup-importer.ts
|
|
14556
|
-
function buildBlobResolver(entries) {
|
|
14557
|
-
const checksumIndex = /* @__PURE__ */ new Map();
|
|
14558
|
-
for (const name of entries.keys()) {
|
|
14559
|
-
if (!name.startsWith(".semiont/")) {
|
|
14560
|
-
const dotIndex = name.lastIndexOf(".");
|
|
14561
|
-
const checksum = dotIndex >= 0 ? name.slice(0, dotIndex) : name;
|
|
14562
|
-
checksumIndex.set(checksum, name);
|
|
14563
|
-
}
|
|
14564
|
-
}
|
|
14565
|
-
return (checksum) => {
|
|
14566
|
-
const entryName = checksumIndex.get(checksum);
|
|
14567
|
-
return entryName ? entries.get(entryName) : void 0;
|
|
14568
|
-
};
|
|
14569
|
-
}
|
|
14570
|
-
async function importBackup(archive, options) {
|
|
14571
|
-
const { eventBus, eventStore, contentStore, logger } = options;
|
|
14572
|
-
const entries = /* @__PURE__ */ new Map();
|
|
14573
|
-
for await (const entry of readTarGz(archive)) {
|
|
14574
|
-
entries.set(entry.name, entry.data);
|
|
14575
|
-
}
|
|
14576
|
-
const manifestData = entries.get(".semiont/manifest.jsonl");
|
|
14577
|
-
if (!manifestData) {
|
|
14578
|
-
throw new Error("Invalid backup: missing .semiont/manifest.jsonl");
|
|
14579
|
-
}
|
|
14580
|
-
const manifestLines = manifestData.toString("utf8").trim().split("\n");
|
|
14581
|
-
const header = JSON.parse(manifestLines[0]);
|
|
14582
|
-
if (!isBackupManifest(header)) {
|
|
14583
|
-
throw new Error(`Invalid backup: expected format "${BACKUP_FORMAT}", got "${header.format}"`);
|
|
14584
|
-
}
|
|
14585
|
-
validateManifestVersion(header.version);
|
|
14586
|
-
const streamSummaries = manifestLines.slice(1).map((line) => JSON.parse(line));
|
|
14587
|
-
logger?.info("Backup import: parsed manifest", {
|
|
14588
|
-
streams: header.stats.streams,
|
|
14589
|
-
events: header.stats.events,
|
|
14590
|
-
blobs: header.stats.blobs
|
|
14591
|
-
});
|
|
14592
|
-
const resolveBlob = buildBlobResolver(entries);
|
|
14593
|
-
const systemData = entries.get(".semiont/events/__system__.jsonl");
|
|
14594
|
-
let stats = { eventsReplayed: 0, resourcesCreated: 0, annotationsCreated: 0, entityTypesAdded: 0 };
|
|
14595
|
-
if (systemData) {
|
|
14596
|
-
const result = await replayEventStream(
|
|
14597
|
-
systemData.toString("utf8"),
|
|
14598
|
-
eventBus,
|
|
14599
|
-
eventStore,
|
|
14600
|
-
resolveBlob,
|
|
14601
|
-
contentStore,
|
|
14602
|
-
logger
|
|
14603
|
-
);
|
|
14604
|
-
stats = mergeStats(stats, result.stats);
|
|
14605
|
-
}
|
|
14606
|
-
for (const summary of streamSummaries) {
|
|
14607
|
-
if (summary.stream === "__system__") continue;
|
|
14608
|
-
const eventData = entries.get(`.semiont/events/${summary.stream}.jsonl`);
|
|
14609
|
-
if (!eventData) {
|
|
14610
|
-
logger?.warn("Backup import: missing event stream", { stream: summary.stream });
|
|
14611
|
-
continue;
|
|
14612
|
-
}
|
|
14613
|
-
const result = await replayEventStream(
|
|
14614
|
-
eventData.toString("utf8"),
|
|
14615
|
-
eventBus,
|
|
14616
|
-
eventStore,
|
|
14617
|
-
resolveBlob,
|
|
14618
|
-
contentStore,
|
|
14619
|
-
logger
|
|
14620
|
-
);
|
|
14621
|
-
stats = mergeStats(stats, result.stats);
|
|
14622
|
-
}
|
|
14623
|
-
logger?.info("Backup import complete", { ...stats });
|
|
14624
|
-
return { manifest: header, stats };
|
|
14625
|
-
}
|
|
14626
|
-
function mergeStats(a, b) {
|
|
14627
|
-
return {
|
|
14628
|
-
eventsReplayed: a.eventsReplayed + b.eventsReplayed,
|
|
14629
|
-
resourcesCreated: a.resourcesCreated + b.resourcesCreated,
|
|
14630
|
-
annotationsCreated: a.annotationsCreated + b.annotationsCreated,
|
|
14631
|
-
entityTypesAdded: a.entityTypesAdded + b.entityTypesAdded
|
|
14632
|
-
};
|
|
14633
|
-
}
|
|
14634
|
-
var SEMIONT_CONTEXT = [
|
|
14635
|
-
"https://schema.org/",
|
|
14636
|
-
"http://www.w3.org/ns/anno.jsonld",
|
|
14637
|
-
{
|
|
14638
|
-
"semiont": "https://semiont.org/vocab/",
|
|
14639
|
-
"entityTypes": "semiont:entityTypes",
|
|
14640
|
-
"archived": "semiont:archived",
|
|
14641
|
-
"representations": { "@id": "semiont:representations", "@container": "@set" },
|
|
14642
|
-
"annotations": { "@id": "semiont:annotations", "@container": "@set" }
|
|
14643
|
-
}
|
|
14644
|
-
];
|
|
14645
|
-
var MANIFEST_CONTEXT = {
|
|
14646
|
-
"semiont": "https://semiont.org/vocab/",
|
|
14647
|
-
"schema": "https://schema.org/",
|
|
14648
|
-
"dct": "http://purl.org/dc/terms/",
|
|
14649
|
-
"prov": "http://www.w3.org/ns/prov#",
|
|
14650
|
-
"void": "http://rdfs.org/ns/void#"
|
|
14651
|
-
};
|
|
14652
|
-
function hydrateAnnotation(annotation, baseUrl) {
|
|
14653
|
-
const hydrated = { ...annotation };
|
|
14654
|
-
if (hydrated.id && !hydrated.id.startsWith("http")) {
|
|
14655
|
-
hydrated.id = `${baseUrl}/annotations/${hydrated.id}`;
|
|
14656
|
-
}
|
|
14657
|
-
if (typeof hydrated.target === "string") {
|
|
14658
|
-
if (!hydrated.target.startsWith("http")) {
|
|
14659
|
-
hydrated.target = `${baseUrl}/resources/${hydrated.target}`;
|
|
14660
|
-
}
|
|
14661
|
-
} else if (hydrated.target && typeof hydrated.target === "object") {
|
|
14662
|
-
const target = { ...hydrated.target };
|
|
14663
|
-
if (target.source && !target.source.startsWith("http")) {
|
|
14664
|
-
target.source = `${baseUrl}/resources/${target.source}`;
|
|
14665
|
-
}
|
|
14666
|
-
hydrated.target = target;
|
|
14667
|
-
}
|
|
14668
|
-
hydrated.body = hydrateBody(hydrated.body, baseUrl);
|
|
14669
|
-
return hydrated;
|
|
14670
|
-
}
|
|
14671
|
-
function hydrateBody(body, baseUrl) {
|
|
14672
|
-
if (Array.isArray(body)) {
|
|
14673
|
-
return body.map((b) => hydrateBodyItem(b, baseUrl));
|
|
14674
|
-
}
|
|
14675
|
-
return hydrateBodyItem(body, baseUrl);
|
|
14676
|
-
}
|
|
14677
|
-
function hydrateBodyItem(item, baseUrl) {
|
|
14678
|
-
if (item && typeof item === "object" && "source" in item) {
|
|
14679
|
-
const source = item.source;
|
|
14680
|
-
if (typeof source === "string" && !source.startsWith("http")) {
|
|
14681
|
-
return { ...item, source: `${baseUrl}/resources/${source}` };
|
|
14682
|
-
}
|
|
14683
|
-
}
|
|
14684
|
-
return item;
|
|
14685
|
-
}
|
|
14686
|
-
async function exportLinkedData(options, output) {
|
|
14687
|
-
const { views, content, sourceUrl, entityTypes, includeArchived, logger } = options;
|
|
14688
|
-
const allViews = await views.getAll();
|
|
14689
|
-
const resourceViews = includeArchived ? allViews : allViews.filter((v) => !v.resource.archived);
|
|
14690
|
-
logger?.info("Linked data export: enumerating resources", { count: resourceViews.length });
|
|
14691
|
-
const contentRefs = /* @__PURE__ */ new Map();
|
|
14692
|
-
for (const view of resourceViews) {
|
|
14693
|
-
collectContentRefsFromResource(view.resource, contentRefs);
|
|
14694
|
-
}
|
|
14695
|
-
const contentBlobs = /* @__PURE__ */ new Map();
|
|
14696
|
-
for (const [storageUri, mediaType] of contentRefs) {
|
|
14697
|
-
try {
|
|
14698
|
-
const data = await content.retrieve(storageUri);
|
|
14699
|
-
const ext = extensionForMediaType(mediaType);
|
|
14700
|
-
contentBlobs.set(storageUri, { data, ext });
|
|
14701
|
-
} catch (err) {
|
|
14702
|
-
logger?.warn("Failed to retrieve content blob", { storageUri, error: String(err) });
|
|
14703
|
-
}
|
|
14704
|
-
}
|
|
14705
|
-
const manifest = {
|
|
14706
|
-
"@context": MANIFEST_CONTEXT,
|
|
14707
|
-
"@type": "void:Dataset",
|
|
14708
|
-
"semiont:format": LINKED_DATA_FORMAT,
|
|
14709
|
-
"semiont:version": FORMAT_VERSION,
|
|
14710
|
-
"dct:created": (/* @__PURE__ */ new Date()).toISOString(),
|
|
14711
|
-
"prov:wasGeneratedBy": {
|
|
14712
|
-
"@type": "prov:Activity",
|
|
14713
|
-
"prov:used": sourceUrl
|
|
14714
|
-
},
|
|
14715
|
-
"semiont:entityTypes": entityTypes,
|
|
14716
|
-
"void:entities": resourceViews.length
|
|
14717
|
-
};
|
|
14718
|
-
async function* generateEntries() {
|
|
14719
|
-
yield {
|
|
14720
|
-
name: ".semiont/manifest.jsonld",
|
|
14721
|
-
data: Buffer.from(JSON.stringify(manifest, null, 2), "utf8")
|
|
14722
|
-
};
|
|
14723
|
-
for (const view of resourceViews) {
|
|
14724
|
-
const resourceId10 = view.resource["@id"];
|
|
14725
|
-
const jsonld = buildResourceJsonLd(view.resource, view.annotations.annotations, sourceUrl);
|
|
14726
|
-
yield {
|
|
14727
|
-
name: `.semiont/resources/${resourceId10}.jsonld`,
|
|
14728
|
-
data: Buffer.from(JSON.stringify(jsonld, null, 2), "utf8")
|
|
14729
|
-
};
|
|
14730
|
-
}
|
|
14731
|
-
for (const [storageUri, { data, ext }] of contentBlobs) {
|
|
14732
|
-
const blobPath = storageUri.startsWith("file://") ? storageUri.slice(7) : storageUri;
|
|
14733
|
-
yield { name: `${blobPath}${ext}`, data };
|
|
14734
|
-
}
|
|
14735
|
-
}
|
|
14736
|
-
await writeTarGz(generateEntries(), output);
|
|
14737
|
-
logger?.info("Linked data export complete", {
|
|
14738
|
-
resources: resourceViews.length,
|
|
14739
|
-
blobs: contentBlobs.size
|
|
14740
|
-
});
|
|
14741
|
-
return manifest;
|
|
14742
|
-
}
|
|
14743
|
-
function buildResourceJsonLd(resource, annotations, sourceUrl) {
|
|
14744
|
-
const resourceId10 = resource["@id"];
|
|
14745
|
-
const resourceUri = resourceId10.startsWith("http") ? resourceId10 : `${sourceUrl}/resources/${resourceId10}`;
|
|
14746
|
-
const doc = {
|
|
14747
|
-
"@context": SEMIONT_CONTEXT,
|
|
14748
|
-
"@id": resourceUri,
|
|
14749
|
-
"@type": resource["@type"] ?? "DigitalDocument",
|
|
14750
|
-
"name": resource.name
|
|
14751
|
-
};
|
|
14752
|
-
if (resource.dateCreated) doc["dateCreated"] = resource.dateCreated;
|
|
14753
|
-
if (resource.dateModified) doc["dateModified"] = resource.dateModified;
|
|
14754
|
-
if (resource.description) doc["description"] = resource.description;
|
|
14755
|
-
const reps = normalizeRepresentations(resource.representations);
|
|
14756
|
-
if (reps.length > 0) {
|
|
14757
|
-
const primary = reps[0];
|
|
14758
|
-
if (primary.language) doc["inLanguage"] = primary.language;
|
|
14759
|
-
if (primary.mediaType) doc["encodingFormat"] = primary.mediaType;
|
|
14760
|
-
}
|
|
14761
|
-
if (resource.entityTypes && resource.entityTypes.length > 0) doc["entityTypes"] = resource.entityTypes;
|
|
14762
|
-
if (resource.archived) doc["archived"] = resource.archived;
|
|
14763
|
-
if (resource.wasDerivedFrom) doc["wasDerivedFrom"] = resource.wasDerivedFrom;
|
|
14764
|
-
if (resource.wasAttributedTo) doc["wasAttributedTo"] = resource.wasAttributedTo;
|
|
14765
|
-
if (resource.sameAs && resource.sameAs.length > 0) doc["sameAs"] = resource.sameAs;
|
|
14766
|
-
if (resource.isPartOf && resource.isPartOf.length > 0) doc["isPartOf"] = resource.isPartOf;
|
|
14767
|
-
if (resource.hasPart && resource.hasPart.length > 0) doc["hasPart"] = resource.hasPart;
|
|
14768
|
-
if (reps.length > 0) {
|
|
14769
|
-
doc["representations"] = reps.map((rep) => {
|
|
14770
|
-
const mediaObj = {
|
|
14771
|
-
"@type": "schema:MediaObject",
|
|
14772
|
-
"encodingFormat": rep.mediaType
|
|
14773
|
-
};
|
|
14774
|
-
if (rep.byteSize !== void 0) mediaObj["contentSize"] = rep.byteSize;
|
|
14775
|
-
if (rep.checksum) {
|
|
14776
|
-
const rawChecksum = rep.checksum.startsWith("sha256:") ? rep.checksum.slice(7) : rep.checksum;
|
|
14777
|
-
mediaObj["sha256"] = rawChecksum;
|
|
14778
|
-
const ext = extensionForMediaType(rep.mediaType);
|
|
14779
|
-
mediaObj["name"] = `${rawChecksum}${ext}`;
|
|
14780
|
-
}
|
|
14781
|
-
if (rep.language) mediaObj["inLanguage"] = rep.language;
|
|
14782
|
-
return mediaObj;
|
|
14783
|
-
});
|
|
14784
|
-
}
|
|
14785
|
-
if (annotations.length > 0) {
|
|
14786
|
-
doc["annotations"] = annotations.map((ann) => hydrateAnnotation(ann, sourceUrl));
|
|
14787
|
-
}
|
|
14788
|
-
return doc;
|
|
14789
|
-
}
|
|
14790
|
-
function normalizeRepresentations(reps) {
|
|
14791
|
-
if (!reps) return [];
|
|
14792
|
-
if (Array.isArray(reps)) return reps;
|
|
14793
|
-
return [reps];
|
|
14794
|
-
}
|
|
14795
|
-
function collectContentRefsFromResource(resource, refs) {
|
|
14796
|
-
if (resource.storageUri) {
|
|
14797
|
-
const reps = normalizeRepresentations(resource.representations);
|
|
14798
|
-
const mediaType = reps[0]?.mediaType ?? "application/octet-stream";
|
|
14799
|
-
refs.set(resource.storageUri, mediaType);
|
|
14800
|
-
}
|
|
14801
|
-
}
|
|
14802
|
-
var IMPORT_TIMEOUT_MS = 3e4;
|
|
14803
|
-
function stripUriToId(uri) {
|
|
14804
|
-
if (!uri.includes("/")) return uri;
|
|
14805
|
-
const lastSlash = uri.lastIndexOf("/");
|
|
14806
|
-
return uri.slice(lastSlash + 1);
|
|
14807
|
-
}
|
|
14808
|
-
function dehydrateAnnotation(annotation) {
|
|
14809
|
-
const dehydrated = { ...annotation };
|
|
14810
|
-
if (dehydrated.id) {
|
|
14811
|
-
dehydrated.id = annotationId(stripUriToId(dehydrated.id));
|
|
14812
|
-
}
|
|
14813
|
-
if (typeof dehydrated.target === "string") {
|
|
14814
|
-
dehydrated.target = stripUriToId(dehydrated.target);
|
|
14815
|
-
} else if (dehydrated.target && typeof dehydrated.target === "object") {
|
|
14816
|
-
const target = { ...dehydrated.target };
|
|
14817
|
-
if (target.source) {
|
|
14818
|
-
target.source = stripUriToId(target.source);
|
|
14819
|
-
}
|
|
14820
|
-
dehydrated.target = target;
|
|
14821
|
-
}
|
|
14822
|
-
dehydrated.body = dehydrateBody(dehydrated.body);
|
|
14823
|
-
return dehydrated;
|
|
14824
|
-
}
|
|
14825
|
-
function dehydrateBody(body) {
|
|
14826
|
-
if (Array.isArray(body)) {
|
|
14827
|
-
return body.map((b) => dehydrateBodyItem(b));
|
|
14828
|
-
}
|
|
14829
|
-
return dehydrateBodyItem(body);
|
|
14830
|
-
}
|
|
14831
|
-
function dehydrateBodyItem(item) {
|
|
14832
|
-
if (item && typeof item === "object" && "source" in item) {
|
|
14833
|
-
const source = item.source;
|
|
14834
|
-
if (typeof source === "string" && source.includes("/")) {
|
|
14835
|
-
return { ...item, source: stripUriToId(source) };
|
|
14836
|
-
}
|
|
14837
|
-
}
|
|
14838
|
-
return item;
|
|
14839
|
-
}
|
|
14840
|
-
function buildBlobResolver2(entries) {
|
|
14841
|
-
const checksumIndex = /* @__PURE__ */ new Map();
|
|
14842
|
-
for (const name of entries.keys()) {
|
|
14843
|
-
if (!name.startsWith(".semiont/")) {
|
|
14844
|
-
const dotIndex = name.lastIndexOf(".");
|
|
14845
|
-
const checksum = dotIndex >= 0 ? name.slice(0, dotIndex) : name;
|
|
14846
|
-
checksumIndex.set(checksum, name);
|
|
14847
|
-
}
|
|
14848
|
-
}
|
|
14849
|
-
return (checksum) => {
|
|
14850
|
-
const entryName = checksumIndex.get(checksum);
|
|
14851
|
-
return entryName ? entries.get(entryName) : void 0;
|
|
14852
|
-
};
|
|
14853
|
-
}
|
|
14854
|
-
async function importLinkedData(archive, options) {
|
|
14855
|
-
const { eventBus, contentStore, userId: userId3, logger } = options;
|
|
14856
|
-
const entries = /* @__PURE__ */ new Map();
|
|
14857
|
-
for await (const entry of readTarGz(archive)) {
|
|
14858
|
-
entries.set(entry.name, entry.data);
|
|
14859
|
-
}
|
|
14860
|
-
const manifestData = entries.get(".semiont/manifest.jsonld");
|
|
14861
|
-
if (!manifestData) {
|
|
14862
|
-
throw new Error("Invalid linked data archive: missing .semiont/manifest.jsonld");
|
|
14863
|
-
}
|
|
14864
|
-
const manifest = JSON.parse(manifestData.toString("utf8"));
|
|
14865
|
-
if (!isLinkedDataManifest(manifest)) {
|
|
14866
|
-
throw new Error(
|
|
14867
|
-
`Invalid linked data archive: expected format "${LINKED_DATA_FORMAT}", got "${manifest["semiont:format"]}"`
|
|
14868
|
-
);
|
|
14869
|
-
}
|
|
14870
|
-
validateManifestVersion(manifest["semiont:version"]);
|
|
14871
|
-
logger?.info("Linked data import: parsed manifest", {
|
|
14872
|
-
entityTypes: manifest["semiont:entityTypes"].length,
|
|
14873
|
-
resources: manifest["void:entities"]
|
|
14874
|
-
});
|
|
14875
|
-
const resolveBlob = buildBlobResolver2(entries);
|
|
14876
|
-
let entityTypesAdded = 0;
|
|
14877
|
-
for (const entityType2 of manifest["semiont:entityTypes"]) {
|
|
14878
|
-
await addEntityType(entityType2, userId3, eventBus, logger);
|
|
14879
|
-
entityTypesAdded++;
|
|
14880
|
-
}
|
|
14881
|
-
const resourceEntries = [...entries.keys()].filter((name) => name.startsWith(".semiont/resources/") && name.endsWith(".jsonld")).sort();
|
|
14882
|
-
let resourcesCreated = 0;
|
|
14883
|
-
let annotationsCreated = 0;
|
|
14884
|
-
for (const entryName of resourceEntries) {
|
|
14885
|
-
const resourceDoc = JSON.parse(entries.get(entryName).toString("utf8"));
|
|
14886
|
-
const result = await importResource(resourceDoc, userId3, eventBus, contentStore, resolveBlob, logger);
|
|
14887
|
-
resourcesCreated++;
|
|
14888
|
-
annotationsCreated += result.annotationsCreated;
|
|
14889
|
-
}
|
|
14890
|
-
logger?.info("Linked data import complete", {
|
|
14891
|
-
resourcesCreated,
|
|
14892
|
-
annotationsCreated,
|
|
14893
|
-
entityTypesAdded
|
|
14894
|
-
});
|
|
14895
|
-
return {
|
|
14896
|
-
manifest,
|
|
14897
|
-
resourcesCreated,
|
|
14898
|
-
annotationsCreated,
|
|
14899
|
-
entityTypesAdded
|
|
14900
|
-
};
|
|
14901
|
-
}
|
|
14902
|
-
async function addEntityType(entityType2, userId3, eventBus, logger) {
|
|
14903
|
-
await busRequest(
|
|
14904
|
-
asBusRequestPrimitive(eventBus),
|
|
14905
|
-
"frame:add-entity-type",
|
|
14906
|
-
{ tag: entityType2, _userId: userId3 },
|
|
14907
|
-
IMPORT_TIMEOUT_MS
|
|
14908
|
-
);
|
|
14909
|
-
logger?.debug("Added entity type", { entityType: entityType2 });
|
|
14910
|
-
}
|
|
14911
|
-
async function importResource(doc, userId3, eventBus, contentStore, resolveBlob, logger) {
|
|
14912
|
-
const name = doc["name"];
|
|
14913
|
-
const representations = doc["representations"];
|
|
14914
|
-
const annotations = doc["annotations"];
|
|
14915
|
-
const entityTypes = doc["entityTypes"];
|
|
14916
|
-
let format = "text/markdown";
|
|
14917
|
-
let language;
|
|
14918
|
-
let contentChecksum;
|
|
14919
|
-
if (representations && representations.length > 0) {
|
|
14920
|
-
const primary = representations[0];
|
|
14921
|
-
if (primary["encodingFormat"]) format = primary["encodingFormat"];
|
|
14922
|
-
if (primary["inLanguage"]) language = primary["inLanguage"];
|
|
14923
|
-
if (primary["sha256"]) contentChecksum = primary["sha256"];
|
|
14924
|
-
}
|
|
14925
|
-
if (!contentChecksum) {
|
|
14926
|
-
throw new Error(`Resource "${name}" has no content checksum in representations`);
|
|
14927
|
-
}
|
|
14928
|
-
const blob = resolveBlob(contentChecksum);
|
|
14929
|
-
if (!blob) {
|
|
14930
|
-
throw new Error(`Missing content blob for checksum ${contentChecksum} (resource "${name}")`);
|
|
14931
|
-
}
|
|
14932
|
-
const base = baseMediaType(format);
|
|
14933
|
-
const resolvedUri = deriveStorageUri(name, isSupportedMediaType(base) ? base : "application/octet-stream");
|
|
14934
|
-
const stored = await contentStore.store(blob, resolvedUri);
|
|
14935
|
-
const { resourceId: createdId } = await busRequest(
|
|
14936
|
-
asBusRequestPrimitive(eventBus),
|
|
14937
|
-
"yield:create",
|
|
14938
|
-
{
|
|
14939
|
-
name,
|
|
14940
|
-
storageUri: resolvedUri,
|
|
14941
|
-
contentChecksum: stored.checksum,
|
|
14942
|
-
byteSize: stored.byteSize,
|
|
14943
|
-
format,
|
|
14944
|
-
_userId: userId3,
|
|
14945
|
-
language,
|
|
14946
|
-
entityTypes: entityTypes ?? []
|
|
14947
|
-
},
|
|
14948
|
-
IMPORT_TIMEOUT_MS
|
|
14949
|
-
);
|
|
14950
|
-
const resourceId10 = resourceId(createdId);
|
|
14951
|
-
logger?.debug("Created resource from JSON-LD", { name, resourceId: resourceId10 });
|
|
14952
|
-
let annotationsCreated = 0;
|
|
14953
|
-
if (annotations && annotations.length > 0) {
|
|
14954
|
-
for (const annotation of annotations) {
|
|
14955
|
-
await createAnnotation(annotation, resourceId10, userId3, eventBus, logger);
|
|
14956
|
-
annotationsCreated++;
|
|
14957
|
-
}
|
|
14958
|
-
}
|
|
14959
|
-
return { annotationsCreated };
|
|
14960
|
-
}
|
|
14961
|
-
async function createAnnotation(annotation, resourceId10, userId3, eventBus, logger) {
|
|
14962
|
-
await awaitReply(
|
|
14963
|
-
eventBus,
|
|
14964
|
-
"mark:create",
|
|
14965
|
-
{
|
|
14966
|
-
annotation: dehydrateAnnotation(annotation),
|
|
14967
|
-
_userId: userId3,
|
|
14968
|
-
resourceId: resourceId10
|
|
14969
|
-
},
|
|
14970
|
-
"mark:create-ok",
|
|
14971
|
-
"mark:create-failed",
|
|
14972
|
-
IMPORT_TIMEOUT_MS
|
|
14973
|
-
);
|
|
14974
|
-
logger?.debug("Created annotation", { annotationId: annotation.id });
|
|
14975
|
-
}
|
|
14976
|
-
var AnnotationOperations = class {
|
|
14977
|
-
/**
|
|
14978
|
-
* Create a new annotation via EventBus → Stower
|
|
14979
|
-
*/
|
|
14980
|
-
static async createAnnotation(request, userId3, creator, eventBus) {
|
|
14981
|
-
const { annotation } = assembleAnnotation(request, creator);
|
|
14982
|
-
const resId = resourceId(request.target.source);
|
|
14983
|
-
eventBus.get("mark:create").next({
|
|
14984
|
-
annotation,
|
|
14985
|
-
_userId: userId3,
|
|
14986
|
-
resourceId: resId
|
|
14987
|
-
});
|
|
14988
|
-
return { annotation };
|
|
14989
|
-
}
|
|
14990
|
-
/**
|
|
14991
|
-
* Update annotation body via EventBus → Stower
|
|
14992
|
-
*/
|
|
14993
|
-
static async updateAnnotationBody(id, request, userId3, eventBus, kb) {
|
|
14994
|
-
const resId = resourceId(request.resourceId);
|
|
14995
|
-
const annotation = await AnnotationContext.getAnnotation(
|
|
14996
|
-
annotationId(id),
|
|
14997
|
-
resId,
|
|
14998
|
-
kb
|
|
14999
|
-
);
|
|
15000
|
-
if (!annotation) {
|
|
15001
|
-
throw new Error("Annotation not found");
|
|
15002
|
-
}
|
|
15003
|
-
eventBus.get("mark:update-body").next({
|
|
15004
|
-
annotationId: annotationId(id),
|
|
15005
|
-
_userId: userId3,
|
|
15006
|
-
resourceId: resId,
|
|
15007
|
-
operations: request.operations
|
|
15008
|
-
});
|
|
15009
|
-
const updatedBody = applyBodyOperations(annotation.body, request.operations);
|
|
15010
|
-
return {
|
|
15011
|
-
annotation: {
|
|
15012
|
-
...annotation,
|
|
15013
|
-
body: updatedBody
|
|
15014
|
-
}
|
|
15015
|
-
};
|
|
15016
|
-
}
|
|
15017
|
-
/**
|
|
15018
|
-
* Delete an annotation via EventBus → Stower
|
|
15019
|
-
*/
|
|
15020
|
-
static async deleteAnnotation(id, resourceIdStr, userId3, eventBus, kb, logger) {
|
|
15021
|
-
const resId = resourceId(resourceIdStr);
|
|
15022
|
-
const projection = await AnnotationContext.getResourceAnnotations(resId, kb);
|
|
15023
|
-
const annotation = projection.annotations.find((a) => a.id === id);
|
|
15024
|
-
if (!annotation) {
|
|
15025
|
-
throw new Error("Annotation not found in resource");
|
|
15026
|
-
}
|
|
15027
|
-
logger?.debug("Removing annotation via EventBus", { annotationId: id });
|
|
15028
|
-
eventBus.get("mark:delete").next({
|
|
15029
|
-
annotationId: annotationId(id),
|
|
15030
|
-
_userId: userId3,
|
|
15031
|
-
resourceId: resId
|
|
15032
|
-
});
|
|
15033
|
-
logger?.debug("Annotation delete event emitted");
|
|
15034
|
-
}
|
|
15035
|
-
};
|
|
15036
14652
|
|
|
15037
|
-
export { AnnotationContext, AnnotationOperations,
|
|
14653
|
+
export { AnnotationContext, AnnotationOperations, Browser, CloneTokenManager, Gatherer, GraphContext, LLMContext, LocalContentTransport, LocalTransport, Matcher, ResourceContext, ResourceOperations, Smelter, Stower, asBusRequestPrimitive, bootstrapEntityTypes, createArchivistServer, createKnowledgeBase, createSmelterActorStateUnit, generateReferenceSuggestions, generateResourceSummary, makeMeaningConfigFrom, readEntityTypesProjection, registerAnnotationAssemblyHandler, registerAnnotationContextHandler, registerBindUpdateBodyHandler, registerBusHandlers, registerGatherSummaryHandler, registerJobCommandHandlers, requireKBName, startMakeMeaning, startMakeMeaningGateway, stopKnowledgeSystem };
|
|
15038
14654
|
//# sourceMappingURL=index.js.map
|
|
15039
14655
|
//# sourceMappingURL=index.js.map
|