@semiont/make-meaning 0.3.7 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +14 -8
- package/dist/index.js +167 -72
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -5,7 +5,7 @@ import { GraphServiceConfig, Logger, StoredEvent, ResourceId, EventBus, componen
|
|
|
5
5
|
export { AssembledAnnotation, applyBodyOperations, assembleAnnotation } from '@semiont/core';
|
|
6
6
|
import { InferenceClient } from '@semiont/inference';
|
|
7
7
|
import { GraphDatabase } from '@semiont/graph';
|
|
8
|
-
import {
|
|
8
|
+
import { WorkingTreeStore } from '@semiont/content';
|
|
9
9
|
import { Writable, Readable } from 'node:stream';
|
|
10
10
|
|
|
11
11
|
/**
|
|
@@ -151,7 +151,7 @@ declare class GraphDBConsumer {
|
|
|
151
151
|
*
|
|
152
152
|
* - Event Log (immutable append-only) — via EventStore
|
|
153
153
|
* - Materialized Views (fast single-doc queries) — via ViewStorage
|
|
154
|
-
* - Content Store (
|
|
154
|
+
* - Content Store (working-tree files, URI-addressed) — via WorkingTreeStore
|
|
155
155
|
* - Graph (eventually consistent relationship projection) — via GraphDatabase
|
|
156
156
|
*
|
|
157
157
|
* The Gatherer and Matcher are the only actors that read from these stores directly.
|
|
@@ -160,8 +160,9 @@ declare class GraphDBConsumer {
|
|
|
160
160
|
interface KnowledgeBase {
|
|
161
161
|
eventStore: EventStore;
|
|
162
162
|
views: ViewStorage;
|
|
163
|
-
content:
|
|
163
|
+
content: WorkingTreeStore;
|
|
164
164
|
graph: GraphDatabase;
|
|
165
|
+
projectionsDir: string;
|
|
165
166
|
}
|
|
166
167
|
declare function createKnowledgeBase(eventStore: EventStore, project: SemiontProject, graphDb: GraphDatabase, logger: Logger): KnowledgeBase;
|
|
167
168
|
|
|
@@ -289,10 +290,12 @@ declare class Matcher {
|
|
|
289
290
|
*
|
|
290
291
|
* Subscriptions:
|
|
291
292
|
* - yield:create → resource.created (+ content store) → yield:created / yield:create-failed
|
|
293
|
+
* - yield:update → resource.updated (+ content store) → yield:updated / yield:update-failed
|
|
294
|
+
* - yield:mv → resource.moved (+ working tree move) → yield:moved / yield:move-failed
|
|
292
295
|
* - mark:create → annotation.added → mark:created / mark:create-failed
|
|
293
296
|
* - mark:delete → annotation.removed → mark:deleted / mark:delete-failed
|
|
294
297
|
* - mark:update-body → annotation.body.updated → (no result event yet)
|
|
295
|
-
* - mark:archive → resource.archived
|
|
298
|
+
* - mark:archive → resource.archived (+ file removal) (resource-scoped, no result event)
|
|
296
299
|
* - mark:unarchive → resource.unarchived (resource-scoped, no result event)
|
|
297
300
|
* - mark:add-entity-type → entitytype.added → mark:entity-type-added / mark:entity-type-add-failed
|
|
298
301
|
* - mark:update-entity-types → entitytag.added / entitytag.removed
|
|
@@ -315,6 +318,8 @@ declare class Stower {
|
|
|
315
318
|
constructor(kb: KnowledgeBase, eventBus: EventBus, logger: Logger);
|
|
316
319
|
initialize(): Promise<void>;
|
|
317
320
|
private handleYieldCreate;
|
|
321
|
+
private handleYieldUpdate;
|
|
322
|
+
private handleYieldMv;
|
|
318
323
|
private handleMarkCreate;
|
|
319
324
|
private handleMarkDelete;
|
|
320
325
|
private handleMarkUpdateBody;
|
|
@@ -487,9 +492,9 @@ interface BackupEventStoreReader {
|
|
|
487
492
|
getEvents(resourceId: ResourceId): Promise<StoredEvent[]>;
|
|
488
493
|
};
|
|
489
494
|
}
|
|
490
|
-
/** Subset of
|
|
495
|
+
/** Subset of WorkingTreeStore used by the backup exporter. */
|
|
491
496
|
interface BackupContentReader {
|
|
492
|
-
retrieve(
|
|
497
|
+
retrieve(storageUri: string): Promise<Buffer>;
|
|
493
498
|
}
|
|
494
499
|
interface BackupExporterOptions {
|
|
495
500
|
eventStore: BackupEventStoreReader;
|
|
@@ -591,9 +596,9 @@ interface LinkedDataViewReader {
|
|
|
591
596
|
};
|
|
592
597
|
}>>;
|
|
593
598
|
}
|
|
594
|
-
/** Subset of
|
|
599
|
+
/** Subset of WorkingTreeStore used by the linked-data exporter. */
|
|
595
600
|
interface LinkedDataContentReader {
|
|
596
|
-
retrieve(
|
|
601
|
+
retrieve(storageUri: string): Promise<Buffer>;
|
|
597
602
|
}
|
|
598
603
|
interface LinkedDataExporterOptions {
|
|
599
604
|
views: LinkedDataViewReader;
|
|
@@ -674,6 +679,7 @@ interface CreateResourceInput {
|
|
|
674
679
|
language?: string;
|
|
675
680
|
entityTypes?: string[];
|
|
676
681
|
creationMethod?: CreationMethod;
|
|
682
|
+
storageUri?: string;
|
|
677
683
|
}
|
|
678
684
|
declare class ResourceOperations {
|
|
679
685
|
/**
|
package/dist/index.js
CHANGED
|
@@ -9945,7 +9945,6 @@ var require_operators = __commonJS({
|
|
|
9945
9945
|
// src/service.ts
|
|
9946
9946
|
import { JobQueue } from "@semiont/jobs";
|
|
9947
9947
|
import { createEventStore as createEventStoreCore } from "@semiont/event-sourcing";
|
|
9948
|
-
import { getPrimaryRepresentation as getPrimaryRepresentation4 } from "@semiont/api-client";
|
|
9949
9948
|
|
|
9950
9949
|
// src/config.ts
|
|
9951
9950
|
function resolveActorInference(config, actor) {
|
|
@@ -10454,14 +10453,14 @@ function resetBootstrap() {
|
|
|
10454
10453
|
|
|
10455
10454
|
// src/knowledge-base.ts
|
|
10456
10455
|
import { FilesystemViewStorage } from "@semiont/event-sourcing";
|
|
10457
|
-
import {
|
|
10456
|
+
import { WorkingTreeStore } from "@semiont/content";
|
|
10458
10457
|
function createKnowledgeBase(eventStore, project, graphDb, logger) {
|
|
10459
10458
|
const views = new FilesystemViewStorage(project);
|
|
10460
|
-
const content = new
|
|
10459
|
+
const content = new WorkingTreeStore(
|
|
10461
10460
|
project,
|
|
10462
|
-
logger.child({ component: "
|
|
10461
|
+
logger.child({ component: "working-tree-store" })
|
|
10463
10462
|
);
|
|
10464
|
-
return { eventStore, views, content, graph: graphDb };
|
|
10463
|
+
return { eventStore, views, content, graph: graphDb, projectionsDir: project.projectionsDir };
|
|
10465
10464
|
}
|
|
10466
10465
|
|
|
10467
10466
|
// src/gatherer.ts
|
|
@@ -10556,10 +10555,10 @@ var ResourceContext = class {
|
|
|
10556
10555
|
return await Promise.all(
|
|
10557
10556
|
resources.map(async (doc) => {
|
|
10558
10557
|
try {
|
|
10559
|
-
|
|
10560
|
-
|
|
10561
|
-
const
|
|
10562
|
-
const contentPreview = decodeRepresentation(contentBuffer, primaryRep
|
|
10558
|
+
if (doc.storageUri) {
|
|
10559
|
+
const contentBuffer = await kb.content.retrieve(doc.storageUri);
|
|
10560
|
+
const primaryRep = getPrimaryRepresentation(doc);
|
|
10561
|
+
const contentPreview = decodeRepresentation(contentBuffer, primaryRep?.mediaType ?? "text/plain").slice(0, 200);
|
|
10563
10562
|
return { ...doc, content: contentPreview };
|
|
10564
10563
|
}
|
|
10565
10564
|
return { ...doc, content: "" };
|
|
@@ -10574,10 +10573,10 @@ var ResourceContext = class {
|
|
|
10574
10573
|
* Retrieves and decodes the primary representation
|
|
10575
10574
|
*/
|
|
10576
10575
|
static async getResourceContent(resource, kb) {
|
|
10577
|
-
|
|
10578
|
-
|
|
10579
|
-
const
|
|
10580
|
-
return decodeRepresentation(contentBuffer, primaryRep
|
|
10576
|
+
if (resource.storageUri) {
|
|
10577
|
+
const contentBuffer = await kb.content.retrieve(resource.storageUri);
|
|
10578
|
+
const primaryRep = getPrimaryRepresentation(resource);
|
|
10579
|
+
return decodeRepresentation(contentBuffer, primaryRep?.mediaType ?? "text/plain");
|
|
10581
10580
|
}
|
|
10582
10581
|
return void 0;
|
|
10583
10582
|
}
|
|
@@ -10644,12 +10643,12 @@ var AnnotationContext = class {
|
|
|
10644
10643
|
}
|
|
10645
10644
|
let sourceContext;
|
|
10646
10645
|
if (includeSourceContext) {
|
|
10647
|
-
|
|
10648
|
-
|
|
10649
|
-
throw new Error("Source content not found");
|
|
10646
|
+
if (!sourceDoc.storageUri) {
|
|
10647
|
+
throw new Error("Source content not found: no storageUri");
|
|
10650
10648
|
}
|
|
10651
|
-
const
|
|
10652
|
-
const
|
|
10649
|
+
const primaryRep = getPrimaryRepresentation2(sourceDoc);
|
|
10650
|
+
const sourceContent = await kb.content.retrieve(sourceDoc.storageUri);
|
|
10651
|
+
const contentStr = decodeRepresentation2(sourceContent, primaryRep?.mediaType ?? "text/plain");
|
|
10653
10652
|
const targetSelectorRaw = getTargetSelector(annotation.target);
|
|
10654
10653
|
const targetSelector = Array.isArray(targetSelectorRaw) ? targetSelectorRaw[0] : targetSelectorRaw;
|
|
10655
10654
|
logger?.debug("Target selector", { type: targetSelector?.type });
|
|
@@ -10685,10 +10684,10 @@ var AnnotationContext = class {
|
|
|
10685
10684
|
}
|
|
10686
10685
|
let targetContext;
|
|
10687
10686
|
if (includeTargetContext && targetDoc) {
|
|
10688
|
-
|
|
10689
|
-
|
|
10690
|
-
const targetContent = await kb.content.retrieve(
|
|
10691
|
-
const contentStr = decodeRepresentation2(targetContent, targetRep
|
|
10687
|
+
if (targetDoc.storageUri) {
|
|
10688
|
+
const targetRep = getPrimaryRepresentation2(targetDoc);
|
|
10689
|
+
const targetContent = await kb.content.retrieve(targetDoc.storageUri);
|
|
10690
|
+
const contentStr = decodeRepresentation2(targetContent, targetRep?.mediaType ?? "text/plain");
|
|
10692
10691
|
targetContext = {
|
|
10693
10692
|
content: contentStr.slice(0, contextWindow * 2),
|
|
10694
10693
|
summary: inferenceClient ? await generateResourceSummary(targetDoc.name, contentStr, getResourceEntityTypes(targetDoc), inferenceClient) : void 0
|
|
@@ -10989,13 +10988,13 @@ Summary:`;
|
|
|
10989
10988
|
/**
|
|
10990
10989
|
* Get resource content as string
|
|
10991
10990
|
*/
|
|
10992
|
-
static async getResourceContent(resource,
|
|
10993
|
-
|
|
10994
|
-
|
|
10995
|
-
throw new Error("Resource content not found");
|
|
10991
|
+
static async getResourceContent(resource, content) {
|
|
10992
|
+
if (!resource.storageUri) {
|
|
10993
|
+
throw new Error("Resource content not found: no storageUri");
|
|
10996
10994
|
}
|
|
10997
|
-
const
|
|
10998
|
-
|
|
10995
|
+
const primaryRep = getPrimaryRepresentation2(resource);
|
|
10996
|
+
const buf = await content.retrieve(resource.storageUri);
|
|
10997
|
+
return decodeRepresentation2(buf, primaryRep?.mediaType ?? "text/plain");
|
|
10999
10998
|
}
|
|
11000
10999
|
/**
|
|
11001
11000
|
* Extract annotation context from resource content
|
|
@@ -11882,7 +11881,10 @@ No explanations.`;
|
|
|
11882
11881
|
// src/stower.ts
|
|
11883
11882
|
var import_rxjs5 = __toESM(require_cjs(), 1);
|
|
11884
11883
|
var import_operators5 = __toESM(require_operators(), 1);
|
|
11884
|
+
import { promises as fs3 } from "fs";
|
|
11885
11885
|
import { resourceId as resourceId3, annotationId as makeAnnotationId3, CREATION_METHODS, generateUuid } from "@semiont/core";
|
|
11886
|
+
import { resolveStorageUri } from "@semiont/event-sourcing";
|
|
11887
|
+
import { getExtensionForMimeType } from "@semiont/content";
|
|
11886
11888
|
var Stower = class {
|
|
11887
11889
|
constructor(kb, eventBus, logger) {
|
|
11888
11890
|
this.kb = kb;
|
|
@@ -11896,6 +11898,8 @@ var Stower = class {
|
|
|
11896
11898
|
const pipe = (event, handler) => this.eventBus.get(event).pipe((0, import_operators5.concatMap)((e) => (0, import_rxjs5.from)(handler(e))));
|
|
11897
11899
|
this.subscription = (0, import_rxjs5.merge)(
|
|
11898
11900
|
pipe("yield:create", (e) => this.handleYieldCreate(e)),
|
|
11901
|
+
pipe("yield:update", (e) => this.handleYieldUpdate(e)),
|
|
11902
|
+
pipe("yield:mv", (e) => this.handleYieldMv(e)),
|
|
11899
11903
|
pipe("mark:create", (e) => this.handleMarkCreate(e)),
|
|
11900
11904
|
pipe("mark:delete", (e) => this.handleMarkDelete(e)),
|
|
11901
11905
|
pipe("mark:update-body", (e) => this.handleMarkUpdateBody(e)),
|
|
@@ -11917,11 +11921,21 @@ var Stower = class {
|
|
|
11917
11921
|
async handleYieldCreate(event) {
|
|
11918
11922
|
try {
|
|
11919
11923
|
const rId = resourceId3(generateUuid());
|
|
11920
|
-
|
|
11921
|
-
|
|
11922
|
-
|
|
11923
|
-
|
|
11924
|
-
|
|
11924
|
+
let checksum;
|
|
11925
|
+
let byteSize;
|
|
11926
|
+
const resolvedStorageUri = event.storageUri ?? deriveStorageUri(event.name, event.format);
|
|
11927
|
+
if (event.content !== void 0) {
|
|
11928
|
+
const stored = await this.kb.content.store(event.content, resolvedStorageUri, { noGit: event.noGit });
|
|
11929
|
+
checksum = stored.checksum;
|
|
11930
|
+
byteSize = stored.byteSize;
|
|
11931
|
+
} else {
|
|
11932
|
+
if (!event.storageUri) {
|
|
11933
|
+
throw new Error("yield:create without content requires storageUri");
|
|
11934
|
+
}
|
|
11935
|
+
const stored = await this.kb.content.register(resolvedStorageUri, event.contentChecksum, { noGit: event.noGit });
|
|
11936
|
+
checksum = stored.checksum;
|
|
11937
|
+
byteSize = stored.byteSize;
|
|
11938
|
+
}
|
|
11925
11939
|
const validCreationMethods = Object.values(CREATION_METHODS);
|
|
11926
11940
|
const validatedCreationMethod = event.creationMethod && validCreationMethods.includes(event.creationMethod) ? event.creationMethod : CREATION_METHODS.API;
|
|
11927
11941
|
await this.kb.eventStore.appendEvent({
|
|
@@ -11932,8 +11946,9 @@ var Stower = class {
|
|
|
11932
11946
|
payload: {
|
|
11933
11947
|
name: event.name,
|
|
11934
11948
|
format: event.format,
|
|
11935
|
-
contentChecksum:
|
|
11936
|
-
contentByteSize:
|
|
11949
|
+
contentChecksum: checksum,
|
|
11950
|
+
contentByteSize: byteSize,
|
|
11951
|
+
storageUri: resolvedStorageUri,
|
|
11937
11952
|
creationMethod: validatedCreationMethod,
|
|
11938
11953
|
entityTypes: event.entityTypes || [],
|
|
11939
11954
|
language: event.language || void 0,
|
|
@@ -11949,14 +11964,16 @@ var Stower = class {
|
|
|
11949
11964
|
archived: false,
|
|
11950
11965
|
entityTypes: event.entityTypes || [],
|
|
11951
11966
|
creationMethod: validatedCreationMethod,
|
|
11967
|
+
storageUri: event.storageUri,
|
|
11968
|
+
currentChecksum: checksum,
|
|
11952
11969
|
dateCreated: (/* @__PURE__ */ new Date()).toISOString(),
|
|
11953
11970
|
representations: [
|
|
11954
11971
|
{
|
|
11955
|
-
mediaType:
|
|
11956
|
-
checksum
|
|
11957
|
-
byteSize
|
|
11972
|
+
mediaType: event.format,
|
|
11973
|
+
checksum,
|
|
11974
|
+
byteSize,
|
|
11958
11975
|
rel: "original",
|
|
11959
|
-
language:
|
|
11976
|
+
language: event.language
|
|
11960
11977
|
}
|
|
11961
11978
|
]
|
|
11962
11979
|
};
|
|
@@ -11968,6 +11985,69 @@ var Stower = class {
|
|
|
11968
11985
|
});
|
|
11969
11986
|
}
|
|
11970
11987
|
}
|
|
11988
|
+
async handleYieldUpdate(event) {
|
|
11989
|
+
try {
|
|
11990
|
+
let byteSize;
|
|
11991
|
+
if (event.content) {
|
|
11992
|
+
const stored = await this.kb.content.store(event.content, event.storageUri, { noGit: event.noGit });
|
|
11993
|
+
byteSize = stored.byteSize;
|
|
11994
|
+
} else {
|
|
11995
|
+
const stored = await this.kb.content.register(event.storageUri, event.contentChecksum, { noGit: event.noGit });
|
|
11996
|
+
byteSize = stored.byteSize;
|
|
11997
|
+
}
|
|
11998
|
+
await this.kb.eventStore.appendEvent({
|
|
11999
|
+
type: "resource.updated",
|
|
12000
|
+
resourceId: event.resourceId,
|
|
12001
|
+
userId: event.userId,
|
|
12002
|
+
version: 1,
|
|
12003
|
+
payload: {
|
|
12004
|
+
contentChecksum: event.contentChecksum,
|
|
12005
|
+
contentByteSize: byteSize
|
|
12006
|
+
}
|
|
12007
|
+
});
|
|
12008
|
+
this.eventBus.get("yield:updated").next({ resourceId: event.resourceId });
|
|
12009
|
+
} catch (error) {
|
|
12010
|
+
this.logger.error("Failed to update resource", { error });
|
|
12011
|
+
this.eventBus.get("yield:update-failed").next({
|
|
12012
|
+
resourceId: event.resourceId,
|
|
12013
|
+
error: error instanceof Error ? error : new Error(String(error))
|
|
12014
|
+
});
|
|
12015
|
+
}
|
|
12016
|
+
}
|
|
12017
|
+
async handleYieldMv(event) {
|
|
12018
|
+
let rId;
|
|
12019
|
+
try {
|
|
12020
|
+
const resolved = await resolveStorageUri(this.kb.projectionsDir, event.fromUri);
|
|
12021
|
+
rId = resolved;
|
|
12022
|
+
} catch (error) {
|
|
12023
|
+
this.logger.error("Failed to resolve resource for move", { fromUri: event.fromUri, error });
|
|
12024
|
+
this.eventBus.get("yield:move-failed").next({
|
|
12025
|
+
fromUri: event.fromUri,
|
|
12026
|
+
error: error instanceof Error ? error : new Error(String(error))
|
|
12027
|
+
});
|
|
12028
|
+
return;
|
|
12029
|
+
}
|
|
12030
|
+
try {
|
|
12031
|
+
await this.kb.content.move(event.fromUri, event.toUri, { noGit: event.noGit });
|
|
12032
|
+
await this.kb.eventStore.appendEvent({
|
|
12033
|
+
type: "resource.moved",
|
|
12034
|
+
resourceId: rId,
|
|
12035
|
+
userId: event.userId,
|
|
12036
|
+
version: 1,
|
|
12037
|
+
payload: {
|
|
12038
|
+
fromUri: event.fromUri,
|
|
12039
|
+
toUri: event.toUri
|
|
12040
|
+
}
|
|
12041
|
+
});
|
|
12042
|
+
this.eventBus.get("yield:moved").next({ resourceId: rId });
|
|
12043
|
+
} catch (error) {
|
|
12044
|
+
this.logger.error("Failed to move resource", { error });
|
|
12045
|
+
this.eventBus.get("yield:move-failed").next({
|
|
12046
|
+
fromUri: event.fromUri,
|
|
12047
|
+
error: error instanceof Error ? error : new Error(String(error))
|
|
12048
|
+
});
|
|
12049
|
+
}
|
|
12050
|
+
}
|
|
11971
12051
|
async handleMarkCreate(event) {
|
|
11972
12052
|
try {
|
|
11973
12053
|
this.logger.debug("Stowing annotation", { annotationId: event.annotation.id });
|
|
@@ -12027,6 +12107,9 @@ var Stower = class {
|
|
|
12027
12107
|
if (!event || typeof event !== "object" || !("userId" in event) || !("resourceId" in event) || !event.resourceId) {
|
|
12028
12108
|
return;
|
|
12029
12109
|
}
|
|
12110
|
+
if (event.storageUri) {
|
|
12111
|
+
await this.kb.content.remove(event.storageUri, { keepFile: event.keepFile, noGit: event.noGit });
|
|
12112
|
+
}
|
|
12030
12113
|
await this.kb.eventStore.appendEvent({
|
|
12031
12114
|
type: "resource.archived",
|
|
12032
12115
|
resourceId: event.resourceId,
|
|
@@ -12039,6 +12122,15 @@ var Stower = class {
|
|
|
12039
12122
|
if (!event || typeof event !== "object" || !("userId" in event) || !("resourceId" in event) || !event.resourceId) {
|
|
12040
12123
|
return;
|
|
12041
12124
|
}
|
|
12125
|
+
if (event.storageUri) {
|
|
12126
|
+
const absPath = this.kb.content.resolveUri(event.storageUri);
|
|
12127
|
+
try {
|
|
12128
|
+
await fs3.access(absPath);
|
|
12129
|
+
} catch {
|
|
12130
|
+
this.logger.warn("Unarchive failed: file not found at storageUri", { storageUri: event.storageUri });
|
|
12131
|
+
return;
|
|
12132
|
+
}
|
|
12133
|
+
}
|
|
12042
12134
|
await this.kb.eventStore.appendEvent({
|
|
12043
12135
|
type: "resource.unarchived",
|
|
12044
12136
|
resourceId: event.resourceId,
|
|
@@ -12140,6 +12232,11 @@ var Stower = class {
|
|
|
12140
12232
|
this.logger.info("Stower actor stopped");
|
|
12141
12233
|
}
|
|
12142
12234
|
};
|
|
12235
|
+
function deriveStorageUri(name, format) {
|
|
12236
|
+
const slug = name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
|
|
12237
|
+
const ext = getExtensionForMimeType(format);
|
|
12238
|
+
return `file://${slug}${ext}`;
|
|
12239
|
+
}
|
|
12143
12240
|
|
|
12144
12241
|
// src/clone-token-manager.ts
|
|
12145
12242
|
var import_rxjs7 = __toESM(require_cjs(), 1);
|
|
@@ -12175,7 +12272,8 @@ var ResourceOperations = class {
|
|
|
12175
12272
|
userId: userId2,
|
|
12176
12273
|
language: input.language,
|
|
12177
12274
|
entityTypes: input.entityTypes,
|
|
12178
|
-
creationMethod: input.creationMethod
|
|
12275
|
+
creationMethod: input.creationMethod,
|
|
12276
|
+
storageUri: input.storageUri
|
|
12179
12277
|
});
|
|
12180
12278
|
const outcome = await (0, import_rxjs6.firstValueFrom)(result$);
|
|
12181
12279
|
if (!outcome.ok) {
|
|
@@ -12243,8 +12341,7 @@ var CloneTokenManager = class {
|
|
|
12243
12341
|
});
|
|
12244
12342
|
return;
|
|
12245
12343
|
}
|
|
12246
|
-
|
|
12247
|
-
if (!primaryRep?.checksum || !primaryRep?.mediaType) {
|
|
12344
|
+
if (!resource.storageUri) {
|
|
12248
12345
|
this.eventBus.get("yield:clone-token-failed").next({
|
|
12249
12346
|
correlationId: event.correlationId,
|
|
12250
12347
|
error: new Error("Resource content not found")
|
|
@@ -12252,7 +12349,7 @@ var CloneTokenManager = class {
|
|
|
12252
12349
|
return;
|
|
12253
12350
|
}
|
|
12254
12351
|
try {
|
|
12255
|
-
await this.kb.content.retrieve(
|
|
12352
|
+
await this.kb.content.retrieve(resource.storageUri);
|
|
12256
12353
|
} catch {
|
|
12257
12354
|
this.eventBus.get("yield:clone-token-failed").next({
|
|
12258
12355
|
correlationId: event.correlationId,
|
|
@@ -12444,7 +12541,7 @@ async function startMakeMeaning(project, config, eventBus, logger) {
|
|
|
12444
12541
|
error: (err) => jobQueueLogger.error("Job status pipeline error", { error: err })
|
|
12445
12542
|
});
|
|
12446
12543
|
const eventStoreLogger = logger.child({ component: "event-store" });
|
|
12447
|
-
const eventStore = createEventStoreCore(project,
|
|
12544
|
+
const eventStore = createEventStoreCore(project, eventBus, eventStoreLogger);
|
|
12448
12545
|
const gathererInferenceClient = createInferenceClient(
|
|
12449
12546
|
resolveActorInference(config, "gatherer"),
|
|
12450
12547
|
logger.child({ component: "inference-client-gatherer" })
|
|
@@ -12493,10 +12590,8 @@ async function startMakeMeaning(project, config, eventBus, logger) {
|
|
|
12493
12590
|
await cloneTokenManager.initialize();
|
|
12494
12591
|
const contentFetcher = async (resourceId4) => {
|
|
12495
12592
|
const view = await kb.views.get(resourceId4);
|
|
12496
|
-
if (!view) return null;
|
|
12497
|
-
const
|
|
12498
|
-
if (!primaryRep?.checksum || !primaryRep?.mediaType) return null;
|
|
12499
|
-
const buffer = await kb.content.retrieve(primaryRep.checksum, primaryRep.mediaType);
|
|
12593
|
+
if (!view?.resource.storageUri) return null;
|
|
12594
|
+
const buffer = await kb.content.retrieve(view.resource.storageUri);
|
|
12500
12595
|
if (!buffer) return null;
|
|
12501
12596
|
return Readable.from([buffer]);
|
|
12502
12597
|
};
|
|
@@ -12567,7 +12662,7 @@ async function startMakeMeaning(project, config, eventBus, logger) {
|
|
|
12567
12662
|
}
|
|
12568
12663
|
|
|
12569
12664
|
// src/exchange/backup-exporter.ts
|
|
12570
|
-
import { getExtensionForMimeType } from "@semiont/content";
|
|
12665
|
+
import { getExtensionForMimeType as getExtensionForMimeType2 } from "@semiont/content";
|
|
12571
12666
|
|
|
12572
12667
|
// src/exchange/tar.ts
|
|
12573
12668
|
import { createGzip, createGunzip } from "zlib";
|
|
@@ -12691,10 +12786,10 @@ async function exportBackup(options, output) {
|
|
|
12691
12786
|
});
|
|
12692
12787
|
const contentBlobs = /* @__PURE__ */ new Map();
|
|
12693
12788
|
let totalContentBytes = 0;
|
|
12694
|
-
for (const [
|
|
12695
|
-
const data = await content.retrieve(
|
|
12696
|
-
const ext =
|
|
12697
|
-
contentBlobs.set(
|
|
12789
|
+
for (const [storageUri, mediaType] of contentRefs) {
|
|
12790
|
+
const data = await content.retrieve(storageUri);
|
|
12791
|
+
const ext = getExtensionForMimeType2(mediaType);
|
|
12792
|
+
contentBlobs.set(storageUri, { data, ext });
|
|
12698
12793
|
totalContentBytes += data.length;
|
|
12699
12794
|
}
|
|
12700
12795
|
const streamSummaries = [];
|
|
@@ -12729,8 +12824,9 @@ async function exportBackup(options, output) {
|
|
|
12729
12824
|
const jsonl = events.map((e) => JSON.stringify(e)).join("\n") + "\n";
|
|
12730
12825
|
yield { name: fileName, data: Buffer.from(jsonl, "utf8") };
|
|
12731
12826
|
}
|
|
12732
|
-
for (const [
|
|
12733
|
-
|
|
12827
|
+
for (const [storageUri, { data, ext }] of contentBlobs) {
|
|
12828
|
+
const blobPath = storageUri.startsWith("file://") ? storageUri.slice(7) : storageUri;
|
|
12829
|
+
yield { name: `${blobPath}${ext}`, data };
|
|
12734
12830
|
}
|
|
12735
12831
|
}
|
|
12736
12832
|
await writeTarGz(generateEntries(), output);
|
|
@@ -12748,8 +12844,8 @@ function collectContentRefs(streamData) {
|
|
|
12748
12844
|
for (const stored of events) {
|
|
12749
12845
|
if (stored.event.type === "resource.created") {
|
|
12750
12846
|
const payload = stored.event.payload;
|
|
12751
|
-
if (payload.
|
|
12752
|
-
refs.set(payload.
|
|
12847
|
+
if (payload.storageUri && payload.format) {
|
|
12848
|
+
refs.set(payload.storageUri, payload.format);
|
|
12753
12849
|
}
|
|
12754
12850
|
}
|
|
12755
12851
|
}
|
|
@@ -13054,7 +13150,7 @@ function mergeStats(a, b) {
|
|
|
13054
13150
|
}
|
|
13055
13151
|
|
|
13056
13152
|
// src/exchange/linked-data-exporter.ts
|
|
13057
|
-
import { getExtensionForMimeType as
|
|
13153
|
+
import { getExtensionForMimeType as getExtensionForMimeType3 } from "@semiont/content";
|
|
13058
13154
|
var SEMIONT_CONTEXT = [
|
|
13059
13155
|
"https://schema.org/",
|
|
13060
13156
|
"http://www.w3.org/ns/anno.jsonld",
|
|
@@ -13118,13 +13214,13 @@ async function exportLinkedData(options, output) {
|
|
|
13118
13214
|
collectContentRefsFromResource(view.resource, contentRefs);
|
|
13119
13215
|
}
|
|
13120
13216
|
const contentBlobs = /* @__PURE__ */ new Map();
|
|
13121
|
-
for (const [
|
|
13217
|
+
for (const [storageUri, mediaType] of contentRefs) {
|
|
13122
13218
|
try {
|
|
13123
|
-
const data = await content.retrieve(
|
|
13124
|
-
const ext =
|
|
13125
|
-
contentBlobs.set(
|
|
13219
|
+
const data = await content.retrieve(storageUri);
|
|
13220
|
+
const ext = getExtensionForMimeType3(mediaType);
|
|
13221
|
+
contentBlobs.set(storageUri, { data, ext });
|
|
13126
13222
|
} catch (err) {
|
|
13127
|
-
logger?.warn("Failed to retrieve content blob", {
|
|
13223
|
+
logger?.warn("Failed to retrieve content blob", { storageUri, error: String(err) });
|
|
13128
13224
|
}
|
|
13129
13225
|
}
|
|
13130
13226
|
const manifest = {
|
|
@@ -13153,8 +13249,9 @@ async function exportLinkedData(options, output) {
|
|
|
13153
13249
|
data: Buffer.from(JSON.stringify(jsonld, null, 2), "utf8")
|
|
13154
13250
|
};
|
|
13155
13251
|
}
|
|
13156
|
-
for (const [
|
|
13157
|
-
|
|
13252
|
+
for (const [storageUri, { data, ext }] of contentBlobs) {
|
|
13253
|
+
const blobPath = storageUri.startsWith("file://") ? storageUri.slice(7) : storageUri;
|
|
13254
|
+
yield { name: `${blobPath}${ext}`, data };
|
|
13158
13255
|
}
|
|
13159
13256
|
}
|
|
13160
13257
|
await writeTarGz(generateEntries(), output);
|
|
@@ -13200,7 +13297,7 @@ function buildResourceJsonLd(resource, annotations, sourceUrl) {
|
|
|
13200
13297
|
if (rep.checksum) {
|
|
13201
13298
|
const rawChecksum = rep.checksum.startsWith("sha256:") ? rep.checksum.slice(7) : rep.checksum;
|
|
13202
13299
|
mediaObj["sha256"] = rawChecksum;
|
|
13203
|
-
const ext =
|
|
13300
|
+
const ext = getExtensionForMimeType3(rep.mediaType);
|
|
13204
13301
|
mediaObj["name"] = `${rawChecksum}${ext}`;
|
|
13205
13302
|
}
|
|
13206
13303
|
if (rep.language) mediaObj["inLanguage"] = rep.language;
|
|
@@ -13218,12 +13315,10 @@ function normalizeRepresentations(reps) {
|
|
|
13218
13315
|
return [reps];
|
|
13219
13316
|
}
|
|
13220
13317
|
function collectContentRefsFromResource(resource, refs) {
|
|
13221
|
-
|
|
13222
|
-
|
|
13223
|
-
|
|
13224
|
-
|
|
13225
|
-
refs.set(rawChecksum, rep.mediaType);
|
|
13226
|
-
}
|
|
13318
|
+
if (resource.storageUri) {
|
|
13319
|
+
const reps = normalizeRepresentations(resource.representations);
|
|
13320
|
+
const mediaType = reps[0]?.mediaType ?? "application/octet-stream";
|
|
13321
|
+
refs.set(resource.storageUri, mediaType);
|
|
13227
13322
|
}
|
|
13228
13323
|
}
|
|
13229
13324
|
|