@semiont/make-meaning 0.5.29 → 0.5.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -10
- package/dist/archivist-main.js +236 -131
- package/dist/archivist-main.js.map +1 -1
- package/dist/index.d.ts +19 -0
- package/dist/index.js +100 -4
- package/dist/index.js.map +1 -1
- package/dist/librarian-main.js +119 -50
- package/dist/librarian-main.js.map +1 -1
- package/dist/smelter-main.js +125 -25
- package/dist/smelter-main.js.map +1 -1
- package/dist/weaver-main.js +130 -28
- package/dist/weaver-main.js.map +1 -1
- package/package.json +12 -12
package/dist/smelter-main.js
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import { archivistContentReads, createAnchoredTextStore, EXTRACTORS, calculateChecksum } from '@semiont/content';
|
|
2
|
-
import { createTomlConfigLoader, accessToken, baseUrl as baseUrl$1, retryWithBackoff, isTransientFetchError, STARTUP_FETCH_RETRY, burstBuffer, errField, resourceId, textExtractionOf, busRequest, getResourceEntityTypes, chunkText, getTargetSelector, getExactText, annotationId, getPrimaryMediaType, getPrimaryRepresentation } from '@semiont/core';
|
|
2
|
+
import { replyChannelsFor, createTomlConfigLoader, accessToken, baseUrl as baseUrl$1, retryWithBackoff, isTransientFetchError, STARTUP_FETCH_RETRY, burstBuffer, errField, resourceId, textExtractionOf, busRequest, getResourceEntityTypes, chunkText, getTargetSelector, getExactText, annotationId, getPrimaryMediaType, getPrimaryRepresentation } from '@semiont/core';
|
|
3
3
|
import { registerVectorIndexSizeProvider, withActorSpan } from '@semiont/observability';
|
|
4
|
-
import
|
|
4
|
+
import '@semiont/ontology';
|
|
5
|
+
import '@semiont/graph';
|
|
5
6
|
import { createEmbeddingProvider, createVectorStore } from '@semiont/vectors';
|
|
7
|
+
import '@semiont/event-sourcing';
|
|
8
|
+
import { HttpTransport } from '@semiont/http-transport';
|
|
6
9
|
import { createServer } from 'http';
|
|
7
10
|
import { existsSync, readFileSync } from 'fs';
|
|
8
11
|
import { homedir } from 'os';
|
|
@@ -10092,14 +10095,14 @@ var Smelter = class _Smelter {
|
|
|
10092
10095
|
* null (logged) when the resource doesn't decode as text, is unavailable,
|
|
10093
10096
|
* or is empty — callers skip it.
|
|
10094
10097
|
*/
|
|
10095
|
-
async fetchEmbeddableText(
|
|
10098
|
+
async fetchEmbeddableText(resourceId6) {
|
|
10096
10099
|
try {
|
|
10097
|
-
const { data, contentType } = await this.content.getBinary(resourceId(
|
|
10100
|
+
const { data, contentType } = await this.content.getBinary(resourceId(resourceId6));
|
|
10098
10101
|
const bytes = Buffer.from(data);
|
|
10099
10102
|
const checksum = calculateChecksum(bytes);
|
|
10100
10103
|
const extractor = EXTRACTORS[textExtractionOf(contentType)];
|
|
10101
10104
|
if (!extractor) {
|
|
10102
|
-
this.logger.debug("Skipping resource with no extractor for its media type", { resourceId:
|
|
10105
|
+
this.logger.debug("Skipping resource with no extractor for its media type", { resourceId: resourceId6, contentType });
|
|
10103
10106
|
return { kind: "skipped", checksum, reason: "no-extractor" };
|
|
10104
10107
|
}
|
|
10105
10108
|
const extracted = await extractor.extract(bytes, contentType, {
|
|
@@ -10107,26 +10110,26 @@ var Smelter = class _Smelter {
|
|
|
10107
10110
|
store: this.anchoredStore
|
|
10108
10111
|
});
|
|
10109
10112
|
if (extracted.kind === "declined") {
|
|
10110
|
-
this.logger.debug("Extractor declined", { resourceId:
|
|
10113
|
+
this.logger.debug("Extractor declined", { resourceId: resourceId6, contentType, reason: extracted.declined });
|
|
10111
10114
|
return { kind: "skipped", checksum, reason: extracted.declined };
|
|
10112
10115
|
}
|
|
10113
10116
|
if (extracted.ocrConfidence && extracted.ocrConfidence.lowConfidenceWords > 0) {
|
|
10114
10117
|
this.logger.info("OCR read words it was unsure of", {
|
|
10115
|
-
resourceId:
|
|
10118
|
+
resourceId: resourceId6,
|
|
10116
10119
|
contentType,
|
|
10117
10120
|
...extracted.ocrConfidence
|
|
10118
10121
|
});
|
|
10119
10122
|
}
|
|
10120
10123
|
if (extracted.unreadPages?.length) {
|
|
10121
10124
|
this.logger.info("Partial extraction coverage", {
|
|
10122
|
-
resourceId:
|
|
10125
|
+
resourceId: resourceId6,
|
|
10123
10126
|
contentType,
|
|
10124
10127
|
unreadPages: extracted.unreadPages
|
|
10125
10128
|
});
|
|
10126
10129
|
}
|
|
10127
10130
|
return extracted.text.trim() ? { kind: "text", text: extracted.text, checksum, machineRead: extracted.method === "ocr" } : { kind: "skipped", checksum, reason: "empty" };
|
|
10128
10131
|
} catch (error) {
|
|
10129
|
-
this.logger.warn("Content unavailable for embedding", { resourceId:
|
|
10132
|
+
this.logger.warn("Content unavailable for embedding", { resourceId: resourceId6, error: errField(error) });
|
|
10130
10133
|
return { kind: "unavailable" };
|
|
10131
10134
|
}
|
|
10132
10135
|
}
|
|
@@ -10136,11 +10139,11 @@ var Smelter = class _Smelter {
|
|
|
10136
10139
|
* fold. Best-effort — waiters degrade to their bounded timeout; a signal
|
|
10137
10140
|
* failure must never fail the embed.
|
|
10138
10141
|
*/
|
|
10139
|
-
async emitSettled(
|
|
10142
|
+
async emitSettled(resourceId6, contentChecksum, outcome, reason) {
|
|
10140
10143
|
try {
|
|
10141
|
-
await this.bus.emit("smelt:settled", { resourceId, contentChecksum, outcome, ...reason ? { reason } : {} });
|
|
10144
|
+
await this.bus.emit("smelt:settled", { resourceId: resourceId6, contentChecksum, outcome, ...reason ? { reason } : {} });
|
|
10142
10145
|
} catch (error) {
|
|
10143
|
-
this.logger.warn("Failed to emit smelt:settled", { resourceId, outcome, error: errField(error) });
|
|
10146
|
+
this.logger.warn("Failed to emit smelt:settled", { resourceId: resourceId6, outcome, error: errField(error) });
|
|
10144
10147
|
}
|
|
10145
10148
|
}
|
|
10146
10149
|
/**
|
|
@@ -10152,11 +10155,11 @@ var Smelter = class _Smelter {
|
|
|
10152
10155
|
* propagates to the pipeline's per-resource error handler (reconcile heals),
|
|
10153
10156
|
* rather than silently stamping `[]` and letting the resource leak into recall.
|
|
10154
10157
|
*/
|
|
10155
|
-
async resolveEntityTypes(
|
|
10158
|
+
async resolveEntityTypes(resourceId6) {
|
|
10156
10159
|
const { resource } = await busRequest(
|
|
10157
10160
|
this.bus,
|
|
10158
10161
|
"browse:resource-requested",
|
|
10159
|
-
{ resourceId }
|
|
10162
|
+
{ resourceId: resourceId6 }
|
|
10160
10163
|
);
|
|
10161
10164
|
return getResourceEntityTypes(resource);
|
|
10162
10165
|
}
|
|
@@ -10240,11 +10243,11 @@ var Smelter = class _Smelter {
|
|
|
10240
10243
|
this.logger.info("Indexed annotation", { annotationId: String(aid) });
|
|
10241
10244
|
}
|
|
10242
10245
|
async handleAnnotationRemoved(event) {
|
|
10243
|
-
const
|
|
10244
|
-
if (!
|
|
10245
|
-
const aid = annotationId(
|
|
10246
|
+
const annotationId3 = event.payload.annotationId;
|
|
10247
|
+
if (!annotationId3) return;
|
|
10248
|
+
const aid = annotationId(annotationId3);
|
|
10246
10249
|
await this.vectorStore.deleteAnnotationVector(aid);
|
|
10247
|
-
this.logger.info("Deleted annotation vector", { annotationId:
|
|
10250
|
+
this.logger.info("Deleted annotation vector", { annotationId: annotationId3 });
|
|
10248
10251
|
}
|
|
10249
10252
|
/**
|
|
10250
10253
|
* Batch-embed chunks from multiple yield:created events in a single
|
|
@@ -10483,16 +10486,16 @@ var Smelter = class _Smelter {
|
|
|
10483
10486
|
* resource (S1/S2).
|
|
10484
10487
|
*/
|
|
10485
10488
|
async rebuildAnchors(command) {
|
|
10486
|
-
const { correlationId, resourceId } = command;
|
|
10489
|
+
const { correlationId, resourceId: resourceId6 } = command;
|
|
10487
10490
|
try {
|
|
10488
10491
|
let work;
|
|
10489
|
-
if (
|
|
10490
|
-
work = [{ type: "smelt:reanchor", resourceId, payload: {} }];
|
|
10492
|
+
if (resourceId6) {
|
|
10493
|
+
work = [{ type: "smelt:reanchor", resourceId: resourceId6, payload: {} }];
|
|
10491
10494
|
} else {
|
|
10492
10495
|
const resources = await this.listAllResources();
|
|
10493
10496
|
work = [...this.classifyEmbeddable(resources)].filter(([, catalog]) => catalog.yieldsGeometry).map(([rid]) => ({ type: "smelt:reanchor", resourceId: rid, payload: {} }));
|
|
10494
10497
|
}
|
|
10495
|
-
this.logger.info("Anchored-text rebuild started", { scoped:
|
|
10498
|
+
this.logger.info("Anchored-text rebuild started", { scoped: resourceId6 ?? null, resources: work.length });
|
|
10496
10499
|
const failed = await this.drain(work);
|
|
10497
10500
|
if (failed > 0) {
|
|
10498
10501
|
await this.bus.emit("smelt:rebuild-anchors-failed", {
|
|
@@ -10502,7 +10505,7 @@ var Smelter = class _Smelter {
|
|
|
10502
10505
|
return;
|
|
10503
10506
|
}
|
|
10504
10507
|
await this.bus.emit("smelt:rebuild-anchors-ok", correlationId ? { correlationId } : {});
|
|
10505
|
-
this.logger.info("Anchored-text rebuild complete", { scoped:
|
|
10508
|
+
this.logger.info("Anchored-text rebuild complete", { scoped: resourceId6 ?? null, resources: work.length });
|
|
10506
10509
|
} catch (error) {
|
|
10507
10510
|
this.logger.error("Anchored-text rebuild failed", { error: errField(error) });
|
|
10508
10511
|
try {
|
|
@@ -10555,6 +10558,98 @@ var Smelter = class _Smelter {
|
|
|
10555
10558
|
}
|
|
10556
10559
|
}
|
|
10557
10560
|
};
|
|
10561
|
+
var MATCHER_CHANNELS = [
|
|
10562
|
+
"match:search-requested"
|
|
10563
|
+
];
|
|
10564
|
+
|
|
10565
|
+
// src/gatherer.ts
|
|
10566
|
+
var GATHERER_CHANNELS = [
|
|
10567
|
+
"gather:requested",
|
|
10568
|
+
"gather:resource-requested"
|
|
10569
|
+
];
|
|
10570
|
+
var STOWER_CHANNELS = [
|
|
10571
|
+
"yield:create",
|
|
10572
|
+
"yield:clone-persist",
|
|
10573
|
+
"yield:update",
|
|
10574
|
+
"yield:mv",
|
|
10575
|
+
"mark:create",
|
|
10576
|
+
"mark:commit",
|
|
10577
|
+
"mark:delete",
|
|
10578
|
+
"mark:update-body",
|
|
10579
|
+
"frame:add-entity-type",
|
|
10580
|
+
"frame:add-tag-schema",
|
|
10581
|
+
"mark:archive",
|
|
10582
|
+
"mark:unarchive",
|
|
10583
|
+
"mark:update-entity-types",
|
|
10584
|
+
"job:start",
|
|
10585
|
+
"job:complete",
|
|
10586
|
+
"job:fail"
|
|
10587
|
+
];
|
|
10588
|
+
|
|
10589
|
+
// src/browser.ts
|
|
10590
|
+
var BROWSER_CHANNELS = [
|
|
10591
|
+
"browse:resource-requested",
|
|
10592
|
+
"browse:anchored-text-requested",
|
|
10593
|
+
"browse:anchored-text-by-checksum-requested",
|
|
10594
|
+
"browse:resources-requested",
|
|
10595
|
+
"browse:annotations-requested",
|
|
10596
|
+
"browse:annotation-requested",
|
|
10597
|
+
"browse:events-requested",
|
|
10598
|
+
"browse:annotation-history-requested",
|
|
10599
|
+
"browse:referenced-by-requested",
|
|
10600
|
+
"browse:entity-types-requested",
|
|
10601
|
+
"browse:tag-schemas-requested",
|
|
10602
|
+
"browse:agents-requested",
|
|
10603
|
+
"browse:directory-requested"
|
|
10604
|
+
];
|
|
10605
|
+
|
|
10606
|
+
// src/clone-token-manager.ts
|
|
10607
|
+
var CLONE_TOKEN_CHANNELS = [
|
|
10608
|
+
"yield:clone-token-requested",
|
|
10609
|
+
"yield:clone-resource-requested",
|
|
10610
|
+
"yield:clone-create"
|
|
10611
|
+
];
|
|
10612
|
+
|
|
10613
|
+
// src/service-channels.ts
|
|
10614
|
+
var SMELTER_AWAITED_OPERATIONS = [
|
|
10615
|
+
"browse:resource-requested",
|
|
10616
|
+
"browse:annotations-requested",
|
|
10617
|
+
"browse:resources-requested"
|
|
10618
|
+
];
|
|
10619
|
+
var SMELTER_REPLY_CHANNELS = replyChannelsFor(SMELTER_AWAITED_OPERATIONS);
|
|
10620
|
+
var WEAVER_AWAITED_OPERATIONS = [
|
|
10621
|
+
"browse:resources-requested",
|
|
10622
|
+
"browse:events-requested",
|
|
10623
|
+
"browse:annotations-requested"
|
|
10624
|
+
];
|
|
10625
|
+
replyChannelsFor(WEAVER_AWAITED_OPERATIONS);
|
|
10626
|
+
var LIBRARIAN_INBOUND_CHANNELS = [
|
|
10627
|
+
...MATCHER_CHANNELS,
|
|
10628
|
+
...GATHERER_CHANNELS,
|
|
10629
|
+
"gather:summary-requested",
|
|
10630
|
+
"weave:applied",
|
|
10631
|
+
"smelt:settled"
|
|
10632
|
+
];
|
|
10633
|
+
replyChannelsFor(LIBRARIAN_INBOUND_CHANNELS);
|
|
10634
|
+
var ARCHIVIST_INBOUND_CHANNELS = [
|
|
10635
|
+
...STOWER_CHANNELS,
|
|
10636
|
+
...BROWSER_CHANNELS,
|
|
10637
|
+
...CLONE_TOKEN_CHANNELS,
|
|
10638
|
+
"mark:create-request",
|
|
10639
|
+
"smelt:settled",
|
|
10640
|
+
// The annotation-context read moved here with the bytes (SINGLE-KB-MOUNT D5).
|
|
10641
|
+
"browse:annotation-context-requested"
|
|
10642
|
+
];
|
|
10643
|
+
var ARCHIVIST_OUTBOUND_STRAYS = [
|
|
10644
|
+
"mark:body-update-failed",
|
|
10645
|
+
// op keyed 'bind:update-body' (gateway handler re-emits mark:update-body)
|
|
10646
|
+
"yield:move-failed"
|
|
10647
|
+
// yield:mv has no registered operation; failure is direct-subscribed
|
|
10648
|
+
];
|
|
10649
|
+
[
|
|
10650
|
+
...ARCHIVIST_OUTBOUND_STRAYS,
|
|
10651
|
+
...replyChannelsFor(ARCHIVIST_INBOUND_CHANNELS)
|
|
10652
|
+
];
|
|
10558
10653
|
var configPath = join(homedir(), ".semiontconfig");
|
|
10559
10654
|
var tomlReader = {
|
|
10560
10655
|
readIfExists: (p) => existsSync(p) ? readFileSync(p, "utf-8") : null
|
|
@@ -10590,7 +10685,7 @@ var chunkingConfig = {
|
|
|
10590
10685
|
overlap: embedding.chunking?.overlap ?? 64
|
|
10591
10686
|
};
|
|
10592
10687
|
var workerSecret = process.env.SEMIONT_WORKER_SECRET ?? "";
|
|
10593
|
-
var healthPort =
|
|
10688
|
+
var healthPort = 24101;
|
|
10594
10689
|
var logger = createProcessLogger("smelter");
|
|
10595
10690
|
async function authenticate() {
|
|
10596
10691
|
if (!workerSecret) {
|
|
@@ -10661,7 +10756,12 @@ async function main() {
|
|
|
10661
10756
|
const httpTransport = new HttpTransport({
|
|
10662
10757
|
baseUrl: baseUrl$1(baseUrl),
|
|
10663
10758
|
token$: tokenSubject,
|
|
10664
|
-
tokenRefresher: refreshToken
|
|
10759
|
+
tokenRefresher: refreshToken,
|
|
10760
|
+
// Only the reply channels this process awaits — not the full bridged
|
|
10761
|
+
// set, whose global reply fan-out is the worker-OOM failure mode. See
|
|
10762
|
+
// SMELTER_AWAITED_OPERATIONS. The domain-event channels are added by
|
|
10763
|
+
// the actor state unit's start() below.
|
|
10764
|
+
channels: SMELTER_REPLY_CHANNELS
|
|
10665
10765
|
});
|
|
10666
10766
|
const actorStateUnit = createSmelterActorStateUnit({
|
|
10667
10767
|
bus: httpTransport.actor
|