@semiont/make-meaning 0.5.28 → 0.5.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -15
- package/dist/archivist-main.d.ts +2 -0
- package/dist/archivist-main.js +12673 -0
- package/dist/archivist-main.js.map +1 -0
- package/dist/index.d.ts +668 -460
- package/dist/index.js +684 -1068
- package/dist/index.js.map +1 -1
- package/dist/librarian-main.d.ts +2 -0
- package/dist/librarian-main.js +11518 -0
- package/dist/librarian-main.js.map +1 -0
- package/dist/smelter-main.js +145 -39
- package/dist/smelter-main.js.map +1 -1
- package/dist/weaver-main.d.ts +2 -0
- package/dist/weaver-main.js +143 -36
- package/dist/weaver-main.js.map +1 -1
- package/package.json +27 -16
package/dist/weaver-main.js
CHANGED
|
@@ -1,9 +1,14 @@
|
|
|
1
|
-
import { createTomlConfigLoader, accessToken, baseUrl as baseUrl$1, retryWithBackoff, isTransientFetchError, STARTUP_FETCH_RETRY, errField, burstBuffer, busRequest, resourceId, didToAgent, annotationId, findBodyItem } from '@semiont/core';
|
|
1
|
+
import { replyChannelsFor, createTomlConfigLoader, accessToken, baseUrl as baseUrl$1, retryWithBackoff, isTransientFetchError, STARTUP_FETCH_RETRY, errField, burstBuffer, busRequest, resourceId, didToAgent, annotationId, findBodyItem } from '@semiont/core';
|
|
2
2
|
import { existsSync, readFileSync, promises } from 'fs';
|
|
3
3
|
import * as path from 'path';
|
|
4
4
|
import { join } from 'path';
|
|
5
|
-
import
|
|
5
|
+
import '@semiont/observability';
|
|
6
|
+
import '@semiont/ontology';
|
|
7
|
+
import '@semiont/content';
|
|
6
8
|
import { getGraphDatabase } from '@semiont/graph';
|
|
9
|
+
import '@semiont/vectors';
|
|
10
|
+
import '@semiont/event-sourcing';
|
|
11
|
+
import { HttpTransport } from '@semiont/http-transport';
|
|
7
12
|
import { createServer } from 'http';
|
|
8
13
|
import { homedir } from 'os';
|
|
9
14
|
import { createProcessLogger } from '@semiont/observability/process-logger';
|
|
@@ -9910,18 +9915,18 @@ var Weaver = class _Weaver {
|
|
|
9910
9915
|
set.add(seq);
|
|
9911
9916
|
}
|
|
9912
9917
|
}
|
|
9913
|
-
noteApplied(
|
|
9914
|
-
const failed = this.failedSeqs.get(
|
|
9918
|
+
noteApplied(resourceId6, sequenceNumber) {
|
|
9919
|
+
const failed = this.failedSeqs.get(resourceId6);
|
|
9915
9920
|
let capped = sequenceNumber;
|
|
9916
9921
|
if (failed && failed.size > 0) {
|
|
9917
9922
|
capped = Math.min(capped, Math.min(...failed) - 1);
|
|
9918
9923
|
}
|
|
9919
|
-
const current = this.lastProcessed.get(
|
|
9924
|
+
const current = this.lastProcessed.get(resourceId6);
|
|
9920
9925
|
if (current !== void 0 && current >= capped) return;
|
|
9921
|
-
this.lastProcessed.set(
|
|
9926
|
+
this.lastProcessed.set(resourceId6, capped);
|
|
9922
9927
|
this.checkpointDirty = true;
|
|
9923
|
-
this.bus.emit("weave:applied", { resourceId, sequenceNumber: capped }).catch((err) => {
|
|
9924
|
-
this.logger.warn("weave:applied emit failed", { resourceId, sequenceNumber: capped, error: errField(err) });
|
|
9928
|
+
this.bus.emit("weave:applied", { resourceId: resourceId6, sequenceNumber: capped }).catch((err) => {
|
|
9929
|
+
this.logger.warn("weave:applied emit failed", { resourceId: resourceId6, sequenceNumber: capped, error: errField(err) });
|
|
9925
9930
|
});
|
|
9926
9931
|
}
|
|
9927
9932
|
async flushCheckpoint() {
|
|
@@ -9967,8 +9972,8 @@ var Weaver = class _Weaver {
|
|
|
9967
9972
|
return resources;
|
|
9968
9973
|
}
|
|
9969
9974
|
/** A resource's full event history over the bus, sorted by sequence. */
|
|
9970
|
-
async fetchResourceEvents(
|
|
9971
|
-
const reply = await busRequest(this.bus, "browse:events-requested", { resourceId });
|
|
9975
|
+
async fetchResourceEvents(resourceId6) {
|
|
9976
|
+
const reply = await busRequest(this.bus, "browse:events-requested", { resourceId: resourceId6 });
|
|
9972
9977
|
return [...reply.events].sort((a, b) => a.metadata.sequenceNumber - b.metadata.sequenceNumber);
|
|
9973
9978
|
}
|
|
9974
9979
|
async catchUp() {
|
|
@@ -10323,12 +10328,17 @@ var Weaver = class _Weaver {
|
|
|
10323
10328
|
representations: [{
|
|
10324
10329
|
mediaType: event.payload.format,
|
|
10325
10330
|
checksum: event.payload.contentChecksum,
|
|
10326
|
-
rel: "original"
|
|
10331
|
+
rel: "original",
|
|
10332
|
+
// The URI lives on the REPRESENTATION (STORAGE-URI-ONE-HOME D1).
|
|
10333
|
+
// `ResourceDescriptor` is `additionalProperties: true`, so writing it
|
|
10334
|
+
// at the top level typechecks and then reads back as undefined
|
|
10335
|
+
// through `getStorageUri` — which is how the graph projection silently
|
|
10336
|
+
// lost all 57 of its URIs until the P2 live gate caught it.
|
|
10337
|
+
...event.payload.storageUri ? { storageUri: event.payload.storageUri } : {}
|
|
10327
10338
|
}],
|
|
10328
10339
|
archived: false,
|
|
10329
10340
|
dateCreated: (/* @__PURE__ */ new Date()).toISOString(),
|
|
10330
|
-
wasAttributedTo: didToAgent(event.userId)
|
|
10331
|
-
...event.payload.storageUri ? { storageUri: event.payload.storageUri } : {}
|
|
10341
|
+
wasAttributedTo: didToAgent(event.userId)
|
|
10332
10342
|
};
|
|
10333
10343
|
}
|
|
10334
10344
|
/**
|
|
@@ -10461,29 +10471,29 @@ var Weaver = class _Weaver {
|
|
|
10461
10471
|
* Rebuild entire resource from events.
|
|
10462
10472
|
* Bypasses the live pipeline — reads directly from event store.
|
|
10463
10473
|
*/
|
|
10464
|
-
async rebuildResource(
|
|
10474
|
+
async rebuildResource(resourceId6) {
|
|
10465
10475
|
const graphDb = this.ensureInitialized();
|
|
10466
|
-
this.logger.info("Rebuilding resource from events", { resourceId });
|
|
10476
|
+
this.logger.info("Rebuilding resource from events", { resourceId: resourceId6 });
|
|
10467
10477
|
try {
|
|
10468
|
-
await graphDb.deleteResource(
|
|
10478
|
+
await graphDb.deleteResource(resourceId6);
|
|
10469
10479
|
} catch (error) {
|
|
10470
|
-
this.logger.debug("No existing resource to delete", { resourceId });
|
|
10480
|
+
this.logger.debug("No existing resource to delete", { resourceId: resourceId6 });
|
|
10471
10481
|
}
|
|
10472
|
-
const events = await this.fetchResourceEvents(String(
|
|
10482
|
+
const events = await this.fetchResourceEvents(String(resourceId6));
|
|
10473
10483
|
let eventsFailed = 0;
|
|
10474
10484
|
for (const storedEvent of events) {
|
|
10475
10485
|
if (!await this.safeApplyEvent(storedEvent)) eventsFailed++;
|
|
10476
10486
|
}
|
|
10477
10487
|
if (events.length > 0 && eventsFailed === 0) {
|
|
10478
|
-
this.noteApplied(String(
|
|
10488
|
+
this.noteApplied(String(resourceId6), Math.max(...events.map((e) => e.metadata.sequenceNumber)));
|
|
10479
10489
|
}
|
|
10480
10490
|
if (eventsFailed > 0) {
|
|
10481
10491
|
this.logger.error("Resource rebuild dropped events \u2014 graph incomplete for this resource", {
|
|
10482
|
-
resourceId,
|
|
10492
|
+
resourceId: resourceId6,
|
|
10483
10493
|
eventsFailed
|
|
10484
10494
|
});
|
|
10485
10495
|
}
|
|
10486
|
-
this.logger.info("Resource rebuild complete", { resourceId, eventCount: events.length, eventsFailed });
|
|
10496
|
+
this.logger.info("Resource rebuild complete", { resourceId: resourceId6, eventCount: events.length, eventsFailed });
|
|
10487
10497
|
return { eventsApplied: events.length - eventsFailed, eventsFailed };
|
|
10488
10498
|
}
|
|
10489
10499
|
/**
|
|
@@ -10505,15 +10515,15 @@ var Weaver = class _Weaver {
|
|
|
10505
10515
|
this.logger.info("Found resources to rebuild", { count: allResourceIds.length });
|
|
10506
10516
|
const ledger = /* @__PURE__ */ new Map();
|
|
10507
10517
|
this.logger.info("PASS 1: Creating all nodes (resources + annotations)");
|
|
10508
|
-
for (const
|
|
10509
|
-
const events = await this.fetchResourceEvents(String(
|
|
10518
|
+
for (const resourceId6 of allResourceIds) {
|
|
10519
|
+
const events = await this.fetchResourceEvents(String(resourceId6));
|
|
10510
10520
|
if (events.length === 0) continue;
|
|
10511
10521
|
const entry = {
|
|
10512
10522
|
maxSeq: Math.max(...events.map((e) => e.metadata.sequenceNumber)),
|
|
10513
10523
|
attempted: 0,
|
|
10514
10524
|
failed: 0
|
|
10515
10525
|
};
|
|
10516
|
-
ledger.set(String(
|
|
10526
|
+
ledger.set(String(resourceId6), entry);
|
|
10517
10527
|
for (const storedEvent of events) {
|
|
10518
10528
|
if (storedEvent.type === "mark:body-updated") {
|
|
10519
10529
|
continue;
|
|
@@ -10524,11 +10534,11 @@ var Weaver = class _Weaver {
|
|
|
10524
10534
|
}
|
|
10525
10535
|
this.logger.info("Pass 1 complete - all nodes created");
|
|
10526
10536
|
this.logger.info("PASS 2: Creating all REFERENCES edges");
|
|
10527
|
-
for (const
|
|
10528
|
-
const events = await this.fetchResourceEvents(String(
|
|
10537
|
+
for (const resourceId6 of allResourceIds) {
|
|
10538
|
+
const events = await this.fetchResourceEvents(String(resourceId6));
|
|
10529
10539
|
for (const storedEvent of events) {
|
|
10530
10540
|
if (storedEvent.type === "mark:body-updated") {
|
|
10531
|
-
const entry = ledger.get(String(
|
|
10541
|
+
const entry = ledger.get(String(resourceId6));
|
|
10532
10542
|
if (entry) entry.attempted++;
|
|
10533
10543
|
if (!await this.safeApplyEvent(storedEvent)) {
|
|
10534
10544
|
if (entry) entry.failed++;
|
|
@@ -10556,8 +10566,8 @@ var Weaver = class _Weaver {
|
|
|
10556
10566
|
* Get consumer health metrics.
|
|
10557
10567
|
*/
|
|
10558
10568
|
/** Highest applied sequence for a resource, if any — diagnostics/tests. */
|
|
10559
|
-
appliedUpTo(
|
|
10560
|
-
return this.lastProcessed.get(
|
|
10569
|
+
appliedUpTo(resourceId6) {
|
|
10570
|
+
return this.lastProcessed.get(resourceId6);
|
|
10561
10571
|
}
|
|
10562
10572
|
getHealthMetrics() {
|
|
10563
10573
|
return {
|
|
@@ -10609,6 +10619,98 @@ var FileWeaverCheckpoint = class {
|
|
|
10609
10619
|
await promises.rename(tmp, this.filePath);
|
|
10610
10620
|
}
|
|
10611
10621
|
};
|
|
10622
|
+
var MATCHER_CHANNELS = [
|
|
10623
|
+
"match:search-requested"
|
|
10624
|
+
];
|
|
10625
|
+
|
|
10626
|
+
// src/gatherer.ts
|
|
10627
|
+
var GATHERER_CHANNELS = [
|
|
10628
|
+
"gather:requested",
|
|
10629
|
+
"gather:resource-requested"
|
|
10630
|
+
];
|
|
10631
|
+
var STOWER_CHANNELS = [
|
|
10632
|
+
"yield:create",
|
|
10633
|
+
"yield:clone-persist",
|
|
10634
|
+
"yield:update",
|
|
10635
|
+
"yield:mv",
|
|
10636
|
+
"mark:create",
|
|
10637
|
+
"mark:commit",
|
|
10638
|
+
"mark:delete",
|
|
10639
|
+
"mark:update-body",
|
|
10640
|
+
"frame:add-entity-type",
|
|
10641
|
+
"frame:add-tag-schema",
|
|
10642
|
+
"mark:archive",
|
|
10643
|
+
"mark:unarchive",
|
|
10644
|
+
"mark:update-entity-types",
|
|
10645
|
+
"job:start",
|
|
10646
|
+
"job:complete",
|
|
10647
|
+
"job:fail"
|
|
10648
|
+
];
|
|
10649
|
+
|
|
10650
|
+
// src/browser.ts
|
|
10651
|
+
var BROWSER_CHANNELS = [
|
|
10652
|
+
"browse:resource-requested",
|
|
10653
|
+
"browse:anchored-text-requested",
|
|
10654
|
+
"browse:anchored-text-by-checksum-requested",
|
|
10655
|
+
"browse:resources-requested",
|
|
10656
|
+
"browse:annotations-requested",
|
|
10657
|
+
"browse:annotation-requested",
|
|
10658
|
+
"browse:events-requested",
|
|
10659
|
+
"browse:annotation-history-requested",
|
|
10660
|
+
"browse:referenced-by-requested",
|
|
10661
|
+
"browse:entity-types-requested",
|
|
10662
|
+
"browse:tag-schemas-requested",
|
|
10663
|
+
"browse:agents-requested",
|
|
10664
|
+
"browse:directory-requested"
|
|
10665
|
+
];
|
|
10666
|
+
|
|
10667
|
+
// src/clone-token-manager.ts
|
|
10668
|
+
var CLONE_TOKEN_CHANNELS = [
|
|
10669
|
+
"yield:clone-token-requested",
|
|
10670
|
+
"yield:clone-resource-requested",
|
|
10671
|
+
"yield:clone-create"
|
|
10672
|
+
];
|
|
10673
|
+
|
|
10674
|
+
// src/service-channels.ts
|
|
10675
|
+
var SMELTER_AWAITED_OPERATIONS = [
|
|
10676
|
+
"browse:resource-requested",
|
|
10677
|
+
"browse:annotations-requested",
|
|
10678
|
+
"browse:resources-requested"
|
|
10679
|
+
];
|
|
10680
|
+
replyChannelsFor(SMELTER_AWAITED_OPERATIONS);
|
|
10681
|
+
var WEAVER_AWAITED_OPERATIONS = [
|
|
10682
|
+
"browse:resources-requested",
|
|
10683
|
+
"browse:events-requested",
|
|
10684
|
+
"browse:annotations-requested"
|
|
10685
|
+
];
|
|
10686
|
+
var WEAVER_REPLY_CHANNELS = replyChannelsFor(WEAVER_AWAITED_OPERATIONS);
|
|
10687
|
+
var LIBRARIAN_INBOUND_CHANNELS = [
|
|
10688
|
+
...MATCHER_CHANNELS,
|
|
10689
|
+
...GATHERER_CHANNELS,
|
|
10690
|
+
"gather:summary-requested",
|
|
10691
|
+
"weave:applied",
|
|
10692
|
+
"smelt:settled"
|
|
10693
|
+
];
|
|
10694
|
+
replyChannelsFor(LIBRARIAN_INBOUND_CHANNELS);
|
|
10695
|
+
var ARCHIVIST_INBOUND_CHANNELS = [
|
|
10696
|
+
...STOWER_CHANNELS,
|
|
10697
|
+
...BROWSER_CHANNELS,
|
|
10698
|
+
...CLONE_TOKEN_CHANNELS,
|
|
10699
|
+
"mark:create-request",
|
|
10700
|
+
"smelt:settled",
|
|
10701
|
+
// The annotation-context read moved here with the bytes (SINGLE-KB-MOUNT D5).
|
|
10702
|
+
"browse:annotation-context-requested"
|
|
10703
|
+
];
|
|
10704
|
+
var ARCHIVIST_OUTBOUND_STRAYS = [
|
|
10705
|
+
"mark:body-update-failed",
|
|
10706
|
+
// op keyed 'bind:update-body' (gateway handler re-emits mark:update-body)
|
|
10707
|
+
"yield:move-failed"
|
|
10708
|
+
// yield:mv has no registered operation; failure is direct-subscribed
|
|
10709
|
+
];
|
|
10710
|
+
[
|
|
10711
|
+
...ARCHIVIST_OUTBOUND_STRAYS,
|
|
10712
|
+
...replyChannelsFor(ARCHIVIST_INBOUND_CHANNELS)
|
|
10713
|
+
];
|
|
10612
10714
|
var configPath = join(homedir(), ".semiontconfig");
|
|
10613
10715
|
var tomlReader = {
|
|
10614
10716
|
readIfExists: (p) => existsSync(p) ? readFileSync(p, "utf-8") : null
|
|
@@ -10618,11 +10720,11 @@ var envConfig = createTomlConfigLoader(
|
|
|
10618
10720
|
configPath,
|
|
10619
10721
|
process.env
|
|
10620
10722
|
)(null);
|
|
10621
|
-
var
|
|
10622
|
-
if (!
|
|
10623
|
-
throw new Error("services.
|
|
10723
|
+
var gatewayPublicURL = envConfig.services?.gateway?.publicURL;
|
|
10724
|
+
if (!gatewayPublicURL) {
|
|
10725
|
+
throw new Error("services.gateway.publicURL is required in ~/.semiontconfig");
|
|
10624
10726
|
}
|
|
10625
|
-
var baseUrl =
|
|
10727
|
+
var baseUrl = gatewayPublicURL;
|
|
10626
10728
|
var maybeGraphConfig = envConfig.services?.graph;
|
|
10627
10729
|
if (!maybeGraphConfig?.type) {
|
|
10628
10730
|
throw new Error("services.graph.type is required in ~/.semiontconfig");
|
|
@@ -10632,7 +10734,7 @@ if (maybeGraphConfig.type === "memory") {
|
|
|
10632
10734
|
}
|
|
10633
10735
|
var graphConfig = maybeGraphConfig;
|
|
10634
10736
|
var workerSecret = process.env.SEMIONT_WORKER_SECRET ?? "";
|
|
10635
|
-
var healthPort =
|
|
10737
|
+
var healthPort = 24102;
|
|
10636
10738
|
var stateHome = process.env.XDG_STATE_HOME ?? join(homedir(), ".local", "state");
|
|
10637
10739
|
var checkpointPath = join(stateHome, "semiont", "weaver-checkpoint.json");
|
|
10638
10740
|
var logger = createProcessLogger("weaver");
|
|
@@ -10661,7 +10763,7 @@ async function authenticate() {
|
|
|
10661
10763
|
isTransientFetchError,
|
|
10662
10764
|
STARTUP_FETCH_RETRY,
|
|
10663
10765
|
({ attempt, attempts, delayMs, error }) => {
|
|
10664
|
-
logger.warn("
|
|
10766
|
+
logger.warn("Gateway unreachable, retrying authentication", {
|
|
10665
10767
|
attempt,
|
|
10666
10768
|
attempts,
|
|
10667
10769
|
retryInMs: delayMs,
|
|
@@ -10693,7 +10795,12 @@ async function main() {
|
|
|
10693
10795
|
const httpTransport = new HttpTransport({
|
|
10694
10796
|
baseUrl: baseUrl$1(baseUrl),
|
|
10695
10797
|
token$: tokenSubject,
|
|
10696
|
-
tokenRefresher: refreshToken
|
|
10798
|
+
tokenRefresher: refreshToken,
|
|
10799
|
+
// Only the reply channels this process awaits — not the full bridged
|
|
10800
|
+
// set, whose global reply fan-out is the worker-OOM failure mode. See
|
|
10801
|
+
// WEAVER_AWAITED_OPERATIONS. The domain-event channels are added by
|
|
10802
|
+
// the actor state unit's start() below.
|
|
10803
|
+
channels: WEAVER_REPLY_CHANNELS
|
|
10697
10804
|
});
|
|
10698
10805
|
const actorStateUnit = createWeaverActorStateUnit({
|
|
10699
10806
|
bus: httpTransport.actor
|