@semiont/make-meaning 0.5.29 → 0.5.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -10
- package/dist/archivist-main.js +236 -131
- package/dist/archivist-main.js.map +1 -1
- package/dist/index.d.ts +19 -0
- package/dist/index.js +100 -4
- package/dist/index.js.map +1 -1
- package/dist/librarian-main.js +119 -50
- package/dist/librarian-main.js.map +1 -1
- package/dist/smelter-main.js +125 -25
- package/dist/smelter-main.js.map +1 -1
- package/dist/weaver-main.js +130 -28
- package/dist/weaver-main.js.map +1 -1
- package/package.json +12 -12
package/dist/weaver-main.js
CHANGED
|
@@ -1,9 +1,14 @@
|
|
|
1
|
-
import { createTomlConfigLoader, accessToken, baseUrl as baseUrl$1, retryWithBackoff, isTransientFetchError, STARTUP_FETCH_RETRY, errField, burstBuffer, busRequest, resourceId, didToAgent, annotationId, findBodyItem } from '@semiont/core';
|
|
1
|
+
import { replyChannelsFor, createTomlConfigLoader, accessToken, baseUrl as baseUrl$1, retryWithBackoff, isTransientFetchError, STARTUP_FETCH_RETRY, errField, burstBuffer, busRequest, resourceId, didToAgent, annotationId, findBodyItem } from '@semiont/core';
|
|
2
2
|
import { existsSync, readFileSync, promises } from 'fs';
|
|
3
3
|
import * as path from 'path';
|
|
4
4
|
import { join } from 'path';
|
|
5
|
-
import
|
|
5
|
+
import '@semiont/observability';
|
|
6
|
+
import '@semiont/ontology';
|
|
7
|
+
import '@semiont/content';
|
|
6
8
|
import { getGraphDatabase } from '@semiont/graph';
|
|
9
|
+
import '@semiont/vectors';
|
|
10
|
+
import '@semiont/event-sourcing';
|
|
11
|
+
import { HttpTransport } from '@semiont/http-transport';
|
|
7
12
|
import { createServer } from 'http';
|
|
8
13
|
import { homedir } from 'os';
|
|
9
14
|
import { createProcessLogger } from '@semiont/observability/process-logger';
|
|
@@ -9910,18 +9915,18 @@ var Weaver = class _Weaver {
|
|
|
9910
9915
|
set.add(seq);
|
|
9911
9916
|
}
|
|
9912
9917
|
}
|
|
9913
|
-
noteApplied(
|
|
9914
|
-
const failed = this.failedSeqs.get(
|
|
9918
|
+
noteApplied(resourceId6, sequenceNumber) {
|
|
9919
|
+
const failed = this.failedSeqs.get(resourceId6);
|
|
9915
9920
|
let capped = sequenceNumber;
|
|
9916
9921
|
if (failed && failed.size > 0) {
|
|
9917
9922
|
capped = Math.min(capped, Math.min(...failed) - 1);
|
|
9918
9923
|
}
|
|
9919
|
-
const current = this.lastProcessed.get(
|
|
9924
|
+
const current = this.lastProcessed.get(resourceId6);
|
|
9920
9925
|
if (current !== void 0 && current >= capped) return;
|
|
9921
|
-
this.lastProcessed.set(
|
|
9926
|
+
this.lastProcessed.set(resourceId6, capped);
|
|
9922
9927
|
this.checkpointDirty = true;
|
|
9923
|
-
this.bus.emit("weave:applied", { resourceId, sequenceNumber: capped }).catch((err) => {
|
|
9924
|
-
this.logger.warn("weave:applied emit failed", { resourceId, sequenceNumber: capped, error: errField(err) });
|
|
9928
|
+
this.bus.emit("weave:applied", { resourceId: resourceId6, sequenceNumber: capped }).catch((err) => {
|
|
9929
|
+
this.logger.warn("weave:applied emit failed", { resourceId: resourceId6, sequenceNumber: capped, error: errField(err) });
|
|
9925
9930
|
});
|
|
9926
9931
|
}
|
|
9927
9932
|
async flushCheckpoint() {
|
|
@@ -9967,8 +9972,8 @@ var Weaver = class _Weaver {
|
|
|
9967
9972
|
return resources;
|
|
9968
9973
|
}
|
|
9969
9974
|
/** A resource's full event history over the bus, sorted by sequence. */
|
|
9970
|
-
async fetchResourceEvents(
|
|
9971
|
-
const reply = await busRequest(this.bus, "browse:events-requested", { resourceId });
|
|
9975
|
+
async fetchResourceEvents(resourceId6) {
|
|
9976
|
+
const reply = await busRequest(this.bus, "browse:events-requested", { resourceId: resourceId6 });
|
|
9972
9977
|
return [...reply.events].sort((a, b) => a.metadata.sequenceNumber - b.metadata.sequenceNumber);
|
|
9973
9978
|
}
|
|
9974
9979
|
async catchUp() {
|
|
@@ -10466,29 +10471,29 @@ var Weaver = class _Weaver {
|
|
|
10466
10471
|
* Rebuild entire resource from events.
|
|
10467
10472
|
* Bypasses the live pipeline — reads directly from event store.
|
|
10468
10473
|
*/
|
|
10469
|
-
async rebuildResource(
|
|
10474
|
+
async rebuildResource(resourceId6) {
|
|
10470
10475
|
const graphDb = this.ensureInitialized();
|
|
10471
|
-
this.logger.info("Rebuilding resource from events", { resourceId });
|
|
10476
|
+
this.logger.info("Rebuilding resource from events", { resourceId: resourceId6 });
|
|
10472
10477
|
try {
|
|
10473
|
-
await graphDb.deleteResource(
|
|
10478
|
+
await graphDb.deleteResource(resourceId6);
|
|
10474
10479
|
} catch (error) {
|
|
10475
|
-
this.logger.debug("No existing resource to delete", { resourceId });
|
|
10480
|
+
this.logger.debug("No existing resource to delete", { resourceId: resourceId6 });
|
|
10476
10481
|
}
|
|
10477
|
-
const events = await this.fetchResourceEvents(String(
|
|
10482
|
+
const events = await this.fetchResourceEvents(String(resourceId6));
|
|
10478
10483
|
let eventsFailed = 0;
|
|
10479
10484
|
for (const storedEvent of events) {
|
|
10480
10485
|
if (!await this.safeApplyEvent(storedEvent)) eventsFailed++;
|
|
10481
10486
|
}
|
|
10482
10487
|
if (events.length > 0 && eventsFailed === 0) {
|
|
10483
|
-
this.noteApplied(String(
|
|
10488
|
+
this.noteApplied(String(resourceId6), Math.max(...events.map((e) => e.metadata.sequenceNumber)));
|
|
10484
10489
|
}
|
|
10485
10490
|
if (eventsFailed > 0) {
|
|
10486
10491
|
this.logger.error("Resource rebuild dropped events \u2014 graph incomplete for this resource", {
|
|
10487
|
-
resourceId,
|
|
10492
|
+
resourceId: resourceId6,
|
|
10488
10493
|
eventsFailed
|
|
10489
10494
|
});
|
|
10490
10495
|
}
|
|
10491
|
-
this.logger.info("Resource rebuild complete", { resourceId, eventCount: events.length, eventsFailed });
|
|
10496
|
+
this.logger.info("Resource rebuild complete", { resourceId: resourceId6, eventCount: events.length, eventsFailed });
|
|
10492
10497
|
return { eventsApplied: events.length - eventsFailed, eventsFailed };
|
|
10493
10498
|
}
|
|
10494
10499
|
/**
|
|
@@ -10510,15 +10515,15 @@ var Weaver = class _Weaver {
|
|
|
10510
10515
|
this.logger.info("Found resources to rebuild", { count: allResourceIds.length });
|
|
10511
10516
|
const ledger = /* @__PURE__ */ new Map();
|
|
10512
10517
|
this.logger.info("PASS 1: Creating all nodes (resources + annotations)");
|
|
10513
|
-
for (const
|
|
10514
|
-
const events = await this.fetchResourceEvents(String(
|
|
10518
|
+
for (const resourceId6 of allResourceIds) {
|
|
10519
|
+
const events = await this.fetchResourceEvents(String(resourceId6));
|
|
10515
10520
|
if (events.length === 0) continue;
|
|
10516
10521
|
const entry = {
|
|
10517
10522
|
maxSeq: Math.max(...events.map((e) => e.metadata.sequenceNumber)),
|
|
10518
10523
|
attempted: 0,
|
|
10519
10524
|
failed: 0
|
|
10520
10525
|
};
|
|
10521
|
-
ledger.set(String(
|
|
10526
|
+
ledger.set(String(resourceId6), entry);
|
|
10522
10527
|
for (const storedEvent of events) {
|
|
10523
10528
|
if (storedEvent.type === "mark:body-updated") {
|
|
10524
10529
|
continue;
|
|
@@ -10529,11 +10534,11 @@ var Weaver = class _Weaver {
|
|
|
10529
10534
|
}
|
|
10530
10535
|
this.logger.info("Pass 1 complete - all nodes created");
|
|
10531
10536
|
this.logger.info("PASS 2: Creating all REFERENCES edges");
|
|
10532
|
-
for (const
|
|
10533
|
-
const events = await this.fetchResourceEvents(String(
|
|
10537
|
+
for (const resourceId6 of allResourceIds) {
|
|
10538
|
+
const events = await this.fetchResourceEvents(String(resourceId6));
|
|
10534
10539
|
for (const storedEvent of events) {
|
|
10535
10540
|
if (storedEvent.type === "mark:body-updated") {
|
|
10536
|
-
const entry = ledger.get(String(
|
|
10541
|
+
const entry = ledger.get(String(resourceId6));
|
|
10537
10542
|
if (entry) entry.attempted++;
|
|
10538
10543
|
if (!await this.safeApplyEvent(storedEvent)) {
|
|
10539
10544
|
if (entry) entry.failed++;
|
|
@@ -10561,8 +10566,8 @@ var Weaver = class _Weaver {
|
|
|
10561
10566
|
* Get consumer health metrics.
|
|
10562
10567
|
*/
|
|
10563
10568
|
/** Highest applied sequence for a resource, if any — diagnostics/tests. */
|
|
10564
|
-
appliedUpTo(
|
|
10565
|
-
return this.lastProcessed.get(
|
|
10569
|
+
appliedUpTo(resourceId6) {
|
|
10570
|
+
return this.lastProcessed.get(resourceId6);
|
|
10566
10571
|
}
|
|
10567
10572
|
getHealthMetrics() {
|
|
10568
10573
|
return {
|
|
@@ -10614,6 +10619,98 @@ var FileWeaverCheckpoint = class {
|
|
|
10614
10619
|
await promises.rename(tmp, this.filePath);
|
|
10615
10620
|
}
|
|
10616
10621
|
};
|
|
10622
|
+
var MATCHER_CHANNELS = [
|
|
10623
|
+
"match:search-requested"
|
|
10624
|
+
];
|
|
10625
|
+
|
|
10626
|
+
// src/gatherer.ts
|
|
10627
|
+
var GATHERER_CHANNELS = [
|
|
10628
|
+
"gather:requested",
|
|
10629
|
+
"gather:resource-requested"
|
|
10630
|
+
];
|
|
10631
|
+
var STOWER_CHANNELS = [
|
|
10632
|
+
"yield:create",
|
|
10633
|
+
"yield:clone-persist",
|
|
10634
|
+
"yield:update",
|
|
10635
|
+
"yield:mv",
|
|
10636
|
+
"mark:create",
|
|
10637
|
+
"mark:commit",
|
|
10638
|
+
"mark:delete",
|
|
10639
|
+
"mark:update-body",
|
|
10640
|
+
"frame:add-entity-type",
|
|
10641
|
+
"frame:add-tag-schema",
|
|
10642
|
+
"mark:archive",
|
|
10643
|
+
"mark:unarchive",
|
|
10644
|
+
"mark:update-entity-types",
|
|
10645
|
+
"job:start",
|
|
10646
|
+
"job:complete",
|
|
10647
|
+
"job:fail"
|
|
10648
|
+
];
|
|
10649
|
+
|
|
10650
|
+
// src/browser.ts
|
|
10651
|
+
var BROWSER_CHANNELS = [
|
|
10652
|
+
"browse:resource-requested",
|
|
10653
|
+
"browse:anchored-text-requested",
|
|
10654
|
+
"browse:anchored-text-by-checksum-requested",
|
|
10655
|
+
"browse:resources-requested",
|
|
10656
|
+
"browse:annotations-requested",
|
|
10657
|
+
"browse:annotation-requested",
|
|
10658
|
+
"browse:events-requested",
|
|
10659
|
+
"browse:annotation-history-requested",
|
|
10660
|
+
"browse:referenced-by-requested",
|
|
10661
|
+
"browse:entity-types-requested",
|
|
10662
|
+
"browse:tag-schemas-requested",
|
|
10663
|
+
"browse:agents-requested",
|
|
10664
|
+
"browse:directory-requested"
|
|
10665
|
+
];
|
|
10666
|
+
|
|
10667
|
+
// src/clone-token-manager.ts
|
|
10668
|
+
var CLONE_TOKEN_CHANNELS = [
|
|
10669
|
+
"yield:clone-token-requested",
|
|
10670
|
+
"yield:clone-resource-requested",
|
|
10671
|
+
"yield:clone-create"
|
|
10672
|
+
];
|
|
10673
|
+
|
|
10674
|
+
// src/service-channels.ts
|
|
10675
|
+
var SMELTER_AWAITED_OPERATIONS = [
|
|
10676
|
+
"browse:resource-requested",
|
|
10677
|
+
"browse:annotations-requested",
|
|
10678
|
+
"browse:resources-requested"
|
|
10679
|
+
];
|
|
10680
|
+
replyChannelsFor(SMELTER_AWAITED_OPERATIONS);
|
|
10681
|
+
var WEAVER_AWAITED_OPERATIONS = [
|
|
10682
|
+
"browse:resources-requested",
|
|
10683
|
+
"browse:events-requested",
|
|
10684
|
+
"browse:annotations-requested"
|
|
10685
|
+
];
|
|
10686
|
+
var WEAVER_REPLY_CHANNELS = replyChannelsFor(WEAVER_AWAITED_OPERATIONS);
|
|
10687
|
+
var LIBRARIAN_INBOUND_CHANNELS = [
|
|
10688
|
+
...MATCHER_CHANNELS,
|
|
10689
|
+
...GATHERER_CHANNELS,
|
|
10690
|
+
"gather:summary-requested",
|
|
10691
|
+
"weave:applied",
|
|
10692
|
+
"smelt:settled"
|
|
10693
|
+
];
|
|
10694
|
+
replyChannelsFor(LIBRARIAN_INBOUND_CHANNELS);
|
|
10695
|
+
var ARCHIVIST_INBOUND_CHANNELS = [
|
|
10696
|
+
...STOWER_CHANNELS,
|
|
10697
|
+
...BROWSER_CHANNELS,
|
|
10698
|
+
...CLONE_TOKEN_CHANNELS,
|
|
10699
|
+
"mark:create-request",
|
|
10700
|
+
"smelt:settled",
|
|
10701
|
+
// The annotation-context read moved here with the bytes (SINGLE-KB-MOUNT D5).
|
|
10702
|
+
"browse:annotation-context-requested"
|
|
10703
|
+
];
|
|
10704
|
+
var ARCHIVIST_OUTBOUND_STRAYS = [
|
|
10705
|
+
"mark:body-update-failed",
|
|
10706
|
+
// op keyed 'bind:update-body' (gateway handler re-emits mark:update-body)
|
|
10707
|
+
"yield:move-failed"
|
|
10708
|
+
// yield:mv has no registered operation; failure is direct-subscribed
|
|
10709
|
+
];
|
|
10710
|
+
[
|
|
10711
|
+
...ARCHIVIST_OUTBOUND_STRAYS,
|
|
10712
|
+
...replyChannelsFor(ARCHIVIST_INBOUND_CHANNELS)
|
|
10713
|
+
];
|
|
10617
10714
|
var configPath = join(homedir(), ".semiontconfig");
|
|
10618
10715
|
var tomlReader = {
|
|
10619
10716
|
readIfExists: (p) => existsSync(p) ? readFileSync(p, "utf-8") : null
|
|
@@ -10637,7 +10734,7 @@ if (maybeGraphConfig.type === "memory") {
|
|
|
10637
10734
|
}
|
|
10638
10735
|
var graphConfig = maybeGraphConfig;
|
|
10639
10736
|
var workerSecret = process.env.SEMIONT_WORKER_SECRET ?? "";
|
|
10640
|
-
var healthPort =
|
|
10737
|
+
var healthPort = 24102;
|
|
10641
10738
|
var stateHome = process.env.XDG_STATE_HOME ?? join(homedir(), ".local", "state");
|
|
10642
10739
|
var checkpointPath = join(stateHome, "semiont", "weaver-checkpoint.json");
|
|
10643
10740
|
var logger = createProcessLogger("weaver");
|
|
@@ -10698,7 +10795,12 @@ async function main() {
|
|
|
10698
10795
|
const httpTransport = new HttpTransport({
|
|
10699
10796
|
baseUrl: baseUrl$1(baseUrl),
|
|
10700
10797
|
token$: tokenSubject,
|
|
10701
|
-
tokenRefresher: refreshToken
|
|
10798
|
+
tokenRefresher: refreshToken,
|
|
10799
|
+
// Only the reply channels this process awaits — not the full bridged
|
|
10800
|
+
// set, whose global reply fan-out is the worker-OOM failure mode. See
|
|
10801
|
+
// WEAVER_AWAITED_OPERATIONS. The domain-event channels are added by
|
|
10802
|
+
// the actor state unit's start() below.
|
|
10803
|
+
channels: WEAVER_REPLY_CHANNELS
|
|
10702
10804
|
});
|
|
10703
10805
|
const actorStateUnit = createWeaverActorStateUnit({
|
|
10704
10806
|
bus: httpTransport.actor
|