@semiont/make-meaning 0.5.28 → 0.5.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -5
- package/dist/archivist-main.d.ts +2 -0
- package/dist/archivist-main.js +12568 -0
- package/dist/archivist-main.js.map +1 -0
- package/dist/index.d.ts +649 -460
- package/dist/index.js +580 -1060
- package/dist/index.js.map +1 -1
- package/dist/librarian-main.d.ts +2 -0
- package/dist/librarian-main.js +11449 -0
- package/dist/librarian-main.js.map +1 -0
- package/dist/smelter-main.js +21 -15
- package/dist/smelter-main.js.map +1 -1
- package/dist/weaver-main.d.ts +2 -0
- package/dist/weaver-main.js +13 -8
- package/dist/weaver-main.js.map +1 -1
- package/package.json +27 -16
package/dist/smelter-main.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
+
import { archivistContentReads, createAnchoredTextStore, EXTRACTORS, calculateChecksum } from '@semiont/content';
|
|
1
2
|
import { createTomlConfigLoader, accessToken, baseUrl as baseUrl$1, retryWithBackoff, isTransientFetchError, STARTUP_FETCH_RETRY, burstBuffer, errField, resourceId, textExtractionOf, busRequest, getResourceEntityTypes, chunkText, getTargetSelector, getExactText, annotationId, getPrimaryMediaType, getPrimaryRepresentation } from '@semiont/core';
|
|
2
|
-
import { anchoredTextStoreOverTransport, EXTRACTORS, calculateChecksum } from '@semiont/content';
|
|
3
3
|
import { registerVectorIndexSizeProvider, withActorSpan } from '@semiont/observability';
|
|
4
|
-
import { HttpTransport
|
|
4
|
+
import { HttpTransport } from '@semiont/http-transport';
|
|
5
5
|
import { createEmbeddingProvider, createVectorStore } from '@semiont/vectors';
|
|
6
6
|
import { createServer } from 'http';
|
|
7
7
|
import { existsSync, readFileSync } from 'fs';
|
|
@@ -9805,23 +9805,24 @@ function isWorkItem(input) {
|
|
|
9805
9805
|
return WORK_ITEM_TYPES.has(input.type);
|
|
9806
9806
|
}
|
|
9807
9807
|
var Smelter = class _Smelter {
|
|
9808
|
-
constructor(events$, rebuildAnchors$, vectorStore, embeddingProvider, content, bus, chunkingConfig2, timing, logger2) {
|
|
9808
|
+
constructor(events$, rebuildAnchors$, vectorStore, embeddingProvider, content, anchoredStore, bus, chunkingConfig2, timing, logger2) {
|
|
9809
9809
|
this.events$ = events$;
|
|
9810
9810
|
this.rebuildAnchors$ = rebuildAnchors$;
|
|
9811
9811
|
this.vectorStore = vectorStore;
|
|
9812
9812
|
this.embeddingProvider = embeddingProvider;
|
|
9813
9813
|
this.content = content;
|
|
9814
|
+
this.anchoredStore = anchoredStore;
|
|
9814
9815
|
this.bus = bus;
|
|
9815
9816
|
this.chunkingConfig = chunkingConfig2;
|
|
9816
9817
|
this.timing = timing;
|
|
9817
9818
|
this.logger = logger2;
|
|
9818
|
-
this.anchoredStore = anchoredTextStoreOverTransport(content, logger2.child({ component: "anchored-text-cache" }));
|
|
9819
9819
|
}
|
|
9820
9820
|
events$;
|
|
9821
9821
|
rebuildAnchors$;
|
|
9822
9822
|
vectorStore;
|
|
9823
9823
|
embeddingProvider;
|
|
9824
9824
|
content;
|
|
9825
|
+
anchoredStore;
|
|
9825
9826
|
bus;
|
|
9826
9827
|
chunkingConfig;
|
|
9827
9828
|
timing;
|
|
@@ -9845,7 +9846,6 @@ var Smelter = class _Smelter {
|
|
|
9845
9846
|
* OCR pass.
|
|
9846
9847
|
*/
|
|
9847
9848
|
drainChain = Promise.resolve();
|
|
9848
|
-
anchoredStore;
|
|
9849
9849
|
get eventsProcessed() {
|
|
9850
9850
|
return this._eventsProcessed;
|
|
9851
9851
|
}
|
|
@@ -10075,7 +10075,7 @@ var Smelter = class _Smelter {
|
|
|
10075
10075
|
});
|
|
10076
10076
|
return;
|
|
10077
10077
|
}
|
|
10078
|
-
await this.
|
|
10078
|
+
await this.anchoredStore.write(checksum, { ...extracted, items: extracted.items });
|
|
10079
10079
|
this.logger.info("Re-anchored resource", { resourceId: rid, checksum, items: extracted.items.length });
|
|
10080
10080
|
}
|
|
10081
10081
|
async handleResourcePurge(event) {
|
|
@@ -10362,7 +10362,7 @@ var Smelter = class _Smelter {
|
|
|
10362
10362
|
// per resource (PERSIST-ANCHORS P0). Keys are resource ids today;
|
|
10363
10363
|
// P1 rekeys the store by content checksum and this lookup moves
|
|
10364
10364
|
// with it.
|
|
10365
|
-
this.
|
|
10365
|
+
this.anchoredStore.list().then((keys) => new Set(keys))
|
|
10366
10366
|
]);
|
|
10367
10367
|
const resources = await this.listAllResources();
|
|
10368
10368
|
this.logger.info("Reconcile started", {
|
|
@@ -10564,11 +10564,11 @@ var envConfig = createTomlConfigLoader(
|
|
|
10564
10564
|
configPath,
|
|
10565
10565
|
process.env
|
|
10566
10566
|
)(null);
|
|
10567
|
-
var
|
|
10568
|
-
if (!
|
|
10569
|
-
throw new Error("services.
|
|
10567
|
+
var gatewayPublicURL = envConfig.services?.gateway?.publicURL;
|
|
10568
|
+
if (!gatewayPublicURL) {
|
|
10569
|
+
throw new Error("services.gateway.publicURL is required in ~/.semiontconfig");
|
|
10570
10570
|
}
|
|
10571
|
-
var baseUrl =
|
|
10571
|
+
var baseUrl = gatewayPublicURL;
|
|
10572
10572
|
var embedding = envConfig.services?.embedding;
|
|
10573
10573
|
if (!embedding?.type || !embedding?.model) {
|
|
10574
10574
|
throw new Error("services.embedding.{type,model} are required in ~/.semiontconfig");
|
|
@@ -10617,7 +10617,7 @@ async function authenticate() {
|
|
|
10617
10617
|
isTransientFetchError,
|
|
10618
10618
|
STARTUP_FETCH_RETRY,
|
|
10619
10619
|
({ attempt, attempts, delayMs, error }) => {
|
|
10620
|
-
logger.warn("
|
|
10620
|
+
logger.warn("Gateway unreachable, retrying authentication", {
|
|
10621
10621
|
attempt,
|
|
10622
10622
|
attempts,
|
|
10623
10623
|
retryInMs: delayMs,
|
|
@@ -10666,14 +10666,20 @@ async function main() {
|
|
|
10666
10666
|
const actorStateUnit = createSmelterActorStateUnit({
|
|
10667
10667
|
bus: httpTransport.actor
|
|
10668
10668
|
});
|
|
10669
|
-
const
|
|
10670
|
-
logger.info("Content
|
|
10669
|
+
const contentReads = archivistContentReads(envConfig);
|
|
10670
|
+
logger.info("Content reads ready", { via: "archivist" });
|
|
10671
|
+
const anchoredTextDir = process.env.SEMIONT_ANCHORED_TEXT_DIR;
|
|
10672
|
+
if (!anchoredTextDir) {
|
|
10673
|
+
throw new Error("SEMIONT_ANCHORED_TEXT_DIR is required \u2014 the Smelter owns the anchored-text store and has no default for where it lives");
|
|
10674
|
+
}
|
|
10675
|
+
const anchoredStore = createAnchoredTextStore(anchoredTextDir, logger.child({ component: "anchored-text-store" }));
|
|
10671
10676
|
const smelter = new Smelter(
|
|
10672
10677
|
actorStateUnit.events$,
|
|
10673
10678
|
actorStateUnit.rebuildAnchors$,
|
|
10674
10679
|
vectorStore,
|
|
10675
10680
|
embeddingProvider,
|
|
10676
|
-
|
|
10681
|
+
contentReads,
|
|
10682
|
+
anchoredStore,
|
|
10677
10683
|
httpTransport,
|
|
10678
10684
|
chunkingConfig,
|
|
10679
10685
|
{ burstWindowMs: 50, maxBatchSize: 100, idleTimeoutMs: 200 },
|