@semiont/make-meaning 0.5.23 → 0.5.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +138 -5
- package/dist/index.js +322 -46
- package/dist/index.js.map +1 -1
- package/dist/smelter-main.js +236 -44
- package/dist/smelter-main.js.map +1 -1
- package/dist/weaver-main.js.map +1 -1
- package/package.json +13 -13
package/dist/index.js
CHANGED
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
import { STALL_THRESHOLD_MS, FsJobQueue } from '@semiont/jobs';
|
|
2
2
|
import { FilesystemViewStorage, resolveStorageUri, EventQuery, createEventStore } from '@semiont/event-sourcing';
|
|
3
|
-
import { getResourceEntityTypes, getResourceId, getTargetSource, resourceId, getPrimaryRepresentation, decodeRepresentation, getBodySource, getTargetSelector, deriveViews, getTextPositionSelector, annotationId, errField, userId, generateUuid, getExactText, busRequest, cloneToken, baseMediaType, isSupportedMediaType, capabilitiesOf, didToAgent, assembleAnnotation, jobId, entityType, baseUrl, busLog, BRIDGED_CHANNELS, burstBuffer, textExtractionOf, getPrimaryMediaType, extensionForMediaType, applyBodyOperations, softwareToAgent } from '@semiont/core';
|
|
3
|
+
import { getResourceEntityTypes, getResourceId, getTargetSource, resourceId, getPrimaryRepresentation, decodeRepresentation, getBodySource, getTargetSelector, deriveViews, getTextPositionSelector, annotationId, errField, userId, generateUuid, getExactText, busRequest, cloneToken, baseMediaType, isSupportedMediaType, capabilitiesOf, didToAgent, assembleAnnotation, jobId, entityType, baseUrl, busLog, BRIDGED_CHANNELS, burstBuffer, textExtractionOf, chunkText, getPrimaryMediaType, extensionForMediaType, applyBodyOperations, softwareToAgent } from '@semiont/core';
|
|
4
4
|
import { recordGatherDegrade, withActorSpan, recordBusEmit, withSpan, SpanKind, registerJobQueueProvider, registerVectorIndexSizeProvider } from '@semiont/observability';
|
|
5
5
|
import { createInferenceClient } from '@semiont/inference';
|
|
6
6
|
import { getGraphDatabase } from '@semiont/graph';
|
|
7
|
-
import { WorkingTreeStore, deriveStorageUri, calculateChecksum } from '@semiont/content';
|
|
7
|
+
import { WorkingTreeStore, createAnchoredTextStore, deriveStorageUri, anchoredTextStoreOverTransport, EXTRACTORS, calculateChecksum } from '@semiont/content';
|
|
8
8
|
import { getEntityTypes, DEFAULT_ENTITY_TYPES } from '@semiont/ontology';
|
|
9
9
|
import { promises } from 'fs';
|
|
10
10
|
import * as path3 from 'path';
|
|
11
|
-
import { chunkText } from '@semiont/vectors';
|
|
12
11
|
import { createGzip, createGunzip } from 'zlib';
|
|
13
12
|
import { pipeline, Readable } from 'stream';
|
|
14
13
|
import { promisify } from 'util';
|
|
@@ -9889,6 +9888,10 @@ async function createKnowledgeBase(eventStore, project, graphDb, eventBus, logge
|
|
|
9889
9888
|
project,
|
|
9890
9889
|
logger.child({ component: "working-tree-store" })
|
|
9891
9890
|
);
|
|
9891
|
+
const anchoredText = createAnchoredTextStore(
|
|
9892
|
+
project.anchoredTextDir,
|
|
9893
|
+
logger.child({ component: "anchored-text-store" })
|
|
9894
|
+
);
|
|
9892
9895
|
const weaveProgress = createWeaveProgress(eventBus);
|
|
9893
9896
|
const smeltProgress = createSmeltProgress(eventBus);
|
|
9894
9897
|
if (!options?.skipRebuild) {
|
|
@@ -9898,6 +9901,7 @@ async function createKnowledgeBase(eventStore, project, graphDb, eventBus, logge
|
|
|
9898
9901
|
eventStore,
|
|
9899
9902
|
views,
|
|
9900
9903
|
content,
|
|
9904
|
+
anchoredText,
|
|
9901
9905
|
graph: graphDb,
|
|
9902
9906
|
weaveProgress,
|
|
9903
9907
|
smeltProgress,
|
|
@@ -10300,7 +10304,8 @@ Summary:`;
|
|
|
10300
10304
|
resourceId: r.resourceId,
|
|
10301
10305
|
annotationId: r.annotationId,
|
|
10302
10306
|
score: r.score,
|
|
10303
|
-
entityTypes: r.entityTypes
|
|
10307
|
+
entityTypes: r.entityTypes,
|
|
10308
|
+
...r.machineRead ? { machineRead: true } : {}
|
|
10304
10309
|
}))
|
|
10305
10310
|
};
|
|
10306
10311
|
logger?.debug("Semantic context found", { matches: results.length });
|
|
@@ -10642,7 +10647,8 @@ var LLMContext = class {
|
|
|
10642
10647
|
resourceId: m.resourceId,
|
|
10643
10648
|
...m.annotationId ? { annotationId: m.annotationId } : {},
|
|
10644
10649
|
score: m.score,
|
|
10645
|
-
...m.entityTypes ? { entityTypes: m.entityTypes } : {}
|
|
10650
|
+
...m.entityTypes ? { entityTypes: m.entityTypes } : {},
|
|
10651
|
+
...m.machineRead ? { machineRead: true } : {}
|
|
10646
10652
|
})),
|
|
10647
10653
|
...excludeEntityTypes.length ? { excludedEntityTypes: excludeEntityTypes } : {}
|
|
10648
10654
|
};
|
|
@@ -11632,6 +11638,21 @@ var Stower = class {
|
|
|
11632
11638
|
// src/browser.ts
|
|
11633
11639
|
var import_rxjs4 = __toESM(require_cjs());
|
|
11634
11640
|
var import_operators4 = __toESM(require_operators());
|
|
11641
|
+
var ANCHORED_TEXT_SETTLE_TIMEOUT_MS = 15e3;
|
|
11642
|
+
async function readAnchoredText(kb, resourceId10) {
|
|
11643
|
+
const view = await kb.views.get(resourceId(resourceId10));
|
|
11644
|
+
const checksum = getPrimaryRepresentation(view?.resource)?.checksum;
|
|
11645
|
+
if (!checksum) return null;
|
|
11646
|
+
const hit = await kb.anchoredText.read(checksum);
|
|
11647
|
+
if (hit) return hit;
|
|
11648
|
+
try {
|
|
11649
|
+
const outcome = await kb.smeltProgress.whenSettled(resourceId10, checksum, ANCHORED_TEXT_SETTLE_TIMEOUT_MS);
|
|
11650
|
+
return outcome === "indexed" ? kb.anchoredText.read(checksum) : null;
|
|
11651
|
+
} catch (error) {
|
|
11652
|
+
if (error instanceof SmeltProgressTimeout) return null;
|
|
11653
|
+
throw error;
|
|
11654
|
+
}
|
|
11655
|
+
}
|
|
11635
11656
|
async function readTagSchemasProjection(project) {
|
|
11636
11657
|
const tagSchemasPath = path3.join(
|
|
11637
11658
|
project.stateDir,
|
|
@@ -11746,6 +11767,7 @@ var Browser = class {
|
|
|
11746
11767
|
);
|
|
11747
11768
|
this.subscriptions.push(
|
|
11748
11769
|
pipe("browse:resource-requested", (e) => this.handleBrowseResource(e)).subscribe({ error: errorHandler }),
|
|
11770
|
+
pipe("browse:anchored-text-requested", (e) => this.handleAnchoredText(e)).subscribe({ error: errorHandler }),
|
|
11749
11771
|
pipe("browse:resources-requested", (e) => this.handleBrowseResources(e)).subscribe({ error: errorHandler }),
|
|
11750
11772
|
pipe("browse:annotations-requested", (e) => this.handleBrowseAnnotations(e)).subscribe({ error: errorHandler }),
|
|
11751
11773
|
pipe("browse:annotation-requested", (e) => this.handleBrowseAnnotation(e)).subscribe({ error: errorHandler }),
|
|
@@ -11761,6 +11783,34 @@ var Browser = class {
|
|
|
11761
11783
|
// ========================================================================
|
|
11762
11784
|
// KB read handlers
|
|
11763
11785
|
// ========================================================================
|
|
11786
|
+
/**
|
|
11787
|
+
* Serve a resource's derived coordinate map (ANCHORED-TEXT-CACHE Lane 5).
|
|
11788
|
+
*
|
|
11789
|
+
* Read-your-writes on the same barrier `llm-context` uses for vectors: a
|
|
11790
|
+
* caller may arrive before the Smelter has finished the resource it just
|
|
11791
|
+
* uploaded, so a miss waits for that content generation to settle rather than
|
|
11792
|
+
* reporting "no map" for a document that is merely still being read.
|
|
11793
|
+
*
|
|
11794
|
+
* **This path never invokes the engine.** The Smelter is the sole producer.
|
|
11795
|
+
* A miss that survives the barrier answers `null`, and the caller degrades —
|
|
11796
|
+
* for a PDF annotation that means geometry with no quoted text, which is what
|
|
11797
|
+
* shipped before any of this existed. OCR in a request path is precisely what
|
|
11798
|
+
* this design exists to avoid.
|
|
11799
|
+
*/
|
|
11800
|
+
async handleAnchoredText(event) {
|
|
11801
|
+
try {
|
|
11802
|
+
this.eventBus.get("browse:anchored-text-result").next({
|
|
11803
|
+
correlationId: event.correlationId,
|
|
11804
|
+
response: await readAnchoredText(this.kb, event.resourceId)
|
|
11805
|
+
});
|
|
11806
|
+
} catch (error) {
|
|
11807
|
+
this.logger.error("Browse anchored text failed", { resourceId: event.resourceId, error: errField(error) });
|
|
11808
|
+
this.eventBus.get("browse:anchored-text-failed").next({
|
|
11809
|
+
correlationId: event.correlationId,
|
|
11810
|
+
message: error instanceof Error ? error.message : String(error)
|
|
11811
|
+
});
|
|
11812
|
+
}
|
|
11813
|
+
}
|
|
11764
11814
|
async handleBrowseResource(event) {
|
|
11765
11815
|
try {
|
|
11766
11816
|
const response = await assembleResourceGraph(this.kb, resourceId(event.resourceId));
|
|
@@ -12916,6 +12966,7 @@ async function startMakeMeaning(project, config, eventBus, logger, options) {
|
|
|
12916
12966
|
return {
|
|
12917
12967
|
knowledgeSystem,
|
|
12918
12968
|
jobQueue,
|
|
12969
|
+
project,
|
|
12919
12970
|
stop: async () => {
|
|
12920
12971
|
logger.info("Stopping Make-Meaning service");
|
|
12921
12972
|
jobStatusSubscription.unsubscribe();
|
|
@@ -13029,6 +13080,40 @@ var LocalContentTransport = class {
|
|
|
13029
13080
|
"LocalContentTransport does not support putBinary() \u2014 create resources via bus emits (mark/yield namespaces) in local mode"
|
|
13030
13081
|
);
|
|
13031
13082
|
}
|
|
13083
|
+
/**
|
|
13084
|
+
* Store a derived coordinate map under the content checksum the producer
|
|
13085
|
+
* read (PERSIST-ANCHORS decision A — see the interface doc for why the
|
|
13086
|
+
* producer supplies the key). In local mode this is the same store the
|
|
13087
|
+
* HTTP route writes to — one storage authority, reached the same way from
|
|
13088
|
+
* every process (ANCHORED-TEXT-CACHE Lane 5).
|
|
13089
|
+
*/
|
|
13090
|
+
async putAnchoredText(checksum, outcome, _options) {
|
|
13091
|
+
busLog("PUT", "anchored-text", { checksum });
|
|
13092
|
+
await this.ks.kb.anchoredText.write(checksum, outcome);
|
|
13093
|
+
}
|
|
13094
|
+
/** The stored outcome, or null when nothing has derived one — the common case. */
|
|
13095
|
+
async getAnchoredText(resourceId10, _options) {
|
|
13096
|
+
busLog("GET", "anchored-text", { resourceId: resourceId10 });
|
|
13097
|
+
return readAnchoredText(this.ks.kb, resourceId10);
|
|
13098
|
+
}
|
|
13099
|
+
/**
|
|
13100
|
+
* The cache-consult read (PERSIST-ANCHORS P2c), straight from the store —
|
|
13101
|
+
* checksum-addressed, so no view resolution and no settle barrier: the
|
|
13102
|
+
* caller holds the content identity already.
|
|
13103
|
+
*/
|
|
13104
|
+
async getAnchoredTextByChecksum(checksum, _options) {
|
|
13105
|
+
busLog("GET", "anchored-text-by-checksum", { checksum });
|
|
13106
|
+
return this.ks.kb.anchoredText.read(checksum);
|
|
13107
|
+
}
|
|
13108
|
+
/**
|
|
13109
|
+
* The store's would-hit keys, straight from the store — planning data for
|
|
13110
|
+
* the reconcile diff (PERSIST-ANCHORS P0), so no settle barrier applies:
|
|
13111
|
+
* presence is being asked, not content at a moment.
|
|
13112
|
+
*/
|
|
13113
|
+
async listAnchoredTextKeys(_options) {
|
|
13114
|
+
busLog("GET", "anchored-text-keys", {});
|
|
13115
|
+
return this.ks.kb.anchoredText.list();
|
|
13116
|
+
}
|
|
13032
13117
|
async getBinary(resourceId10, _options) {
|
|
13033
13118
|
busLog("GET", "content", { resourceId: resourceId10 });
|
|
13034
13119
|
return withSpan(
|
|
@@ -13111,6 +13196,7 @@ function sameStringSet(a, b) {
|
|
|
13111
13196
|
var WORK_ITEM_TYPES = /* @__PURE__ */ new Set([
|
|
13112
13197
|
"smelt:embed",
|
|
13113
13198
|
"smelt:restamp",
|
|
13199
|
+
"smelt:reanchor",
|
|
13114
13200
|
"smelt:purge",
|
|
13115
13201
|
"smelt:embed-annotation",
|
|
13116
13202
|
"smelt:purge-annotation"
|
|
@@ -13119,8 +13205,9 @@ function isWorkItem(input) {
|
|
|
13119
13205
|
return WORK_ITEM_TYPES.has(input.type);
|
|
13120
13206
|
}
|
|
13121
13207
|
var Smelter = class _Smelter {
|
|
13122
|
-
constructor(events$, vectorStore, embeddingProvider, content, bus, chunkingConfig, timing, logger) {
|
|
13208
|
+
constructor(events$, rebuildAnchors$, vectorStore, embeddingProvider, content, bus, chunkingConfig, timing, logger) {
|
|
13123
13209
|
this.events$ = events$;
|
|
13210
|
+
this.rebuildAnchors$ = rebuildAnchors$;
|
|
13124
13211
|
this.vectorStore = vectorStore;
|
|
13125
13212
|
this.embeddingProvider = embeddingProvider;
|
|
13126
13213
|
this.content = content;
|
|
@@ -13128,8 +13215,10 @@ var Smelter = class _Smelter {
|
|
|
13128
13215
|
this.chunkingConfig = chunkingConfig;
|
|
13129
13216
|
this.timing = timing;
|
|
13130
13217
|
this.logger = logger;
|
|
13218
|
+
this.anchoredStore = anchoredTextStoreOverTransport(content, logger.child({ component: "anchored-text-cache" }));
|
|
13131
13219
|
}
|
|
13132
13220
|
events$;
|
|
13221
|
+
rebuildAnchors$;
|
|
13133
13222
|
vectorStore;
|
|
13134
13223
|
embeddingProvider;
|
|
13135
13224
|
content;
|
|
@@ -13142,11 +13231,21 @@ var Smelter = class _Smelter {
|
|
|
13142
13231
|
static RECONCILE_WAVE = 8;
|
|
13143
13232
|
eventSubject = new import_rxjs9.Subject();
|
|
13144
13233
|
sourceSubscription = null;
|
|
13234
|
+
commandSubscription = null;
|
|
13145
13235
|
pipelineSubscription = null;
|
|
13146
13236
|
_eventsProcessed = 0;
|
|
13147
13237
|
_reconcileState = { phase: "pending" };
|
|
13148
13238
|
workDone = 0;
|
|
13239
|
+
workFailed = 0;
|
|
13149
13240
|
workWaiter = null;
|
|
13241
|
+
/**
|
|
13242
|
+
* Serializes every planner drain (reconcile, anchored-text rebuilds):
|
|
13243
|
+
* there is one waiter slot, and the weave:rebuild rule — rebuilds never
|
|
13244
|
+
* interleave — applies to every unit here being a potential multi-second
|
|
13245
|
+
* OCR pass.
|
|
13246
|
+
*/
|
|
13247
|
+
drainChain = Promise.resolve();
|
|
13248
|
+
anchoredStore;
|
|
13150
13249
|
get eventsProcessed() {
|
|
13151
13250
|
return this._eventsProcessed;
|
|
13152
13251
|
}
|
|
@@ -13174,7 +13273,7 @@ var Smelter = class _Smelter {
|
|
|
13174
13273
|
return (0, import_rxjs9.from)(
|
|
13175
13274
|
withActorSpan("smelter", inputOrBatch.type, async () => {
|
|
13176
13275
|
const ok = await this.safeProcessEvent(inputOrBatch);
|
|
13177
|
-
if (isWorkItem(inputOrBatch)) this.noteWorkDone(1);
|
|
13276
|
+
if (isWorkItem(inputOrBatch)) this.noteWorkDone(1, ok ? 0 : 1);
|
|
13178
13277
|
else if (ok) this._eventsProcessed++;
|
|
13179
13278
|
})
|
|
13180
13279
|
);
|
|
@@ -13188,18 +13287,26 @@ var Smelter = class _Smelter {
|
|
|
13188
13287
|
this.logger.debug("Bus event received", { type: event.type, resourceId: event.resourceId });
|
|
13189
13288
|
this.eventSubject.next(event);
|
|
13190
13289
|
});
|
|
13290
|
+
this.commandSubscription = this.rebuildAnchors$.pipe(
|
|
13291
|
+
(0, import_operators7.concatMap)((command) => (0, import_rxjs9.from)(this.rebuildAnchors(command)))
|
|
13292
|
+
).subscribe({
|
|
13293
|
+
error: (err) => this.logger.error("Smelter command pipeline error", { error: errField(err) })
|
|
13294
|
+
});
|
|
13191
13295
|
this.logger.info("Smelter pipeline initialized");
|
|
13192
13296
|
}
|
|
13193
13297
|
stop() {
|
|
13194
13298
|
this.sourceSubscription?.unsubscribe();
|
|
13195
13299
|
this.sourceSubscription = null;
|
|
13300
|
+
this.commandSubscription?.unsubscribe();
|
|
13301
|
+
this.commandSubscription = null;
|
|
13196
13302
|
this.pipelineSubscription?.unsubscribe();
|
|
13197
13303
|
this.pipelineSubscription = null;
|
|
13198
13304
|
this.eventSubject.complete();
|
|
13199
13305
|
this.logger.info("Smelter stopped");
|
|
13200
13306
|
}
|
|
13201
|
-
noteWorkDone(count) {
|
|
13307
|
+
noteWorkDone(count, failed) {
|
|
13202
13308
|
this.workDone += count;
|
|
13309
|
+
this.workFailed += failed;
|
|
13203
13310
|
if (this.workWaiter && this.workDone >= this.workWaiter.target) {
|
|
13204
13311
|
this.workWaiter.resolve();
|
|
13205
13312
|
this.workWaiter = null;
|
|
@@ -13213,12 +13320,15 @@ var Smelter = class _Smelter {
|
|
|
13213
13320
|
let wireProcessed = 0;
|
|
13214
13321
|
for (const run of partitionByType(events)) {
|
|
13215
13322
|
const workRun = isWorkItem(run[0]);
|
|
13323
|
+
let succeeded = 0;
|
|
13216
13324
|
try {
|
|
13217
13325
|
if (run.length === 1) {
|
|
13218
13326
|
const ok = await this.safeProcessEvent(run[0]);
|
|
13327
|
+
if (ok) succeeded = 1;
|
|
13219
13328
|
if (ok && !workRun) wireProcessed++;
|
|
13220
13329
|
} else {
|
|
13221
13330
|
const processed = await this.applyBatchByType(run);
|
|
13331
|
+
succeeded = processed;
|
|
13222
13332
|
if (!workRun) wireProcessed += processed;
|
|
13223
13333
|
}
|
|
13224
13334
|
} catch (error) {
|
|
@@ -13228,7 +13338,7 @@ var Smelter = class _Smelter {
|
|
|
13228
13338
|
error: errField(error)
|
|
13229
13339
|
});
|
|
13230
13340
|
} finally {
|
|
13231
|
-
if (workRun) this.noteWorkDone(run.length);
|
|
13341
|
+
if (workRun) this.noteWorkDone(run.length, run.length - succeeded);
|
|
13232
13342
|
}
|
|
13233
13343
|
}
|
|
13234
13344
|
return wireProcessed;
|
|
@@ -13300,6 +13410,9 @@ var Smelter = class _Smelter {
|
|
|
13300
13410
|
case "smelt:restamp":
|
|
13301
13411
|
await this.restampResource(event);
|
|
13302
13412
|
break;
|
|
13413
|
+
case "smelt:reanchor":
|
|
13414
|
+
await this.reanchorResource(event);
|
|
13415
|
+
break;
|
|
13303
13416
|
case "smelt:purge":
|
|
13304
13417
|
await this.handleResourcePurge(event);
|
|
13305
13418
|
break;
|
|
@@ -13326,6 +13439,45 @@ var Smelter = class _Smelter {
|
|
|
13326
13439
|
await this.vectorStore.updateResourceEntityTypes(resourceId(rid), entityTypes);
|
|
13327
13440
|
this.logger.info("Restamped resource entity types", { resourceId: rid, entityTypes });
|
|
13328
13441
|
}
|
|
13442
|
+
/**
|
|
13443
|
+
* Re-derive a lost anchored-text artifact from the resource's current
|
|
13444
|
+
* bytes (PERSIST-ANCHORS P0, the third drift class). Extraction is the
|
|
13445
|
+
* cost here — the vectors are already correct, so this NEVER calls the
|
|
13446
|
+
* embedding provider, the vector store, or the settled signal: the index
|
|
13447
|
+
* decision was already made and announced at its checksum; only the map
|
|
13448
|
+
* is missing. Name the work for what it does (the S13 discipline).
|
|
13449
|
+
*
|
|
13450
|
+
* The publish is STRICT, unlike the embed path's best-effort side
|
|
13451
|
+
* publish: here the artifact IS the job, so a store failure must throw —
|
|
13452
|
+
* the pipeline logs and counts it, and the rebuild command's partial-
|
|
13453
|
+
* failure accounting depends on that throw.
|
|
13454
|
+
*/
|
|
13455
|
+
async reanchorResource(event) {
|
|
13456
|
+
const rid = event.resourceId;
|
|
13457
|
+
if (!rid) return;
|
|
13458
|
+
const { data, contentType } = await this.content.getBinary(resourceId(rid));
|
|
13459
|
+
const bytes = Buffer.from(data);
|
|
13460
|
+
const extractor = EXTRACTORS[textExtractionOf(contentType)];
|
|
13461
|
+
if (!extractor?.yieldsGeometry) {
|
|
13462
|
+
this.logger.info("Re-anchor found no geometry-capable extractor", { resourceId: rid, contentType });
|
|
13463
|
+
return;
|
|
13464
|
+
}
|
|
13465
|
+
const checksum = calculateChecksum(bytes);
|
|
13466
|
+
const extracted = await extractor.extract(bytes, contentType, {
|
|
13467
|
+
key: checksum,
|
|
13468
|
+
store: this.anchoredStore
|
|
13469
|
+
});
|
|
13470
|
+
if ("declined" in extracted || !extracted.items?.length) {
|
|
13471
|
+
this.logger.info("Re-anchor extraction yielded no geometry", {
|
|
13472
|
+
resourceId: rid,
|
|
13473
|
+
contentType,
|
|
13474
|
+
..."declined" in extracted ? { declined: extracted.declined } : {}
|
|
13475
|
+
});
|
|
13476
|
+
return;
|
|
13477
|
+
}
|
|
13478
|
+
await this.content.putAnchoredText(checksum, { ...extracted, items: extracted.items });
|
|
13479
|
+
this.logger.info("Re-anchored resource", { resourceId: rid, checksum, items: extracted.items.length });
|
|
13480
|
+
}
|
|
13329
13481
|
async handleResourcePurge(event) {
|
|
13330
13482
|
const rid = event.resourceId;
|
|
13331
13483
|
if (!rid) return;
|
|
@@ -13345,12 +13497,34 @@ var Smelter = class _Smelter {
|
|
|
13345
13497
|
const { data, contentType } = await this.content.getBinary(resourceId(resourceId10));
|
|
13346
13498
|
const bytes = Buffer.from(data);
|
|
13347
13499
|
const checksum = calculateChecksum(bytes);
|
|
13348
|
-
|
|
13349
|
-
|
|
13350
|
-
|
|
13500
|
+
const extractor = EXTRACTORS[textExtractionOf(contentType)];
|
|
13501
|
+
if (!extractor) {
|
|
13502
|
+
this.logger.debug("Skipping resource with no extractor for its media type", { resourceId: resourceId10, contentType });
|
|
13503
|
+
return { kind: "skipped", checksum, reason: "no-extractor" };
|
|
13504
|
+
}
|
|
13505
|
+
const extracted = await extractor.extract(bytes, contentType, {
|
|
13506
|
+
key: checksum,
|
|
13507
|
+
store: this.anchoredStore
|
|
13508
|
+
});
|
|
13509
|
+
if ("declined" in extracted) {
|
|
13510
|
+
this.logger.debug("Extractor declined", { resourceId: resourceId10, contentType, reason: extracted.declined });
|
|
13511
|
+
return { kind: "skipped", checksum, reason: extracted.declined };
|
|
13512
|
+
}
|
|
13513
|
+
if (extracted.ocrConfidence && extracted.ocrConfidence.lowConfidenceWords > 0) {
|
|
13514
|
+
this.logger.info("OCR read words it was unsure of", {
|
|
13515
|
+
resourceId: resourceId10,
|
|
13516
|
+
contentType,
|
|
13517
|
+
...extracted.ocrConfidence
|
|
13518
|
+
});
|
|
13519
|
+
}
|
|
13520
|
+
if (extracted.unreadPages?.length) {
|
|
13521
|
+
this.logger.info("Partial extraction coverage", {
|
|
13522
|
+
resourceId: resourceId10,
|
|
13523
|
+
contentType,
|
|
13524
|
+
unreadPages: extracted.unreadPages
|
|
13525
|
+
});
|
|
13351
13526
|
}
|
|
13352
|
-
|
|
13353
|
-
return text.trim() ? { kind: "text", text, checksum } : { kind: "skipped", checksum };
|
|
13527
|
+
return extracted.text.trim() ? { kind: "text", text: extracted.text, checksum, machineRead: extracted.method === "ocr" } : { kind: "skipped", checksum, reason: "empty" };
|
|
13354
13528
|
} catch (error) {
|
|
13355
13529
|
this.logger.warn("Content unavailable for embedding", { resourceId: resourceId10, error: errField(error) });
|
|
13356
13530
|
return { kind: "unavailable" };
|
|
@@ -13362,9 +13536,9 @@ var Smelter = class _Smelter {
|
|
|
13362
13536
|
* fold. Best-effort — waiters degrade to their bounded timeout; a signal
|
|
13363
13537
|
* failure must never fail the embed.
|
|
13364
13538
|
*/
|
|
13365
|
-
async emitSettled(resourceId10, contentChecksum, outcome) {
|
|
13539
|
+
async emitSettled(resourceId10, contentChecksum, outcome, reason) {
|
|
13366
13540
|
try {
|
|
13367
|
-
await this.bus.emit("smelt:settled", { resourceId: resourceId10, contentChecksum, outcome });
|
|
13541
|
+
await this.bus.emit("smelt:settled", { resourceId: resourceId10, contentChecksum, outcome, ...reason ? { reason } : {} });
|
|
13368
13542
|
} catch (error) {
|
|
13369
13543
|
this.logger.warn("Failed to emit smelt:settled", { resourceId: resourceId10, outcome, error: errField(error) });
|
|
13370
13544
|
}
|
|
@@ -13392,12 +13566,14 @@ var Smelter = class _Smelter {
|
|
|
13392
13566
|
const fetched = await this.fetchEmbeddableText(rid);
|
|
13393
13567
|
if (fetched.kind === "unavailable") return;
|
|
13394
13568
|
if (fetched.kind === "skipped") {
|
|
13395
|
-
await this.
|
|
13569
|
+
await this.vectorStore.deleteResourceVectors(resourceId(rid));
|
|
13570
|
+
await this.emitSettled(rid, fetched.checksum, "skipped", fetched.reason);
|
|
13396
13571
|
return;
|
|
13397
13572
|
}
|
|
13398
13573
|
const chunks = chunkText(fetched.text, this.chunkingConfig);
|
|
13399
13574
|
if (chunks.length === 0) {
|
|
13400
|
-
await this.
|
|
13575
|
+
await this.vectorStore.deleteResourceVectors(resourceId(rid));
|
|
13576
|
+
await this.emitSettled(rid, fetched.checksum, "skipped", "empty");
|
|
13401
13577
|
return;
|
|
13402
13578
|
}
|
|
13403
13579
|
const entityTypes = await this.resolveEntityTypes(rid);
|
|
@@ -13407,7 +13583,7 @@ var Smelter = class _Smelter {
|
|
|
13407
13583
|
text: t,
|
|
13408
13584
|
embedding: embeddings[i]
|
|
13409
13585
|
}));
|
|
13410
|
-
await this.vectorStore.upsertResourceVectors(resourceId(rid), embeddingChunks, fetched.checksum, entityTypes);
|
|
13586
|
+
await this.vectorStore.upsertResourceVectors(resourceId(rid), embeddingChunks, fetched.checksum, entityTypes, fetched.machineRead);
|
|
13411
13587
|
await this.emitSettled(rid, fetched.checksum, "indexed");
|
|
13412
13588
|
this.logger.info(logMessage, { resourceId: rid, chunks: chunks.length });
|
|
13413
13589
|
}
|
|
@@ -13451,12 +13627,14 @@ var Smelter = class _Smelter {
|
|
|
13451
13627
|
if (!exactText?.trim()) return;
|
|
13452
13628
|
const aid = annotationId(annotation.id);
|
|
13453
13629
|
const embedding = await this.embeddingProvider.embed(exactText);
|
|
13630
|
+
const stamp = await this.vectorStore.getResourceStamp(resourceId(rid));
|
|
13454
13631
|
const payload = {
|
|
13455
13632
|
annotationId: aid,
|
|
13456
13633
|
resourceId: resourceId(rid),
|
|
13457
13634
|
motivation: annotation.motivation ?? "",
|
|
13458
13635
|
entityTypes: annotation.entityTypes ?? [],
|
|
13459
|
-
exactText
|
|
13636
|
+
exactText,
|
|
13637
|
+
...stamp?.machineRead ? { machineRead: true } : {}
|
|
13460
13638
|
};
|
|
13461
13639
|
await this.vectorStore.upsertAnnotationVector(aid, embedding, payload);
|
|
13462
13640
|
this.logger.info("Indexed annotation", { annotationId: String(aid) });
|
|
@@ -13481,28 +13659,30 @@ var Smelter = class _Smelter {
|
|
|
13481
13659
|
const fetched = await this.fetchEmbeddableText(rid);
|
|
13482
13660
|
if (fetched.kind === "unavailable") continue;
|
|
13483
13661
|
if (fetched.kind === "skipped") {
|
|
13484
|
-
await this.
|
|
13662
|
+
await this.vectorStore.deleteResourceVectors(resourceId(rid));
|
|
13663
|
+
await this.emitSettled(rid, fetched.checksum, "skipped", fetched.reason);
|
|
13485
13664
|
continue;
|
|
13486
13665
|
}
|
|
13487
13666
|
const chunks = chunkText(fetched.text, this.chunkingConfig);
|
|
13488
13667
|
if (chunks.length === 0) {
|
|
13489
|
-
await this.
|
|
13668
|
+
await this.vectorStore.deleteResourceVectors(resourceId(rid));
|
|
13669
|
+
await this.emitSettled(rid, fetched.checksum, "skipped", "empty");
|
|
13490
13670
|
continue;
|
|
13491
13671
|
}
|
|
13492
13672
|
const entityTypes = await this.resolveEntityTypes(rid);
|
|
13493
|
-
resourceData.push({ rid: resourceId(rid), chunks, checksum: fetched.checksum, entityTypes });
|
|
13673
|
+
resourceData.push({ rid: resourceId(rid), chunks, checksum: fetched.checksum, entityTypes, machineRead: fetched.machineRead });
|
|
13494
13674
|
allChunks.push(...chunks);
|
|
13495
13675
|
}
|
|
13496
13676
|
if (allChunks.length === 0) return events.length;
|
|
13497
13677
|
const allEmbeddings = await this.embeddingProvider.embedBatch(allChunks);
|
|
13498
13678
|
let offset = 0;
|
|
13499
|
-
for (const { rid, chunks, checksum, entityTypes } of resourceData) {
|
|
13679
|
+
for (const { rid, chunks, checksum, entityTypes, machineRead } of resourceData) {
|
|
13500
13680
|
const embeddingChunks = chunks.map((t, i) => ({
|
|
13501
13681
|
chunkIndex: i,
|
|
13502
13682
|
text: t,
|
|
13503
13683
|
embedding: allEmbeddings[offset + i]
|
|
13504
13684
|
}));
|
|
13505
|
-
await this.vectorStore.upsertResourceVectors(rid, embeddingChunks, checksum, entityTypes);
|
|
13685
|
+
await this.vectorStore.upsertResourceVectors(rid, embeddingChunks, checksum, entityTypes, machineRead);
|
|
13506
13686
|
await this.emitSettled(String(rid), checksum, "indexed");
|
|
13507
13687
|
this.logger.info("Batch-indexed resource", { resourceId: String(rid), chunks: chunks.length });
|
|
13508
13688
|
offset += chunks.length;
|
|
@@ -13575,26 +13755,23 @@ var Smelter = class _Smelter {
|
|
|
13575
13755
|
}
|
|
13576
13756
|
this._reconcileState = { phase: "running" };
|
|
13577
13757
|
try {
|
|
13578
|
-
const [indexedResources, indexedAnnotations] = await Promise.all([
|
|
13758
|
+
const [indexedResources, indexedAnnotations, anchoredKeys] = await Promise.all([
|
|
13579
13759
|
this.vectorStore.listResourceStamps(),
|
|
13580
|
-
this.vectorStore.listAnnotationIds()
|
|
13760
|
+
this.vectorStore.listAnnotationIds(),
|
|
13761
|
+
// The artifact store's would-hit keys — one bulk read, never a probe
|
|
13762
|
+
// per resource (PERSIST-ANCHORS P0). Keys are resource ids today;
|
|
13763
|
+
// P1 rekeys the store by content checksum and this lookup moves
|
|
13764
|
+
// with it.
|
|
13765
|
+
this.content.listAnchoredTextKeys().then((keys) => new Set(keys))
|
|
13581
13766
|
]);
|
|
13582
13767
|
const resources = await this.listAllResources();
|
|
13583
13768
|
this.logger.info("Reconcile started", {
|
|
13584
13769
|
indexedResources: indexedResources.size,
|
|
13585
13770
|
indexedAnnotations: indexedAnnotations.size,
|
|
13771
|
+
anchoredArtifacts: anchoredKeys.size,
|
|
13586
13772
|
liveResources: resources.length
|
|
13587
13773
|
});
|
|
13588
|
-
const embeddable =
|
|
13589
|
-
for (const resource of resources) {
|
|
13590
|
-
const mediaType = getPrimaryMediaType(resource);
|
|
13591
|
-
if (resource["@id"] && mediaType && textExtractionOf(mediaType) === "decode") {
|
|
13592
|
-
embeddable.set(resource["@id"], {
|
|
13593
|
-
checksum: getPrimaryRepresentation(resource)?.checksum,
|
|
13594
|
-
entityTypes: getResourceEntityTypes(resource)
|
|
13595
|
-
});
|
|
13596
|
-
}
|
|
13597
|
-
}
|
|
13774
|
+
const embeddable = this.classifyEmbeddable(resources);
|
|
13598
13775
|
const work = [];
|
|
13599
13776
|
for (const rid of indexedResources.keys()) {
|
|
13600
13777
|
if (!embeddable.has(rid)) work.push({ type: "smelt:purge", resourceId: rid, payload: {} });
|
|
@@ -13608,6 +13785,9 @@ var Smelter = class _Smelter {
|
|
|
13608
13785
|
} else if (!sameStringSet(indexed.entityTypes, catalog.entityTypes)) {
|
|
13609
13786
|
work.push({ type: "smelt:restamp", resourceId: rid, payload: {} });
|
|
13610
13787
|
}
|
|
13788
|
+
if (indexed && catalog.checksum !== void 0 && indexed.contentChecksum === catalog.checksum && catalog.yieldsGeometry && !anchoredKeys.has(catalog.checksum)) {
|
|
13789
|
+
work.push({ type: "smelt:reanchor", resourceId: rid, payload: {} });
|
|
13790
|
+
}
|
|
13611
13791
|
}
|
|
13612
13792
|
const liveAnnotationIds = /* @__PURE__ */ new Set();
|
|
13613
13793
|
for (const resource of resources) {
|
|
@@ -13636,9 +13816,12 @@ var Smelter = class _Smelter {
|
|
|
13636
13816
|
const summary = {
|
|
13637
13817
|
resourcesEmbedded: work.filter((w) => w.type === "smelt:embed").length,
|
|
13638
13818
|
resourcesRestamped: work.filter((w) => w.type === "smelt:restamp").length,
|
|
13819
|
+
resourcesReanchored: work.filter((w) => w.type === "smelt:reanchor").length,
|
|
13639
13820
|
resourceVectorsDeleted: work.filter((w) => w.type === "smelt:purge").length,
|
|
13640
13821
|
annotationsEmbedded: work.filter((w) => w.type === "smelt:embed-annotation").length,
|
|
13641
|
-
annotationVectorsDeleted: work.filter((w) => w.type === "smelt:purge-annotation").length
|
|
13822
|
+
annotationVectorsDeleted: work.filter((w) => w.type === "smelt:purge-annotation").length,
|
|
13823
|
+
resourcesEligible: embeddable.size,
|
|
13824
|
+
resourcesIndexed: (await this.vectorStore.listResourceStamps()).size
|
|
13642
13825
|
};
|
|
13643
13826
|
this._reconcileState = { phase: "done", summary };
|
|
13644
13827
|
this.logger.info("Reconcile complete", { ...summary });
|
|
@@ -13657,16 +13840,106 @@ var Smelter = class _Smelter {
|
|
|
13657
13840
|
* completion. The pipeline ticks `noteWorkDone` for every consumed work
|
|
13658
13841
|
* item (success or failure — failures are logged like any live event), so
|
|
13659
13842
|
* each wave's waiter resolves exactly when its items have been processed.
|
|
13843
|
+
*
|
|
13844
|
+
* Serialized through `drainChain`: there is ONE waiter slot, and the
|
|
13845
|
+
* planners that drain (reconcile, `smelt:rebuild-anchors`) must not
|
|
13846
|
+
* interleave — a rebuild command arriving mid-reconcile waits its turn.
|
|
13847
|
+
*
|
|
13848
|
+
* @returns how many of THESE items failed — the rebuild command's
|
|
13849
|
+
* partial-failure accounting (failure detail is in the logs).
|
|
13660
13850
|
*/
|
|
13661
13851
|
async drain(work) {
|
|
13662
|
-
|
|
13663
|
-
|
|
13664
|
-
const
|
|
13665
|
-
|
|
13666
|
-
|
|
13667
|
-
|
|
13668
|
-
|
|
13852
|
+
let failures = 0;
|
|
13853
|
+
const run = this.drainChain.then(async () => {
|
|
13854
|
+
const failedBefore = this.workFailed;
|
|
13855
|
+
for (let i = 0; i < work.length; i += _Smelter.RECONCILE_WAVE) {
|
|
13856
|
+
const wave = work.slice(i, i + _Smelter.RECONCILE_WAVE);
|
|
13857
|
+
const done = new Promise((resolve2) => {
|
|
13858
|
+
this.workWaiter = { target: this.workDone + wave.length, resolve: resolve2 };
|
|
13859
|
+
});
|
|
13860
|
+
for (const item of wave) this.eventSubject.next(item);
|
|
13861
|
+
await done;
|
|
13862
|
+
}
|
|
13863
|
+
failures = this.workFailed - failedBefore;
|
|
13864
|
+
});
|
|
13865
|
+
this.drainChain = run.then(() => void 0, () => void 0);
|
|
13866
|
+
await run;
|
|
13867
|
+
return failures;
|
|
13868
|
+
}
|
|
13869
|
+
/**
|
|
13870
|
+
* `smelt:rebuild-anchors` — the operator's explicit re-derivation of
|
|
13871
|
+
* anchored-text artifacts (PERSIST-ANCHORS P0), shaped after
|
|
13872
|
+
* `weave:rebuild`: optionally scoped, strictly serialized (concatMap on
|
|
13873
|
+
* the command stream + the drain chain), correlated ok/failed replies,
|
|
13874
|
+
* and partial completion FAILS — a rebuild that quietly skipped resources
|
|
13875
|
+
* would present exactly like a document with no text, which is the #845
|
|
13876
|
+
* failure mode wearing different clothes.
|
|
13877
|
+
*
|
|
13878
|
+
* Never destructive: nothing is deleted first, stale entries are simply
|
|
13879
|
+
* overwritten (the W5-frames lesson — a rebuild that clears before it
|
|
13880
|
+
* re-derives turns a partial failure into a loss). Re-anchoring makes
|
|
13881
|
+
* zero embedding calls; work items ride the normal per-resource lanes,
|
|
13882
|
+
* so a rebuild can never interleave with live processing of the same
|
|
13883
|
+
* resource (S1/S2).
|
|
13884
|
+
*/
|
|
13885
|
+
async rebuildAnchors(command) {
|
|
13886
|
+
const { correlationId, resourceId: resourceId10 } = command;
|
|
13887
|
+
try {
|
|
13888
|
+
let work;
|
|
13889
|
+
if (resourceId10) {
|
|
13890
|
+
work = [{ type: "smelt:reanchor", resourceId: resourceId10, payload: {} }];
|
|
13891
|
+
} else {
|
|
13892
|
+
const resources = await this.listAllResources();
|
|
13893
|
+
work = [...this.classifyEmbeddable(resources)].filter(([, catalog]) => catalog.yieldsGeometry).map(([rid]) => ({ type: "smelt:reanchor", resourceId: rid, payload: {} }));
|
|
13894
|
+
}
|
|
13895
|
+
this.logger.info("Anchored-text rebuild started", { scoped: resourceId10 ?? null, resources: work.length });
|
|
13896
|
+
const failed = await this.drain(work);
|
|
13897
|
+
if (failed > 0) {
|
|
13898
|
+
await this.bus.emit("smelt:rebuild-anchors-failed", {
|
|
13899
|
+
...correlationId ? { correlationId } : {},
|
|
13900
|
+
message: `${failed} of ${work.length} resources failed to re-anchor \u2014 see smelter logs`
|
|
13901
|
+
});
|
|
13902
|
+
return;
|
|
13903
|
+
}
|
|
13904
|
+
await this.bus.emit("smelt:rebuild-anchors-ok", correlationId ? { correlationId } : {});
|
|
13905
|
+
this.logger.info("Anchored-text rebuild complete", { scoped: resourceId10 ?? null, resources: work.length });
|
|
13906
|
+
} catch (error) {
|
|
13907
|
+
this.logger.error("Anchored-text rebuild failed", { error: errField(error) });
|
|
13908
|
+
try {
|
|
13909
|
+
await this.bus.emit("smelt:rebuild-anchors-failed", {
|
|
13910
|
+
...correlationId ? { correlationId } : {},
|
|
13911
|
+
message: error instanceof Error ? error.message : String(error)
|
|
13912
|
+
});
|
|
13913
|
+
} catch (emitError) {
|
|
13914
|
+
this.logger.warn("Failed to emit smelt:rebuild-anchors-failed", { error: errField(emitError) });
|
|
13915
|
+
}
|
|
13916
|
+
}
|
|
13917
|
+
}
|
|
13918
|
+
/**
|
|
13919
|
+
* Embeddable live resources, each with the catalog's claims: the primary
|
|
13920
|
+
* representation's checksum (the bytes the smelter would read), the
|
|
13921
|
+
* current entity-type set (the discriminator the stamps must carry), and
|
|
13922
|
+
* whether the media type's extractor derives geometry (whether an
|
|
13923
|
+
* anchored-text artifact should exist). Embeddable ⇔ an extractor exists
|
|
13924
|
+
* for the media type's strategy — the same registry the live fetch
|
|
13925
|
+
* resolves, and `yieldsGeometry` is declared on the extractor itself, so
|
|
13926
|
+
* every gate here and the live fetch's behavior are twins by construction.
|
|
13927
|
+
* Shared by `reconcile()` and the `smelt:rebuild-anchors` planner.
|
|
13928
|
+
*/
|
|
13929
|
+
classifyEmbeddable(resources) {
|
|
13930
|
+
const embeddable = /* @__PURE__ */ new Map();
|
|
13931
|
+
for (const resource of resources) {
|
|
13932
|
+
const mediaType = getPrimaryMediaType(resource);
|
|
13933
|
+
const extractor = mediaType ? EXTRACTORS[textExtractionOf(mediaType)] : null;
|
|
13934
|
+
if (resource["@id"] && extractor) {
|
|
13935
|
+
embeddable.set(resource["@id"], {
|
|
13936
|
+
checksum: getPrimaryRepresentation(resource)?.checksum,
|
|
13937
|
+
entityTypes: getResourceEntityTypes(resource),
|
|
13938
|
+
yieldsGeometry: extractor.yieldsGeometry
|
|
13939
|
+
});
|
|
13940
|
+
}
|
|
13669
13941
|
}
|
|
13942
|
+
return embeddable;
|
|
13670
13943
|
}
|
|
13671
13944
|
/** Page through `browse:resources-requested` until the catalog is exhausted. */
|
|
13672
13945
|
async listAllResources() {
|
|
@@ -13697,6 +13970,7 @@ var SMELTER_CHANNELS = [
|
|
|
13697
13970
|
"mark:entity-tag-added",
|
|
13698
13971
|
"mark:entity-tag-removed"
|
|
13699
13972
|
];
|
|
13973
|
+
var SMELTER_COMMAND_CHANNELS = ["smelt:rebuild-anchors"];
|
|
13700
13974
|
function createSmelterActorStateUnit(options) {
|
|
13701
13975
|
const { bus } = options;
|
|
13702
13976
|
let started = false;
|
|
@@ -13711,12 +13985,14 @@ function createSmelterActorStateUnit(options) {
|
|
|
13711
13985
|
)
|
|
13712
13986
|
)
|
|
13713
13987
|
);
|
|
13988
|
+
const rebuildAnchors$ = bus.on$("smelt:rebuild-anchors");
|
|
13714
13989
|
return {
|
|
13715
13990
|
events$,
|
|
13991
|
+
rebuildAnchors$,
|
|
13716
13992
|
start: () => {
|
|
13717
13993
|
if (started) return;
|
|
13718
13994
|
started = true;
|
|
13719
|
-
bus.addChannels?.([...SMELTER_CHANNELS]);
|
|
13995
|
+
bus.addChannels?.([...SMELTER_CHANNELS, ...SMELTER_COMMAND_CHANNELS]);
|
|
13720
13996
|
},
|
|
13721
13997
|
dispose: () => {
|
|
13722
13998
|
started = false;
|