@semiont/make-meaning 0.5.24 → 0.5.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +159 -20
- package/dist/index.js +357 -93
- package/dist/index.js.map +1 -1
- package/dist/smelter-main.js +236 -44
- package/dist/smelter-main.js.map +1 -1
- package/package.json +13 -13
package/dist/smelter-main.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { createTomlConfigLoader, accessToken, baseUrl as baseUrl$1, retryWithBackoff, isTransientFetchError, STARTUP_FETCH_RETRY, burstBuffer, errField, resourceId, textExtractionOf,
|
|
2
|
-
import { calculateChecksum } from '@semiont/content';
|
|
3
|
-
import { createEmbeddingProvider, createVectorStore, chunkText } from '@semiont/vectors';
|
|
1
|
+
import { createTomlConfigLoader, accessToken, baseUrl as baseUrl$1, retryWithBackoff, isTransientFetchError, STARTUP_FETCH_RETRY, burstBuffer, errField, resourceId, textExtractionOf, busRequest, getResourceEntityTypes, chunkText, getTargetSelector, getExactText, annotationId, getPrimaryMediaType, getPrimaryRepresentation } from '@semiont/core';
|
|
2
|
+
import { anchoredTextStoreOverTransport, EXTRACTORS, calculateChecksum } from '@semiont/content';
|
|
4
3
|
import { registerVectorIndexSizeProvider, withActorSpan } from '@semiont/observability';
|
|
5
4
|
import { HttpTransport, HttpContentTransport } from '@semiont/http-transport';
|
|
5
|
+
import { createEmbeddingProvider, createVectorStore } from '@semiont/vectors';
|
|
6
6
|
import { createServer } from 'http';
|
|
7
7
|
import { existsSync, readFileSync } from 'fs';
|
|
8
8
|
import { homedir } from 'os';
|
|
@@ -9738,6 +9738,7 @@ var SMELTER_CHANNELS = [
|
|
|
9738
9738
|
"mark:entity-tag-added",
|
|
9739
9739
|
"mark:entity-tag-removed"
|
|
9740
9740
|
];
|
|
9741
|
+
var SMELTER_COMMAND_CHANNELS = ["smelt:rebuild-anchors"];
|
|
9741
9742
|
function createSmelterActorStateUnit(options) {
|
|
9742
9743
|
const { bus } = options;
|
|
9743
9744
|
let started = false;
|
|
@@ -9752,12 +9753,14 @@ function createSmelterActorStateUnit(options) {
|
|
|
9752
9753
|
)
|
|
9753
9754
|
)
|
|
9754
9755
|
);
|
|
9756
|
+
const rebuildAnchors$ = bus.on$("smelt:rebuild-anchors");
|
|
9755
9757
|
return {
|
|
9756
9758
|
events$,
|
|
9759
|
+
rebuildAnchors$,
|
|
9757
9760
|
start: () => {
|
|
9758
9761
|
if (started) return;
|
|
9759
9762
|
started = true;
|
|
9760
|
-
bus.addChannels?.([...SMELTER_CHANNELS]);
|
|
9763
|
+
bus.addChannels?.([...SMELTER_CHANNELS, ...SMELTER_COMMAND_CHANNELS]);
|
|
9761
9764
|
},
|
|
9762
9765
|
dispose: () => {
|
|
9763
9766
|
started = false;
|
|
@@ -9793,6 +9796,7 @@ function sameStringSet(a, b) {
|
|
|
9793
9796
|
var WORK_ITEM_TYPES = /* @__PURE__ */ new Set([
|
|
9794
9797
|
"smelt:embed",
|
|
9795
9798
|
"smelt:restamp",
|
|
9799
|
+
"smelt:reanchor",
|
|
9796
9800
|
"smelt:purge",
|
|
9797
9801
|
"smelt:embed-annotation",
|
|
9798
9802
|
"smelt:purge-annotation"
|
|
@@ -9801,8 +9805,9 @@ function isWorkItem(input) {
|
|
|
9801
9805
|
return WORK_ITEM_TYPES.has(input.type);
|
|
9802
9806
|
}
|
|
9803
9807
|
var Smelter = class _Smelter {
|
|
9804
|
-
constructor(events$, vectorStore, embeddingProvider, content, bus, chunkingConfig2, timing, logger2) {
|
|
9808
|
+
constructor(events$, rebuildAnchors$, vectorStore, embeddingProvider, content, bus, chunkingConfig2, timing, logger2) {
|
|
9805
9809
|
this.events$ = events$;
|
|
9810
|
+
this.rebuildAnchors$ = rebuildAnchors$;
|
|
9806
9811
|
this.vectorStore = vectorStore;
|
|
9807
9812
|
this.embeddingProvider = embeddingProvider;
|
|
9808
9813
|
this.content = content;
|
|
@@ -9810,8 +9815,10 @@ var Smelter = class _Smelter {
|
|
|
9810
9815
|
this.chunkingConfig = chunkingConfig2;
|
|
9811
9816
|
this.timing = timing;
|
|
9812
9817
|
this.logger = logger2;
|
|
9818
|
+
this.anchoredStore = anchoredTextStoreOverTransport(content, logger2.child({ component: "anchored-text-cache" }));
|
|
9813
9819
|
}
|
|
9814
9820
|
events$;
|
|
9821
|
+
rebuildAnchors$;
|
|
9815
9822
|
vectorStore;
|
|
9816
9823
|
embeddingProvider;
|
|
9817
9824
|
content;
|
|
@@ -9824,11 +9831,21 @@ var Smelter = class _Smelter {
|
|
|
9824
9831
|
static RECONCILE_WAVE = 8;
|
|
9825
9832
|
eventSubject = new import_rxjs2.Subject();
|
|
9826
9833
|
sourceSubscription = null;
|
|
9834
|
+
commandSubscription = null;
|
|
9827
9835
|
pipelineSubscription = null;
|
|
9828
9836
|
_eventsProcessed = 0;
|
|
9829
9837
|
_reconcileState = { phase: "pending" };
|
|
9830
9838
|
workDone = 0;
|
|
9839
|
+
workFailed = 0;
|
|
9831
9840
|
workWaiter = null;
|
|
9841
|
+
/**
|
|
9842
|
+
* Serializes every planner drain (reconcile, anchored-text rebuilds):
|
|
9843
|
+
* there is one waiter slot, and the weave:rebuild rule — rebuilds never
|
|
9844
|
+
* interleave — applies to every unit here being a potential multi-second
|
|
9845
|
+
* OCR pass.
|
|
9846
|
+
*/
|
|
9847
|
+
drainChain = Promise.resolve();
|
|
9848
|
+
anchoredStore;
|
|
9832
9849
|
get eventsProcessed() {
|
|
9833
9850
|
return this._eventsProcessed;
|
|
9834
9851
|
}
|
|
@@ -9856,7 +9873,7 @@ var Smelter = class _Smelter {
|
|
|
9856
9873
|
return (0, import_rxjs2.from)(
|
|
9857
9874
|
withActorSpan("smelter", inputOrBatch.type, async () => {
|
|
9858
9875
|
const ok = await this.safeProcessEvent(inputOrBatch);
|
|
9859
|
-
if (isWorkItem(inputOrBatch)) this.noteWorkDone(1);
|
|
9876
|
+
if (isWorkItem(inputOrBatch)) this.noteWorkDone(1, ok ? 0 : 1);
|
|
9860
9877
|
else if (ok) this._eventsProcessed++;
|
|
9861
9878
|
})
|
|
9862
9879
|
);
|
|
@@ -9870,18 +9887,26 @@ var Smelter = class _Smelter {
|
|
|
9870
9887
|
this.logger.debug("Bus event received", { type: event.type, resourceId: event.resourceId });
|
|
9871
9888
|
this.eventSubject.next(event);
|
|
9872
9889
|
});
|
|
9890
|
+
this.commandSubscription = this.rebuildAnchors$.pipe(
|
|
9891
|
+
(0, import_operators2.concatMap)((command) => (0, import_rxjs2.from)(this.rebuildAnchors(command)))
|
|
9892
|
+
).subscribe({
|
|
9893
|
+
error: (err) => this.logger.error("Smelter command pipeline error", { error: errField(err) })
|
|
9894
|
+
});
|
|
9873
9895
|
this.logger.info("Smelter pipeline initialized");
|
|
9874
9896
|
}
|
|
9875
9897
|
stop() {
|
|
9876
9898
|
this.sourceSubscription?.unsubscribe();
|
|
9877
9899
|
this.sourceSubscription = null;
|
|
9900
|
+
this.commandSubscription?.unsubscribe();
|
|
9901
|
+
this.commandSubscription = null;
|
|
9878
9902
|
this.pipelineSubscription?.unsubscribe();
|
|
9879
9903
|
this.pipelineSubscription = null;
|
|
9880
9904
|
this.eventSubject.complete();
|
|
9881
9905
|
this.logger.info("Smelter stopped");
|
|
9882
9906
|
}
|
|
9883
|
-
noteWorkDone(count) {
|
|
9907
|
+
noteWorkDone(count, failed) {
|
|
9884
9908
|
this.workDone += count;
|
|
9909
|
+
this.workFailed += failed;
|
|
9885
9910
|
if (this.workWaiter && this.workDone >= this.workWaiter.target) {
|
|
9886
9911
|
this.workWaiter.resolve();
|
|
9887
9912
|
this.workWaiter = null;
|
|
@@ -9895,12 +9920,15 @@ var Smelter = class _Smelter {
|
|
|
9895
9920
|
let wireProcessed = 0;
|
|
9896
9921
|
for (const run of partitionByType(events)) {
|
|
9897
9922
|
const workRun = isWorkItem(run[0]);
|
|
9923
|
+
let succeeded = 0;
|
|
9898
9924
|
try {
|
|
9899
9925
|
if (run.length === 1) {
|
|
9900
9926
|
const ok = await this.safeProcessEvent(run[0]);
|
|
9927
|
+
if (ok) succeeded = 1;
|
|
9901
9928
|
if (ok && !workRun) wireProcessed++;
|
|
9902
9929
|
} else {
|
|
9903
9930
|
const processed = await this.applyBatchByType(run);
|
|
9931
|
+
succeeded = processed;
|
|
9904
9932
|
if (!workRun) wireProcessed += processed;
|
|
9905
9933
|
}
|
|
9906
9934
|
} catch (error) {
|
|
@@ -9910,7 +9938,7 @@ var Smelter = class _Smelter {
|
|
|
9910
9938
|
error: errField(error)
|
|
9911
9939
|
});
|
|
9912
9940
|
} finally {
|
|
9913
|
-
if (workRun) this.noteWorkDone(run.length);
|
|
9941
|
+
if (workRun) this.noteWorkDone(run.length, run.length - succeeded);
|
|
9914
9942
|
}
|
|
9915
9943
|
}
|
|
9916
9944
|
return wireProcessed;
|
|
@@ -9982,6 +10010,9 @@ var Smelter = class _Smelter {
|
|
|
9982
10010
|
case "smelt:restamp":
|
|
9983
10011
|
await this.restampResource(event);
|
|
9984
10012
|
break;
|
|
10013
|
+
case "smelt:reanchor":
|
|
10014
|
+
await this.reanchorResource(event);
|
|
10015
|
+
break;
|
|
9985
10016
|
case "smelt:purge":
|
|
9986
10017
|
await this.handleResourcePurge(event);
|
|
9987
10018
|
break;
|
|
@@ -10008,6 +10039,45 @@ var Smelter = class _Smelter {
|
|
|
10008
10039
|
await this.vectorStore.updateResourceEntityTypes(resourceId(rid), entityTypes);
|
|
10009
10040
|
this.logger.info("Restamped resource entity types", { resourceId: rid, entityTypes });
|
|
10010
10041
|
}
|
|
10042
|
+
/**
|
|
10043
|
+
* Re-derive a lost anchored-text artifact from the resource's current
|
|
10044
|
+
* bytes (PERSIST-ANCHORS P0, the third drift class). Extraction is the
|
|
10045
|
+
* cost here — the vectors are already correct, so this NEVER calls the
|
|
10046
|
+
* embedding provider, the vector store, or the settled signal: the index
|
|
10047
|
+
* decision was already made and announced at its checksum; only the map
|
|
10048
|
+
* is missing. Name the work for what it does (the S13 discipline).
|
|
10049
|
+
*
|
|
10050
|
+
* The publish is STRICT, unlike the embed path's best-effort side
|
|
10051
|
+
* publish: here the artifact IS the job, so a store failure must throw —
|
|
10052
|
+
* the pipeline logs and counts it, and the rebuild command's partial-
|
|
10053
|
+
* failure accounting depends on that throw.
|
|
10054
|
+
*/
|
|
10055
|
+
async reanchorResource(event) {
|
|
10056
|
+
const rid = event.resourceId;
|
|
10057
|
+
if (!rid) return;
|
|
10058
|
+
const { data, contentType } = await this.content.getBinary(resourceId(rid));
|
|
10059
|
+
const bytes = Buffer.from(data);
|
|
10060
|
+
const extractor = EXTRACTORS[textExtractionOf(contentType)];
|
|
10061
|
+
if (!extractor?.yieldsGeometry) {
|
|
10062
|
+
this.logger.info("Re-anchor found no geometry-capable extractor", { resourceId: rid, contentType });
|
|
10063
|
+
return;
|
|
10064
|
+
}
|
|
10065
|
+
const checksum = calculateChecksum(bytes);
|
|
10066
|
+
const extracted = await extractor.extract(bytes, contentType, {
|
|
10067
|
+
key: checksum,
|
|
10068
|
+
store: this.anchoredStore
|
|
10069
|
+
});
|
|
10070
|
+
if ("declined" in extracted || !extracted.items?.length) {
|
|
10071
|
+
this.logger.info("Re-anchor extraction yielded no geometry", {
|
|
10072
|
+
resourceId: rid,
|
|
10073
|
+
contentType,
|
|
10074
|
+
..."declined" in extracted ? { declined: extracted.declined } : {}
|
|
10075
|
+
});
|
|
10076
|
+
return;
|
|
10077
|
+
}
|
|
10078
|
+
await this.content.putAnchoredText(checksum, { ...extracted, items: extracted.items });
|
|
10079
|
+
this.logger.info("Re-anchored resource", { resourceId: rid, checksum, items: extracted.items.length });
|
|
10080
|
+
}
|
|
10011
10081
|
async handleResourcePurge(event) {
|
|
10012
10082
|
const rid = event.resourceId;
|
|
10013
10083
|
if (!rid) return;
|
|
@@ -10027,12 +10097,34 @@ var Smelter = class _Smelter {
|
|
|
10027
10097
|
const { data, contentType } = await this.content.getBinary(resourceId(resourceId$1));
|
|
10028
10098
|
const bytes = Buffer.from(data);
|
|
10029
10099
|
const checksum = calculateChecksum(bytes);
|
|
10030
|
-
|
|
10031
|
-
|
|
10032
|
-
|
|
10100
|
+
const extractor = EXTRACTORS[textExtractionOf(contentType)];
|
|
10101
|
+
if (!extractor) {
|
|
10102
|
+
this.logger.debug("Skipping resource with no extractor for its media type", { resourceId: resourceId$1, contentType });
|
|
10103
|
+
return { kind: "skipped", checksum, reason: "no-extractor" };
|
|
10104
|
+
}
|
|
10105
|
+
const extracted = await extractor.extract(bytes, contentType, {
|
|
10106
|
+
key: checksum,
|
|
10107
|
+
store: this.anchoredStore
|
|
10108
|
+
});
|
|
10109
|
+
if ("declined" in extracted) {
|
|
10110
|
+
this.logger.debug("Extractor declined", { resourceId: resourceId$1, contentType, reason: extracted.declined });
|
|
10111
|
+
return { kind: "skipped", checksum, reason: extracted.declined };
|
|
10112
|
+
}
|
|
10113
|
+
if (extracted.ocrConfidence && extracted.ocrConfidence.lowConfidenceWords > 0) {
|
|
10114
|
+
this.logger.info("OCR read words it was unsure of", {
|
|
10115
|
+
resourceId: resourceId$1,
|
|
10116
|
+
contentType,
|
|
10117
|
+
...extracted.ocrConfidence
|
|
10118
|
+
});
|
|
10033
10119
|
}
|
|
10034
|
-
|
|
10035
|
-
|
|
10120
|
+
if (extracted.unreadPages?.length) {
|
|
10121
|
+
this.logger.info("Partial extraction coverage", {
|
|
10122
|
+
resourceId: resourceId$1,
|
|
10123
|
+
contentType,
|
|
10124
|
+
unreadPages: extracted.unreadPages
|
|
10125
|
+
});
|
|
10126
|
+
}
|
|
10127
|
+
return extracted.text.trim() ? { kind: "text", text: extracted.text, checksum, machineRead: extracted.method === "ocr" } : { kind: "skipped", checksum, reason: "empty" };
|
|
10036
10128
|
} catch (error) {
|
|
10037
10129
|
this.logger.warn("Content unavailable for embedding", { resourceId: resourceId$1, error: errField(error) });
|
|
10038
10130
|
return { kind: "unavailable" };
|
|
@@ -10044,9 +10136,9 @@ var Smelter = class _Smelter {
|
|
|
10044
10136
|
* fold. Best-effort — waiters degrade to their bounded timeout; a signal
|
|
10045
10137
|
* failure must never fail the embed.
|
|
10046
10138
|
*/
|
|
10047
|
-
async emitSettled(resourceId, contentChecksum, outcome) {
|
|
10139
|
+
async emitSettled(resourceId, contentChecksum, outcome, reason) {
|
|
10048
10140
|
try {
|
|
10049
|
-
await this.bus.emit("smelt:settled", { resourceId, contentChecksum, outcome });
|
|
10141
|
+
await this.bus.emit("smelt:settled", { resourceId, contentChecksum, outcome, ...reason ? { reason } : {} });
|
|
10050
10142
|
} catch (error) {
|
|
10051
10143
|
this.logger.warn("Failed to emit smelt:settled", { resourceId, outcome, error: errField(error) });
|
|
10052
10144
|
}
|
|
@@ -10074,12 +10166,14 @@ var Smelter = class _Smelter {
|
|
|
10074
10166
|
const fetched = await this.fetchEmbeddableText(rid);
|
|
10075
10167
|
if (fetched.kind === "unavailable") return;
|
|
10076
10168
|
if (fetched.kind === "skipped") {
|
|
10077
|
-
await this.
|
|
10169
|
+
await this.vectorStore.deleteResourceVectors(resourceId(rid));
|
|
10170
|
+
await this.emitSettled(rid, fetched.checksum, "skipped", fetched.reason);
|
|
10078
10171
|
return;
|
|
10079
10172
|
}
|
|
10080
10173
|
const chunks = chunkText(fetched.text, this.chunkingConfig);
|
|
10081
10174
|
if (chunks.length === 0) {
|
|
10082
|
-
await this.
|
|
10175
|
+
await this.vectorStore.deleteResourceVectors(resourceId(rid));
|
|
10176
|
+
await this.emitSettled(rid, fetched.checksum, "skipped", "empty");
|
|
10083
10177
|
return;
|
|
10084
10178
|
}
|
|
10085
10179
|
const entityTypes = await this.resolveEntityTypes(rid);
|
|
@@ -10089,7 +10183,7 @@ var Smelter = class _Smelter {
|
|
|
10089
10183
|
text: t,
|
|
10090
10184
|
embedding: embeddings[i]
|
|
10091
10185
|
}));
|
|
10092
|
-
await this.vectorStore.upsertResourceVectors(resourceId(rid), embeddingChunks, fetched.checksum, entityTypes);
|
|
10186
|
+
await this.vectorStore.upsertResourceVectors(resourceId(rid), embeddingChunks, fetched.checksum, entityTypes, fetched.machineRead);
|
|
10093
10187
|
await this.emitSettled(rid, fetched.checksum, "indexed");
|
|
10094
10188
|
this.logger.info(logMessage, { resourceId: rid, chunks: chunks.length });
|
|
10095
10189
|
}
|
|
@@ -10133,12 +10227,14 @@ var Smelter = class _Smelter {
|
|
|
10133
10227
|
if (!exactText?.trim()) return;
|
|
10134
10228
|
const aid = annotationId(annotation.id);
|
|
10135
10229
|
const embedding2 = await this.embeddingProvider.embed(exactText);
|
|
10230
|
+
const stamp = await this.vectorStore.getResourceStamp(resourceId(rid));
|
|
10136
10231
|
const payload = {
|
|
10137
10232
|
annotationId: aid,
|
|
10138
10233
|
resourceId: resourceId(rid),
|
|
10139
10234
|
motivation: annotation.motivation ?? "",
|
|
10140
10235
|
entityTypes: annotation.entityTypes ?? [],
|
|
10141
|
-
exactText
|
|
10236
|
+
exactText,
|
|
10237
|
+
...stamp?.machineRead ? { machineRead: true } : {}
|
|
10142
10238
|
};
|
|
10143
10239
|
await this.vectorStore.upsertAnnotationVector(aid, embedding2, payload);
|
|
10144
10240
|
this.logger.info("Indexed annotation", { annotationId: String(aid) });
|
|
@@ -10163,28 +10259,30 @@ var Smelter = class _Smelter {
|
|
|
10163
10259
|
const fetched = await this.fetchEmbeddableText(rid);
|
|
10164
10260
|
if (fetched.kind === "unavailable") continue;
|
|
10165
10261
|
if (fetched.kind === "skipped") {
|
|
10166
|
-
await this.
|
|
10262
|
+
await this.vectorStore.deleteResourceVectors(resourceId(rid));
|
|
10263
|
+
await this.emitSettled(rid, fetched.checksum, "skipped", fetched.reason);
|
|
10167
10264
|
continue;
|
|
10168
10265
|
}
|
|
10169
10266
|
const chunks = chunkText(fetched.text, this.chunkingConfig);
|
|
10170
10267
|
if (chunks.length === 0) {
|
|
10171
|
-
await this.
|
|
10268
|
+
await this.vectorStore.deleteResourceVectors(resourceId(rid));
|
|
10269
|
+
await this.emitSettled(rid, fetched.checksum, "skipped", "empty");
|
|
10172
10270
|
continue;
|
|
10173
10271
|
}
|
|
10174
10272
|
const entityTypes = await this.resolveEntityTypes(rid);
|
|
10175
|
-
resourceData.push({ rid: resourceId(rid), chunks, checksum: fetched.checksum, entityTypes });
|
|
10273
|
+
resourceData.push({ rid: resourceId(rid), chunks, checksum: fetched.checksum, entityTypes, machineRead: fetched.machineRead });
|
|
10176
10274
|
allChunks.push(...chunks);
|
|
10177
10275
|
}
|
|
10178
10276
|
if (allChunks.length === 0) return events.length;
|
|
10179
10277
|
const allEmbeddings = await this.embeddingProvider.embedBatch(allChunks);
|
|
10180
10278
|
let offset = 0;
|
|
10181
|
-
for (const { rid, chunks, checksum, entityTypes } of resourceData) {
|
|
10279
|
+
for (const { rid, chunks, checksum, entityTypes, machineRead } of resourceData) {
|
|
10182
10280
|
const embeddingChunks = chunks.map((t, i) => ({
|
|
10183
10281
|
chunkIndex: i,
|
|
10184
10282
|
text: t,
|
|
10185
10283
|
embedding: allEmbeddings[offset + i]
|
|
10186
10284
|
}));
|
|
10187
|
-
await this.vectorStore.upsertResourceVectors(rid, embeddingChunks, checksum, entityTypes);
|
|
10285
|
+
await this.vectorStore.upsertResourceVectors(rid, embeddingChunks, checksum, entityTypes, machineRead);
|
|
10188
10286
|
await this.emitSettled(String(rid), checksum, "indexed");
|
|
10189
10287
|
this.logger.info("Batch-indexed resource", { resourceId: String(rid), chunks: chunks.length });
|
|
10190
10288
|
offset += chunks.length;
|
|
@@ -10257,26 +10355,23 @@ var Smelter = class _Smelter {
|
|
|
10257
10355
|
}
|
|
10258
10356
|
this._reconcileState = { phase: "running" };
|
|
10259
10357
|
try {
|
|
10260
|
-
const [indexedResources, indexedAnnotations] = await Promise.all([
|
|
10358
|
+
const [indexedResources, indexedAnnotations, anchoredKeys] = await Promise.all([
|
|
10261
10359
|
this.vectorStore.listResourceStamps(),
|
|
10262
|
-
this.vectorStore.listAnnotationIds()
|
|
10360
|
+
this.vectorStore.listAnnotationIds(),
|
|
10361
|
+
// The artifact store's would-hit keys — one bulk read, never a probe
|
|
10362
|
+
// per resource (PERSIST-ANCHORS P0). Keys are resource ids today;
|
|
10363
|
+
// P1 rekeys the store by content checksum and this lookup moves
|
|
10364
|
+
// with it.
|
|
10365
|
+
this.content.listAnchoredTextKeys().then((keys) => new Set(keys))
|
|
10263
10366
|
]);
|
|
10264
10367
|
const resources = await this.listAllResources();
|
|
10265
10368
|
this.logger.info("Reconcile started", {
|
|
10266
10369
|
indexedResources: indexedResources.size,
|
|
10267
10370
|
indexedAnnotations: indexedAnnotations.size,
|
|
10371
|
+
anchoredArtifacts: anchoredKeys.size,
|
|
10268
10372
|
liveResources: resources.length
|
|
10269
10373
|
});
|
|
10270
|
-
const embeddable =
|
|
10271
|
-
for (const resource of resources) {
|
|
10272
|
-
const mediaType = getPrimaryMediaType(resource);
|
|
10273
|
-
if (resource["@id"] && mediaType && textExtractionOf(mediaType) === "decode") {
|
|
10274
|
-
embeddable.set(resource["@id"], {
|
|
10275
|
-
checksum: getPrimaryRepresentation(resource)?.checksum,
|
|
10276
|
-
entityTypes: getResourceEntityTypes(resource)
|
|
10277
|
-
});
|
|
10278
|
-
}
|
|
10279
|
-
}
|
|
10374
|
+
const embeddable = this.classifyEmbeddable(resources);
|
|
10280
10375
|
const work = [];
|
|
10281
10376
|
for (const rid of indexedResources.keys()) {
|
|
10282
10377
|
if (!embeddable.has(rid)) work.push({ type: "smelt:purge", resourceId: rid, payload: {} });
|
|
@@ -10290,6 +10385,9 @@ var Smelter = class _Smelter {
|
|
|
10290
10385
|
} else if (!sameStringSet(indexed.entityTypes, catalog.entityTypes)) {
|
|
10291
10386
|
work.push({ type: "smelt:restamp", resourceId: rid, payload: {} });
|
|
10292
10387
|
}
|
|
10388
|
+
if (indexed && catalog.checksum !== void 0 && indexed.contentChecksum === catalog.checksum && catalog.yieldsGeometry && !anchoredKeys.has(catalog.checksum)) {
|
|
10389
|
+
work.push({ type: "smelt:reanchor", resourceId: rid, payload: {} });
|
|
10390
|
+
}
|
|
10293
10391
|
}
|
|
10294
10392
|
const liveAnnotationIds = /* @__PURE__ */ new Set();
|
|
10295
10393
|
for (const resource of resources) {
|
|
@@ -10318,9 +10416,12 @@ var Smelter = class _Smelter {
|
|
|
10318
10416
|
const summary = {
|
|
10319
10417
|
resourcesEmbedded: work.filter((w) => w.type === "smelt:embed").length,
|
|
10320
10418
|
resourcesRestamped: work.filter((w) => w.type === "smelt:restamp").length,
|
|
10419
|
+
resourcesReanchored: work.filter((w) => w.type === "smelt:reanchor").length,
|
|
10321
10420
|
resourceVectorsDeleted: work.filter((w) => w.type === "smelt:purge").length,
|
|
10322
10421
|
annotationsEmbedded: work.filter((w) => w.type === "smelt:embed-annotation").length,
|
|
10323
|
-
annotationVectorsDeleted: work.filter((w) => w.type === "smelt:purge-annotation").length
|
|
10422
|
+
annotationVectorsDeleted: work.filter((w) => w.type === "smelt:purge-annotation").length,
|
|
10423
|
+
resourcesEligible: embeddable.size,
|
|
10424
|
+
resourcesIndexed: (await this.vectorStore.listResourceStamps()).size
|
|
10324
10425
|
};
|
|
10325
10426
|
this._reconcileState = { phase: "done", summary };
|
|
10326
10427
|
this.logger.info("Reconcile complete", { ...summary });
|
|
@@ -10339,16 +10440,106 @@ var Smelter = class _Smelter {
|
|
|
10339
10440
|
* completion. The pipeline ticks `noteWorkDone` for every consumed work
|
|
10340
10441
|
* item (success or failure — failures are logged like any live event), so
|
|
10341
10442
|
* each wave's waiter resolves exactly when its items have been processed.
|
|
10443
|
+
*
|
|
10444
|
+
* Serialized through `drainChain`: there is ONE waiter slot, and the
|
|
10445
|
+
* planners that drain (reconcile, `smelt:rebuild-anchors`) must not
|
|
10446
|
+
* interleave — a rebuild command arriving mid-reconcile waits its turn.
|
|
10447
|
+
*
|
|
10448
|
+
* @returns how many of THESE items failed — the rebuild command's
|
|
10449
|
+
* partial-failure accounting (failure detail is in the logs).
|
|
10342
10450
|
*/
|
|
10343
10451
|
async drain(work) {
|
|
10344
|
-
|
|
10345
|
-
|
|
10346
|
-
const
|
|
10347
|
-
|
|
10348
|
-
|
|
10349
|
-
|
|
10350
|
-
|
|
10452
|
+
let failures = 0;
|
|
10453
|
+
const run = this.drainChain.then(async () => {
|
|
10454
|
+
const failedBefore = this.workFailed;
|
|
10455
|
+
for (let i = 0; i < work.length; i += _Smelter.RECONCILE_WAVE) {
|
|
10456
|
+
const wave = work.slice(i, i + _Smelter.RECONCILE_WAVE);
|
|
10457
|
+
const done = new Promise((resolve) => {
|
|
10458
|
+
this.workWaiter = { target: this.workDone + wave.length, resolve };
|
|
10459
|
+
});
|
|
10460
|
+
for (const item of wave) this.eventSubject.next(item);
|
|
10461
|
+
await done;
|
|
10462
|
+
}
|
|
10463
|
+
failures = this.workFailed - failedBefore;
|
|
10464
|
+
});
|
|
10465
|
+
this.drainChain = run.then(() => void 0, () => void 0);
|
|
10466
|
+
await run;
|
|
10467
|
+
return failures;
|
|
10468
|
+
}
|
|
10469
|
+
/**
|
|
10470
|
+
* `smelt:rebuild-anchors` — the operator's explicit re-derivation of
|
|
10471
|
+
* anchored-text artifacts (PERSIST-ANCHORS P0), shaped after
|
|
10472
|
+
* `weave:rebuild`: optionally scoped, strictly serialized (concatMap on
|
|
10473
|
+
* the command stream + the drain chain), correlated ok/failed replies,
|
|
10474
|
+
* and partial completion FAILS — a rebuild that quietly skipped resources
|
|
10475
|
+
* would present exactly like a document with no text, which is the #845
|
|
10476
|
+
* failure mode wearing different clothes.
|
|
10477
|
+
*
|
|
10478
|
+
* Never destructive: nothing is deleted first, stale entries are simply
|
|
10479
|
+
* overwritten (the W5-frames lesson — a rebuild that clears before it
|
|
10480
|
+
* re-derives turns a partial failure into a loss). Re-anchoring makes
|
|
10481
|
+
* zero embedding calls; work items ride the normal per-resource lanes,
|
|
10482
|
+
* so a rebuild can never interleave with live processing of the same
|
|
10483
|
+
* resource (S1/S2).
|
|
10484
|
+
*/
|
|
10485
|
+
async rebuildAnchors(command) {
|
|
10486
|
+
const { correlationId, resourceId } = command;
|
|
10487
|
+
try {
|
|
10488
|
+
let work;
|
|
10489
|
+
if (resourceId) {
|
|
10490
|
+
work = [{ type: "smelt:reanchor", resourceId, payload: {} }];
|
|
10491
|
+
} else {
|
|
10492
|
+
const resources = await this.listAllResources();
|
|
10493
|
+
work = [...this.classifyEmbeddable(resources)].filter(([, catalog]) => catalog.yieldsGeometry).map(([rid]) => ({ type: "smelt:reanchor", resourceId: rid, payload: {} }));
|
|
10494
|
+
}
|
|
10495
|
+
this.logger.info("Anchored-text rebuild started", { scoped: resourceId ?? null, resources: work.length });
|
|
10496
|
+
const failed = await this.drain(work);
|
|
10497
|
+
if (failed > 0) {
|
|
10498
|
+
await this.bus.emit("smelt:rebuild-anchors-failed", {
|
|
10499
|
+
...correlationId ? { correlationId } : {},
|
|
10500
|
+
message: `${failed} of ${work.length} resources failed to re-anchor \u2014 see smelter logs`
|
|
10501
|
+
});
|
|
10502
|
+
return;
|
|
10503
|
+
}
|
|
10504
|
+
await this.bus.emit("smelt:rebuild-anchors-ok", correlationId ? { correlationId } : {});
|
|
10505
|
+
this.logger.info("Anchored-text rebuild complete", { scoped: resourceId ?? null, resources: work.length });
|
|
10506
|
+
} catch (error) {
|
|
10507
|
+
this.logger.error("Anchored-text rebuild failed", { error: errField(error) });
|
|
10508
|
+
try {
|
|
10509
|
+
await this.bus.emit("smelt:rebuild-anchors-failed", {
|
|
10510
|
+
...correlationId ? { correlationId } : {},
|
|
10511
|
+
message: error instanceof Error ? error.message : String(error)
|
|
10512
|
+
});
|
|
10513
|
+
} catch (emitError) {
|
|
10514
|
+
this.logger.warn("Failed to emit smelt:rebuild-anchors-failed", { error: errField(emitError) });
|
|
10515
|
+
}
|
|
10516
|
+
}
|
|
10517
|
+
}
|
|
10518
|
+
/**
|
|
10519
|
+
* Embeddable live resources, each with the catalog's claims: the primary
|
|
10520
|
+
* representation's checksum (the bytes the smelter would read), the
|
|
10521
|
+
* current entity-type set (the discriminator the stamps must carry), and
|
|
10522
|
+
* whether the media type's extractor derives geometry (whether an
|
|
10523
|
+
* anchored-text artifact should exist). Embeddable ⇔ an extractor exists
|
|
10524
|
+
* for the media type's strategy — the same registry the live fetch
|
|
10525
|
+
* resolves, and `yieldsGeometry` is declared on the extractor itself, so
|
|
10526
|
+
* every gate here and the live fetch's behavior are twins by construction.
|
|
10527
|
+
* Shared by `reconcile()` and the `smelt:rebuild-anchors` planner.
|
|
10528
|
+
*/
|
|
10529
|
+
classifyEmbeddable(resources) {
|
|
10530
|
+
const embeddable = /* @__PURE__ */ new Map();
|
|
10531
|
+
for (const resource of resources) {
|
|
10532
|
+
const mediaType = getPrimaryMediaType(resource);
|
|
10533
|
+
const extractor = mediaType ? EXTRACTORS[textExtractionOf(mediaType)] : null;
|
|
10534
|
+
if (resource["@id"] && extractor) {
|
|
10535
|
+
embeddable.set(resource["@id"], {
|
|
10536
|
+
checksum: getPrimaryRepresentation(resource)?.checksum,
|
|
10537
|
+
entityTypes: getResourceEntityTypes(resource),
|
|
10538
|
+
yieldsGeometry: extractor.yieldsGeometry
|
|
10539
|
+
});
|
|
10540
|
+
}
|
|
10351
10541
|
}
|
|
10542
|
+
return embeddable;
|
|
10352
10543
|
}
|
|
10353
10544
|
/** Page through `browse:resources-requested` until the catalog is exhausted. */
|
|
10354
10545
|
async listAllResources() {
|
|
@@ -10480,6 +10671,7 @@ async function main() {
|
|
|
10480
10671
|
logger.info("Content transport ready", { mode: "http" });
|
|
10481
10672
|
const smelter = new Smelter(
|
|
10482
10673
|
actorStateUnit.events$,
|
|
10674
|
+
actorStateUnit.rebuildAnchors$,
|
|
10483
10675
|
vectorStore,
|
|
10484
10676
|
embeddingProvider,
|
|
10485
10677
|
contentTransport,
|