@semiont/make-meaning 0.5.24 → 0.5.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,8 @@
1
- import { createTomlConfigLoader, accessToken, baseUrl as baseUrl$1, retryWithBackoff, isTransientFetchError, STARTUP_FETCH_RETRY, burstBuffer, errField, resourceId, textExtractionOf, decodeRepresentation, busRequest, getResourceEntityTypes, getTargetSelector, getExactText, annotationId, getPrimaryMediaType, getPrimaryRepresentation } from '@semiont/core';
2
- import { calculateChecksum } from '@semiont/content';
3
- import { createEmbeddingProvider, createVectorStore, chunkText } from '@semiont/vectors';
1
+ import { createTomlConfigLoader, accessToken, baseUrl as baseUrl$1, retryWithBackoff, isTransientFetchError, STARTUP_FETCH_RETRY, burstBuffer, errField, resourceId, textExtractionOf, busRequest, getResourceEntityTypes, chunkText, getTargetSelector, getExactText, annotationId, getPrimaryMediaType, getPrimaryRepresentation } from '@semiont/core';
2
+ import { anchoredTextStoreOverTransport, EXTRACTORS, calculateChecksum } from '@semiont/content';
4
3
  import { registerVectorIndexSizeProvider, withActorSpan } from '@semiont/observability';
5
4
  import { HttpTransport, HttpContentTransport } from '@semiont/http-transport';
5
+ import { createEmbeddingProvider, createVectorStore } from '@semiont/vectors';
6
6
  import { createServer } from 'http';
7
7
  import { existsSync, readFileSync } from 'fs';
8
8
  import { homedir } from 'os';
@@ -9738,6 +9738,7 @@ var SMELTER_CHANNELS = [
9738
9738
  "mark:entity-tag-added",
9739
9739
  "mark:entity-tag-removed"
9740
9740
  ];
9741
+ var SMELTER_COMMAND_CHANNELS = ["smelt:rebuild-anchors"];
9741
9742
  function createSmelterActorStateUnit(options) {
9742
9743
  const { bus } = options;
9743
9744
  let started = false;
@@ -9752,12 +9753,14 @@ function createSmelterActorStateUnit(options) {
9752
9753
  )
9753
9754
  )
9754
9755
  );
9756
+ const rebuildAnchors$ = bus.on$("smelt:rebuild-anchors");
9755
9757
  return {
9756
9758
  events$,
9759
+ rebuildAnchors$,
9757
9760
  start: () => {
9758
9761
  if (started) return;
9759
9762
  started = true;
9760
- bus.addChannels?.([...SMELTER_CHANNELS]);
9763
+ bus.addChannels?.([...SMELTER_CHANNELS, ...SMELTER_COMMAND_CHANNELS]);
9761
9764
  },
9762
9765
  dispose: () => {
9763
9766
  started = false;
@@ -9793,6 +9796,7 @@ function sameStringSet(a, b) {
9793
9796
  var WORK_ITEM_TYPES = /* @__PURE__ */ new Set([
9794
9797
  "smelt:embed",
9795
9798
  "smelt:restamp",
9799
+ "smelt:reanchor",
9796
9800
  "smelt:purge",
9797
9801
  "smelt:embed-annotation",
9798
9802
  "smelt:purge-annotation"
@@ -9801,8 +9805,9 @@ function isWorkItem(input) {
9801
9805
  return WORK_ITEM_TYPES.has(input.type);
9802
9806
  }
9803
9807
  var Smelter = class _Smelter {
9804
- constructor(events$, vectorStore, embeddingProvider, content, bus, chunkingConfig2, timing, logger2) {
9808
+ constructor(events$, rebuildAnchors$, vectorStore, embeddingProvider, content, bus, chunkingConfig2, timing, logger2) {
9805
9809
  this.events$ = events$;
9810
+ this.rebuildAnchors$ = rebuildAnchors$;
9806
9811
  this.vectorStore = vectorStore;
9807
9812
  this.embeddingProvider = embeddingProvider;
9808
9813
  this.content = content;
@@ -9810,8 +9815,10 @@ var Smelter = class _Smelter {
9810
9815
  this.chunkingConfig = chunkingConfig2;
9811
9816
  this.timing = timing;
9812
9817
  this.logger = logger2;
9818
+ this.anchoredStore = anchoredTextStoreOverTransport(content, logger2.child({ component: "anchored-text-cache" }));
9813
9819
  }
9814
9820
  events$;
9821
+ rebuildAnchors$;
9815
9822
  vectorStore;
9816
9823
  embeddingProvider;
9817
9824
  content;
@@ -9824,11 +9831,21 @@ var Smelter = class _Smelter {
9824
9831
  static RECONCILE_WAVE = 8;
9825
9832
  eventSubject = new import_rxjs2.Subject();
9826
9833
  sourceSubscription = null;
9834
+ commandSubscription = null;
9827
9835
  pipelineSubscription = null;
9828
9836
  _eventsProcessed = 0;
9829
9837
  _reconcileState = { phase: "pending" };
9830
9838
  workDone = 0;
9839
+ workFailed = 0;
9831
9840
  workWaiter = null;
9841
+ /**
9842
+ * Serializes every planner drain (reconcile, anchored-text rebuilds):
9843
+ * there is one waiter slot, and the weave:rebuild rule — rebuilds never
9844
+ * interleave — applies to every unit here being a potential multi-second
9845
+ * OCR pass.
9846
+ */
9847
+ drainChain = Promise.resolve();
9848
+ anchoredStore;
9832
9849
  get eventsProcessed() {
9833
9850
  return this._eventsProcessed;
9834
9851
  }
@@ -9856,7 +9873,7 @@ var Smelter = class _Smelter {
9856
9873
  return (0, import_rxjs2.from)(
9857
9874
  withActorSpan("smelter", inputOrBatch.type, async () => {
9858
9875
  const ok = await this.safeProcessEvent(inputOrBatch);
9859
- if (isWorkItem(inputOrBatch)) this.noteWorkDone(1);
9876
+ if (isWorkItem(inputOrBatch)) this.noteWorkDone(1, ok ? 0 : 1);
9860
9877
  else if (ok) this._eventsProcessed++;
9861
9878
  })
9862
9879
  );
@@ -9870,18 +9887,26 @@ var Smelter = class _Smelter {
9870
9887
  this.logger.debug("Bus event received", { type: event.type, resourceId: event.resourceId });
9871
9888
  this.eventSubject.next(event);
9872
9889
  });
9890
+ this.commandSubscription = this.rebuildAnchors$.pipe(
9891
+ (0, import_operators2.concatMap)((command) => (0, import_rxjs2.from)(this.rebuildAnchors(command)))
9892
+ ).subscribe({
9893
+ error: (err) => this.logger.error("Smelter command pipeline error", { error: errField(err) })
9894
+ });
9873
9895
  this.logger.info("Smelter pipeline initialized");
9874
9896
  }
9875
9897
  stop() {
9876
9898
  this.sourceSubscription?.unsubscribe();
9877
9899
  this.sourceSubscription = null;
9900
+ this.commandSubscription?.unsubscribe();
9901
+ this.commandSubscription = null;
9878
9902
  this.pipelineSubscription?.unsubscribe();
9879
9903
  this.pipelineSubscription = null;
9880
9904
  this.eventSubject.complete();
9881
9905
  this.logger.info("Smelter stopped");
9882
9906
  }
9883
- noteWorkDone(count) {
9907
+ noteWorkDone(count, failed) {
9884
9908
  this.workDone += count;
9909
+ this.workFailed += failed;
9885
9910
  if (this.workWaiter && this.workDone >= this.workWaiter.target) {
9886
9911
  this.workWaiter.resolve();
9887
9912
  this.workWaiter = null;
@@ -9895,12 +9920,15 @@ var Smelter = class _Smelter {
9895
9920
  let wireProcessed = 0;
9896
9921
  for (const run of partitionByType(events)) {
9897
9922
  const workRun = isWorkItem(run[0]);
9923
+ let succeeded = 0;
9898
9924
  try {
9899
9925
  if (run.length === 1) {
9900
9926
  const ok = await this.safeProcessEvent(run[0]);
9927
+ if (ok) succeeded = 1;
9901
9928
  if (ok && !workRun) wireProcessed++;
9902
9929
  } else {
9903
9930
  const processed = await this.applyBatchByType(run);
9931
+ succeeded = processed;
9904
9932
  if (!workRun) wireProcessed += processed;
9905
9933
  }
9906
9934
  } catch (error) {
@@ -9910,7 +9938,7 @@ var Smelter = class _Smelter {
9910
9938
  error: errField(error)
9911
9939
  });
9912
9940
  } finally {
9913
- if (workRun) this.noteWorkDone(run.length);
9941
+ if (workRun) this.noteWorkDone(run.length, run.length - succeeded);
9914
9942
  }
9915
9943
  }
9916
9944
  return wireProcessed;
@@ -9982,6 +10010,9 @@ var Smelter = class _Smelter {
9982
10010
  case "smelt:restamp":
9983
10011
  await this.restampResource(event);
9984
10012
  break;
10013
+ case "smelt:reanchor":
10014
+ await this.reanchorResource(event);
10015
+ break;
9985
10016
  case "smelt:purge":
9986
10017
  await this.handleResourcePurge(event);
9987
10018
  break;
@@ -10008,6 +10039,45 @@ var Smelter = class _Smelter {
10008
10039
  await this.vectorStore.updateResourceEntityTypes(resourceId(rid), entityTypes);
10009
10040
  this.logger.info("Restamped resource entity types", { resourceId: rid, entityTypes });
10010
10041
  }
10042
+ /**
10043
+ * Re-derive a lost anchored-text artifact from the resource's current
10044
+ * bytes (PERSIST-ANCHORS P0, the third drift class). Extraction is the
10045
+ * cost here — the vectors are already correct, so this NEVER calls the
10046
+ * embedding provider, the vector store, or the settled signal: the index
10047
+ * decision was already made and announced at its checksum; only the map
10048
+ * is missing. Name the work for what it does (the S13 discipline).
10049
+ *
10050
+ * The publish is STRICT, unlike the embed path's best-effort side
10051
+ * publish: here the artifact IS the job, so a store failure must throw —
10052
+ * the pipeline logs and counts it, and the rebuild command's partial-
10053
+ * failure accounting depends on that throw.
10054
+ */
10055
+ async reanchorResource(event) {
10056
+ const rid = event.resourceId;
10057
+ if (!rid) return;
10058
+ const { data, contentType } = await this.content.getBinary(resourceId(rid));
10059
+ const bytes = Buffer.from(data);
10060
+ const extractor = EXTRACTORS[textExtractionOf(contentType)];
10061
+ if (!extractor?.yieldsGeometry) {
10062
+ this.logger.info("Re-anchor found no geometry-capable extractor", { resourceId: rid, contentType });
10063
+ return;
10064
+ }
10065
+ const checksum = calculateChecksum(bytes);
10066
+ const extracted = await extractor.extract(bytes, contentType, {
10067
+ key: checksum,
10068
+ store: this.anchoredStore
10069
+ });
10070
+ if ("declined" in extracted || !extracted.items?.length) {
10071
+ this.logger.info("Re-anchor extraction yielded no geometry", {
10072
+ resourceId: rid,
10073
+ contentType,
10074
+ ..."declined" in extracted ? { declined: extracted.declined } : {}
10075
+ });
10076
+ return;
10077
+ }
10078
+ await this.content.putAnchoredText(checksum, { ...extracted, items: extracted.items });
10079
+ this.logger.info("Re-anchored resource", { resourceId: rid, checksum, items: extracted.items.length });
10080
+ }
10011
10081
  async handleResourcePurge(event) {
10012
10082
  const rid = event.resourceId;
10013
10083
  if (!rid) return;
@@ -10027,12 +10097,34 @@ var Smelter = class _Smelter {
10027
10097
  const { data, contentType } = await this.content.getBinary(resourceId(resourceId$1));
10028
10098
  const bytes = Buffer.from(data);
10029
10099
  const checksum = calculateChecksum(bytes);
10030
- if (textExtractionOf(contentType) !== "decode") {
10031
- this.logger.debug("Skipping resource that does not decode as text", { resourceId: resourceId$1, contentType });
10032
- return { kind: "skipped", checksum };
10100
+ const extractor = EXTRACTORS[textExtractionOf(contentType)];
10101
+ if (!extractor) {
10102
+ this.logger.debug("Skipping resource with no extractor for its media type", { resourceId: resourceId$1, contentType });
10103
+ return { kind: "skipped", checksum, reason: "no-extractor" };
10104
+ }
10105
+ const extracted = await extractor.extract(bytes, contentType, {
10106
+ key: checksum,
10107
+ store: this.anchoredStore
10108
+ });
10109
+ if ("declined" in extracted) {
10110
+ this.logger.debug("Extractor declined", { resourceId: resourceId$1, contentType, reason: extracted.declined });
10111
+ return { kind: "skipped", checksum, reason: extracted.declined };
10112
+ }
10113
+ if (extracted.ocrConfidence && extracted.ocrConfidence.lowConfidenceWords > 0) {
10114
+ this.logger.info("OCR read words it was unsure of", {
10115
+ resourceId: resourceId$1,
10116
+ contentType,
10117
+ ...extracted.ocrConfidence
10118
+ });
10033
10119
  }
10034
- const text = decodeRepresentation(bytes, contentType);
10035
- return text.trim() ? { kind: "text", text, checksum } : { kind: "skipped", checksum };
10120
+ if (extracted.unreadPages?.length) {
10121
+ this.logger.info("Partial extraction coverage", {
10122
+ resourceId: resourceId$1,
10123
+ contentType,
10124
+ unreadPages: extracted.unreadPages
10125
+ });
10126
+ }
10127
+ return extracted.text.trim() ? { kind: "text", text: extracted.text, checksum, machineRead: extracted.method === "ocr" } : { kind: "skipped", checksum, reason: "empty" };
10036
10128
  } catch (error) {
10037
10129
  this.logger.warn("Content unavailable for embedding", { resourceId: resourceId$1, error: errField(error) });
10038
10130
  return { kind: "unavailable" };
@@ -10044,9 +10136,9 @@ var Smelter = class _Smelter {
10044
10136
  * fold. Best-effort — waiters degrade to their bounded timeout; a signal
10045
10137
  * failure must never fail the embed.
10046
10138
  */
10047
- async emitSettled(resourceId, contentChecksum, outcome) {
10139
+ async emitSettled(resourceId, contentChecksum, outcome, reason) {
10048
10140
  try {
10049
- await this.bus.emit("smelt:settled", { resourceId, contentChecksum, outcome });
10141
+ await this.bus.emit("smelt:settled", { resourceId, contentChecksum, outcome, ...reason ? { reason } : {} });
10050
10142
  } catch (error) {
10051
10143
  this.logger.warn("Failed to emit smelt:settled", { resourceId, outcome, error: errField(error) });
10052
10144
  }
@@ -10074,12 +10166,14 @@ var Smelter = class _Smelter {
10074
10166
  const fetched = await this.fetchEmbeddableText(rid);
10075
10167
  if (fetched.kind === "unavailable") return;
10076
10168
  if (fetched.kind === "skipped") {
10077
- await this.emitSettled(rid, fetched.checksum, "skipped");
10169
+ await this.vectorStore.deleteResourceVectors(resourceId(rid));
10170
+ await this.emitSettled(rid, fetched.checksum, "skipped", fetched.reason);
10078
10171
  return;
10079
10172
  }
10080
10173
  const chunks = chunkText(fetched.text, this.chunkingConfig);
10081
10174
  if (chunks.length === 0) {
10082
- await this.emitSettled(rid, fetched.checksum, "skipped");
10175
+ await this.vectorStore.deleteResourceVectors(resourceId(rid));
10176
+ await this.emitSettled(rid, fetched.checksum, "skipped", "empty");
10083
10177
  return;
10084
10178
  }
10085
10179
  const entityTypes = await this.resolveEntityTypes(rid);
@@ -10089,7 +10183,7 @@ var Smelter = class _Smelter {
10089
10183
  text: t,
10090
10184
  embedding: embeddings[i]
10091
10185
  }));
10092
- await this.vectorStore.upsertResourceVectors(resourceId(rid), embeddingChunks, fetched.checksum, entityTypes);
10186
+ await this.vectorStore.upsertResourceVectors(resourceId(rid), embeddingChunks, fetched.checksum, entityTypes, fetched.machineRead);
10093
10187
  await this.emitSettled(rid, fetched.checksum, "indexed");
10094
10188
  this.logger.info(logMessage, { resourceId: rid, chunks: chunks.length });
10095
10189
  }
@@ -10133,12 +10227,14 @@ var Smelter = class _Smelter {
10133
10227
  if (!exactText?.trim()) return;
10134
10228
  const aid = annotationId(annotation.id);
10135
10229
  const embedding2 = await this.embeddingProvider.embed(exactText);
10230
+ const stamp = await this.vectorStore.getResourceStamp(resourceId(rid));
10136
10231
  const payload = {
10137
10232
  annotationId: aid,
10138
10233
  resourceId: resourceId(rid),
10139
10234
  motivation: annotation.motivation ?? "",
10140
10235
  entityTypes: annotation.entityTypes ?? [],
10141
- exactText
10236
+ exactText,
10237
+ ...stamp?.machineRead ? { machineRead: true } : {}
10142
10238
  };
10143
10239
  await this.vectorStore.upsertAnnotationVector(aid, embedding2, payload);
10144
10240
  this.logger.info("Indexed annotation", { annotationId: String(aid) });
@@ -10163,28 +10259,30 @@ var Smelter = class _Smelter {
10163
10259
  const fetched = await this.fetchEmbeddableText(rid);
10164
10260
  if (fetched.kind === "unavailable") continue;
10165
10261
  if (fetched.kind === "skipped") {
10166
- await this.emitSettled(rid, fetched.checksum, "skipped");
10262
+ await this.vectorStore.deleteResourceVectors(resourceId(rid));
10263
+ await this.emitSettled(rid, fetched.checksum, "skipped", fetched.reason);
10167
10264
  continue;
10168
10265
  }
10169
10266
  const chunks = chunkText(fetched.text, this.chunkingConfig);
10170
10267
  if (chunks.length === 0) {
10171
- await this.emitSettled(rid, fetched.checksum, "skipped");
10268
+ await this.vectorStore.deleteResourceVectors(resourceId(rid));
10269
+ await this.emitSettled(rid, fetched.checksum, "skipped", "empty");
10172
10270
  continue;
10173
10271
  }
10174
10272
  const entityTypes = await this.resolveEntityTypes(rid);
10175
- resourceData.push({ rid: resourceId(rid), chunks, checksum: fetched.checksum, entityTypes });
10273
+ resourceData.push({ rid: resourceId(rid), chunks, checksum: fetched.checksum, entityTypes, machineRead: fetched.machineRead });
10176
10274
  allChunks.push(...chunks);
10177
10275
  }
10178
10276
  if (allChunks.length === 0) return events.length;
10179
10277
  const allEmbeddings = await this.embeddingProvider.embedBatch(allChunks);
10180
10278
  let offset = 0;
10181
- for (const { rid, chunks, checksum, entityTypes } of resourceData) {
10279
+ for (const { rid, chunks, checksum, entityTypes, machineRead } of resourceData) {
10182
10280
  const embeddingChunks = chunks.map((t, i) => ({
10183
10281
  chunkIndex: i,
10184
10282
  text: t,
10185
10283
  embedding: allEmbeddings[offset + i]
10186
10284
  }));
10187
- await this.vectorStore.upsertResourceVectors(rid, embeddingChunks, checksum, entityTypes);
10285
+ await this.vectorStore.upsertResourceVectors(rid, embeddingChunks, checksum, entityTypes, machineRead);
10188
10286
  await this.emitSettled(String(rid), checksum, "indexed");
10189
10287
  this.logger.info("Batch-indexed resource", { resourceId: String(rid), chunks: chunks.length });
10190
10288
  offset += chunks.length;
@@ -10257,26 +10355,23 @@ var Smelter = class _Smelter {
10257
10355
  }
10258
10356
  this._reconcileState = { phase: "running" };
10259
10357
  try {
10260
- const [indexedResources, indexedAnnotations] = await Promise.all([
10358
+ const [indexedResources, indexedAnnotations, anchoredKeys] = await Promise.all([
10261
10359
  this.vectorStore.listResourceStamps(),
10262
- this.vectorStore.listAnnotationIds()
10360
+ this.vectorStore.listAnnotationIds(),
10361
+ // The artifact store's would-hit keys — one bulk read, never a probe
10362
+ // per resource (PERSIST-ANCHORS P0). Keys are resource ids today;
10363
+ // P1 rekeys the store by content checksum and this lookup moves
10364
+ // with it.
10365
+ this.content.listAnchoredTextKeys().then((keys) => new Set(keys))
10263
10366
  ]);
10264
10367
  const resources = await this.listAllResources();
10265
10368
  this.logger.info("Reconcile started", {
10266
10369
  indexedResources: indexedResources.size,
10267
10370
  indexedAnnotations: indexedAnnotations.size,
10371
+ anchoredArtifacts: anchoredKeys.size,
10268
10372
  liveResources: resources.length
10269
10373
  });
10270
- const embeddable = /* @__PURE__ */ new Map();
10271
- for (const resource of resources) {
10272
- const mediaType = getPrimaryMediaType(resource);
10273
- if (resource["@id"] && mediaType && textExtractionOf(mediaType) === "decode") {
10274
- embeddable.set(resource["@id"], {
10275
- checksum: getPrimaryRepresentation(resource)?.checksum,
10276
- entityTypes: getResourceEntityTypes(resource)
10277
- });
10278
- }
10279
- }
10374
+ const embeddable = this.classifyEmbeddable(resources);
10280
10375
  const work = [];
10281
10376
  for (const rid of indexedResources.keys()) {
10282
10377
  if (!embeddable.has(rid)) work.push({ type: "smelt:purge", resourceId: rid, payload: {} });
@@ -10290,6 +10385,9 @@ var Smelter = class _Smelter {
10290
10385
  } else if (!sameStringSet(indexed.entityTypes, catalog.entityTypes)) {
10291
10386
  work.push({ type: "smelt:restamp", resourceId: rid, payload: {} });
10292
10387
  }
10388
+ if (indexed && catalog.checksum !== void 0 && indexed.contentChecksum === catalog.checksum && catalog.yieldsGeometry && !anchoredKeys.has(catalog.checksum)) {
10389
+ work.push({ type: "smelt:reanchor", resourceId: rid, payload: {} });
10390
+ }
10293
10391
  }
10294
10392
  const liveAnnotationIds = /* @__PURE__ */ new Set();
10295
10393
  for (const resource of resources) {
@@ -10318,9 +10416,12 @@ var Smelter = class _Smelter {
10318
10416
  const summary = {
10319
10417
  resourcesEmbedded: work.filter((w) => w.type === "smelt:embed").length,
10320
10418
  resourcesRestamped: work.filter((w) => w.type === "smelt:restamp").length,
10419
+ resourcesReanchored: work.filter((w) => w.type === "smelt:reanchor").length,
10321
10420
  resourceVectorsDeleted: work.filter((w) => w.type === "smelt:purge").length,
10322
10421
  annotationsEmbedded: work.filter((w) => w.type === "smelt:embed-annotation").length,
10323
- annotationVectorsDeleted: work.filter((w) => w.type === "smelt:purge-annotation").length
10422
+ annotationVectorsDeleted: work.filter((w) => w.type === "smelt:purge-annotation").length,
10423
+ resourcesEligible: embeddable.size,
10424
+ resourcesIndexed: (await this.vectorStore.listResourceStamps()).size
10324
10425
  };
10325
10426
  this._reconcileState = { phase: "done", summary };
10326
10427
  this.logger.info("Reconcile complete", { ...summary });
@@ -10339,16 +10440,106 @@ var Smelter = class _Smelter {
10339
10440
  * completion. The pipeline ticks `noteWorkDone` for every consumed work
10340
10441
  * item (success or failure — failures are logged like any live event), so
10341
10442
  * each wave's waiter resolves exactly when its items have been processed.
10443
+ *
10444
+ * Serialized through `drainChain`: there is ONE waiter slot, and the
10445
+ * planners that drain (reconcile, `smelt:rebuild-anchors`) must not
10446
+ * interleave — a rebuild command arriving mid-reconcile waits its turn.
10447
+ *
10448
+ * @returns how many of THESE items failed — the rebuild command's
10449
+ * partial-failure accounting (failure detail is in the logs).
10342
10450
  */
10343
10451
  async drain(work) {
10344
- for (let i = 0; i < work.length; i += _Smelter.RECONCILE_WAVE) {
10345
- const wave = work.slice(i, i + _Smelter.RECONCILE_WAVE);
10346
- const done = new Promise((resolve) => {
10347
- this.workWaiter = { target: this.workDone + wave.length, resolve };
10348
- });
10349
- for (const item of wave) this.eventSubject.next(item);
10350
- await done;
10452
+ let failures = 0;
10453
+ const run = this.drainChain.then(async () => {
10454
+ const failedBefore = this.workFailed;
10455
+ for (let i = 0; i < work.length; i += _Smelter.RECONCILE_WAVE) {
10456
+ const wave = work.slice(i, i + _Smelter.RECONCILE_WAVE);
10457
+ const done = new Promise((resolve) => {
10458
+ this.workWaiter = { target: this.workDone + wave.length, resolve };
10459
+ });
10460
+ for (const item of wave) this.eventSubject.next(item);
10461
+ await done;
10462
+ }
10463
+ failures = this.workFailed - failedBefore;
10464
+ });
10465
+ this.drainChain = run.then(() => void 0, () => void 0);
10466
+ await run;
10467
+ return failures;
10468
+ }
10469
+ /**
10470
+ * `smelt:rebuild-anchors` — the operator's explicit re-derivation of
10471
+ * anchored-text artifacts (PERSIST-ANCHORS P0), shaped after
10472
+ * `weave:rebuild`: optionally scoped, strictly serialized (concatMap on
10473
+ * the command stream + the drain chain), correlated ok/failed replies,
10474
+ * and partial completion FAILS — a rebuild that quietly skipped resources
10475
+ * would present exactly like a document with no text, which is the #845
10476
+ * failure mode wearing different clothes.
10477
+ *
10478
+ * Never destructive: nothing is deleted first, stale entries are simply
10479
+ * overwritten (the W5-frames lesson — a rebuild that clears before it
10480
+ * re-derives turns a partial failure into a loss). Re-anchoring makes
10481
+ * zero embedding calls; work items ride the normal per-resource lanes,
10482
+ * so a rebuild can never interleave with live processing of the same
10483
+ * resource (S1/S2).
10484
+ */
10485
+ async rebuildAnchors(command) {
10486
+ const { correlationId, resourceId } = command;
10487
+ try {
10488
+ let work;
10489
+ if (resourceId) {
10490
+ work = [{ type: "smelt:reanchor", resourceId, payload: {} }];
10491
+ } else {
10492
+ const resources = await this.listAllResources();
10493
+ work = [...this.classifyEmbeddable(resources)].filter(([, catalog]) => catalog.yieldsGeometry).map(([rid]) => ({ type: "smelt:reanchor", resourceId: rid, payload: {} }));
10494
+ }
10495
+ this.logger.info("Anchored-text rebuild started", { scoped: resourceId ?? null, resources: work.length });
10496
+ const failed = await this.drain(work);
10497
+ if (failed > 0) {
10498
+ await this.bus.emit("smelt:rebuild-anchors-failed", {
10499
+ ...correlationId ? { correlationId } : {},
10500
+ message: `${failed} of ${work.length} resources failed to re-anchor \u2014 see smelter logs`
10501
+ });
10502
+ return;
10503
+ }
10504
+ await this.bus.emit("smelt:rebuild-anchors-ok", correlationId ? { correlationId } : {});
10505
+ this.logger.info("Anchored-text rebuild complete", { scoped: resourceId ?? null, resources: work.length });
10506
+ } catch (error) {
10507
+ this.logger.error("Anchored-text rebuild failed", { error: errField(error) });
10508
+ try {
10509
+ await this.bus.emit("smelt:rebuild-anchors-failed", {
10510
+ ...correlationId ? { correlationId } : {},
10511
+ message: error instanceof Error ? error.message : String(error)
10512
+ });
10513
+ } catch (emitError) {
10514
+ this.logger.warn("Failed to emit smelt:rebuild-anchors-failed", { error: errField(emitError) });
10515
+ }
10516
+ }
10517
+ }
10518
+ /**
10519
+ * Embeddable live resources, each with the catalog's claims: the primary
10520
+ * representation's checksum (the bytes the smelter would read), the
10521
+ * current entity-type set (the discriminator the stamps must carry), and
10522
+ * whether the media type's extractor derives geometry (whether an
10523
+ * anchored-text artifact should exist). Embeddable ⇔ an extractor exists
10524
+ * for the media type's strategy — the same registry the live fetch
10525
+ * resolves, and `yieldsGeometry` is declared on the extractor itself, so
10526
+ * every gate here and the live fetch's behavior are twins by construction.
10527
+ * Shared by `reconcile()` and the `smelt:rebuild-anchors` planner.
10528
+ */
10529
+ classifyEmbeddable(resources) {
10530
+ const embeddable = /* @__PURE__ */ new Map();
10531
+ for (const resource of resources) {
10532
+ const mediaType = getPrimaryMediaType(resource);
10533
+ const extractor = mediaType ? EXTRACTORS[textExtractionOf(mediaType)] : null;
10534
+ if (resource["@id"] && extractor) {
10535
+ embeddable.set(resource["@id"], {
10536
+ checksum: getPrimaryRepresentation(resource)?.checksum,
10537
+ entityTypes: getResourceEntityTypes(resource),
10538
+ yieldsGeometry: extractor.yieldsGeometry
10539
+ });
10540
+ }
10351
10541
  }
10542
+ return embeddable;
10352
10543
  }
10353
10544
  /** Page through `browse:resources-requested` until the catalog is exhausted. */
10354
10545
  async listAllResources() {
@@ -10480,6 +10671,7 @@ async function main() {
10480
10671
  logger.info("Content transport ready", { mode: "http" });
10481
10672
  const smelter = new Smelter(
10482
10673
  actorStateUnit.events$,
10674
+ actorStateUnit.rebuildAnchors$,
10483
10675
  vectorStore,
10484
10676
  embeddingProvider,
10485
10677
  contentTransport,