@semiont/make-meaning 0.5.23 → 0.5.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,14 +1,13 @@
1
1
  import { STALL_THRESHOLD_MS, FsJobQueue } from '@semiont/jobs';
2
2
  import { FilesystemViewStorage, resolveStorageUri, EventQuery, createEventStore } from '@semiont/event-sourcing';
3
- import { getResourceEntityTypes, getResourceId, getTargetSource, resourceId, getPrimaryRepresentation, decodeRepresentation, getBodySource, getTargetSelector, deriveViews, getTextPositionSelector, annotationId, errField, userId, generateUuid, getExactText, busRequest, cloneToken, baseMediaType, isSupportedMediaType, capabilitiesOf, didToAgent, assembleAnnotation, jobId, entityType, baseUrl, busLog, BRIDGED_CHANNELS, burstBuffer, textExtractionOf, getPrimaryMediaType, extensionForMediaType, applyBodyOperations, softwareToAgent } from '@semiont/core';
3
+ import { getResourceEntityTypes, getResourceId, getTargetSource, resourceId, getPrimaryRepresentation, decodeRepresentation, getBodySource, getTargetSelector, deriveViews, getTextPositionSelector, annotationId, errField, userId, generateUuid, getExactText, busRequest, cloneToken, baseMediaType, isSupportedMediaType, capabilitiesOf, didToAgent, assembleAnnotation, jobId, entityType, baseUrl, busLog, BRIDGED_CHANNELS, burstBuffer, textExtractionOf, chunkText, getPrimaryMediaType, extensionForMediaType, applyBodyOperations, softwareToAgent } from '@semiont/core';
4
4
  import { recordGatherDegrade, withActorSpan, recordBusEmit, withSpan, SpanKind, registerJobQueueProvider, registerVectorIndexSizeProvider } from '@semiont/observability';
5
5
  import { createInferenceClient } from '@semiont/inference';
6
6
  import { getGraphDatabase } from '@semiont/graph';
7
- import { WorkingTreeStore, deriveStorageUri, calculateChecksum } from '@semiont/content';
7
+ import { WorkingTreeStore, createAnchoredTextStore, deriveStorageUri, anchoredTextStoreOverTransport, EXTRACTORS, calculateChecksum } from '@semiont/content';
8
8
  import { getEntityTypes, DEFAULT_ENTITY_TYPES } from '@semiont/ontology';
9
9
  import { promises } from 'fs';
10
10
  import * as path3 from 'path';
11
- import { chunkText } from '@semiont/vectors';
12
11
  import { createGzip, createGunzip } from 'zlib';
13
12
  import { pipeline, Readable } from 'stream';
14
13
  import { promisify } from 'util';
@@ -9889,6 +9888,10 @@ async function createKnowledgeBase(eventStore, project, graphDb, eventBus, logge
9889
9888
  project,
9890
9889
  logger.child({ component: "working-tree-store" })
9891
9890
  );
9891
+ const anchoredText = createAnchoredTextStore(
9892
+ project.anchoredTextDir,
9893
+ logger.child({ component: "anchored-text-store" })
9894
+ );
9892
9895
  const weaveProgress = createWeaveProgress(eventBus);
9893
9896
  const smeltProgress = createSmeltProgress(eventBus);
9894
9897
  if (!options?.skipRebuild) {
@@ -9898,6 +9901,7 @@ async function createKnowledgeBase(eventStore, project, graphDb, eventBus, logge
9898
9901
  eventStore,
9899
9902
  views,
9900
9903
  content,
9904
+ anchoredText,
9901
9905
  graph: graphDb,
9902
9906
  weaveProgress,
9903
9907
  smeltProgress,
@@ -10300,7 +10304,8 @@ Summary:`;
10300
10304
  resourceId: r.resourceId,
10301
10305
  annotationId: r.annotationId,
10302
10306
  score: r.score,
10303
- entityTypes: r.entityTypes
10307
+ entityTypes: r.entityTypes,
10308
+ ...r.machineRead ? { machineRead: true } : {}
10304
10309
  }))
10305
10310
  };
10306
10311
  logger?.debug("Semantic context found", { matches: results.length });
@@ -10642,7 +10647,8 @@ var LLMContext = class {
10642
10647
  resourceId: m.resourceId,
10643
10648
  ...m.annotationId ? { annotationId: m.annotationId } : {},
10644
10649
  score: m.score,
10645
- ...m.entityTypes ? { entityTypes: m.entityTypes } : {}
10650
+ ...m.entityTypes ? { entityTypes: m.entityTypes } : {},
10651
+ ...m.machineRead ? { machineRead: true } : {}
10646
10652
  })),
10647
10653
  ...excludeEntityTypes.length ? { excludedEntityTypes: excludeEntityTypes } : {}
10648
10654
  };
@@ -11632,6 +11638,21 @@ var Stower = class {
11632
11638
  // src/browser.ts
11633
11639
  var import_rxjs4 = __toESM(require_cjs());
11634
11640
  var import_operators4 = __toESM(require_operators());
11641
+ var ANCHORED_TEXT_SETTLE_TIMEOUT_MS = 15e3;
11642
+ async function readAnchoredText(kb, resourceId10) {
11643
+ const view = await kb.views.get(resourceId(resourceId10));
11644
+ const checksum = getPrimaryRepresentation(view?.resource)?.checksum;
11645
+ if (!checksum) return null;
11646
+ const hit = await kb.anchoredText.read(checksum);
11647
+ if (hit) return hit;
11648
+ try {
11649
+ const outcome = await kb.smeltProgress.whenSettled(resourceId10, checksum, ANCHORED_TEXT_SETTLE_TIMEOUT_MS);
11650
+ return outcome === "indexed" ? kb.anchoredText.read(checksum) : null;
11651
+ } catch (error) {
11652
+ if (error instanceof SmeltProgressTimeout) return null;
11653
+ throw error;
11654
+ }
11655
+ }
11635
11656
  async function readTagSchemasProjection(project) {
11636
11657
  const tagSchemasPath = path3.join(
11637
11658
  project.stateDir,
@@ -11746,6 +11767,7 @@ var Browser = class {
11746
11767
  );
11747
11768
  this.subscriptions.push(
11748
11769
  pipe("browse:resource-requested", (e) => this.handleBrowseResource(e)).subscribe({ error: errorHandler }),
11770
+ pipe("browse:anchored-text-requested", (e) => this.handleAnchoredText(e)).subscribe({ error: errorHandler }),
11749
11771
  pipe("browse:resources-requested", (e) => this.handleBrowseResources(e)).subscribe({ error: errorHandler }),
11750
11772
  pipe("browse:annotations-requested", (e) => this.handleBrowseAnnotations(e)).subscribe({ error: errorHandler }),
11751
11773
  pipe("browse:annotation-requested", (e) => this.handleBrowseAnnotation(e)).subscribe({ error: errorHandler }),
@@ -11761,6 +11783,34 @@ var Browser = class {
11761
11783
  // ========================================================================
11762
11784
  // KB read handlers
11763
11785
  // ========================================================================
11786
+ /**
11787
+ * Serve a resource's derived coordinate map (ANCHORED-TEXT-CACHE Lane 5).
11788
+ *
11789
+ * Read-your-writes on the same barrier `llm-context` uses for vectors: a
11790
+ * caller may arrive before the Smelter has finished the resource it just
11791
+ * uploaded, so a miss waits for that content generation to settle rather than
11792
+ * reporting "no map" for a document that is merely still being read.
11793
+ *
11794
+ * **This path never invokes the engine.** The Smelter is the sole producer.
11795
+ * A miss that survives the barrier answers `null`, and the caller degrades —
11796
+ * for a PDF annotation that means geometry with no quoted text, which is what
11797
+ * shipped before any of this existed. OCR in a request path is precisely what
11798
+ * this design exists to avoid.
11799
+ */
11800
+ async handleAnchoredText(event) {
11801
+ try {
11802
+ this.eventBus.get("browse:anchored-text-result").next({
11803
+ correlationId: event.correlationId,
11804
+ response: await readAnchoredText(this.kb, event.resourceId)
11805
+ });
11806
+ } catch (error) {
11807
+ this.logger.error("Browse anchored text failed", { resourceId: event.resourceId, error: errField(error) });
11808
+ this.eventBus.get("browse:anchored-text-failed").next({
11809
+ correlationId: event.correlationId,
11810
+ message: error instanceof Error ? error.message : String(error)
11811
+ });
11812
+ }
11813
+ }
11764
11814
  async handleBrowseResource(event) {
11765
11815
  try {
11766
11816
  const response = await assembleResourceGraph(this.kb, resourceId(event.resourceId));
@@ -12916,6 +12966,7 @@ async function startMakeMeaning(project, config, eventBus, logger, options) {
12916
12966
  return {
12917
12967
  knowledgeSystem,
12918
12968
  jobQueue,
12969
+ project,
12919
12970
  stop: async () => {
12920
12971
  logger.info("Stopping Make-Meaning service");
12921
12972
  jobStatusSubscription.unsubscribe();
@@ -13029,6 +13080,40 @@ var LocalContentTransport = class {
13029
13080
  "LocalContentTransport does not support putBinary() \u2014 create resources via bus emits (mark/yield namespaces) in local mode"
13030
13081
  );
13031
13082
  }
13083
+ /**
13084
+ * Store a derived coordinate map under the content checksum the producer
13085
+ * read (PERSIST-ANCHORS decision A — see the interface doc for why the
13086
+ * producer supplies the key). In local mode this is the same store the
13087
+ * HTTP route writes to — one storage authority, reached the same way from
13088
+ * every process (ANCHORED-TEXT-CACHE Lane 5).
13089
+ */
13090
+ async putAnchoredText(checksum, outcome, _options) {
13091
+ busLog("PUT", "anchored-text", { checksum });
13092
+ await this.ks.kb.anchoredText.write(checksum, outcome);
13093
+ }
13094
+ /** The stored outcome, or null when nothing has derived one — the common case. */
13095
+ async getAnchoredText(resourceId10, _options) {
13096
+ busLog("GET", "anchored-text", { resourceId: resourceId10 });
13097
+ return readAnchoredText(this.ks.kb, resourceId10);
13098
+ }
13099
+ /**
13100
+ * The cache-consult read (PERSIST-ANCHORS P2c), straight from the store —
13101
+ * checksum-addressed, so no view resolution and no settle barrier: the
13102
+ * caller holds the content identity already.
13103
+ */
13104
+ async getAnchoredTextByChecksum(checksum, _options) {
13105
+ busLog("GET", "anchored-text-by-checksum", { checksum });
13106
+ return this.ks.kb.anchoredText.read(checksum);
13107
+ }
13108
+ /**
13109
+ * The store's would-hit keys, straight from the store — planning data for
13110
+ * the reconcile diff (PERSIST-ANCHORS P0), so no settle barrier applies:
13111
+ * presence is being asked, not content at a moment.
13112
+ */
13113
+ async listAnchoredTextKeys(_options) {
13114
+ busLog("GET", "anchored-text-keys", {});
13115
+ return this.ks.kb.anchoredText.list();
13116
+ }
13032
13117
  async getBinary(resourceId10, _options) {
13033
13118
  busLog("GET", "content", { resourceId: resourceId10 });
13034
13119
  return withSpan(
@@ -13111,6 +13196,7 @@ function sameStringSet(a, b) {
13111
13196
  var WORK_ITEM_TYPES = /* @__PURE__ */ new Set([
13112
13197
  "smelt:embed",
13113
13198
  "smelt:restamp",
13199
+ "smelt:reanchor",
13114
13200
  "smelt:purge",
13115
13201
  "smelt:embed-annotation",
13116
13202
  "smelt:purge-annotation"
@@ -13119,8 +13205,9 @@ function isWorkItem(input) {
13119
13205
  return WORK_ITEM_TYPES.has(input.type);
13120
13206
  }
13121
13207
  var Smelter = class _Smelter {
13122
- constructor(events$, vectorStore, embeddingProvider, content, bus, chunkingConfig, timing, logger) {
13208
+ constructor(events$, rebuildAnchors$, vectorStore, embeddingProvider, content, bus, chunkingConfig, timing, logger) {
13123
13209
  this.events$ = events$;
13210
+ this.rebuildAnchors$ = rebuildAnchors$;
13124
13211
  this.vectorStore = vectorStore;
13125
13212
  this.embeddingProvider = embeddingProvider;
13126
13213
  this.content = content;
@@ -13128,8 +13215,10 @@ var Smelter = class _Smelter {
13128
13215
  this.chunkingConfig = chunkingConfig;
13129
13216
  this.timing = timing;
13130
13217
  this.logger = logger;
13218
+ this.anchoredStore = anchoredTextStoreOverTransport(content, logger.child({ component: "anchored-text-cache" }));
13131
13219
  }
13132
13220
  events$;
13221
+ rebuildAnchors$;
13133
13222
  vectorStore;
13134
13223
  embeddingProvider;
13135
13224
  content;
@@ -13142,11 +13231,21 @@ var Smelter = class _Smelter {
13142
13231
  static RECONCILE_WAVE = 8;
13143
13232
  eventSubject = new import_rxjs9.Subject();
13144
13233
  sourceSubscription = null;
13234
+ commandSubscription = null;
13145
13235
  pipelineSubscription = null;
13146
13236
  _eventsProcessed = 0;
13147
13237
  _reconcileState = { phase: "pending" };
13148
13238
  workDone = 0;
13239
+ workFailed = 0;
13149
13240
  workWaiter = null;
13241
+ /**
13242
+ * Serializes every planner drain (reconcile, anchored-text rebuilds):
13243
+ * there is one waiter slot, and the weave:rebuild rule — rebuilds never
13244
+ * interleave — applies to every unit here being a potential multi-second
13245
+ * OCR pass.
13246
+ */
13247
+ drainChain = Promise.resolve();
13248
+ anchoredStore;
13150
13249
  get eventsProcessed() {
13151
13250
  return this._eventsProcessed;
13152
13251
  }
@@ -13174,7 +13273,7 @@ var Smelter = class _Smelter {
13174
13273
  return (0, import_rxjs9.from)(
13175
13274
  withActorSpan("smelter", inputOrBatch.type, async () => {
13176
13275
  const ok = await this.safeProcessEvent(inputOrBatch);
13177
- if (isWorkItem(inputOrBatch)) this.noteWorkDone(1);
13276
+ if (isWorkItem(inputOrBatch)) this.noteWorkDone(1, ok ? 0 : 1);
13178
13277
  else if (ok) this._eventsProcessed++;
13179
13278
  })
13180
13279
  );
@@ -13188,18 +13287,26 @@ var Smelter = class _Smelter {
13188
13287
  this.logger.debug("Bus event received", { type: event.type, resourceId: event.resourceId });
13189
13288
  this.eventSubject.next(event);
13190
13289
  });
13290
+ this.commandSubscription = this.rebuildAnchors$.pipe(
13291
+ (0, import_operators7.concatMap)((command) => (0, import_rxjs9.from)(this.rebuildAnchors(command)))
13292
+ ).subscribe({
13293
+ error: (err) => this.logger.error("Smelter command pipeline error", { error: errField(err) })
13294
+ });
13191
13295
  this.logger.info("Smelter pipeline initialized");
13192
13296
  }
13193
13297
  stop() {
13194
13298
  this.sourceSubscription?.unsubscribe();
13195
13299
  this.sourceSubscription = null;
13300
+ this.commandSubscription?.unsubscribe();
13301
+ this.commandSubscription = null;
13196
13302
  this.pipelineSubscription?.unsubscribe();
13197
13303
  this.pipelineSubscription = null;
13198
13304
  this.eventSubject.complete();
13199
13305
  this.logger.info("Smelter stopped");
13200
13306
  }
13201
- noteWorkDone(count) {
13307
+ noteWorkDone(count, failed) {
13202
13308
  this.workDone += count;
13309
+ this.workFailed += failed;
13203
13310
  if (this.workWaiter && this.workDone >= this.workWaiter.target) {
13204
13311
  this.workWaiter.resolve();
13205
13312
  this.workWaiter = null;
@@ -13213,12 +13320,15 @@ var Smelter = class _Smelter {
13213
13320
  let wireProcessed = 0;
13214
13321
  for (const run of partitionByType(events)) {
13215
13322
  const workRun = isWorkItem(run[0]);
13323
+ let succeeded = 0;
13216
13324
  try {
13217
13325
  if (run.length === 1) {
13218
13326
  const ok = await this.safeProcessEvent(run[0]);
13327
+ if (ok) succeeded = 1;
13219
13328
  if (ok && !workRun) wireProcessed++;
13220
13329
  } else {
13221
13330
  const processed = await this.applyBatchByType(run);
13331
+ succeeded = processed;
13222
13332
  if (!workRun) wireProcessed += processed;
13223
13333
  }
13224
13334
  } catch (error) {
@@ -13228,7 +13338,7 @@ var Smelter = class _Smelter {
13228
13338
  error: errField(error)
13229
13339
  });
13230
13340
  } finally {
13231
- if (workRun) this.noteWorkDone(run.length);
13341
+ if (workRun) this.noteWorkDone(run.length, run.length - succeeded);
13232
13342
  }
13233
13343
  }
13234
13344
  return wireProcessed;
@@ -13300,6 +13410,9 @@ var Smelter = class _Smelter {
13300
13410
  case "smelt:restamp":
13301
13411
  await this.restampResource(event);
13302
13412
  break;
13413
+ case "smelt:reanchor":
13414
+ await this.reanchorResource(event);
13415
+ break;
13303
13416
  case "smelt:purge":
13304
13417
  await this.handleResourcePurge(event);
13305
13418
  break;
@@ -13326,6 +13439,45 @@ var Smelter = class _Smelter {
13326
13439
  await this.vectorStore.updateResourceEntityTypes(resourceId(rid), entityTypes);
13327
13440
  this.logger.info("Restamped resource entity types", { resourceId: rid, entityTypes });
13328
13441
  }
13442
+ /**
13443
+ * Re-derive a lost anchored-text artifact from the resource's current
13444
+ * bytes (PERSIST-ANCHORS P0, the third drift class). Extraction is the
13445
+ * cost here — the vectors are already correct, so this NEVER calls the
13446
+ * embedding provider, the vector store, or the settled signal: the index
13447
+ * decision was already made and announced at its checksum; only the map
13448
+ * is missing. Name the work for what it does (the S13 discipline).
13449
+ *
13450
+ * The publish is STRICT, unlike the embed path's best-effort side
13451
+ * publish: here the artifact IS the job, so a store failure must throw —
13452
+ * the pipeline logs and counts it, and the rebuild command's partial-
13453
+ * failure accounting depends on that throw.
13454
+ */
13455
+ async reanchorResource(event) {
13456
+ const rid = event.resourceId;
13457
+ if (!rid) return;
13458
+ const { data, contentType } = await this.content.getBinary(resourceId(rid));
13459
+ const bytes = Buffer.from(data);
13460
+ const extractor = EXTRACTORS[textExtractionOf(contentType)];
13461
+ if (!extractor?.yieldsGeometry) {
13462
+ this.logger.info("Re-anchor found no geometry-capable extractor", { resourceId: rid, contentType });
13463
+ return;
13464
+ }
13465
+ const checksum = calculateChecksum(bytes);
13466
+ const extracted = await extractor.extract(bytes, contentType, {
13467
+ key: checksum,
13468
+ store: this.anchoredStore
13469
+ });
13470
+ if ("declined" in extracted || !extracted.items?.length) {
13471
+ this.logger.info("Re-anchor extraction yielded no geometry", {
13472
+ resourceId: rid,
13473
+ contentType,
13474
+ ..."declined" in extracted ? { declined: extracted.declined } : {}
13475
+ });
13476
+ return;
13477
+ }
13478
+ await this.content.putAnchoredText(checksum, { ...extracted, items: extracted.items });
13479
+ this.logger.info("Re-anchored resource", { resourceId: rid, checksum, items: extracted.items.length });
13480
+ }
13329
13481
  async handleResourcePurge(event) {
13330
13482
  const rid = event.resourceId;
13331
13483
  if (!rid) return;
@@ -13345,12 +13497,34 @@ var Smelter = class _Smelter {
13345
13497
  const { data, contentType } = await this.content.getBinary(resourceId(resourceId10));
13346
13498
  const bytes = Buffer.from(data);
13347
13499
  const checksum = calculateChecksum(bytes);
13348
- if (textExtractionOf(contentType) !== "decode") {
13349
- this.logger.debug("Skipping resource that does not decode as text", { resourceId: resourceId10, contentType });
13350
- return { kind: "skipped", checksum };
13500
+ const extractor = EXTRACTORS[textExtractionOf(contentType)];
13501
+ if (!extractor) {
13502
+ this.logger.debug("Skipping resource with no extractor for its media type", { resourceId: resourceId10, contentType });
13503
+ return { kind: "skipped", checksum, reason: "no-extractor" };
13504
+ }
13505
+ const extracted = await extractor.extract(bytes, contentType, {
13506
+ key: checksum,
13507
+ store: this.anchoredStore
13508
+ });
13509
+ if ("declined" in extracted) {
13510
+ this.logger.debug("Extractor declined", { resourceId: resourceId10, contentType, reason: extracted.declined });
13511
+ return { kind: "skipped", checksum, reason: extracted.declined };
13512
+ }
13513
+ if (extracted.ocrConfidence && extracted.ocrConfidence.lowConfidenceWords > 0) {
13514
+ this.logger.info("OCR read words it was unsure of", {
13515
+ resourceId: resourceId10,
13516
+ contentType,
13517
+ ...extracted.ocrConfidence
13518
+ });
13519
+ }
13520
+ if (extracted.unreadPages?.length) {
13521
+ this.logger.info("Partial extraction coverage", {
13522
+ resourceId: resourceId10,
13523
+ contentType,
13524
+ unreadPages: extracted.unreadPages
13525
+ });
13351
13526
  }
13352
- const text = decodeRepresentation(bytes, contentType);
13353
- return text.trim() ? { kind: "text", text, checksum } : { kind: "skipped", checksum };
13527
+ return extracted.text.trim() ? { kind: "text", text: extracted.text, checksum, machineRead: extracted.method === "ocr" } : { kind: "skipped", checksum, reason: "empty" };
13354
13528
  } catch (error) {
13355
13529
  this.logger.warn("Content unavailable for embedding", { resourceId: resourceId10, error: errField(error) });
13356
13530
  return { kind: "unavailable" };
@@ -13362,9 +13536,9 @@ var Smelter = class _Smelter {
13362
13536
  * fold. Best-effort — waiters degrade to their bounded timeout; a signal
13363
13537
  * failure must never fail the embed.
13364
13538
  */
13365
- async emitSettled(resourceId10, contentChecksum, outcome) {
13539
+ async emitSettled(resourceId10, contentChecksum, outcome, reason) {
13366
13540
  try {
13367
- await this.bus.emit("smelt:settled", { resourceId: resourceId10, contentChecksum, outcome });
13541
+ await this.bus.emit("smelt:settled", { resourceId: resourceId10, contentChecksum, outcome, ...reason ? { reason } : {} });
13368
13542
  } catch (error) {
13369
13543
  this.logger.warn("Failed to emit smelt:settled", { resourceId: resourceId10, outcome, error: errField(error) });
13370
13544
  }
@@ -13392,12 +13566,14 @@ var Smelter = class _Smelter {
13392
13566
  const fetched = await this.fetchEmbeddableText(rid);
13393
13567
  if (fetched.kind === "unavailable") return;
13394
13568
  if (fetched.kind === "skipped") {
13395
- await this.emitSettled(rid, fetched.checksum, "skipped");
13569
+ await this.vectorStore.deleteResourceVectors(resourceId(rid));
13570
+ await this.emitSettled(rid, fetched.checksum, "skipped", fetched.reason);
13396
13571
  return;
13397
13572
  }
13398
13573
  const chunks = chunkText(fetched.text, this.chunkingConfig);
13399
13574
  if (chunks.length === 0) {
13400
- await this.emitSettled(rid, fetched.checksum, "skipped");
13575
+ await this.vectorStore.deleteResourceVectors(resourceId(rid));
13576
+ await this.emitSettled(rid, fetched.checksum, "skipped", "empty");
13401
13577
  return;
13402
13578
  }
13403
13579
  const entityTypes = await this.resolveEntityTypes(rid);
@@ -13407,7 +13583,7 @@ var Smelter = class _Smelter {
13407
13583
  text: t,
13408
13584
  embedding: embeddings[i]
13409
13585
  }));
13410
- await this.vectorStore.upsertResourceVectors(resourceId(rid), embeddingChunks, fetched.checksum, entityTypes);
13586
+ await this.vectorStore.upsertResourceVectors(resourceId(rid), embeddingChunks, fetched.checksum, entityTypes, fetched.machineRead);
13411
13587
  await this.emitSettled(rid, fetched.checksum, "indexed");
13412
13588
  this.logger.info(logMessage, { resourceId: rid, chunks: chunks.length });
13413
13589
  }
@@ -13451,12 +13627,14 @@ var Smelter = class _Smelter {
13451
13627
  if (!exactText?.trim()) return;
13452
13628
  const aid = annotationId(annotation.id);
13453
13629
  const embedding = await this.embeddingProvider.embed(exactText);
13630
+ const stamp = await this.vectorStore.getResourceStamp(resourceId(rid));
13454
13631
  const payload = {
13455
13632
  annotationId: aid,
13456
13633
  resourceId: resourceId(rid),
13457
13634
  motivation: annotation.motivation ?? "",
13458
13635
  entityTypes: annotation.entityTypes ?? [],
13459
- exactText
13636
+ exactText,
13637
+ ...stamp?.machineRead ? { machineRead: true } : {}
13460
13638
  };
13461
13639
  await this.vectorStore.upsertAnnotationVector(aid, embedding, payload);
13462
13640
  this.logger.info("Indexed annotation", { annotationId: String(aid) });
@@ -13481,28 +13659,30 @@ var Smelter = class _Smelter {
13481
13659
  const fetched = await this.fetchEmbeddableText(rid);
13482
13660
  if (fetched.kind === "unavailable") continue;
13483
13661
  if (fetched.kind === "skipped") {
13484
- await this.emitSettled(rid, fetched.checksum, "skipped");
13662
+ await this.vectorStore.deleteResourceVectors(resourceId(rid));
13663
+ await this.emitSettled(rid, fetched.checksum, "skipped", fetched.reason);
13485
13664
  continue;
13486
13665
  }
13487
13666
  const chunks = chunkText(fetched.text, this.chunkingConfig);
13488
13667
  if (chunks.length === 0) {
13489
- await this.emitSettled(rid, fetched.checksum, "skipped");
13668
+ await this.vectorStore.deleteResourceVectors(resourceId(rid));
13669
+ await this.emitSettled(rid, fetched.checksum, "skipped", "empty");
13490
13670
  continue;
13491
13671
  }
13492
13672
  const entityTypes = await this.resolveEntityTypes(rid);
13493
- resourceData.push({ rid: resourceId(rid), chunks, checksum: fetched.checksum, entityTypes });
13673
+ resourceData.push({ rid: resourceId(rid), chunks, checksum: fetched.checksum, entityTypes, machineRead: fetched.machineRead });
13494
13674
  allChunks.push(...chunks);
13495
13675
  }
13496
13676
  if (allChunks.length === 0) return events.length;
13497
13677
  const allEmbeddings = await this.embeddingProvider.embedBatch(allChunks);
13498
13678
  let offset = 0;
13499
- for (const { rid, chunks, checksum, entityTypes } of resourceData) {
13679
+ for (const { rid, chunks, checksum, entityTypes, machineRead } of resourceData) {
13500
13680
  const embeddingChunks = chunks.map((t, i) => ({
13501
13681
  chunkIndex: i,
13502
13682
  text: t,
13503
13683
  embedding: allEmbeddings[offset + i]
13504
13684
  }));
13505
- await this.vectorStore.upsertResourceVectors(rid, embeddingChunks, checksum, entityTypes);
13685
+ await this.vectorStore.upsertResourceVectors(rid, embeddingChunks, checksum, entityTypes, machineRead);
13506
13686
  await this.emitSettled(String(rid), checksum, "indexed");
13507
13687
  this.logger.info("Batch-indexed resource", { resourceId: String(rid), chunks: chunks.length });
13508
13688
  offset += chunks.length;
@@ -13575,26 +13755,23 @@ var Smelter = class _Smelter {
13575
13755
  }
13576
13756
  this._reconcileState = { phase: "running" };
13577
13757
  try {
13578
- const [indexedResources, indexedAnnotations] = await Promise.all([
13758
+ const [indexedResources, indexedAnnotations, anchoredKeys] = await Promise.all([
13579
13759
  this.vectorStore.listResourceStamps(),
13580
- this.vectorStore.listAnnotationIds()
13760
+ this.vectorStore.listAnnotationIds(),
13761
+ // The artifact store's would-hit keys — one bulk read, never a probe
13762
+ // per resource (PERSIST-ANCHORS P0). Keys are resource ids today;
13763
+ // P1 rekeys the store by content checksum and this lookup moves
13764
+ // with it.
13765
+ this.content.listAnchoredTextKeys().then((keys) => new Set(keys))
13581
13766
  ]);
13582
13767
  const resources = await this.listAllResources();
13583
13768
  this.logger.info("Reconcile started", {
13584
13769
  indexedResources: indexedResources.size,
13585
13770
  indexedAnnotations: indexedAnnotations.size,
13771
+ anchoredArtifacts: anchoredKeys.size,
13586
13772
  liveResources: resources.length
13587
13773
  });
13588
- const embeddable = /* @__PURE__ */ new Map();
13589
- for (const resource of resources) {
13590
- const mediaType = getPrimaryMediaType(resource);
13591
- if (resource["@id"] && mediaType && textExtractionOf(mediaType) === "decode") {
13592
- embeddable.set(resource["@id"], {
13593
- checksum: getPrimaryRepresentation(resource)?.checksum,
13594
- entityTypes: getResourceEntityTypes(resource)
13595
- });
13596
- }
13597
- }
13774
+ const embeddable = this.classifyEmbeddable(resources);
13598
13775
  const work = [];
13599
13776
  for (const rid of indexedResources.keys()) {
13600
13777
  if (!embeddable.has(rid)) work.push({ type: "smelt:purge", resourceId: rid, payload: {} });
@@ -13608,6 +13785,9 @@ var Smelter = class _Smelter {
13608
13785
  } else if (!sameStringSet(indexed.entityTypes, catalog.entityTypes)) {
13609
13786
  work.push({ type: "smelt:restamp", resourceId: rid, payload: {} });
13610
13787
  }
13788
+ if (indexed && catalog.checksum !== void 0 && indexed.contentChecksum === catalog.checksum && catalog.yieldsGeometry && !anchoredKeys.has(catalog.checksum)) {
13789
+ work.push({ type: "smelt:reanchor", resourceId: rid, payload: {} });
13790
+ }
13611
13791
  }
13612
13792
  const liveAnnotationIds = /* @__PURE__ */ new Set();
13613
13793
  for (const resource of resources) {
@@ -13636,9 +13816,12 @@ var Smelter = class _Smelter {
13636
13816
  const summary = {
13637
13817
  resourcesEmbedded: work.filter((w) => w.type === "smelt:embed").length,
13638
13818
  resourcesRestamped: work.filter((w) => w.type === "smelt:restamp").length,
13819
+ resourcesReanchored: work.filter((w) => w.type === "smelt:reanchor").length,
13639
13820
  resourceVectorsDeleted: work.filter((w) => w.type === "smelt:purge").length,
13640
13821
  annotationsEmbedded: work.filter((w) => w.type === "smelt:embed-annotation").length,
13641
- annotationVectorsDeleted: work.filter((w) => w.type === "smelt:purge-annotation").length
13822
+ annotationVectorsDeleted: work.filter((w) => w.type === "smelt:purge-annotation").length,
13823
+ resourcesEligible: embeddable.size,
13824
+ resourcesIndexed: (await this.vectorStore.listResourceStamps()).size
13642
13825
  };
13643
13826
  this._reconcileState = { phase: "done", summary };
13644
13827
  this.logger.info("Reconcile complete", { ...summary });
@@ -13657,16 +13840,106 @@ var Smelter = class _Smelter {
13657
13840
  * completion. The pipeline ticks `noteWorkDone` for every consumed work
13658
13841
  * item (success or failure — failures are logged like any live event), so
13659
13842
  * each wave's waiter resolves exactly when its items have been processed.
13843
+ *
13844
+ * Serialized through `drainChain`: there is ONE waiter slot, and the
13845
+ * planners that drain (reconcile, `smelt:rebuild-anchors`) must not
13846
+ * interleave — a rebuild command arriving mid-reconcile waits its turn.
13847
+ *
13848
+ * @returns how many of THESE items failed — the rebuild command's
13849
+ * partial-failure accounting (failure detail is in the logs).
13660
13850
  */
13661
13851
  async drain(work) {
13662
- for (let i = 0; i < work.length; i += _Smelter.RECONCILE_WAVE) {
13663
- const wave = work.slice(i, i + _Smelter.RECONCILE_WAVE);
13664
- const done = new Promise((resolve2) => {
13665
- this.workWaiter = { target: this.workDone + wave.length, resolve: resolve2 };
13666
- });
13667
- for (const item of wave) this.eventSubject.next(item);
13668
- await done;
13852
+ let failures = 0;
13853
+ const run = this.drainChain.then(async () => {
13854
+ const failedBefore = this.workFailed;
13855
+ for (let i = 0; i < work.length; i += _Smelter.RECONCILE_WAVE) {
13856
+ const wave = work.slice(i, i + _Smelter.RECONCILE_WAVE);
13857
+ const done = new Promise((resolve2) => {
13858
+ this.workWaiter = { target: this.workDone + wave.length, resolve: resolve2 };
13859
+ });
13860
+ for (const item of wave) this.eventSubject.next(item);
13861
+ await done;
13862
+ }
13863
+ failures = this.workFailed - failedBefore;
13864
+ });
13865
+ this.drainChain = run.then(() => void 0, () => void 0);
13866
+ await run;
13867
+ return failures;
13868
+ }
13869
+ /**
13870
+ * `smelt:rebuild-anchors` — the operator's explicit re-derivation of
13871
+ * anchored-text artifacts (PERSIST-ANCHORS P0), shaped after
13872
+ * `weave:rebuild`: optionally scoped, strictly serialized (concatMap on
13873
+ * the command stream + the drain chain), correlated ok/failed replies,
13874
+ * and partial completion FAILS — a rebuild that quietly skipped resources
13875
+ * would present exactly like a document with no text, which is the #845
13876
+ * failure mode wearing different clothes.
13877
+ *
13878
+ * Never destructive: nothing is deleted first, stale entries are simply
13879
+ * overwritten (the W5-frames lesson — a rebuild that clears before it
13880
+ * re-derives turns a partial failure into a loss). Re-anchoring makes
13881
+ * zero embedding calls; work items ride the normal per-resource lanes,
13882
+ * so a rebuild can never interleave with live processing of the same
13883
+ * resource (S1/S2).
13884
+ */
13885
+ async rebuildAnchors(command) {
13886
+ const { correlationId, resourceId: resourceId10 } = command;
13887
+ try {
13888
+ let work;
13889
+ if (resourceId10) {
13890
+ work = [{ type: "smelt:reanchor", resourceId: resourceId10, payload: {} }];
13891
+ } else {
13892
+ const resources = await this.listAllResources();
13893
+ work = [...this.classifyEmbeddable(resources)].filter(([, catalog]) => catalog.yieldsGeometry).map(([rid]) => ({ type: "smelt:reanchor", resourceId: rid, payload: {} }));
13894
+ }
13895
+ this.logger.info("Anchored-text rebuild started", { scoped: resourceId10 ?? null, resources: work.length });
13896
+ const failed = await this.drain(work);
13897
+ if (failed > 0) {
13898
+ await this.bus.emit("smelt:rebuild-anchors-failed", {
13899
+ ...correlationId ? { correlationId } : {},
13900
+ message: `${failed} of ${work.length} resources failed to re-anchor \u2014 see smelter logs`
13901
+ });
13902
+ return;
13903
+ }
13904
+ await this.bus.emit("smelt:rebuild-anchors-ok", correlationId ? { correlationId } : {});
13905
+ this.logger.info("Anchored-text rebuild complete", { scoped: resourceId10 ?? null, resources: work.length });
13906
+ } catch (error) {
13907
+ this.logger.error("Anchored-text rebuild failed", { error: errField(error) });
13908
+ try {
13909
+ await this.bus.emit("smelt:rebuild-anchors-failed", {
13910
+ ...correlationId ? { correlationId } : {},
13911
+ message: error instanceof Error ? error.message : String(error)
13912
+ });
13913
+ } catch (emitError) {
13914
+ this.logger.warn("Failed to emit smelt:rebuild-anchors-failed", { error: errField(emitError) });
13915
+ }
13916
+ }
13917
+ }
13918
+ /**
13919
+ * Embeddable live resources, each with the catalog's claims: the primary
13920
+ * representation's checksum (the bytes the smelter would read), the
13921
+ * current entity-type set (the discriminator the stamps must carry), and
13922
+ * whether the media type's extractor derives geometry (whether an
13923
+ * anchored-text artifact should exist). Embeddable ⇔ an extractor exists
13924
+ * for the media type's strategy — the same registry the live fetch
13925
+ * resolves, and `yieldsGeometry` is declared on the extractor itself, so
13926
+ * every gate here and the live fetch's behavior are twins by construction.
13927
+ * Shared by `reconcile()` and the `smelt:rebuild-anchors` planner.
13928
+ */
13929
+ classifyEmbeddable(resources) {
13930
+ const embeddable = /* @__PURE__ */ new Map();
13931
+ for (const resource of resources) {
13932
+ const mediaType = getPrimaryMediaType(resource);
13933
+ const extractor = mediaType ? EXTRACTORS[textExtractionOf(mediaType)] : null;
13934
+ if (resource["@id"] && extractor) {
13935
+ embeddable.set(resource["@id"], {
13936
+ checksum: getPrimaryRepresentation(resource)?.checksum,
13937
+ entityTypes: getResourceEntityTypes(resource),
13938
+ yieldsGeometry: extractor.yieldsGeometry
13939
+ });
13940
+ }
13669
13941
  }
13942
+ return embeddable;
13670
13943
  }
13671
13944
  /** Page through `browse:resources-requested` until the catalog is exhausted. */
13672
13945
  async listAllResources() {
@@ -13697,6 +13970,7 @@ var SMELTER_CHANNELS = [
13697
13970
  "mark:entity-tag-added",
13698
13971
  "mark:entity-tag-removed"
13699
13972
  ];
13973
+ var SMELTER_COMMAND_CHANNELS = ["smelt:rebuild-anchors"];
13700
13974
  function createSmelterActorStateUnit(options) {
13701
13975
  const { bus } = options;
13702
13976
  let started = false;
@@ -13711,12 +13985,14 @@ function createSmelterActorStateUnit(options) {
13711
13985
  )
13712
13986
  )
13713
13987
  );
13988
+ const rebuildAnchors$ = bus.on$("smelt:rebuild-anchors");
13714
13989
  return {
13715
13990
  events$,
13991
+ rebuildAnchors$,
13716
13992
  start: () => {
13717
13993
  if (started) return;
13718
13994
  started = true;
13719
- bus.addChannels?.([...SMELTER_CHANNELS]);
13995
+ bus.addChannels?.([...SMELTER_CHANNELS, ...SMELTER_COMMAND_CHANNELS]);
13720
13996
  },
13721
13997
  dispose: () => {
13722
13998
  started = false;