@semiont/make-meaning 0.5.5 → 0.5.7

package/dist/smelter-main.js

@@ -1,6 +1,9 @@
-import { HttpTransport } from '@semiont/api-client';
-import { createTomlConfigLoader, accessToken, baseUrl as baseUrl$1, burstBuffer, resourceId, getTargetSelector, getExactText, annotationId } from '@semiont/core';
+import { createTomlConfigLoader, accessToken, baseUrl as baseUrl$1, burstBuffer, errField, resourceId, textExtractionOf, decodeRepresentation, getTargetSelector, getExactText, annotationId, getPrimaryMediaType, getPrimaryRepresentation } from '@semiont/core';
+import { calculateChecksum } from '@semiont/content';
 import { createEmbeddingProvider, createVectorStore, chunkText } from '@semiont/vectors';
+import { registerVectorIndexSizeProvider, withActorSpan } from '@semiont/observability';
+import { busRequest } from '@semiont/sdk';
+import { HttpTransport, HttpContentTransport } from '@semiont/http-transport';
 import { createServer } from 'http';
 import { existsSync, readFileSync } from 'fs';
 import { homedir } from 'os';
@@ -699,19 +702,19 @@ var require_Observable = __commonJS({
     var config_1 = require_config();
     var isFunction_1 = require_isFunction();
     var errorContext_1 = require_errorContext();
-    var Observable2 = (function() {
-      function Observable3(subscribe) {
+    var Observable3 = (function() {
+      function Observable4(subscribe) {
         if (subscribe) {
           this._subscribe = subscribe;
         }
       }
-      Observable3.prototype.lift = function(operator) {
-        var observable = new Observable3();
+      Observable4.prototype.lift = function(operator) {
+        var observable = new Observable4();
         observable.source = this;
         observable.operator = operator;
         return observable;
       };
-      Observable3.prototype.subscribe = function(observerOrNext, error, complete) {
+      Observable4.prototype.subscribe = function(observerOrNext, error, complete) {
         var _this = this;
         var subscriber = isSubscriber(observerOrNext) ? observerOrNext : new Subscriber_1.SafeSubscriber(observerOrNext, error, complete);
         errorContext_1.errorContext(function() {
@@ -720,14 +723,14 @@ var require_Observable = __commonJS({
         });
         return subscriber;
       };
-      Observable3.prototype._trySubscribe = function(sink) {
+      Observable4.prototype._trySubscribe = function(sink) {
         try {
           return this._subscribe(sink);
         } catch (err) {
           sink.error(err);
         }
       };
-      Observable3.prototype.forEach = function(next, promiseCtor) {
+      Observable4.prototype.forEach = function(next, promiseCtor) {
         var _this = this;
         promiseCtor = getPromiseCtor(promiseCtor);
         return new promiseCtor(function(resolve, reject) {
@@ -746,21 +749,21 @@ var require_Observable = __commonJS({
           _this.subscribe(subscriber);
         });
       };
-      Observable3.prototype._subscribe = function(subscriber) {
+      Observable4.prototype._subscribe = function(subscriber) {
         var _a;
         return (_a = this.source) === null || _a === void 0 ? void 0 : _a.subscribe(subscriber);
       };
-      Observable3.prototype[observable_1.observable] = function() {
+      Observable4.prototype[observable_1.observable] = function() {
         return this;
       };
-      Observable3.prototype.pipe = function() {
+      Observable4.prototype.pipe = function() {
         var operations = [];
         for (var _i = 0; _i < arguments.length; _i++) {
           operations[_i] = arguments[_i];
         }
         return pipe_1.pipeFromArray(operations)(this);
       };
-      Observable3.prototype.toPromise = function(promiseCtor) {
+      Observable4.prototype.toPromise = function(promiseCtor) {
         var _this = this;
         promiseCtor = getPromiseCtor(promiseCtor);
         return new promiseCtor(function(resolve, reject) {
@@ -774,12 +777,12 @@ var require_Observable = __commonJS({
           });
         });
       };
-      Observable3.create = function(subscribe) {
-        return new Observable3(subscribe);
+      Observable4.create = function(subscribe) {
+        return new Observable4(subscribe);
       };
-      return Observable3;
+      return Observable4;
     })();
-    exports.Observable = Observable2;
+    exports.Observable = Observable3;
     function getPromiseCtor(promiseCtor) {
       var _a;
       return (_a = promiseCtor !== null && promiseCtor !== void 0 ? promiseCtor : config_1.config.Promise) !== null && _a !== void 0 ? _a : Promise;
@@ -9716,8 +9719,7 @@ var require_operators = __commonJS({
 });
 
 // src/smelter-main.ts
-var import_rxjs2 = __toESM(require_cjs());
-var import_operators2 = __toESM(require_operators());
+var import_rxjs3 = __toESM(require_cjs());
 
 // src/smelter-actor-state-unit.ts
 var import_rxjs = __toESM(require_cjs());
@@ -9746,7 +9748,6 @@ function createSmelterActorStateUnit(options) {
   );
   return {
     events$,
-    emit: (channel, payload) => bus.emit(channel, payload),
     start: () => {
       if (started) return;
       started = true;
@@ -9758,8 +9759,493 @@ function createSmelterActorStateUnit(options) {
   };
 }
 
-// src/smelter-main.ts
-var import_rxjs3 = __toESM(require_cjs());
+// src/smelter.ts
+var import_rxjs2 = __toESM(require_cjs());
+var import_operators2 = __toESM(require_operators());
+
+// src/batch-utils.ts
+function partitionByType(events) {
+  const runs = [];
+  let currentRun = [];
+  for (const event of events) {
+    if (currentRun.length > 0 && currentRun[0].type !== event.type) {
+      runs.push(currentRun);
+      currentRun = [];
+    }
+    currentRun.push(event);
+  }
+  if (currentRun.length > 0) runs.push(currentRun);
+  return runs;
+}
+
+// src/smelter.ts
+function isWorkItem(input) {
+  return input.type.startsWith("smelt:");
+}
+var Smelter = class _Smelter {
+  constructor(events$, vectorStore, embeddingProvider, content, bus, chunkingConfig2, timing, logger2) {
+    this.events$ = events$;
+    this.vectorStore = vectorStore;
+    this.embeddingProvider = embeddingProvider;
+    this.content = content;
+    this.bus = bus;
+    this.chunkingConfig = chunkingConfig2;
+    this.timing = timing;
+    this.logger = logger2;
+  }
+  events$;
+  vectorStore;
+  embeddingProvider;
+  content;
+  bus;
+  chunkingConfig;
+  timing;
+  logger;
+  static RECONCILE_PAGE_SIZE = 200;
+  /** Bound on concurrently in-flight reconcile work — a cold rebuild must not fan out unbounded embedding calls. */
+  static RECONCILE_WAVE = 8;
+  eventSubject = new import_rxjs2.Subject();
+  sourceSubscription = null;
+  pipelineSubscription = null;
+  _eventsProcessed = 0;
+  _reconcileState = { phase: "pending" };
+  workDone = 0;
+  workWaiter = null;
+  get eventsProcessed() {
+    return this._eventsProcessed;
+  }
+  get reconcileState() {
+    return this._reconcileState;
+  }
+  initialize() {
+    this.pipelineSubscription = this.eventSubject.pipe(
+      (0, import_operators2.groupBy)((e) => e.resourceId ?? "__unknown__"),
+      (0, import_operators2.mergeMap)(
+        (group) => group.pipe(
+          burstBuffer({
+            burstWindowMs: this.timing.burstWindowMs,
+            maxBatchSize: this.timing.maxBatchSize,
+            idleTimeoutMs: this.timing.idleTimeoutMs
+          }),
+          (0, import_operators2.concatMap)((inputOrBatch) => {
+            if (Array.isArray(inputOrBatch)) {
+              return (0, import_rxjs2.from)(
+                withActorSpan("smelter", "batch", async () => {
+                  this._eventsProcessed += await this.processBatch(inputOrBatch);
+                }, { "batch.size": inputOrBatch.length })
+              );
+            }
+            return (0, import_rxjs2.from)(
+              withActorSpan("smelter", inputOrBatch.type, async () => {
+                const ok = await this.safeProcessEvent(inputOrBatch);
+                if (isWorkItem(inputOrBatch)) this.noteWorkDone(1);
+                else if (ok) this._eventsProcessed++;
+              })
+            );
+          })
+        )
+      )
+    ).subscribe({
+      error: (err) => this.logger.error("Smelter pipeline error", { error: errField(err) })
+    });
+    this.sourceSubscription = this.events$.subscribe((event) => {
+      this.logger.debug("Bus event received", { type: event.type, resourceId: event.resourceId });
+      this.eventSubject.next(event);
+    });
+    this.logger.info("Smelter pipeline initialized");
+  }
+  stop() {
+    this.sourceSubscription?.unsubscribe();
+    this.sourceSubscription = null;
+    this.pipelineSubscription?.unsubscribe();
+    this.pipelineSubscription = null;
+    this.eventSubject.complete();
+    this.logger.info("Smelter stopped");
+  }
+  noteWorkDone(count) {
+    this.workDone += count;
+    if (this.workWaiter && this.workDone >= this.workWaiter.target) {
+      this.workWaiter.resolve();
+      this.workWaiter = null;
+    }
+  }
+  /**
+   * Returns the number of WIRE events processed without error (the S9b
+   * oracle) — `smelt:*` work-item runs tick the drain counter instead.
+   */
+  async processBatch(events) {
+    let wireProcessed = 0;
+    for (const run of partitionByType(events)) {
+      const workRun = isWorkItem(run[0]);
+      try {
+        if (run.length === 1) {
+          const ok = await this.safeProcessEvent(run[0]);
+          if (ok && !workRun) wireProcessed++;
+        } else {
+          const processed = await this.applyBatchByType(run);
+          if (!workRun) wireProcessed += processed;
+        }
+      } catch (error) {
+        this.logger.error("Smelter failed to process batch run", {
+          eventType: run[0].type,
+          runSize: run.length,
+          error: errField(error)
+        });
+      } finally {
+        if (workRun) this.noteWorkDone(run.length);
+      }
+    }
+    return wireProcessed;
+  }
+  /**
+   * Batch-optimized processing for consecutive events of the same type.
+   * Returns the number of events processed without error.
+   */
+  async applyBatchByType(events) {
+    switch (events[0].type) {
+      case "yield:created":
+      case "smelt:embed":
+        return this.batchResourceCreated(events);
+      case "mark:added":
+      case "smelt:embed-annotation":
+        return this.batchAnnotationAdded(events);
+      default: {
+        let processed = 0;
+        for (const event of events) {
+          if (await this.safeProcessEvent(event)) processed++;
+        }
+        return processed;
+      }
+    }
+  }
+  /** Returns true if the input was processed without error. */
+  async safeProcessEvent(event) {
+    try {
+      await this.processEvent(event);
+      return true;
+    } catch (err) {
+      this.logger.error("Smelter failed to process event", {
+        type: event.type,
+        resourceId: event.resourceId,
+        error: errField(err)
+      });
+      return false;
+    }
+  }
+  async processEvent(event) {
+    switch (event.type) {
+      case "yield:created":
+        await this.embedResource(event, "Indexed resource");
+        break;
+      case "yield:updated":
+      case "yield:representation-added":
+        await this.embedResource(event, "Re-embedded resource");
+        break;
+      case "mark:archived":
+        await this.handleResourceArchived(event);
+        break;
+      case "mark:added":
+        await this.handleAnnotationAdded(event);
+        break;
+      case "mark:removed":
+        await this.handleAnnotationRemoved(event);
+        break;
+      // Reconcile work items — same handlers, distinct provenance.
+      case "smelt:embed":
+        await this.embedResource(event, "Reconcile-indexed resource");
+        break;
+      case "smelt:purge":
+        await this.handleResourcePurge(event);
+        break;
+      case "smelt:embed-annotation":
+        await this.handleAnnotationAdded(event);
+        break;
+      case "smelt:purge-annotation":
+        await this.handleAnnotationRemoved(event);
+        break;
+    }
+  }
+  async handleResourcePurge(event) {
+    const rid = event.resourceId;
+    if (!rid) return;
+    await this.vectorStore.deleteResourceVectors(resourceId(rid));
+    this.logger.info("Reconcile deleted orphan resource vectors", { resourceId: rid });
+  }
+  /**
+   * Resolve a resource's embeddable text: bytes via the content transport,
+   * gated to media types that decode as text, decoded charset-aware. The
+   * checksum is over the raw bytes actually read — stamped onto the vectors
+   * so reconciliation can compare against the catalog's claim (S12). Returns
+   * null (logged) when the resource doesn't decode as text, is unavailable,
+   * or is empty — callers skip it.
+   */
+  async fetchEmbeddableText(resourceId$1) {
+    try {
+      const { data, contentType } = await this.content.getBinary(resourceId(resourceId$1));
+      if (textExtractionOf(contentType) !== "decode") {
+        this.logger.debug("Skipping resource that does not decode as text", { resourceId: resourceId$1, contentType });
+        return null;
+      }
+      const bytes = Buffer.from(data);
+      const text = decodeRepresentation(bytes, contentType);
+      return text.trim() ? { text, checksum: calculateChecksum(bytes) } : null;
+    } catch (error) {
+      this.logger.warn("Content unavailable for embedding", { resourceId: resourceId$1, error: errField(error) });
+      return null;
+    }
+  }
+  async embedResource(event, logMessage) {
+    const rid = event.resourceId;
+    if (!rid) return;
+    const fetched = await this.fetchEmbeddableText(rid);
+    if (!fetched) return;
+    const chunks = chunkText(fetched.text, this.chunkingConfig);
+    if (chunks.length === 0) return;
+    const embeddings = await this.embeddingProvider.embedBatch(chunks);
+    const embeddingChunks = chunks.map((t, i) => ({
+      chunkIndex: i,
+      text: t,
+      embedding: embeddings[i]
+    }));
+    await this.vectorStore.upsertResourceVectors(resourceId(rid), embeddingChunks, fetched.checksum);
+    this.logger.info(logMessage, { resourceId: rid, chunks: chunks.length });
+  }
+  async handleResourceArchived(event) {
+    const rid = event.resourceId;
+    if (!rid) return;
+    await this.vectorStore.deleteResourceVectors(resourceId(rid));
+    await this.vectorStore.deleteAnnotationVectorsForResource(resourceId(rid));
+    this.logger.info("Deleted vectors for archived resource", { resourceId: rid });
+  }
+  async handleAnnotationAdded(event) {
+    const annotation = event.payload.annotation;
+    if (!annotation?.id) return;
+    const rid = event.resourceId;
+    if (!rid) return;
+    const selector = getTargetSelector(annotation.target);
+    const exactText = getExactText(selector);
+    if (!exactText?.trim()) return;
+    const aid = annotationId(annotation.id);
+    const embedding2 = await this.embeddingProvider.embed(exactText);
+    const payload = {
+      annotationId: aid,
+      resourceId: resourceId(rid),
+      motivation: annotation.motivation ?? "",
+      entityTypes: annotation.entityTypes ?? [],
+      exactText
+    };
+    await this.vectorStore.upsertAnnotationVector(aid, embedding2, payload);
+    this.logger.info("Indexed annotation", { annotationId: String(aid) });
+  }
+  async handleAnnotationRemoved(event) {
+    const annotationId$1 = event.payload.annotationId;
+    if (!annotationId$1) return;
+    const aid = annotationId(annotationId$1);
+    await this.vectorStore.deleteAnnotationVector(aid);
+    this.logger.info("Deleted annotation vector", { annotationId: annotationId$1 });
+  }
+  /**
+   * Batch-embed chunks from multiple yield:created events in a single
+   * embedBatch() call, then index per resource.
+   */
+  async batchResourceCreated(events) {
+    const resourceData = [];
+    const allChunks = [];
+    for (const event of events) {
+      const rid = event.resourceId;
+      if (!rid) continue;
+      const fetched = await this.fetchEmbeddableText(rid);
+      if (!fetched) continue;
+      const chunks = chunkText(fetched.text, this.chunkingConfig);
+      if (chunks.length === 0) continue;
+      resourceData.push({ rid: resourceId(rid), chunks, checksum: fetched.checksum });
+      allChunks.push(...chunks);
+    }
+    if (allChunks.length === 0) return events.length;
+    const allEmbeddings = await this.embeddingProvider.embedBatch(allChunks);
+    let offset = 0;
+    for (const { rid, chunks, checksum } of resourceData) {
+      const embeddingChunks = chunks.map((t, i) => ({
+        chunkIndex: i,
+        text: t,
+        embedding: allEmbeddings[offset + i]
+      }));
+      await this.vectorStore.upsertResourceVectors(rid, embeddingChunks, checksum);
+      this.logger.info("Batch-indexed resource", { resourceId: String(rid), chunks: chunks.length });
+      offset += chunks.length;
+    }
+    return events.length;
+  }
+  /**
+   * Batch-embed exact texts from multiple mark:added events in a single
+   * embedBatch() call, then index per annotation.
+   */
+  async batchAnnotationAdded(events) {
+    const annotationData = [];
+    for (const event of events) {
+      const annotation = event.payload.annotation;
+      if (!annotation?.id) continue;
+      const rid = event.resourceId;
+      if (!rid) continue;
+      const selector = getTargetSelector(annotation.target);
+      const exactText = getExactText(selector);
+      if (!exactText?.trim()) continue;
+      annotationData.push({
+        rid: resourceId(rid),
+        aid: annotationId(annotation.id),
+        exactText,
+        motivation: annotation.motivation ?? "",
+        entityTypes: annotation.entityTypes ?? []
+      });
+    }
+    if (annotationData.length === 0) return events.length;
+    const allEmbeddings = await this.embeddingProvider.embedBatch(
+      annotationData.map((a) => a.exactText)
+    );
+    for (let i = 0; i < annotationData.length; i++) {
+      const { rid, aid, exactText, motivation, entityTypes } = annotationData[i];
+      const payload = {
+        annotationId: aid,
+        resourceId: rid,
+        motivation,
+        entityTypes,
+        exactText
+      };
+      await this.vectorStore.upsertAnnotationVector(aid, allEmbeddings[i], payload);
+      this.logger.info("Batch-indexed annotation", { annotationId: String(aid) });
+    }
+    return events.length;
+  }
+  // ── Reconciliation ───────────────────────────────────────────────────
+  /**
+   * Reconcile the vector store against the KS catalog.
+   *
+   * Lists what IS indexed (via the store's id enumeration) and what SHOULD
+   * be (non-archived resources with embeddable media types, plus their
+   * exact-text annotations, via the `browse:*` RPC channels), then plans the
+   * diff as `smelt:*` work items — embeds for what's missing, purges for
+   * what shouldn't be there — and drains them through the pipeline mailbox.
+   * Work items share the per-resource lanes with live events, so a reconcile
+   * re-embed can never interleave with (or stale-overwrite) live processing
+   * of the same resource (axioms S1/S2). Waves of RECONCILE_WAVE bound how
+   * many embedding calls a cold rebuild has in flight.
+   *
+   * Call after the live subscription is attached so nothing falls in the
+   * gap. The index snapshot is taken BEFORE the catalog listing so a
+   * resource indexed by a live event mid-reconcile is never mistaken for an
+   * orphan; convergence holds because every upsert replaces a resource's
+   * full vector set from current content.
+   */
+  async reconcile() {
+    if (!this.pipelineSubscription) {
+      throw new Error("Smelter.reconcile() requires initialize() \u2014 work items drain through the pipeline");
+    }
+    this._reconcileState = { phase: "running" };
+    try {
+      const [indexedResources, indexedAnnotations] = await Promise.all([
+        this.vectorStore.listResourceChecksums(),
+        this.vectorStore.listAnnotationIds()
+      ]);
+      const resources = await this.listAllResources();
+      this.logger.info("Reconcile started", {
+        indexedResources: indexedResources.size,
+        indexedAnnotations: indexedAnnotations.size,
+        liveResources: resources.length
+      });
+      const embeddable = /* @__PURE__ */ new Map();
+      for (const resource of resources) {
+        const mediaType = getPrimaryMediaType(resource);
+        if (resource["@id"] && mediaType && textExtractionOf(mediaType) === "decode") {
+          embeddable.set(resource["@id"], getPrimaryRepresentation(resource)?.checksum);
+        }
+      }
+      const work = [];
+      for (const rid of indexedResources.keys()) {
+        if (!embeddable.has(rid)) work.push({ type: "smelt:purge", resourceId: rid, payload: {} });
+      }
+      for (const [rid, catalogChecksum] of embeddable) {
+        if (!indexedResources.has(rid)) {
+          work.push({ type: "smelt:embed", resourceId: rid, payload: {} });
+        } else if (catalogChecksum !== void 0 && indexedResources.get(rid) !== catalogChecksum) {
+          work.push({ type: "smelt:embed", resourceId: rid, payload: {} });
+        }
+      }
+      const liveAnnotationIds = /* @__PURE__ */ new Set();
+      for (const resource of resources) {
+        const rid = resource["@id"];
+        if (!rid) continue;
+        const { annotations } = await busRequest(
+          this.bus,
+          "browse:annotations-requested",
+          { resourceId: rid },
+          "browse:annotations-result",
+          "browse:annotations-failed"
+        );
+        for (const annotation of annotations) {
+          const exactText = getExactText(getTargetSelector(annotation.target));
+          if (!annotation.id || !exactText?.trim()) continue;
+          liveAnnotationIds.add(annotation.id);
+          if (!indexedAnnotations.has(annotation.id)) {
+            work.push({ type: "smelt:embed-annotation", resourceId: rid, payload: { annotation } });
+          }
+        }
+      }
+      for (const aid of indexedAnnotations) {
+        if (!liveAnnotationIds.has(aid)) {
+          work.push({ type: "smelt:purge-annotation", resourceId: aid, payload: { annotationId: aid } });
+        }
+      }
+      await this.drain(work);
+      const summary = {
+        resourcesEmbedded: work.filter((w) => w.type === "smelt:embed").length,
+        resourceVectorsDeleted: work.filter((w) => w.type === "smelt:purge").length,
+        annotationsEmbedded: work.filter((w) => w.type === "smelt:embed-annotation").length,
+        annotationVectorsDeleted: work.filter((w) => w.type === "smelt:purge-annotation").length
+      };
+      this._reconcileState = { phase: "done", summary };
+      this.logger.info("Reconcile complete", { ...summary });
+      return summary;
+    } catch (error) {
+      this._reconcileState = {
+        phase: "failed",
+        error: error instanceof Error ? error.message : String(error)
+      };
+      this.logger.error("Reconcile failed", { error: errField(error) });
+      throw error;
+    }
+  }
+  /**
+   * Enqueue planner work through the mailbox in bounded waves and await
+   * completion. The pipeline ticks `noteWorkDone` for every consumed work
+   * item (success or failure — failures are logged like any live event), so
+   * each wave's waiter resolves exactly when its items have been processed.
+   */
+  async drain(work) {
+    for (let i = 0; i < work.length; i += _Smelter.RECONCILE_WAVE) {
+      const wave = work.slice(i, i + _Smelter.RECONCILE_WAVE);
+      const done = new Promise((resolve) => {
+        this.workWaiter = { target: this.workDone + wave.length, resolve };
+      });
+      for (const item of wave) this.eventSubject.next(item);
+      await done;
+    }
+  }
+  /** Page through `browse:resources-requested` until the catalog is exhausted. */
+  async listAllResources() {
+    const all = [];
+    for (; ; ) {
+      const page = await busRequest(
+        this.bus,
+        "browse:resources-requested",
+        { archived: false, offset: all.length, limit: _Smelter.RECONCILE_PAGE_SIZE },
+        "browse:resources-result",
+        "browse:resources-failed"
+      );
+      all.push(...page.resources);
+      if (page.resources.length === 0 || all.length >= page.total) return all;
+    }
+  }
+};
 var configPath = join(homedir(), ".semiontconfig");
 var tomlReader = {
   readIfExists: (p) => existsSync(p) ? readFileSync(p, "utf-8") : null
@@ -9796,9 +10282,6 @@ var chunkingConfig = {
 };
 var workerSecret = process.env.SEMIONT_WORKER_SECRET ?? "";
 var healthPort = 9091;
-var BURST_WINDOW_MS = 50;
-var MAX_BATCH_SIZE = 100;
-var IDLE_TIMEOUT_MS = 200;
 var logger = createProcessLogger("smelter");
 async function authenticate() {
   if (!workerSecret) {
@@ -9820,250 +10303,70 @@ async function authenticate() {
   const { token } = await response.json();
   return token;
 }
-var authToken = "";
-async function fetchContent(resourceId) {
-  try {
-    const response = await fetch(`${baseUrl}/api/resources/${resourceId}`, {
-      headers: {
-        Authorization: `Bearer ${authToken}`,
-        Accept: "text/plain"
-      }
-    });
-    if (!response.ok) return null;
-    return response.text();
-  } catch {
-    return null;
-  }
-}
-var vectorStore;
-var embeddingProvider;
-var eventsProcessed = 0;
-async function processEvent(event) {
-  try {
-    switch (event.type) {
-      case "yield:created":
-        await handleResourceCreated(event);
-        break;
-      case "yield:updated":
-      case "yield:representation-added":
-        await handleResourceReembed(event);
-        break;
-      case "mark:archived":
-        await handleResourceArchived(event);
-        break;
-      case "mark:added":
-        await handleAnnotationAdded(event);
-        break;
-      case "mark:removed":
-        await handleAnnotationRemoved(event);
-        break;
-    }
-    eventsProcessed++;
-  } catch (err) {
-    logger.error("Failed to process event", { type: event.type, resourceId: event.resourceId, error: err instanceof Error ? err.message : String(err) });
-  }
-}
-async function handleResourceCreated(event) {
-  const rid = event.resourceId;
-  if (!rid) return;
-  const text = await fetchContent(rid);
-  if (!text?.trim()) return;
-  const chunks = chunkText(text, chunkingConfig);
-  if (chunks.length === 0) return;
-  const embeddings = await embeddingProvider.embedBatch(chunks);
-  const embeddingChunks = chunks.map((t, i) => ({
-    chunkIndex: i,
-    text: t,
-    embedding: embeddings[i]
-  }));
-  await vectorStore.upsertResourceVectors(resourceId(rid), embeddingChunks);
-  logger.info("Indexed resource", { resourceId: rid, chunks: chunks.length });
-}
-async function handleResourceReembed(event) {
-  const rid = event.resourceId;
-  if (!rid) return;
-  const text = await fetchContent(rid);
-  if (!text?.trim()) return;
-  const chunks = chunkText(text, chunkingConfig);
-  if (chunks.length === 0) return;
-  const embeddings = await embeddingProvider.embedBatch(chunks);
-  const embeddingChunks = chunks.map((t, i) => ({
-    chunkIndex: i,
-    text: t,
-    embedding: embeddings[i]
-  }));
-  await vectorStore.deleteResourceVectors(resourceId(rid));
-  await vectorStore.upsertResourceVectors(resourceId(rid), embeddingChunks);
-  logger.info("Re-embedded resource", { resourceId: rid, chunks: chunks.length });
-}
-async function handleResourceArchived(event) {
-  const rid = event.resourceId;
-  if (!rid) return;
-  await vectorStore.deleteResourceVectors(resourceId(rid));
-  logger.info("Deleted vectors for archived resource", { resourceId: rid });
-}
-async function handleAnnotationAdded(event) {
-  const annotation = event.payload.annotation;
-  if (!annotation?.id) return;
-  const rid = event.resourceId;
-  if (!rid) return;
-  const selector = getTargetSelector(annotation.target);
-  const exactText = getExactText(selector);
-  if (!exactText?.trim()) return;
-  const aid = annotationId(annotation.id);
-  const embedding2 = await embeddingProvider.embed(exactText);
-  const payload = {
-    annotationId: aid,
-    resourceId: resourceId(rid),
-    motivation: annotation.motivation ?? "",
-    entityTypes: annotation.entityTypes ?? [],
-    exactText
-  };
-  await vectorStore.upsertAnnotationVector(aid, embedding2, payload);
-  logger.info("Indexed annotation", { annotationId: String(aid) });
-}
-async function handleAnnotationRemoved(event) {
-  const annotationId$1 = event.payload.annotationId;
-  if (!annotationId$1) return;
-  const aid = annotationId(annotationId$1);
-  await vectorStore.deleteAnnotationVector(aid);
-  logger.info("Deleted annotation vector", { annotationId: annotationId$1 });
-}
-async function processBatch(events) {
-  const type = events[0].type;
-  if (type === "yield:created") {
-    await batchResourceCreated(events);
-  } else if (type === "mark:added") {
-    await batchAnnotationAdded(events);
-  } else {
-    for (const event of events) {
-      await processEvent(event);
-    }
-  }
-}
-async function batchResourceCreated(events) {
-  const resourceData = [];
-  const allChunks = [];
-  for (const event of events) {
-    const rid = event.resourceId;
-    if (!rid) continue;
-    const text = await fetchContent(rid);
-    if (!text?.trim()) continue;
-    const chunks = chunkText(text, chunkingConfig);
-    if (chunks.length === 0) continue;
-    resourceData.push({ rid: resourceId(rid), chunks });
-    allChunks.push(...chunks);
-  }
-  if (allChunks.length === 0) return;
-  const allEmbeddings = await embeddingProvider.embedBatch(allChunks);
-  let offset = 0;
-  for (const { rid, chunks } of resourceData) {
-    const embeddingChunks = chunks.map((t, i) => ({
-      chunkIndex: i,
-      text: t,
-      embedding: allEmbeddings[offset + i]
-    }));
-    await vectorStore.upsertResourceVectors(rid, embeddingChunks);
-    logger.info("Batch-indexed resource", { resourceId: String(rid), chunks: chunks.length });
-    offset += chunks.length;
-  }
-  eventsProcessed += events.length;
-}
-async function batchAnnotationAdded(events) {
-  const annotationData = [];
-  for (const event of events) {
-    const annotation = event.payload.annotation;
-    if (!annotation?.id) continue;
-    const rid = event.resourceId;
-    if (!rid) continue;
-    const selector = getTargetSelector(annotation.target);
-    const exactText = getExactText(selector);
-    if (!exactText?.trim()) continue;
-    annotationData.push({
-      rid: resourceId(rid),
-      aid: annotationId(annotation.id),
-      exactText,
-      motivation: annotation.motivation ?? "",
-      entityTypes: annotation.entityTypes ?? []
-    });
-  }
-  if (annotationData.length === 0) return;
-  const allEmbeddings = await embeddingProvider.embedBatch(
-    annotationData.map((a) => a.exactText)
-  );
-  for (let i = 0; i < annotationData.length; i++) {
-    const { rid, aid, exactText, motivation, entityTypes } = annotationData[i];
-    const payload = {
-      annotationId: aid,
-      resourceId: rid,
-      motivation,
-      entityTypes,
-      exactText
-    };
-    await vectorStore.upsertAnnotationVector(aid, allEmbeddings[i], payload);
-    logger.info("Batch-indexed annotation", { annotationId: String(aid) });
-  }
-  eventsProcessed += events.length;
-}
 async function main() {
   const { initObservabilityNode } = await import('@semiont/observability/node');
   initObservabilityNode({ serviceName: "semiont-smelter" });
   logger.info("Authenticating", { baseUrl });
-  authToken = await authenticate();
+  const tokenSubject = new import_rxjs3.BehaviorSubject(accessToken(await authenticate()));
   logger.info("Authenticated");
-  embeddingProvider = await createEmbeddingProvider({
+  const refreshToken = async () => {
+    const token = await authenticate();
+    tokenSubject.next(accessToken(token));
+    return token;
+  };
+  const reauthTimer = setInterval(() => {
+    refreshToken().catch((error) => {
+      logger.error("Proactive re-authentication failed", {
+        error: error instanceof Error ? error.message : String(error)
+      });
+    });
+  }, 12 * 60 * 60 * 1e3);
+  const embeddingProvider = await createEmbeddingProvider({
     type: embeddingType,
     model: embeddingModel,
     baseURL: embeddingBaseURL
   });
   logger.info("Embedding provider ready", { type: embeddingType, model: embeddingModel });
   const dimensions = embeddingProvider.dimensions();
-  vectorStore = await createVectorStore({
+  const vectorStore = await createVectorStore({
     type: "qdrant",
     host: qdrantHost,
     port: qdrantPort,
     dimensions
   });
   logger.info("Vector store ready", { host: qdrantHost, port: qdrantPort, dimensions });
-  const tokenSubject = new import_rxjs3.BehaviorSubject(accessToken(authToken));
+  registerVectorIndexSizeProvider(() => vectorStore.count());
   const httpTransport = new HttpTransport({
     baseUrl: baseUrl$1(baseUrl),
-    token$: tokenSubject
+    token$: tokenSubject,
+    tokenRefresher: refreshToken
   });
   const actorStateUnit = createSmelterActorStateUnit({
     bus: httpTransport.actor
   });
-  const eventSubject = new import_rxjs2.Subject();
-  const pipelineSubscription = eventSubject.pipe(
-    (0, import_operators2.groupBy)((e) => e.resourceId ?? "__unknown__"),
-    (0, import_operators2.mergeMap)(
-      (group) => group.pipe(
-        burstBuffer({
-          burstWindowMs: BURST_WINDOW_MS,
-          maxBatchSize: MAX_BATCH_SIZE,
-          idleTimeoutMs: IDLE_TIMEOUT_MS
-        }),
-        (0, import_operators2.concatMap)((eventOrBatch) => {
-          if (Array.isArray(eventOrBatch)) {
-            return (0, import_rxjs2.from)(processBatch(eventOrBatch));
-          }
-          return (0, import_rxjs2.from)(processEvent(eventOrBatch));
-        })
-      )
-    )
-  ).subscribe({
-    error: (err) => logger.error("Pipeline error", { error: err instanceof Error ? err.message : String(err) })
-  });
-  actorStateUnit.events$.subscribe((event) => {
-    logger.debug("Bus event received", { type: event.type, resourceId: event.resourceId });
-    eventSubject.next(event);
-  });
+  const contentTransport = new HttpContentTransport(httpTransport);
+  logger.info("Content transport ready", { mode: "http" });
+  const smelter = new Smelter(
+    actorStateUnit.events$,
+    vectorStore,
+    embeddingProvider,
+    contentTransport,
+    httpTransport,
+    chunkingConfig,
+    { burstWindowMs: 50, maxBatchSize: 100, idleTimeoutMs: 200 },
+    logger
+  );
+  smelter.initialize();
   actorStateUnit.start();
   logger.info("Subscribed to domain events");
   const health = createServer((req, res) => {
     if (req.url === "/health") {
       res.writeHead(200, { "Content-Type": "application/json" });
-      res.end(JSON.stringify({ status: "ok", eventsProcessed }));
+      res.end(JSON.stringify({
+        status: "ok",
+        eventsProcessed: smelter.eventsProcessed,
+        reconcile: smelter.reconcileState
+      }));
     } else {
       res.writeHead(404);
       res.end();
@@ -10074,15 +10377,16 @@ async function main() {
   });
   const shutdown = () => {
     logger.info("Shutting down");
+    clearInterval(reauthTimer);
     actorStateUnit.dispose();
     httpTransport.dispose();
-    pipelineSubscription.unsubscribe();
-    eventSubject.complete();
+    smelter.stop();
     health.close();
     process.exit(0);
   };
   process.on("SIGTERM", shutdown);
   process.on("SIGINT", shutdown);
+  await smelter.reconcile();
 }
 main().catch((error) => {
   logger.error("Fatal", { error: error instanceof Error ? error.message : String(error), stack: error instanceof Error ? error.stack : void 0 });