@remnic/core 9.3.640 → 9.3.642

package/dist/{chunk-WAKPKNEX.js → chunk-3UR4SBZF.js}

@@ -29,7 +29,7 @@ import {
 import {
   CompoundingEngine,
   defaultTierMigrationCycleBudget
-} from "./chunk-SBHQBC37.js";
+} from "./chunk-ECMMM2VE.js";
 import {
   SharedContextManager
 } from "./chunk-DRD2Q7HQ.js";
@@ -157,7 +157,7 @@ import {
 } from "./chunk-M75TBFKQ.js";
 import {
   ExtractionEngine
-} from "./chunk-SEOTBRO4.js";
+} from "./chunk-U3GQ33JC.js";
 import {
   parseMemoryActionEligibilityContext
 } from "./chunk-ROZJACKP.js";
@@ -175,7 +175,7 @@ import {
   buildEntityRecallSection,
   entityRecentTranscriptLookbackHours,
   readRecentEntityTranscriptEntries
-} from "./chunk-A3UPMOJZ.js";
+} from "./chunk-NV2MGIW7.js";
 import {
   buildEventOrderRecallSection,
   shouldRecallEventOrderEvidence
@@ -210,7 +210,7 @@ import {
   materializeAfterSemanticConsolidation,
   parseConsolidationResponse,
   parseOperatorAwareConsolidationResponse
-} from "./chunk-H4RWKZE2.js";
+} from "./chunk-TKNTAUA3.js";
 import {
   normalizeReplaySessionKey
 } from "./chunk-2PRQG7PV.js";
@@ -219,13 +219,13 @@ import {
 } from "./chunk-X6IRLNOO.js";
 import {
   searchVerifiedEpisodes
-} from "./chunk-RJGPH2Q3.js";
+} from "./chunk-4HYLRROV.js";
 import {
   ThreadingManager
 } from "./chunk-W4RVMTHR.js";
 import {
   searchVerifiedSemanticRules
-} from "./chunk-C6XO6ONP.js";
+} from "./chunk-SZL2GGFY.js";
 import {
   searchWorkProductLedgerEntries
 } from "./chunk-ZRWB5D4H.js";
@@ -234,7 +234,7 @@ import {
 } from "./chunk-IHG27THY.js";
 import {
   NamespaceStorageRouter
-} from "./chunk-IXSTQGYV.js";
+} from "./chunk-RREI7CNM.js";
 import {
   PolicyRuntimeManager
 } from "./chunk-33JBK2XP.js";
@@ -314,7 +314,7 @@ import {
 } from "./chunk-FF4KLI5W.js";
 import {
   buildXraySnapshot
-} from "./chunk-CMZLMIZI.js";
+} from "./chunk-BEMWL2FZ.js";
 import {
   NamespaceSearchRouter
 } from "./chunk-HRXCPKN4.js";
@@ -352,7 +352,7 @@ import {
 } from "./chunk-Y56J7CXW.js";
 import {
   buildChainFollowupGenerator
-} from "./chunk-J2KFC47S.js";
+} from "./chunk-RFSCIBCB.js";
 import {
   ContentHashIndex,
   StorageManager,
@@ -362,7 +362,7 @@ import {
   normalizeEntityName,
   parseEntityFile,
   stripAttributesSuffix
-} from "./chunk-SHHT4JTC.js";
+} from "./chunk-GYITUZFT.js";
 import {
   isValidTranscriptDate,
   loadSpeakerRegistry,
@@ -377,7 +377,7 @@ import {
 } from "./chunk-J6A3CX5N.js";
 import {
   confidenceTier
-} from "./chunk-3F7RMHC4.js";
+} from "./chunk-C43KEWEV.js";
 import {
   inferMemoryStatus,
   isActiveMemoryStatus
@@ -1541,6 +1541,16 @@ function extractSnippet(body, index, matchLength) {
 }
 
 // src/orchestrator.ts
+var BulkImportBatchPartialFailureError = class extends Error {
+  partialResult;
+  originalError;
+  constructor(message, partialResult, originalError) {
+    super(message);
+    this.name = "BulkImportBatchPartialFailureError";
+    this.partialResult = partialResult;
+    this.originalError = originalError;
+  }
+};
 function dedupeEntitySynthesisEvidenceEntries(entries) {
   const dedupedEvidenceEntries = [];
   const evidenceByFact = /* @__PURE__ */ new Map();
@@ -1862,7 +1872,7 @@ function sortSourceValidAtSlicesChronologically(slices) {
     return a.order < b.order ? -1 : 1;
   }).map((slice) => slice.turns);
 }
-function splitTurnsBySourceValidAt(turns) {
+function splitTurnsBySourceValidAt(turns, options = {}) {
   if (turns.length === 0) return [];
   if (!turns.some((turn) => sourceValidAtMs(turn) !== null)) {
     return [[...turns]];
@@ -1876,13 +1886,9 @@ function splitTurnsBySourceValidAt(turns) {
     while (end < turns.length && sourceValidAtSliceKey(turns[end], end) === activeKey) {
       end += 1;
     }
+    const contextTurns = options.includeContext === false ? [] : sourceValidAtContextTurns(turns, start, end, targetValidAtMs);
     slices.push([
-      ...sourceValidAtContextTurns(
-        turns,
-        start,
-        end,
-        targetValidAtMs
-      ),
+      ...contextTurns,
       ...turns.slice(start, end).map(asExtractionTargetTurn)
     ]);
     start = end;
@@ -9539,8 +9545,30 @@ _Context: ${topQuestion.context}_`
    * the CLI surface does not yet expose a `--namespace` flag.
    */
   async ingestBulkImportBatch(turns, options = {}) {
-    if (!Array.isArray(turns) || turns.length === 0) return;
-    const sessionKey = `bulk-import:batch:${Date.now().toString(36)}-` + randomBytes(6).toString("hex");
+    if (!Array.isArray(turns) || turns.length === 0) {
+      return {
+        attemptedTurnCount: 0,
+        extractionCount: 0,
+        persistedCount: 0,
+        durableOutputCount: 0,
+        skippedCount: 0,
+        failedCount: 0,
+        postPersistMetadataFailureCount: 0,
+        processedTurnCount: 0
+      };
+    }
+    const shouldUseStableBatchKey = turns.some(
+      (turn) => turn.persistProcessedFingerprint === true || typeof turn.turnFingerprint === "string" && turn.turnFingerprint.length > 0
+    );
+    const stableBatchFingerprint = shouldUseStableBatchKey ? createHash("sha256").update(
+      turns.map(
+        (turn) => [
+          turn.role,
+          typeof turn.turnFingerprint === "string" && turn.turnFingerprint.length > 0 ? turn.turnFingerprint : turn.content.replace(/\s+/g, " ").trim()
+        ].join(":")
+      ).join("\n")
+    ).digest("hex").slice(0, 32) : void 0;
+    const sessionKey = stableBatchFingerprint ? `bulk-import:batch:${stableBatchFingerprint}` : `bulk-import:batch:${Date.now().toString(36)}-${randomBytes(6).toString("hex")}`;
     const sessionTurns = [];
     for (const turn of turns) {
       if (turn.role !== "user" && turn.role !== "assistant") continue;
@@ -9553,10 +9581,23 @@ _Context: ${topQuestion.context}_`
         parts: turn.parts,
         rawContent: turn.rawContent,
         sourceFormat: turn.sourceFormat,
-        importProvenance: turn.importProvenance
+        importProvenance: turn.importProvenance,
+        turnFingerprint: turn.turnFingerprint,
+        persistProcessedFingerprint: turn.persistProcessedFingerprint === true
       });
     }
-    if (sessionTurns.length === 0) return;
+    if (sessionTurns.length === 0) {
+      return {
+        attemptedTurnCount: 0,
+        extractionCount: 0,
+        persistedCount: 0,
+        durableOutputCount: 0,
+        skippedCount: 0,
+        failedCount: 0,
+        postPersistMetadataFailureCount: 0,
+        processedTurnCount: 0
+      };
+    }
     if (this.lcmEngine?.enabled) {
       await this.lcmEngine.observeMessages(
         sessionKey,
@@ -9569,27 +9610,75 @@ _Context: ${topQuestion.context}_`
         }))
       );
     }
-    const importTasks = splitTurnsBySourceValidAt(sessionTurns).map(
-      (sessionSlice) => new Promise((resolve, reject) => {
-        void this.queueBufferedExtraction(sessionSlice, "trigger_mode", {
-          skipDedupeCheck: true,
-          clearBufferAfterExtraction: false,
-          skipCharThreshold: true,
-          skipUserTurnThreshold: true,
-          bufferKey: sessionKey,
-          extractionDeadlineMs: options.deadlineMs,
-          writeNamespaceOverride: this.bulkImportWriteNamespace(),
-          onTaskSettled: (err) => err ? reject(err) : resolve()
-        }).catch(reject);
-      })
-    );
-    const settled = await Promise.allSettled(importTasks);
-    const firstRejected = settled.find(
-      (result) => result.status === "rejected"
-    );
+    const sessionSlices = splitTurnsBySourceValidAt(sessionTurns, {
+      includeContext: options.includeSourceValidAtContext !== false
+    });
+    const results = [];
+    let processedTurnCount = 0;
+    let firstRejected;
+    for (const sessionSlice of sessionSlices) {
+      try {
+        const result = await new Promise(
+          (resolve, reject) => {
+            void this.queueBufferedExtraction(sessionSlice, "trigger_mode", {
+              skipDedupeCheck: true,
+              clearBufferAfterExtraction: false,
+              skipCharThreshold: true,
+              skipUserTurnThreshold: true,
+              bufferKey: sessionKey,
+              extractionDeadlineMs: options.deadlineMs,
+              failOnExtractionFailure: options.failOnExtractionFailure === true,
+              writeNamespaceOverride: this.bulkImportWriteNamespace(),
+              onTaskSettled: (err, result2) => err ? reject(err) : resolve(
+                result2 ?? {
+                  status: "skipped",
+                  reason: "missing_extraction_result",
+                  persistedCount: 0,
+                  durableOutputCount: 0
+                }
+              )
+            }).catch(reject);
+          }
+        );
+        results.push(result);
+        processedTurnCount += sessionSlice.filter(
+          (turn) => turn.extractionContextOnly !== true
+        ).length;
+      } catch (err) {
+        firstRejected = err;
+        break;
+      }
+    }
+    const rejectedCount = firstRejected ? 1 : 0;
+    const ingestResult = {
+      attemptedTurnCount: sessionTurns.length,
+      extractionCount: results.length,
+      persistedCount: results.reduce(
+        (sum, result) => sum + result.persistedCount,
+        0
+      ),
+      durableOutputCount: results.reduce(
+        (sum, result) => sum + result.durableOutputCount,
+        0
+      ),
+      skippedCount: results.filter((result) => result.status === "skipped").length,
+      failedCount: rejectedCount,
+      postPersistMetadataFailureCount: results.filter(
+        (result) => result.postPersistMetadataFailed === true
+      ).length,
+      processedTurnCount: rejectedCount === 0 ? sessionTurns.length : processedTurnCount
+    };
     if (firstRejected) {
-      throw firstRejected.reason;
+      if (processedTurnCount > 0) {
+        throw new BulkImportBatchPartialFailureError(
+          "bulk import failed after partial processing",
+          ingestResult,
+          firstRejected
+        );
+      }
+      throw firstRejected;
     }
+    return ingestResult;
   }
   async observeSessionHeartbeat(sessionKey, options = {}) {
     if (this.config.sessionObserverEnabled !== true) return;
@@ -9647,24 +9736,63 @@ _Context: ${topQuestion.context}_`
     const bufferKey = options.bufferKey ?? turnsToExtract[0]?.sessionKey ?? "default";
     if (!options.skipDedupeCheck && !this.shouldQueueExtraction(turnsToExtract, { bufferKey })) {
       log.debug(`extraction dedupe skip: preserving buffer (${reason})`);
-      options.onTaskSettled?.();
+      options.onTaskSettled?.(void 0, {
+        status: "skipped",
+        reason: "dedupe",
+        persistedCount: 0,
+        durableOutputCount: 0
+      });
       return;
     }
+    const extractionDeadlineMs = typeof options.extractionDeadlineMs === "number" && Number.isFinite(options.extractionDeadlineMs) ? options.extractionDeadlineMs : void 0;
+    let timeout;
+    let settled = false;
+    const clearQueueWaitTimer = () => {
+      if (timeout) {
+        clearTimeout(timeout);
+        timeout = void 0;
+      }
+    };
+    const settleTask = (error, result) => {
+      if (settled) return false;
+      settled = true;
+      clearQueueWaitTimer();
+      options.onTaskSettled?.(error, result);
+      return true;
+    };
+    if (typeof extractionDeadlineMs === "number") {
+      const remainingMs = extractionDeadlineMs - Date.now();
+      if (remainingMs <= 0) {
+        settleTask(new Error("replay extraction deadline exceeded (queue_wait)"));
+        return;
+      }
+      timeout = setTimeout(() => {
+        settleTask(new Error("replay extraction deadline exceeded (queue_wait)"));
+      }, remainingMs);
+    }
     this.extractionQueue.push(async () => {
+      if (settled) return;
+      if (typeof extractionDeadlineMs === "number" && extractionDeadlineMs <= Date.now()) {
+        settleTask(new Error("replay extraction deadline exceeded (queue_wait)"));
+        return;
+      }
+      clearQueueWaitTimer();
       try {
-        await this.runExtraction(turnsToExtract, {
+        const result = await this.runExtraction(turnsToExtract, {
           clearBufferAfterExtraction: options.clearBufferAfterExtraction ?? true,
           skipCharThreshold: options.skipCharThreshold ?? false,
           skipUserTurnThreshold: options.skipUserTurnThreshold ?? false,
-          deadlineMs: options.extractionDeadlineMs,
+          deadlineMs: extractionDeadlineMs,
           bufferKey,
           abortSignal: options.abortSignal,
+          failOnExtractionFailure: options.failOnExtractionFailure === true,
           writeNamespaceOverride: options.writeNamespaceOverride
         });
-        options.onTaskSettled?.();
+        settleTask(void 0, result);
       } catch (err) {
-        options.onTaskSettled?.(err);
-        throw err;
+        if (settleTask(err)) {
+          throw err;
+        }
       }
     });
     if (!this.queueProcessing) {
@@ -9787,7 +9915,12 @@ ${normalized}`).digest("hex");
     if (sessionKey.includes(":cron:")) {
       log.debug(`skipping extraction for cron session: ${sessionKey}`);
       await clearBuffer();
-      return;
+      return {
+        status: "skipped",
+        reason: "cron_session",
+        persistedCount: 0,
+        durableOutputCount: 0
+      };
     }
     const normalizedTurns = turns.filter(
       (t) => (t.role === "user" || t.role === "assistant") && typeof t.content === "string"
@@ -9801,7 +9934,12 @@ ${normalized}`).digest("hex");
     if (targetTurns.length === 0) {
       log.debug("skipping extraction: no non-context turns after normalization");
       await clearBuffer();
-      return;
+      return {
+        status: "skipped",
+        reason: "empty_normalized_turns",
+        persistedCount: 0,
+        durableOutputCount: 0
+      };
     }
     const sourceValidAt = latestSourceValidAtFromTurns(targetTurns);
     throwIfDeadlineExceeded("before_extract");
@@ -9818,7 +9956,12 @@ ${normalized}`).digest("hex");
         `skipping extraction: below threshold (totalChars=${totalChars}, userTurns=${userTurns.length})`
       );
       await clearBuffer();
-      return;
+      return {
+        status: "skipped",
+        reason: "below_threshold",
+        persistedCount: 0,
+        durableOutputCount: 0
+      };
     }
     const principal = resolvePrincipal(sessionKey, this.config);
     const selfNamespace = typeof options.writeNamespaceOverride === "string" && options.writeNamespaceOverride.length > 0 ? options.writeNamespaceOverride : this.applyCodingNamespaceOverlay(
@@ -9841,7 +9984,12 @@ ${normalized}`).digest("hex");
         `runExtraction: skipping already-processed extraction fingerprint for ${bufferKey}`
       );
       await clearBuffer();
-      return;
+      return {
+        status: "skipped",
+        reason: "processed_fingerprint",
+        persistedCount: 0,
+        durableOutputCount: 0
+      };
     }
     const existingEntities = await storage.listEntityNames();
     const result = await raceRecallAbort(
@@ -9856,23 +10004,74 @@ ${normalized}`).digest("hex");
     throwIfAborted2("before_persist");
     if (!result) {
       log.warn("runExtraction: extraction returned null/undefined");
+      if (options.failOnExtractionFailure) {
+        throw new Error("extraction failed: invalid_extraction_result");
+      }
       await clearBuffer();
-      return;
+      return {
+        status: "skipped",
+        reason: "invalid_extraction_result",
+        persistedCount: 0,
+        durableOutputCount: 0
+      };
     }
-    if (!Array.isArray(result.facts)) {
+    const invalidExtractionResultFields = [
+      ["facts", result.facts],
+      ["entities", result.entities],
+      ["questions", result.questions],
+      ["profileUpdates", result.profileUpdates]
+    ].filter(([, value]) => !Array.isArray(value)).map(([field]) => field);
+    if (invalidExtractionResultFields.length > 0) {
       log.warn(
-        "runExtraction: extraction returned invalid facts (not an array)",
-        { factsType: typeof result.facts, resultKeys: Object.keys(result) }
+        "runExtraction: extraction returned invalid collection fields",
+        {
+          invalidFields: invalidExtractionResultFields,
+          resultKeys: typeof result === "object" && result !== null ? Object.keys(result) : []
+        }
       );
+      if (options.failOnExtractionFailure) {
+        throw new Error("extraction failed: invalid_extraction_result");
+      }
       await clearBuffer();
-      return;
+      return {
+        status: "skipped",
+        reason: "invalid_extraction_result",
+        persistedCount: 0,
+        durableOutputCount: 0
+      };
+    }
+    const extractionFailure = typeof result.extractionFailure === "string" && result.extractionFailure.trim().length > 0 ? result.extractionFailure : void 0;
+    if (options.failOnExtractionFailure && extractionFailure) {
+      throw new Error(`extraction failed: ${extractionFailure}`);
     }
     if (result.facts.length === 0 && result.entities.length === 0 && result.questions.length === 0 && result.profileUpdates.length === 0) {
       log.debug(
         "runExtraction: extraction produced no durable outputs; skipping persistence"
       );
+      if (extractionFailure) {
+        log.warn(
+          "runExtraction: extraction reported failure with no durable outputs; not marking fingerprint processed",
+          { extractionFailure }
+        );
+      }
+      if (extractionFingerprint && shouldPersistProcessedFingerprint && !extractionFailure) {
+        meta ??= await storage.loadMeta();
+        await this.recordProcessedExtractionFingerprint(
+          storage,
+          extractionFingerprint,
+          meta
+        );
+        meta.extractionCount += 1;
+        meta.lastExtractionAt = (/* @__PURE__ */ new Date()).toISOString();
+        await storage.saveMeta(meta);
+      }
       await clearBuffer();
-      return;
+      return {
+        status: "skipped",
+        reason: "empty_extraction_result",
+        persistedCount: 0,
+        durableOutputCount: 0
+      };
     }
     let threadIdForExtraction = null;
     if (this.config.threadingEnabled && turns.length > 0) {
@@ -9892,6 +10091,7 @@ ${normalized}`).digest("hex");
       threadIdForExtraction,
       { sessionKey, principal, validAt: sourceValidAt }
     );
+    let postPersistMetadataFailed = false;
     meta ??= await storage.loadMeta();
     if (extractionFingerprint && shouldPersistProcessedFingerprint) {
       try {
@@ -9905,18 +10105,23 @@ ${normalized}`).digest("hex");
           "runExtraction: failed to persist processed extraction fingerprint; continuing with buffer clear",
           error
         );
+        postPersistMetadataFailed = true;
       }
     }
     meta.extractionCount += 1;
     meta.lastExtractionAt = (/* @__PURE__ */ new Date()).toISOString();
     meta.totalMemories += Array.isArray(result?.facts) ? result.facts.length : 0;
     meta.totalEntities += Array.isArray(result?.entities) ? result.entities.length : 0;
-    let postPersistMetaError;
     try {
       await storage.saveMeta(meta);
     } catch (error) {
-      postPersistMetaError = error;
+      log.warn(
+        "runExtraction: failed to save extraction metadata after durable persistence; continuing with buffer clear",
+        error
+      );
+      postPersistMetadataFailed = true;
     }
+    const durableOutputCount = result.facts.length + result.entities.length + result.questions.length + result.profileUpdates.length;
     try {
       if (clearBufferAfterExtraction && !this.config.extractionJudgeShadow) {
         const deferredCount = this.lastPersistExtractionDeferredCount;
@@ -9964,19 +10169,50 @@ ${normalized}`).digest("hex");
     }
     if (this.config.threadingEnabled && threadIdForExtraction) {
       const conversationContent = turns.map((t) => t.content).join(" ");
-      await this.threading.updateThreadTitle(
-        threadIdForExtraction,
-        conversationContent
+      try {
+        await this.threading.updateThreadTitle(
+          threadIdForExtraction,
+          conversationContent
+        );
+      } catch (err) {
+        log.warn(
+          "[threading] updateThreadTitle failed after persistence (non-fatal)",
+          err
+        );
+      }
+    }
+    const nonZeroExtraction = durableOutputCount > 0;
+    try {
+      if (nonZeroExtraction) this.nonZeroExtractionsSinceConsolidation += 1;
+      this.maybeScheduleConsolidation(nonZeroExtraction);
+    } catch (err) {
+      log.warn(
+        "runExtraction: consolidation scheduling failed after persistence (non-fatal)",
+        err
       );
     }
-    const nonZeroExtraction = result.facts.length > 0 || result.entities.length > 0 || result.questions.length > 0 || result.profileUpdates.length > 0;
-    if (nonZeroExtraction) this.nonZeroExtractionsSinceConsolidation += 1;
-    this.maybeScheduleConsolidation(nonZeroExtraction);
-    this.requestQmdMaintenance();
-    await this.runTierMigrationCycle(storage, "extraction");
-    if (postPersistMetaError) {
-      throw postPersistMetaError;
+    try {
+      this.requestQmdMaintenance();
+    } catch (err) {
+      log.warn(
+        "runExtraction: QMD maintenance scheduling failed after persistence (non-fatal)",
+        err
+      );
     }
+    try {
+      await this.runTierMigrationCycle(storage, "extraction");
+    } catch (err) {
+      log.warn(
+        "runExtraction: tier migration failed after persistence (non-fatal)",
+        err
+      );
+    }
+    return {
+      status: "completed",
+      persistedCount: persistedIds.length,
+      durableOutputCount,
+      postPersistMetadataFailed
+    };
   }
   async recordProcessedExtractionFingerprint(storage, fingerprint, preloadedMeta) {
     const meta = preloadedMeta ?? await storage.loadMeta();
@@ -14036,6 +14272,7 @@ export {
   ensureBuiltInWearableConnectors,
   WearablesService,
   locateTranscriptPath,
+  BulkImportBatchPartialFailureError,
   dedupeEntitySynthesisEvidenceEntries,
   defaultWorkspaceDir,
   sanitizeSessionKeyForFilename,
@@ -14065,4 +14302,4 @@ export {
   resolvePersistedMemoryRelativePath,
   Orchestrator
 };
-//# sourceMappingURL=chunk-WAKPKNEX.js.map
+//# sourceMappingURL=chunk-3UR4SBZF.js.map