@remnic/core 9.45.3 → 9.45.4

package/src/orchestration/extraction-watermark.ts

@@ -36,12 +36,11 @@ export interface AggregateExtractionWatermarkOptions {
   caps?: TokenCapabilities;
 }
 
-function readFailure(reason: string, rootStats?: ExtractionRootStats): ExtractionWatermarkRead {
+function readFailure(reason: string): ExtractionWatermarkRead {
   return {
     lastExtractionAt: null,
     readFailed: true,
     readError: reason,
-    ...(rootStats ? { rootStats } : {}),
   };
 }
 
@@ -49,12 +48,101 @@ function errorMessage(error: unknown): string {
   return error instanceof Error ? error.message : String(error);
 }
 
+const ISO_TIMESTAMP_PATTERN =
+  /^([+-]?\d{4,6})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(?::(\d{2}))?(?:\.(\d+))?(Z|[+-]\d{2}:?\d{2})$/i;
+
+function validatedTimestampMs(value: string): number | null {
+  const match = ISO_TIMESTAMP_PATTERN.exec(value);
+  const parsed = Date.parse(value);
+  if (!match || !Number.isFinite(parsed)) return null;
+  const [, yearText, monthText, dayText, hourText, minuteText, secondText, fraction, zone] = match;
+  const normalizedZone = zone.toUpperCase();
+  const offsetMinuteText = normalizedZone.includes(":")
+    ? normalizedZone.slice(4, 6)
+    : normalizedZone.slice(3, 5);
+  const year = Number(yearText);
+  const month = Number(monthText);
+  const day = Number(dayText);
+  const hour = Number(hourText);
+  const minute = Number(minuteText);
+  const second = secondText !== undefined ? Number(secondText) : 0;
+  const leapYear = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0);
+  const daysInMonth = [31, leapYear ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
+  if (month < 1 || month > 12 || day < 1 || day > daysInMonth[month - 1]!) return null;
+  if (hour === 24) {
+    const isEndOfDay = minute === 0 && second === 0 && /^0*$/.test(fraction ?? "");
+    return isEndOfDay ? parsed : null;
+  }
+  const zoneSign = normalizedZone === "Z" || normalizedZone[0] === "+" ? 1 : -1;
+  const offsetMinutes =
+    normalizedZone === "Z"
+      ? 0
+      : zoneSign * (Number(normalizedZone.slice(1, 3)) * 60 + Number(offsetMinuteText));
+  const representedCalendar = new Date(parsed + offsetMinutes * 60_000);
+  const millisecond = Number((fraction ?? "").padEnd(3, "0").slice(0, 3));
+  const expected = [
+    year,
+    month,
+    day,
+    hour,
+    minute,
+    second,
+    millisecond,
+  ];
+  const actual = [
+    representedCalendar.getUTCFullYear(),
+    representedCalendar.getUTCMonth() + 1,
+    representedCalendar.getUTCDate(),
+    representedCalendar.getUTCHours(),
+    representedCalendar.getUTCMinutes(),
+    representedCalendar.getUTCSeconds(),
+    representedCalendar.getUTCMilliseconds(),
+  ];
+  return actual.every((part, index) => part === expected[index]) ? parsed : null;
+}
+
+function mergeRootStats(
+  current: ExtractionRootStats | undefined,
+  candidate: ExtractionRootStats | undefined,
+): ExtractionRootStats | undefined {
+  if (!current && !candidate) return undefined;
+  const currentCount = current?.extractionCount;
+  const candidateCount = candidate?.extractionCount;
+  const extractionCount =
+    currentCount === undefined && candidateCount === undefined
+      ? undefined
+      : (currentCount ?? 0) + (candidateCount ?? 0);
+  const currentConsolidation = current?.lastConsolidationAt;
+  const candidateConsolidation = candidate?.lastConsolidationAt;
+  const hasConsolidation =
+    currentConsolidation !== undefined || candidateConsolidation !== undefined;
+  return {
+    ...(extractionCount === undefined ? {} : { extractionCount }),
+    ...(hasConsolidation
+      ? {
+          lastConsolidationAt: newerWatermark(
+            currentConsolidation ?? null,
+            candidateConsolidation ?? null,
+          ),
+        }
+      : {}),
+  };
+}
+
 function watermarkFromMeta(meta: ExtractionWatermarkMeta, name: string): ExtractionWatermarkRead {
-  if (meta.lastExtractionAt !== null && meta.lastExtractionAt !== undefined) {
-    const parsed = Date.parse(meta.lastExtractionAt);
-    if (!Number.isFinite(parsed)) {
-      return readFailure(`${name} watermark timestamp invalid`);
-    }
+  if (
+    meta.lastExtractionAt !== null &&
+    meta.lastExtractionAt !== undefined &&
+    validatedTimestampMs(meta.lastExtractionAt) === null
+  ) {
+    return readFailure(`${name} watermark timestamp invalid`);
+  }
+  if (
+    meta.lastConsolidationAt !== null &&
+    meta.lastConsolidationAt !== undefined &&
+    validatedTimestampMs(meta.lastConsolidationAt) === null
+  ) {
+    return readFailure(`${name} consolidation timestamp invalid`);
   }
   const hasRootStats = meta.extractionCount !== undefined || meta.lastConsolidationAt !== undefined;
   return {
@@ -107,11 +195,11 @@ function isReadFailure(value: CorpusNamespaceRoot[] | ExtractionWatermarkRead):
 
 function newerWatermark(current: string | null, candidate: string | null): string | null {
   if (candidate === null) return current;
-  const candidateMs = Date.parse(candidate);
-  if (!Number.isFinite(candidateMs)) return current;
+  const candidateMs = validatedTimestampMs(candidate);
+  if (candidateMs === null) return current;
   if (current === null) return candidate;
-  const currentMs = Date.parse(current);
-  return !Number.isFinite(currentMs) || candidateMs > currentMs ? candidate : current;
+  const currentMs = validatedTimestampMs(current);
+  return currentMs === null || candidateMs > currentMs ? candidate : current;
 }
 
 export async function readAggregateExtractionWatermark(
@@ -155,10 +243,13 @@ export async function readAggregateExtractionWatermark(
     (!options.caps || capabilityAllowsNamespace(options.caps, defaultNamespace));
   let rootRead: ExtractionWatermarkRead = { lastExtractionAt: null, readFailed: false };
   if (canAccessRoot) {
-    rootRead = await readWatermark(options.rootStorage, "root store");
+    rootRead = options.rootMeta
+      ? watermarkFromMeta(options.rootMeta, "root store")
+      : await readWatermark(options.rootStorage, "root store");
     if (rootRead.readFailed) return rootRead;
   }
   let lastExtractionAt = rootRead.lastExtractionAt;
+  let rootStats = rootRead.rootStats;
 
   const reads = await Promise.all(
     targets.map(async (target) => {
@@ -173,16 +264,14 @@ export async function readAggregateExtractionWatermark(
 
   for (const read of reads) {
     if (read.readFailed) {
-      return readFailure(
-        read.readError ?? "namespace watermark unreadable",
-        rootRead.rootStats
-      );
+      return readFailure(read.readError ?? "namespace watermark unreadable");
     }
     lastExtractionAt = newerWatermark(lastExtractionAt, read.lastExtractionAt);
+    rootStats = mergeRootStats(rootStats, read.rootStats);
   }
   return {
     lastExtractionAt,
     readFailed: false,
-    ...(rootRead.rootStats ? { rootStats: rootRead.rootStats } : {}),
+    ...(rootStats ? { rootStats } : {}),
   };
 }

package/src/orchestration/generic-recall-paths.ts

@@ -1,5 +1,7 @@
 import path from "node:path";
 import type { QmdSearchResult } from "../types.js";
+import { isExternalWikiCollectionName } from "../external-wiki-guard.js";
+export { isExternalWikiCollectionName } from "../external-wiki-guard.js";
 import { ALL_CATEGORY_DIRS } from "../utils/category-dir.js";
 import {
   isActivityDigestPath,
@@ -81,12 +83,24 @@ export function isTopLevelArchivePath(
   return /^(?:archive|namespaces[\\/][^\\/]+[\\/]archive)(?:[\\/]|$)/i.test(normalized);
 }
 
+function isExternalWikiQmdPath(
+  filePath: string,
+  source: GenericRecallPathSource,
+): boolean {
+  if (source !== "qmd") return false;
+  const normalized = normalizeQmdUriPath(filePath, source).replace(/\\/g, "/");
+  const slashIndex = normalized.indexOf("/");
+  const collection = slashIndex === -1 ? normalized : normalized.slice(0, slashIndex);
+  return isExternalWikiCollectionName(collection);
+}
+
 export function isGenericRecallExcludedPath(
   filePath: string,
   policy: GenericRecallPathPolicy = {},
   source: GenericRecallPathSource = "filesystem",
 ): boolean {
   return (
+    isExternalWikiQmdPath(filePath, source) ||
     isArtifactMemoryPath(filePath) ||
     isActivityDigestPath(filePath, policy.memoryDir) ||
     isTopLevelArchivePath(filePath, policy, source) ||

package/src/orchestration/orchestrator-helpers.ts

@@ -197,6 +197,8 @@ export interface RecallInvocationOptions {
   topK?: number;
   mode?: RecallPlanMode;
   abortSignal?: AbortSignal;
+  /** Server-resolved identity of the connector performing this recall. */
+  sourceConnector?: string;
   /**
    * Capture a `RecallXraySnapshot` for this recall (issue #570).  When
    * `true`, the orchestrator builds a snapshot from the data it has

package/src/orchestration/orchestrator-init.ts

@@ -26,6 +26,7 @@ import type { ConversationIndexBackend } from "../conversation-index/backend.js"
 import type { CorrectionService } from "../correction/correction-service.js";
 import { EmbeddingFallback } from "../embedding-fallback.js";
 import { ContentHashIndex, StorageManager } from "../index.js";
+import { assertExternalWikiRootOutsideMemoryDirCanonical } from "../external-wiki-guard.js";
 import { log } from "../logger.js";
 import { migrateFromEngram } from "../migrate/from-engram.js";
 import { NamespaceCatalog } from "../namespaces/catalog.js";
@@ -118,6 +119,14 @@ export class OrchestratorInitCoordinator {
       });
       await this.deps.storage.loadAliases();
       await this.deps.storage.ensureDirectories();
+      await Promise.all(
+        this.deps.config.externalWikis.map((wiki) =>
+          assertExternalWikiRootOutsideMemoryDirCanonical(
+            this.deps.config.memoryDir,
+            wiki.rootDir,
+          ),
+        ),
+      );
       if (resolveNamespaceCapabilities(this.deps.config).namespaces) {
         const namespaces = new Set<string>([
           this.deps.config.defaultNamespace,

package/src/orchestration/recall-internal-deps.ts

@@ -34,6 +34,7 @@ import type { CodingContext, EngramTraceEvent, IdentityInjectionMode, MemoryFile
 import type { VerifiedEpisodeResult } from "../verified-recall.js";
 import type { WorkProductLedgerSearchResult } from "../work-product-ledger.js";
 import type { GraphRecallRankedResult, GraphRecallShadowComparison } from "./graph-recall-coordinator.js";
+import type { RecallRerankCoordinator, RecallResultPartitionSink } from "./recall-rerank-coordinator.js";
 import type { RecallSectionAppendOptions, RecallSectionBuckets } from "./recall-section-coordinator.js";
 
 export interface RecallInternalDeps {
@@ -64,6 +65,7 @@ export interface RecallInternalDeps {
     onDegradation?: (degradation: SearchDegradation) => void;
     /** Issue #680 — historical recall point in ms-since-epoch. */
     asOfMs?: number;
+    requestingConnector?: string;
     /**
      * Optional out-parameter that receives the pre-MMR / pre-truncation
      * pool size captured inside the pipeline (issue #570 PR 1).  The
@@ -73,6 +75,7 @@ export interface RecallInternalDeps {
      * Unset by default so existing call sites are unaffected.
      */
     xrayPoolSizeSink?: { size: number };
+    resultPartitionSink?: RecallResultPartitionSink;
     /**
      * Issue #1577 — out-parameter that receives the TrustScore stage's
      * per-path trust map (admitted + quarantined) when the cold path runs
@@ -126,6 +129,7 @@ export interface RecallInternalDeps {
        * string at the input boundary (CLI / HTTP / MCP).
        */
       asOfMs?: number;
+      requestingConnector?: string;
     },
   ): Promise<QmdSearchResult[]>;
   boxBuilderFor(storage: StorageManager): BoxBuilder;
@@ -177,6 +181,7 @@ export interface RecallInternalDeps {
   ): string[];
   readonly compounding?: CompoundingEngine;
   readonly config: PluginConfig;
+  readonly recallRerankCoordinator: Pick<RecallRerankCoordinator, "diversifyRecallResultsWithHeadroom">;
   currentPolicyVersion(): string;
   diversifyAndLimitRecallResults(
     sectionId: string,
@@ -234,6 +239,7 @@ export interface RecallInternalDeps {
       abortSignal?: AbortSignal;
       dropUnresolved?: boolean;
       recallNamespaces?: readonly string[];
+      requestingConnector?: string;
     },
   ): Promise<{ results: QmdSearchResult[]; memoryByPath: Map<string, MemoryFile> }>;
   formatCausalTrajectoryResults(