@remnic/core 9.40.0 → 9.42.0

package/dist/chunk-OYFDEJZB.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/corpus-watermark.ts"],"sourcesContent":["/**\n * Corpus watermark (issue #2149) — a cheap, comparable fingerprint of a\n * daemon's active-memory corpus.\n *\n * Deployment shape this addresses: two daemons behind an active/backup VIP,\n * each owning its own memory directory. Nothing compared the two corpora, so a\n * pair could diverge for months (e.g. one holding ~190k files, its backup\n * ~340k with different histories) with no signal until a manual audit. This\n * module ships the WATERMARK primitive and the read surfaces that expose it;\n * peer polling and divergence alerting are a follow-up.\n *\n * The digest is a CENSUS fingerprint (per day-partition file counts), NOT a\n * content hash — two daemons that agree on how many memory FILES live in\n * each `<tier>:<category>/<day>` partition produce the same digest, so a\n * differing digest is a cheap divergence signal without reading or hashing file\n * bodies. Buckets are tier-aware (`hot:` / `cold:`): demoted memories stay\n * reachable via cold recall, so the census counts BOTH tiers and a hot/cold\n * split shows in the digest rather than being silently folded together (issue\n * #2156 finding D). `newestWriteAt` stays scoped to the newest HOT partition\n * only, so the freshness probe stays O(one day's active files) even on a 100k+\n * corpus.\n *\n * Robustness (issue #2156 review rounds):\n * - `/health` and `remnic doctor` resolve their namespace set through the ONE\n *   {@link resolveCorpusNamespaceRoots} helper — config-driven enumeration\n *   unioned with the PERSISTED namespace catalog read from config — so both\n *   surfaces enumerate the same tenants and cannot drift.\n * - The `/health` builder serves through a stale-while-revalidate\n *   {@link CorpusWatermarkCache}, so a probe NEVER awaits the recursive corpus\n *   scan (it returns the cached/stale value and refreshes in the background).\n * - Each namespace's census brackets its hot+cold walk with a corpus-mutation\n *   sentinel and retries when a tier migration races it, so a transient\n *   double-count/miss is never cached as a false divergence.\n * - An unreadable corpus root (EACCES) is OMITTED, not published as a false\n *   empty; a missing root (ENOENT — a not-yet-created namespace) is a genuine\n *   empty corpus.\n */\n\nimport { createHash } from \"node:crypto\";\nimport { stat } from \"node:fs/promises\";\nimport path from \"node:path\";\nimport { capabilityAllowsNamespace, type TokenCapabilities } from \"./access-token-capabilities.js\";\nimport { listNamespaces } from \"./namespaces/migrate.js\";\nimport type { PluginConfig } from \"./types.js\";\n\nexport interface CorpusWatermark {\n  namespace: string;\n  /**\n   * Count of memory files under the census scan roots (hot + cold). This is a\n   * FILE census, not a status-filtered active count: reading each file's\n   * frontmatter status would defeat the cheap-probe design, so an in-place\n   * status change (e.g. archived-in-place) is not reflected here.\n   */\n  memoryFileCount: number;\n  /** Newest `YYYY-MM-DD` day-partition seen in the HOT tier, or null when none is dated. */\n  newestPartition: string | null;\n  /** ISO max mtime within the newest HOT partition, or null when it has no files. */\n  newestWriteAt: string | null;\n  /** sha256 hex over the deterministic per-partition census. */\n  digest: string;\n  computedAt: string;\n}\n\n/** Explicit bucket for paths that do not match the `<category>/<day>` shape. */\nexport const UNPARTITIONED_BUCKET = \"unpartitioned\";\n\n/** Storage tier a memory file lives in. */\nexport type CorpusTier = \"hot\" | \"cold\";\n\nconst DAY_PARTITION_RE = /^\\d{4}-\\d{2}-\\d{2}$/;\n/** Cold-tier root segment (relative to a namespace base dir): `<baseDir>/cold/...`. */\nconst COLD_TIER_DIR = \"cold\";\n\n/**\n * True when `token` is not just `YYYY-MM-DD`-shaped but a REAL calendar day.\n * The regex alone accepts impossible dates (e.g. `2026-99-99`) that would then\n * sort lexically above every valid partition and corrupt `newestPartition`\n * (issue #2156 round-6); round-tripping through a UTC Date rejects month/day\n * overflow. A malformed dir falls back to the unpartitioned bucket.\n */\nfunction isRealCalendarDay(token: string): boolean {\n  if (!DAY_PARTITION_RE.test(token)) return false;\n  const [year, month, day] = token.split(\"-\").map(Number);\n  const date = new Date(Date.UTC(year, month - 1, day));\n  return date.getUTCFullYear() === year && date.getUTCMonth() === month - 1 && date.getUTCDate() === day;\n}\n\ninterface ParsedPath {\n  readonly bucket: string;\n  readonly date: string | null;\n  readonly tier: CorpusTier;\n}\n\nfunction parsePartition(baseDir: string, filePath: string): ParsedPath {\n  const rel = path.relative(baseDir, filePath).split(path.sep).join(\"/\");\n  const parts = rel.split(\"/\");\n  // Cold-tier files live under `<baseDir>/cold/<category>/<day>/...`; hot files\n  // sit directly under `<baseDir>/<category>/<day>/...`. Strip the leading\n  // `cold/` segment so the category/day parse is tier-agnostic, then prefix the\n  // bucket with the tier so a hot and cold partition of the same day never\n  // collide (they must diverge the digest, not fold together).\n  const tier: CorpusTier = parts[0] === COLD_TIER_DIR ? \"cold\" : \"hot\";\n  const categoryParts = tier === \"cold\" ? parts.slice(1) : parts;\n  if (categoryParts.length >= 2 && isRealCalendarDay(categoryParts[1])) {\n    return { bucket: `${tier}:${categoryParts[0]}/${categoryParts[1]}`, date: categoryParts[1], tier };\n  }\n  return { bucket: `${tier}:${UNPARTITIONED_BUCKET}`, date: null, tier };\n}\n\n/**\n * Build the per-partition census: bucket key -> file count. Bucket keys are\n * tier-aware (`hot:`/`cold:`). Paths not matching a day-partition land in the\n * explicit tier {@link UNPARTITIONED_BUCKET}; they are never silently dropped.\n * Pure — no filesystem access.\n */\nexport function buildPartitionCensus(paths: readonly string[], baseDir: string): Map<string, number> {\n  const census = new Map<string, number>();\n  for (const filePath of paths) {\n    const { bucket } = parsePartition(baseDir, filePath);\n    census.set(bucket, (census.get(bucket) ?? 0) + 1);\n  }\n  return census;\n}\n\n/**\n * Deterministic census -> digest. Bucket keys are SORTED before serialization\n * (AGENTS.md pattern 26: insertion order must not affect a hash), then emitted\n * as `\"<bucket>:<count>\"` lines joined by `\\n` and hashed with sha256. Pure —\n * separately exported so it can be unit-tested without a filesystem.\n */\nexport function digestPartitionCensus(census: ReadonlyMap<string, number> | Record<string, number>): string {\n  const entries = census instanceof Map ? [...census.entries()] : Object.entries(census);\n  const serialized = entries\n    .sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0))\n    .map(([bucket, count]) => `${bucket}:${count}`)\n    .join(\"\\n\");\n  return createHash(\"sha256\").update(serialized).digest(\"hex\");\n}\n\n/**\n * Compute a watermark for a single namespace from an already-collected path\n * list (both tiers, under `baseDir`) plus its memory base directory. Never\n * throws on an empty list — an empty corpus yields the digest of the empty\n * census and null timestamps. Only the newest HOT partition's files are\n * `stat`-ed, so freshness stays cheap even on a 100k+ corpus.\n */\nexport async function computeCorpusWatermark(input: {\n  namespace: string;\n  paths: readonly string[];\n  baseDir: string;\n  now?: Date;\n}): Promise<CorpusWatermark> {\n  const { namespace, paths, baseDir } = input;\n\n  const census = new Map<string, number>();\n  let newestPartition: string | null = null;\n  const parsedByPath: ParsedPath[] = [];\n  for (const filePath of paths) {\n    const parsed = parsePartition(baseDir, filePath);\n    parsedByPath.push(parsed);\n    census.set(parsed.bucket, (census.get(parsed.bucket) ?? 0) + 1);\n    // Freshness is the newest ACTIVE (hot) write. Cold memories are demoted and\n    // keep their original (older) partition date, so they never advance the\n    // freshness probe and their files are never stat-ed below.\n    if (parsed.tier === \"hot\" && parsed.date !== null && (newestPartition === null || parsed.date > newestPartition)) {\n      newestPartition = parsed.date;\n    }\n  }\n\n  const digest = digestPartitionCensus(census);\n\n  let newestWriteAt: string | null = null;\n  if (newestPartition !== null) {\n    let maxMtimeMs = Number.NEGATIVE_INFINITY;\n    for (let i = 0; i < paths.length; i += 1) {\n      if (parsedByPath[i].tier !== \"hot\" || parsedByPath[i].date !== newestPartition) continue;\n      try {\n        const st = await stat(paths[i]);\n        if (st.mtimeMs > maxMtimeMs) maxMtimeMs = st.mtimeMs;\n      } catch (err) {\n        // A file vanishing between scan and stat (ENOENT) is an expected race —\n        // skip it. A backend read failure (EACCES/EIO/…) means the corpus is\n        // unreadable: propagate so the namespace is omitted, not published with\n        // a stale/null freshness (issue #2156 round-6).\n        if ((err as NodeJS.ErrnoException).code !== \"ENOENT\") throw err;\n      }\n    }\n    if (maxMtimeMs > Number.NEGATIVE_INFINITY) {\n      newestWriteAt = new Date(maxMtimeMs).toISOString();\n    }\n  }\n\n  // Stamp AFTER the (possibly slow) stat loop so an age-based HA consumer never\n  // sees a timestamp earlier than when the scan actually finished (finding\n  // round-4). An explicit `now` (tests / determinism) still wins.\n  const computedAt = (input.now ?? new Date()).toISOString();\n\n  return {\n    namespace,\n    memoryFileCount: paths.length,\n    newestPartition,\n    newestWriteAt,\n    digest,\n    computedAt,\n  };\n}\n\n/** Minimal storage surface a watermark needs — satisfied by StorageManager. */\nexport interface CorpusStorage {\n  readonly dir: string;\n  /** Hot-tier active memory paths (the recall corpus). */\n  collectActiveMemoryPaths(options?: { propagateReadErrors?: boolean }): Promise<string[]>;\n  /** Cold-tier (demoted-but-reachable) memory paths (issue #2156 finding D). */\n  collectColdMemoryPaths(options?: { propagateReadErrors?: boolean }): Promise<string[]>;\n  /**\n   * Optional combined hot+cold corpus-mutation sentinel. When present, the\n   * census brackets its scan with it and retries on change so a tier migration\n   * racing the walkers is not cached as a false divergence. Absent (test fakes)\n   * ⇒ single-pass.\n   */\n  getCorpusScanVersion?(): string | Promise<string>;\n}\n\n/** Max census re-scans when a corpus mutation keeps racing the walkers. */\nconst CENSUS_RACE_MAX_ATTEMPTS = 3;\n\n/**\n * Compute one namespace's watermark from its storage, spanning BOTH tiers so a\n * cold-tier divergence is caught. Reads are STRICT: the collectors propagate a\n * backend read failure (EACCES on the root OR any nested category dir, ENOTDIR,\n * …) so an unreadable corpus is OMITTED by the caller rather than published as a\n * false empty census; a missing root (ENOENT — a not-yet-created namespace)\n * stays a genuine empty corpus. The hot+cold walk is bracketed by the\n * corpus-mutation sentinel (when the storage exposes one) and retried on change;\n * if it never stabilizes (sustained tier churn) this THROWS rather than caching\n * a transient double-count/miss as a false divergence.\n */\nasync function computeNamespaceWatermark(\n  namespace: string,\n  storage: CorpusStorage,\n  now?: Date,\n): Promise<CorpusWatermark> {\n  const readVersion = storage.getCorpusScanVersion?.bind(storage);\n  const maxAttempts = readVersion ? CENSUS_RACE_MAX_ATTEMPTS : 1;\n  for (let attempt = 0; attempt < maxAttempts; attempt += 1) {\n    const before = readVersion ? await readVersion() : null;\n    const [hotPaths, coldPaths] = await Promise.all([\n      storage.collectActiveMemoryPaths({ propagateReadErrors: true }),\n      storage.collectColdMemoryPaths({ propagateReadErrors: true }),\n    ]);\n    const result = await computeCorpusWatermark({ namespace, paths: [...hotPaths, ...coldPaths], baseDir: storage.dir, now });\n    const after = readVersion ? await readVersion() : null;\n    if (before === after) return result;\n    // A tier write (e.g. hot→cold migration) raced the two walkers, so the\n    // count/digest may double-count or miss the in-flight memory. Retry for a\n    // consistent snapshot.\n  }\n  throw new Error(`corpus census for namespace \"${namespace}\" did not stabilize after ${maxAttempts} attempts`);\n}\n\n/**\n * Compute watermarks for a set of namespaces. A namespace whose storage is\n * unavailable (or unreadable) is skipped rather than failing the whole read, so\n * a single bad namespace never blanks the health/doctor payload.\n */\nexport async function computeCorpusWatermarks(\n  namespaces: readonly string[],\n  storageFor: (namespace: string) => CorpusStorage | Promise<CorpusStorage>,\n  now?: Date,\n): Promise<CorpusWatermark[]> {\n  const watermarks: CorpusWatermark[] = [];\n  for (const namespace of namespaces) {\n    try {\n      watermarks.push(await computeNamespaceWatermark(namespace, await storageFor(namespace), now));\n    } catch {\n      // Storage unavailable/unreadable for this namespace — degrade gracefully.\n    }\n  }\n  return watermarks;\n}\n\n/** A namespace and the memory root its corpus lives under. */\nexport interface CorpusNamespaceRoot {\n  /** Representative namespace label for this root (default wins on collapse). */\n  readonly namespace: string;\n  readonly rootDir: string;\n  /**\n   * Every configured namespace name that resolves to `rootDir` (aliases that\n   * collapsed onto one physical root, e.g. `global` + `shared` when namespaces\n   * are disabled). `/health` filters visibility against ALL of these so a token\n   * scoped to a non-representative alias still sees the shared corpus (issue\n   * #2156 round-10). Always includes `namespace`.\n   */\n  readonly namespaces: readonly string[];\n}\n\n/**\n * Resolve the namespace set the corpus census covers, shared by the `/health`\n * builder AND the `remnic doctor` corpus check (issue #2156 finding C) so the\n * two surfaces enumerate the SAME tenants — and, critically, resolve each\n * namespace's ROOT the SAME way — and cannot drift. Enumeration is purely\n * config-driven (`listNamespaces({ config })`): it discovers the default,\n * shared, policy, and on-disk `namespaces/<ns>` tenants and resolves every root\n * through the namespace storage router, exactly as the live daemon's\n * `getStorage(namespace)` does. It still works when the namespace catalog is\n * opted out. The catalog is deliberately NOT unioned in: a catalog-registered\n * tenant with no directory has an empty corpus (nothing to fingerprint), and\n * trusting `catalog.storageDir` here could make the doctor scan a different root\n * than `/health` for the same namespace (issue #2156 round-8). Deduped by\n * resolved root so a namespaces-disabled / flat-root deployment reports its\n * single shared corpus once, under the default-namespace label.\n */\nexport async function resolveCorpusNamespaceRoots(options: {\n  config: PluginConfig;\n  propagateDiscoveryErrors?: boolean;\n}): Promise<CorpusNamespaceRoot[]> {\n  const { config } = options;\n  const configDriven = await listNamespaces({\n    config,\n    propagateDiscoveryErrors: options.propagateDiscoveryErrors,\n  });\n  // Default namespace first so it wins the representative label when several\n  // configured names collapse onto one root (namespaces disabled → memoryDir).\n  const ordered = [\n    ...configDriven.filter((entry) => entry.namespace === config.defaultNamespace),\n    ...configDriven.filter((entry) => entry.namespace !== config.defaultNamespace),\n  ];\n  const seenNamespaces = new Set<string>();\n  const byRoot = new Map<string, { namespace: string; rootDir: string; namespaces: string[] }>();\n  for (const entry of ordered) {\n    if (seenNamespaces.has(entry.namespace)) continue;\n    seenNamespaces.add(entry.namespace);\n    const existing = byRoot.get(entry.rootDir);\n    if (existing) {\n      existing.namespaces.push(entry.namespace); // an alias collapsing onto a seen root\n    } else {\n      byRoot.set(entry.rootDir, { namespace: entry.namespace, rootDir: entry.rootDir, namespaces: [entry.namespace] });\n    }\n  }\n  const roots: CorpusNamespaceRoot[] = [...byRoot.values()];\n  if (roots.length === 0) {\n    roots.push({ namespace: config.defaultNamespace, rootDir: config.memoryDir, namespaces: [config.defaultNamespace] });\n  }\n  return roots;\n}\n\n/**\n * Divergence detection tolerates a watermark that is stale by up to a minute\n * (peers are compared on a slow cadence); a health/readiness probe does NOT\n * tolerate a full corpus scan (a recursive realpath-per-Markdown-file walk) on\n * every request. This TTL is that trade-off: recompute a namespace's watermark\n * at most once per window; probes in between serve the cached census (whose\n * `computedAt` shows its staleness).\n */\nexport const WATERMARK_CACHE_TTL_MS = 60_000;\n\n/**\n * Cap on concurrent background corpus scans. A probe that finds many namespaces\n * cold/expired would otherwise fan out one recursive hot/cold walk PER namespace\n * at once, a periodic unbounded filesystem-work burst that can starve recall and\n * storage (issue #2156 round-9). Excess refreshes queue and start as slots free;\n * single-flight still dedupes duplicate requests for the same namespace.\n */\nconst MAX_CONCURRENT_WATERMARK_REFRESHES = 4;\n\n/**\n * Instance-scoped (AGENTS.md pattern 5 — never a bare module global)\n * stale-while-revalidate cache in front of the per-namespace corpus scan.\n * {@link get} NEVER awaits the scan: it returns the freshest already-computed\n * watermark (stale allowed, or undefined for a cold namespace) and triggers a\n * single-flight background refresh when the entry is missing or expired. So a\n * `/health` probe is always O(1) — it never blocks on a recursive corpus walk,\n * even on the first request after startup or after a TTL expiry. Owned by the\n * long-lived access service instance.\n */\nexport class CorpusWatermarkCache {\n  private readonly entries = new Map<string, { value: CorpusWatermark; expiresAt: number }>();\n  private readonly inFlight = new Map<string, Promise<void>>();\n  private readonly ttlMs: number;\n  private readonly clock: () => number;\n  private rootsEntry: { value: CorpusNamespaceRoot[]; expiresAt: number } | undefined;\n  private rootsInFlight: Promise<void> | undefined;\n  private rootsRefreshError: unknown;\n  private readonly queued = new Map<string, () => void>();\n  private activeRefreshes = 0;\n  private readonly maxConcurrentRefreshes: number;\n\n  constructor(options: { ttlMs?: number; clock?: () => number; maxConcurrentRefreshes?: number } = {}) {\n    this.ttlMs = options.ttlMs ?? WATERMARK_CACHE_TTL_MS;\n    this.clock = options.clock ?? Date.now;\n    this.maxConcurrentRefreshes = Math.max(1, options.maxConcurrentRefreshes ?? MAX_CONCURRENT_WATERMARK_REFRESHES);\n  }\n\n  get(namespace: string, compute: () => Promise<CorpusWatermark>): CorpusWatermark | undefined {\n    const cached = this.entries.get(namespace);\n    const fresh = cached !== undefined && this.clock() < cached.expiresAt;\n    if (!fresh) this.refresh(namespace, compute);\n    return cached?.value;\n  }\n\n  private refresh(namespace: string, compute: () => Promise<CorpusWatermark>): void {\n    // Single-flight + not-already-queued: one background scan per namespace.\n    if (this.inFlight.has(namespace) || this.queued.has(namespace)) return;\n    const run = (): void => {\n      this.activeRefreshes += 1;\n      const pending = compute()\n        .then((value) => {\n          this.entries.set(namespace, { value, expiresAt: this.clock() + this.ttlMs });\n        })\n        .catch(() => {\n          // Failed/unreadable scan: never cache it; keep serving any stale value.\n        })\n        .finally(() => {\n          this.inFlight.delete(namespace);\n          this.activeRefreshes -= 1;\n          this.drainQueue();\n        });\n      this.inFlight.set(namespace, pending);\n    };\n    // Bound the concurrent fan-out: run now if a slot is free, else queue.\n    if (this.activeRefreshes < this.maxConcurrentRefreshes) {\n      run();\n    } else {\n      this.queued.set(namespace, run);\n    }\n  }\n\n  private drainQueue(): void {\n    while (this.activeRefreshes < this.maxConcurrentRefreshes && this.queued.size > 0) {\n      const next = this.queued.entries().next().value;\n      if (!next) break;\n      const [namespace, run] = next;\n      this.queued.delete(namespace);\n      run();\n    }\n  }\n\n\n  /**\n   * Stale-while-revalidate the resolved namespace roots (issue #2156 round-7).\n   * Namespace enumeration (config scan + persisted-catalog parse) is O(tenants)\n   * filesystem work; without this it ran on every probe ahead of the watermark\n   * cache. NEVER awaits: returns the cached/stale roots (or undefined when cold)\n   * and single-flights a background refresh.\n   */\n  getResolvedRoots(compute: () => Promise<CorpusNamespaceRoot[]>): CorpusNamespaceRoot[] | undefined {\n    const fresh = this.rootsEntry !== undefined && this.clock() < this.rootsEntry.expiresAt;\n    if (!fresh && this.rootsInFlight === undefined) {\n      this.rootsInFlight = compute()\n        .then((value) => {\n          this.rootsEntry = { value, expiresAt: this.clock() + this.ttlMs };\n          this.rootsRefreshError = undefined;\n        })\n        .catch((error: unknown) => {\n          this.rootsRefreshError = error;\n          // Keep serving any stale roots; callers that require completeness\n          // inspect the refresh status before trusting them.\n        })\n        .finally(() => {\n          this.rootsInFlight = undefined;\n        });\n    }\n    return this.rootsEntry?.value;\n  }\n\n  getResolvedRootsStatus(compute: () => Promise<CorpusNamespaceRoot[]>): {\n    roots: CorpusNamespaceRoot[] | undefined;\n    refreshError: unknown;\n  } {\n    const roots = this.getResolvedRoots(compute);\n    return { roots, refreshError: this.rootsRefreshError };\n  }\n  /** Await every background refresh — in-flight AND queued (issue #2156 round-9) —\n   *  plus any roots refresh (shutdown / deterministic tests). */\n  async whenIdle(): Promise<void> {\n    while (this.inFlight.size > 0 || this.rootsInFlight !== undefined || this.queued.size > 0) {\n      const pending: Promise<unknown>[] = [...this.inFlight.values()];\n      if (this.rootsInFlight) pending.push(this.rootsInFlight);\n      if (pending.length === 0) {\n        this.drainQueue(); // queued work with no running peer — start it\n        await Promise.resolve();\n        continue;\n      }\n      await Promise.all(pending);\n    }\n  }\n}\n\n/** Orchestrator surface the service watermark builder reads — satisfied by Orchestrator. */\nexport interface CorpusWatermarkHost {\n  config: PluginConfig;\n  getStorage(namespace: string): CorpusStorage | Promise<CorpusStorage>;\n}\n\nexport interface ServiceCorpusWatermarkOptions {\n  /** Stale-while-revalidate cache so probes never await the corpus scan. */\n  cache?: CorpusWatermarkCache;\n  /**\n   * Presenting token capabilities. When the token is namespace-restricted the\n   * fleet view is filtered to the namespaces it may access (issue #2156 finding\n   * B); an unrestricted/legacy token keeps the full fleet view.\n   */\n  caps?: TokenCapabilities | null;\n  now?: Date;\n}\n\n/**\n * Build the per-namespace watermark array served on `/health`. Namespaces are\n * resolved through {@link resolveCorpusNamespaceRoots} (shared with the doctor),\n * filtered to the caller's capabilities, then served through the (optional)\n * stale-while-revalidate cache so a probe never blocks on a scan. Degrades to an\n * empty array — never throws — if enumeration is unavailable; an individual\n * namespace whose scan fails is omitted, not allowed to blank the payload.\n */\nexport async function computeServiceCorpusWatermarks(\n  host: CorpusWatermarkHost,\n  options: ServiceCorpusWatermarkOptions = {},\n): Promise<CorpusWatermark[]> {\n  return (await computeServiceCorpusCensus(host, options)).watermarks;\n}\n\n/**\n * Watermarks PLUS whether every configured namespace made it into the set.\n *\n * Completeness is returned ATOMICALLY with the data rather than stashed in a\n * side channel: every early exit here (enumeration threw, roots still warming,\n * a per-namespace scan failed, a cache entry not yet populated) yields an\n * incomplete set, and a caller that certifies convergence against a partial\n * census would hide the divergence of whatever was dropped (issue #2149).\n */\nexport async function computeServiceCorpusCensus(\n  host: CorpusWatermarkHost,\n  options: ServiceCorpusWatermarkOptions = {},\n): Promise<{ watermarks: CorpusWatermark[]; complete: boolean }> {\n  let roots: CorpusNamespaceRoot[] | undefined;\n  if (options.cache) {\n    const { roots: cachedRoots, refreshError } = options.cache.getResolvedRootsStatus(() =>\n      resolveCorpusNamespaceRoots({ config: host.config, propagateDiscoveryErrors: true }),\n    );\n    if (refreshError !== undefined) return { watermarks: [], complete: false };\n    roots = cachedRoots;\n  } else {\n    try {\n      roots = await resolveCorpusNamespaceRoots({\n        config: host.config,\n        propagateDiscoveryErrors: true,\n      });\n    } catch {\n      return { watermarks: [], complete: false }; // never enumerated — not an empty corpus\n    }\n  }\n  // Enumeration still warming (cold): served on a later probe, and NOT complete.\n  if (!roots) return { watermarks: [], complete: false };\n  const visible = roots.filter((root) => root.namespaces.some((ns) => capabilityAllowsNamespace(options.caps, ns)));\n  const watermarks: CorpusWatermark[] = [];\n  let complete = true;\n  for (const { namespace } of visible) {\n    const compute = async (): Promise<CorpusWatermark> =>\n      computeNamespaceWatermark(namespace, await host.getStorage(namespace), options.now);\n    if (options.cache) {\n      const cached = options.cache.get(namespace, compute);\n      if (cached) watermarks.push(cached);\n      else complete = false; // still warming — the set is not the full census yet\n    } else {\n      try {\n        watermarks.push(await compute());\n      } catch {\n        // One tenant's storage/scan failed — omit just that namespace.\n        complete = false;\n      }\n    }\n  }\n  return { watermarks, complete };\n}\n\n\n"],"mappings":";;;;;;;;AAsCA,SAAS,kBAAkB;AAC3B,SAAS,YAAY;AACrB,OAAO,UAAU;AAwBV,IAAM,uBAAuB;AAKpC,IAAM,mBAAmB;AAEzB,IAAM,gBAAgB;AAStB,SAAS,kBAAkB,OAAwB;AACjD,MAAI,CAAC,iBAAiB,KAAK,KAAK,EAAG,QAAO;AAC1C,QAAM,CAAC,MAAM,OAAO,GAAG,IAAI,MAAM,MAAM,GAAG,EAAE,IAAI,MAAM;AACtD,QAAM,OAAO,IAAI,KAAK,KAAK,IAAI,MAAM,QAAQ,GAAG,GAAG,CAAC;AACpD,SAAO,KAAK,eAAe,MAAM,QAAQ,KAAK,YAAY,MAAM,QAAQ,KAAK,KAAK,WAAW,MAAM;AACrG;AAQA,SAAS,eAAe,SAAiB,UAA8B;AACrE,QAAM,MAAM,KAAK,SAAS,SAAS,QAAQ,EAAE,MAAM,KAAK,GAAG,EAAE,KAAK,GAAG;AACrE,QAAM,QAAQ,IAAI,MAAM,GAAG;AAM3B,QAAM,OAAmB,MAAM,CAAC,MAAM,gBAAgB,SAAS;AAC/D,QAAM,gBAAgB,SAAS,SAAS,MAAM,MAAM,CAAC,IAAI;AACzD,MAAI,cAAc,UAAU,KAAK,kBAAkB,cAAc,CAAC,CAAC,GAAG;AACpE,WAAO,EAAE,QAAQ,GAAG,IAAI,IAAI,cAAc,CAAC,CAAC,IAAI,cAAc,CAAC,CAAC,IAAI,MAAM,cAAc,CAAC,GAAG,KAAK;AAAA,EACnG;AACA,SAAO,EAAE,QAAQ,GAAG,IAAI,IAAI,oBAAoB,IAAI,MAAM,MAAM,KAAK;AACvE;AAQO,SAAS,qBAAqB,OAA0B,SAAsC;AACnG,QAAM,SAAS,oBAAI,IAAoB;AACvC,aAAW,YAAY,OAAO;AAC5B,UAAM,EAAE,OAAO,IAAI,eAAe,SAAS,QAAQ;AACnD,WAAO,IAAI,SAAS,OAAO,IAAI,MAAM,KAAK,KAAK,CAAC;AAAA,EAClD;AACA,SAAO;AACT;AAQO,SAAS,sBAAsB,QAAsE;AAC1G,QAAM,UAAU,kBAAkB,MAAM,CAAC,GAAG,OAAO,QAAQ,CAAC,IAAI,OAAO,QAAQ,MAAM;AACrF,QAAM,aAAa,QAChB,KAAK,CAAC,GAAG,MAAO,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,IAAI,CAAE,EACvD,IAAI,CAAC,CAAC,QAAQ,KAAK,MAAM,GAAG,MAAM,IAAI,KAAK,EAAE,EAC7C,KAAK,IAAI;AACZ,SAAO,WAAW,QAAQ,EAAE,OAAO,UAAU,EAAE,OAAO,KAAK;AAC7D;AASA,eAAsB,uBAAuB,OAKhB;AAC3B,QAAM,EAAE,WAAW,OAAO,QAAQ,IAAI;AAEtC,QAAM,SAAS,oBAAI,IAAoB;AACvC,MAAI,kBAAiC;AACrC,QAAM,eAA6B,CAAC;AACpC,aAAW,YAAY,OAAO;AAC5B,UAAM,SAAS,eAAe,SAAS,QAAQ;AAC/C,iBAAa,KAAK,MAAM;AACxB,WAAO,IAAI,OAAO,SAAS,OAAO,IAAI,OAAO,MAAM,KAAK,KAAK,CAAC;AAI9D,QAAI,OAAO,SAAS,SAAS,OAAO,SAAS,SAAS,oBAAoB,QAAQ,OAAO,OAAO,kBAAkB;AAChH,wBAAkB,OAAO;AAAA,IAC3B;AAAA,EACF;AAEA,QAAM,SAAS,sBAAsB,MAAM;AAE3C,MAAI,gBAA+B;AACnC,MAAI,oBAAoB,MAAM;AAC5B,QAAI,aAAa,OAAO;AACxB,aAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,GAAG;AACxC,UAAI,aAAa,CAAC,EAAE,SAAS,SAAS,aAAa,CAAC,EAAE,SAAS,gBAAiB;AAChF,UAAI;AACF,cAAM,KAAK,MAAM,KAAK,MAAM,CAAC,CAAC;AAC9B,YAAI,GAAG,UAAU,WAAY,cAAa,GAAG;AAAA,MAC/C,SAAS,KAAK;AAKZ,YAAK,IAA8B,SAAS,SAAU,OAAM;AAAA,MAC9D;AAAA,IACF;AACA,QAAI,aAAa,OAAO,mBAAmB;AACzC,sBAAgB,IAAI,KAAK,UAAU,EAAE,YAAY;AAAA,IACnD;AAAA,EACF;AAKA,QAAM,cAAc,MAAM,OAAO,oBAAI,KAAK,GAAG,YAAY;AAEzD,SAAO;AAAA,IACL;AAAA,IACA,iBAAiB,MAAM;AAAA,IACvB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAmBA,IAAM,2BAA2B;AAajC,eAAe,0BACb,WACA,SACA,KAC0B;AAC1B,QAAM,cAAc,QAAQ,sBAAsB,KAAK,OAAO;AAC9D,QAAM,cAAc,cAAc,2BAA2B;AAC7D,WAAS,UAAU,GAAG,UAAU,aAAa,WAAW,GAAG;AACzD,UAAM,SAAS,cAAc,MAAM,YAAY,IAAI;AACnD,UAAM,CAAC,UAAU,SAAS,IAAI,MAAM,QAAQ,IAAI;AAAA,MAC9C,QAAQ,yBAAyB,EAAE,qBAAqB,KAAK,CAAC;AAAA,MAC9D,QAAQ,uBAAuB,EAAE,qBAAqB,KAAK,CAAC;AAAA,IAC9D,CAAC;AACD,UAAM,SAAS,MAAM,uBAAuB,EAAE,WAAW,OAAO,CAAC,GAAG,UAAU,GAAG,SAAS,GAAG,SAAS,QAAQ,KAAK,IAAI,CAAC;AACxH,UAAM,QAAQ,cAAc,MAAM,YAAY,IAAI;AAClD,QAAI,WAAW,MAAO,QAAO;AAAA,EAI/B;AACA,QAAM,IAAI,MAAM,gCAAgC,SAAS,6BAA6B,WAAW,WAAW;AAC9G;AAOA,eAAsB,wBACpB,YACA,YACA,KAC4B;AAC5B,QAAM,aAAgC,CAAC;AACvC,aAAW,aAAa,YAAY;AAClC,QAAI;AACF,iBAAW,KAAK,MAAM,0BAA0B,WAAW,MAAM,WAAW,SAAS,GAAG,GAAG,CAAC;AAAA,IAC9F,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAiCA,eAAsB,4BAA4B,SAGf;AACjC,QAAM,EAAE,OAAO,IAAI;AACnB,QAAM,eAAe,MAAM,eAAe;AAAA,IACxC;AAAA,IACA,0BAA0B,QAAQ;AAAA,EACpC,CAAC;AAGD,QAAM,UAAU;AAAA,IACd,GAAG,aAAa,OAAO,CAAC,UAAU,MAAM,cAAc,OAAO,gBAAgB;AAAA,IAC7E,GAAG,aAAa,OAAO,CAAC,UAAU,MAAM,cAAc,OAAO,gBAAgB;AAAA,EAC/E;AACA,QAAM,iBAAiB,oBAAI,IAAY;AACvC,QAAM,SAAS,oBAAI,IAA0E;AAC7F,aAAW,SAAS,SAAS;AAC3B,QAAI,eAAe,IAAI,MAAM,SAAS,EAAG;AACzC,mBAAe,IAAI,MAAM,SAAS;AAClC,UAAM,WAAW,OAAO,IAAI,MAAM,OAAO;AACzC,QAAI,UAAU;AACZ,eAAS,WAAW,KAAK,MAAM,SAAS;AAAA,IAC1C,OAAO;AACL,aAAO,IAAI,MAAM,SAAS,EAAE,WAAW,MAAM,WAAW,SAAS,MAAM,SAAS,YAAY,CAAC,MAAM,SAAS,EAAE,CAAC;AAAA,IACjH;AAAA,EACF;AACA,QAAM,QAA+B,CAAC,GAAG,OAAO,OAAO,CAAC;AACxD,MAAI,MAAM,WAAW,GAAG;AACtB,UAAM,KAAK,EAAE,WAAW,OAAO,kBAAkB,SAAS,OAAO,WAAW,YAAY,CAAC,OAAO,gBAAgB,EAAE,CAAC;AAAA,EACrH;AACA,SAAO;AACT;AAUO,IAAM,yBAAyB;AAStC,IAAM,qCAAqC;AAYpC,IAAM,uBAAN,MAA2B;AAAA,EACf,UAAU,oBAAI,IAA2D;AAAA,EACzE,WAAW,oBAAI,IAA2B;AAAA,EAC1C;AAAA,EACA;AAAA,EACT;AAAA,EACA;AAAA,EACA;AAAA,EACS,SAAS,oBAAI,IAAwB;AAAA,EAC9C,kBAAkB;AAAA,EACT;AAAA,EAEjB,YAAY,UAAqF,CAAC,GAAG;AACnG,SAAK,QAAQ,QAAQ,SAAS;AAC9B,SAAK,QAAQ,QAAQ,SAAS,KAAK;AACnC,SAAK,yBAAyB,KAAK,IAAI,GAAG,QAAQ,0BAA0B,kCAAkC;AAAA,EAChH;AAAA,EAEA,IAAI,WAAmB,SAAsE;AAC3F,UAAM,SAAS,KAAK,QAAQ,IAAI,SAAS;AACzC,UAAM,QAAQ,WAAW,UAAa,KAAK,MAAM,IAAI,OAAO;AAC5D,QAAI,CAAC,MAAO,MAAK,QAAQ,WAAW,OAAO;AAC3C,WAAO,QAAQ;AAAA,EACjB;AAAA,EAEQ,QAAQ,WAAmB,SAA+C;AAEhF,QAAI,KAAK,SAAS,IAAI,SAAS,KAAK,KAAK,OAAO,IAAI,SAAS,EAAG;AAChE,UAAM,MAAM,MAAY;AACtB,WAAK,mBAAmB;AACxB,YAAM,UAAU,QAAQ,EACrB,KAAK,CAAC,UAAU;AACf,aAAK,QAAQ,IAAI,WAAW,EAAE,OAAO,WAAW,KAAK,MAAM,IAAI,KAAK,MAAM,CAAC;AAAA,MAC7E,CAAC,EACA,MAAM,MAAM;AAAA,MAEb,CAAC,EACA,QAAQ,MAAM;AACb,aAAK,SAAS,OAAO,SAAS;AAC9B,aAAK,mBAAmB;AACxB,aAAK,WAAW;AAAA,MAClB,CAAC;AACH,WAAK,SAAS,IAAI,WAAW,OAAO;AAAA,IACtC;AAEA,QAAI,KAAK,kBAAkB,KAAK,wBAAwB;AACtD,UAAI;AAAA,IACN,OAAO;AACL,WAAK,OAAO,IAAI,WAAW,GAAG;AAAA,IAChC;AAAA,EACF;AAAA,EAEQ,aAAmB;AACzB,WAAO,KAAK,kBAAkB,KAAK,0BAA0B,KAAK,OAAO,OAAO,GAAG;AACjF,YAAM,OAAO,KAAK,OAAO,QAAQ,EAAE,KAAK,EAAE;AAC1C,UAAI,CAAC,KAAM;AACX,YAAM,CAAC,WAAW,GAAG,IAAI;AACzB,WAAK,OAAO,OAAO,SAAS;AAC5B,UAAI;AAAA,IACN;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,iBAAiB,SAAkF;AACjG,UAAM,QAAQ,KAAK,eAAe,UAAa,KAAK,MAAM,IAAI,KAAK,WAAW;AAC9E,QAAI,CAAC,SAAS,KAAK,kBAAkB,QAAW;AAC9C,WAAK,gBAAgB,QAAQ,EAC1B,KAAK,CAAC,UAAU;AACf,aAAK,aAAa,EAAE,OAAO,WAAW,KAAK,MAAM,IAAI,KAAK,MAAM;AAChE,aAAK,oBAAoB;AAAA,MAC3B,CAAC,EACA,MAAM,CAAC,UAAmB;AACzB,aAAK,oBAAoB;AAAA,MAG3B,CAAC,EACA,QAAQ,MAAM;AACb,aAAK,gBAAgB;AAAA,MACvB,CAAC;AAAA,IACL;AACA,WAAO,KAAK,YAAY;AAAA,EAC1B;AAAA,EAEA,uBAAuB,SAGrB;AACA,UAAM,QAAQ,KAAK,iBAAiB,OAAO;AAC3C,WAAO,EAAE,OAAO,cAAc,KAAK,kBAAkB;AAAA,EACvD;AAAA;AAAA;AAAA,EAGA,MAAM,WAA0B;AAC9B,WAAO,KAAK,SAAS,OAAO,KAAK,KAAK,kBAAkB,UAAa,KAAK,OAAO,OAAO,GAAG;AACzF,YAAM,UAA8B,CAAC,GAAG,KAAK,SAAS,OAAO,CAAC;AAC9D,UAAI,KAAK,cAAe,SAAQ,KAAK,KAAK,aAAa;AACvD,UAAI,QAAQ,WAAW,GAAG;AACxB,aAAK,WAAW;AAChB,cAAM,QAAQ,QAAQ;AACtB;AAAA,MACF;AACA,YAAM,QAAQ,IAAI,OAAO;AAAA,IAC3B;AAAA,EACF;AACF;AA4BA,eAAsB,+BACpB,MACA,UAAyC,CAAC,GACd;AAC5B,UAAQ,MAAM,2BAA2B,MAAM,OAAO,GAAG;AAC3D;AAWA,eAAsB,2BACpB,MACA,UAAyC,CAAC,GACqB;AAC/D,MAAI;AACJ,MAAI,QAAQ,OAAO;AACjB,UAAM,EAAE,OAAO,aAAa,aAAa,IAAI,QAAQ,MAAM;AAAA,MAAuB,MAChF,4BAA4B,EAAE,QAAQ,KAAK,QAAQ,0BAA0B,KAAK,CAAC;AAAA,IACrF;AACA,QAAI,iBAAiB,OAAW,QAAO,EAAE,YAAY,CAAC,GAAG,UAAU,MAAM;AACzE,YAAQ;AAAA,EACV,OAAO;AACL,QAAI;AACF,cAAQ,MAAM,4BAA4B;AAAA,QACxC,QAAQ,KAAK;AAAA,QACb,0BAA0B;AAAA,MAC5B,CAAC;AAAA,IACH,QAAQ;AACN,aAAO,EAAE,YAAY,CAAC,GAAG,UAAU,MAAM;AAAA,IAC3C;AAAA,EACF;AAEA,MAAI,CAAC,MAAO,QAAO,EAAE,YAAY,CAAC,GAAG,UAAU,MAAM;AACrD,QAAM,UAAU,MAAM,OAAO,CAAC,SAAS,KAAK,WAAW,KAAK,CAAC,OAAO,0BAA0B,QAAQ,MAAM,EAAE,CAAC,CAAC;AAChH,QAAM,aAAgC,CAAC;AACvC,MAAI,WAAW;AACf,aAAW,EAAE,UAAU,KAAK,SAAS;AACnC,UAAM,UAAU,YACd,0BAA0B,WAAW,MAAM,KAAK,WAAW,SAAS,GAAG,QAAQ,GAAG;AACpF,QAAI,QAAQ,OAAO;AACjB,YAAM,SAAS,QAAQ,MAAM,IAAI,WAAW,OAAO;AACnD,UAAI,OAAQ,YAAW,KAAK,MAAM;AAAA,UAC7B,YAAW;AAAA,IAClB,OAAO;AACL,UAAI;AACF,mBAAW,KAAK,MAAM,QAAQ,CAAC;AAAA,MACjC,QAAQ;AAEN,mBAAW;AAAA,MACb;AAAA,IACF;AAAA,EACF;AACA,SAAO,EAAE,YAAY,SAAS;AAChC;","names":[]}

package/dist/{chunk-SD4A6IZY.js → chunk-PWKZV4A3.js}

@@ -28,7 +28,7 @@ import {
   defineOperation,
   parseNamespacePreflightWriteOp,
   resolveAuthorizedNamespaceWritablePreflight
-} from "./chunk-XYXYEDPL.js";
+} from "./chunk-Q4XL7LJW.js";
 import {
   projectTagProjectId
 } from "./chunk-BUE5E2HL.js";
@@ -494,4 +494,4 @@ defineOperation({ name: "graph_events", description: "SSE graph events.", schema
 defineOperation({ name: "chat_message", description: "Chat message.", schema: strictSchema({ message: S.str, chatSessionId: S.str }), handler: async (_i, _ctx) => ({ result: { ok: true } }) });
 defineOperation({ name: "chat_events", description: "SSE chat events.", schema: strictSchema({}), handler: async (_i, _ctx) => ({ result: { stream: "sse" } }) });
 defineOperation({ name: "adapters_status", description: "Adapter registry metadata (issue #1850 round 5). Marker op: the HTTP transport builds the response from its own adapterRegistry, but the op still must be registered so it appears in listRegisteredOperations and enforceTokenOp can gate it for scoped tokens.", schema: strictSchema({}), handler: async (_i, _ctx) => ({ result: { ok: true } }) });
-//# sourceMappingURL=chunk-SD4A6IZY.js.map
+//# sourceMappingURL=chunk-PWKZV4A3.js.map

package/dist/{chunk-XYXYEDPL.js → chunk-Q4XL7LJW.js}

@@ -43,6 +43,9 @@ import {
   boundRecallContextComposition,
   composeRecallContext
 } from "./chunk-AHVEHR6K.js";
+import {
+  readAggregateExtractionWatermark
+} from "./chunk-7VIB7PAE.js";
 import {
   buildProposedActions,
   buildQualityScore,
@@ -101,7 +104,7 @@ import {
 import {
   CorpusWatermarkCache,
   computeServiceCorpusCensus
-} from "./chunk-AXHMCRS3.js";
+} from "./chunk-OYFDEJZB.js";
 import {
   namespaceCollectionName
 } from "./chunk-IOQFPVGY.js";
@@ -211,7 +214,7 @@ import {
 import {
   ExtractionLivenessWarnThrottle,
   computeExtractionLivenessStatus
-} from "./chunk-443XCR34.js";
+} from "./chunk-3SG4FGDV.js";
 import {
   sanitizeMemoryContent
 } from "./chunk-5GKPFL3S.js";
@@ -10437,8 +10440,26 @@ var EngramAccessService = class {
     const storage = await this.orchestrator.getStorage(resolvedNamespace);
     const searchBackend = this.orchestrator.config.searchBackend ?? "qmd";
     const qmdEnabled = resolveQmdCapabilities(this.orchestrator.config).qmd === true;
-    const extraction = await computeExtractionLivenessStatus(this.orchestrator, this.extractionLivenessWarn);
     const caps = tokenCapabilityStore.getStore();
+    const extractionWatermark = await readAggregateExtractionWatermark({
+      config: this.orchestrator.config,
+      rootStorage: this.orchestrator.storage,
+      storageForNamespace: (candidate) => this.orchestrator.getStorage(candidate),
+      rootsCache: this.corpusWatermarkCache
+    });
+    const extraction = await computeExtractionLivenessStatus(
+      this.orchestrator,
+      extractionWatermark,
+      caps?.namespaces !== void 0 ? void 0 : this.extractionLivenessWarn
+    );
+    const visibleExtraction = caps?.namespaces !== void 0 ? {
+      ...extraction,
+      lastExtractionAt: null,
+      bufferedSessionCount: 0,
+      pendingTurnCount: 0,
+      oldestBufferedTurnAgeMs: null,
+      degradedReason: extraction.degraded ? "daemon extraction pipeline degraded" : null
+    } : extraction;
     const corpusCensus = await computeServiceCorpusCensus(this.orchestrator, {
       cache: this.corpusWatermarkCache,
       caps
@@ -10465,7 +10486,7 @@ var EngramAccessService = class {
       ),
       nativeKnowledgeEnabled: this.orchestrator.config.nativeKnowledge?.enabled === true,
       projectionAvailable,
-      extraction,
+      extraction: visibleExtraction,
       corpus: corpusCensus.watermarks,
       // Describes THIS response's array (round 8): a separate scan could report
       // complete while the shipped corpus was partial, and peers trust it.
@@ -13323,4 +13344,4 @@ export {
   EngramAccessService,
   AccessAdminOpsSurface
 };
-//# sourceMappingURL=chunk-XYXYEDPL.js.map
+//# sourceMappingURL=chunk-Q4XL7LJW.js.map