@remnic/core 9.3.700 → 9.3.702

package/src/summary-snapshot.ts

@@ -1,18 +1,14 @@
-import {
-  mkdir,
-  open,
-  readFile,
-  stat,
-  unlink,
-  utimes,
-  writeFile,
-} from "node:fs/promises";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { z } from "zod";
 import {
   encodeStoragePathSegment,
   resolvePathInsideStorageRoot,
 } from "./storage-paths.js";
+import {
+  serializeMutations,
+  withHeldFileLock,
+} from "./utils/serialize-mutations.js";
 import type { HourlySummary } from "./types.js";
 
 const summarySnapshotSchemaVersion = 1;
@@ -34,7 +30,9 @@ const SummarySnapshotSchema = z.object({
 
 type SummarySnapshot = z.infer<typeof SummarySnapshotSchema>;
 
-const summarySnapshotUpserts = new Map<string, Promise<void>>();
+// Lock timings for the cross-process summary-snapshot file lock. These are
+// passed straight through to the shared `withHeldFileLock` utility (issue
+// #1524 adoption); the previous bespoke lock used the same numbers.
 const summarySnapshotLockTimeoutMs = 5_000;
 const summarySnapshotLockStaleMs = 30_000;
 const summarySnapshotLockHeartbeatMs = Math.max(
@@ -137,31 +135,65 @@ export async function writeSummarySnapshot(
   await writeFile(filePath, JSON.stringify(payload, null, 2), "utf-8");
 }
 
+// ── Concurrency primitives (issue #1524 adoption) ─────────────────────────
+//
+// Summary-snapshot upserts were previously guarded by TWO bespoke serializers:
+//   1. an in-process `summarySnapshotUpserts` map keyed by sessionKey that
+//      chained each upsert off the prior one's release (mirroring
+//      `serializeMutations`), and
+//   2. an on-disk `withExclusiveSummarySnapshotFileLock` that re-implemented
+//      `open(path, "wx")` acquire, mtime heartbeat, stale-break, and
+//      ownership-checked release.
+// Both now delegate to the shared utility so there is ONE home for each
+// primitive (`serializeMutations` for in-process, `withHeldFileLock` for
+// cross-process). The behavior contract is preserved:
+//   - in-process upserts for the SAME sessionKey are strictly serialized
+//     (read-merge-write cannot interleave);
+//   - a cross-process holder blocks up to `summarySnapshotLockTimeoutMs`, then
+//     the upsert FAILS (a snapshot is advisory; we never race a read-merge-write
+//     unlocked). To preserve this strictness on top of a best-effort utility,
+//     the work callback throws when `acquired === false`.
+
 async function withSummarySnapshotLock<T>(
   memoryDir: string,
   sessionKey: string,
   work: () => Promise<T>,
 ): Promise<T> {
-  const previous = summarySnapshotUpserts.get(sessionKey) ?? Promise.resolve();
-  let release!: () => void;
-  const current = new Promise<void>((resolve) => {
-    release = resolve;
-  });
-  const chained = previous.then(() => current);
-  summarySnapshotUpserts.set(sessionKey, chained);
-
-  await previous;
-  try {
-    return await withExclusiveSummarySnapshotFileLock(
+  // In-process serialization (one upsert per sessionKey at a time). The
+  // serializer recovers from rejection, so a failed upsert never poisons the
+  // next one — matching the previous chain's `.then(noop, noop)` recovery.
+  return serializeMutations(`summary-snapshot:${sessionKey}`, () =>
+    // Cross-process mutex via the shared utility. The lock path, stale
+    // threshold, bounded wait, and heartbeat cadence all flow through; the
+    // utility's replacement-safe stale break (NG7Bg) and ownership-checked
+    // release are stronger than the bare `unlink` this module used before.
+    withHeldFileLock(
       summarySnapshotLockPath(memoryDir, sessionKey),
-      work,
-    );
-  } finally {
-    release();
-    if (summarySnapshotUpserts.get(sessionKey) === chained) {
-      summarySnapshotUpserts.delete(sessionKey);
-    }
-  }
+      {
+        staleMs: summarySnapshotLockStaleMs,
+        maxWaitMs: summarySnapshotLockTimeoutMs,
+        heartbeatMs: summarySnapshotLockHeartbeatMs,
+      },
+      async (acquired) => {
+        // Strict-fail when the lock could not be acquired: the upsert is a
+        // read-merge-write, so a best-effort unlocked run would clobber a
+        // concurrent writer. The utility's `acquired === false` covers BOTH a
+        // genuine contention timeout AND a filesystem acquire failure (lock-dir
+        // mkdir/open/permission errors — the advisory lock is best-effort, so
+        // the util degrades rather than throwing). The bespoke lock this
+        // replaced propagated fs errors verbatim and reserved the timeout
+        // message for contention; we no longer claim "timed out" for an fs
+        // failure (cursor Low 25143f4f) — the message names both causes so
+        // upstream fail-open (runHourly) is unchanged but debugging is honest.
+        if (!acquired) {
+          throw new Error(
+            "could not acquire summary snapshot lock (contention timeout or filesystem error)",
+          );
+        }
+        return work();
+      },
+    ),
+  );
 }
 
 export async function upsertSummarySnapshot(
@@ -185,54 +217,3 @@ export async function upsertSummarySnapshot(
     await writeSummarySnapshot(memoryDir, summary.sessionKey, next);
   });
 }
-
-async function withExclusiveSummarySnapshotFileLock<T>(
-  lockPath: string,
-  callback: () => Promise<T>,
-): Promise<T> {
-  await mkdir(path.dirname(lockPath), { recursive: true });
-  const startedAt = Date.now();
-
-  while (true) {
-    try {
-      const handle = await open(lockPath, "wx");
-      let heartbeat: NodeJS.Timeout | null = null;
-      if (summarySnapshotLockHeartbeatMs > 0) {
-        heartbeat = setInterval(() => {
-          void utimes(lockPath, new Date(), new Date()).catch(() => undefined);
-        }, summarySnapshotLockHeartbeatMs);
-        heartbeat.unref?.();
-      }
-      try {
-        return await callback();
-      } finally {
-        if (heartbeat) clearInterval(heartbeat);
-        await handle.close().catch(() => undefined);
-        await unlink(lockPath).catch(() => undefined);
-      }
-    } catch (error) {
-      if (!isAlreadyExistsError(error)) throw error;
-      try {
-        const lockStat = await stat(lockPath);
-        if (Date.now() - lockStat.mtimeMs > summarySnapshotLockStaleMs) {
-          await unlink(lockPath).catch(() => undefined);
-          continue;
-        }
-      } catch {
-        continue;
-      }
-      if (Date.now() - startedAt > summarySnapshotLockTimeoutMs) {
-        throw new Error("timed out acquiring summary snapshot lock");
-      }
-      await sleep(10);
-    }
-  }
-}
-
-function isAlreadyExistsError(error: unknown): boolean {
-  return typeof error === "object" && error !== null && "code" in error && error.code === "EEXIST";
-}
-
-function sleep(ms: number): Promise<void> {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}

package/src/targeted-fact-recall.ts

@@ -9,6 +9,11 @@ import {
   resolveLcmReadSessionIds,
 } from "./lcm-fallback-read.js";
 
+import {
+  unifiedDedupeAndRank,
+  type RankedEvidenceItem,
+} from "./recall-pipeline-stages.js";
+
 export interface TargetedFactRecallOptions {
   engine: ExplicitCueRecallEngine | null | undefined;
   sessionId?: string;
@@ -27,10 +32,6 @@ export interface TargetedFactRecallOptions {
   title?: string;
 }
 
-interface RankedEvidenceItem extends EvidencePackItem {
-  rank: number;
-}
-
 const DEFAULT_MAX_SEARCH_RESULTS = 48;
 const DEFAULT_SCAN_WINDOW_TURNS = 8;
 const DEFAULT_SCAN_WINDOW_TOKENS = 12_000;
@@ -210,38 +211,30 @@ async function collectTargetedFactScanItems(
   return items;
 }
 
+/**
+ * Issue #1539 PR3: targeted-fact now routes dedup/score/threshold/sort through
+ * the unified spine (`unifiedDedupeAndRank`). The divergence that lived inline
+ * here — append-normalized-numeric-cues content transform, relevance scoring on
+ * ORIGINAL content, default DESC turn-index ordering with -1 sentinel for
+ * missing turns — is now declared config on the spine. The score and
+ * appendNormalizedNumericCues helpers stay in this module because they are
+ * genuinely per-tier policy; only the duplicated dedup/sort/threshold machinery
+ * moved out.
+ *
+ * Behavior is byte-for-byte identical to the inline implementation it replaces
+ * (the spine was extracted to match it exactly; see recall-pipeline-stages.ts).
+ * The targeted-fact-recall.test.ts suite (42 cases, incl. the search/scan
+ * ordering and summary insertion tests) is the characterization.
+ */
 function rankAndDedupeTargetedFactItems(
   items: EvidencePackItem[],
   query: string,
 ): RankedEvidenceItem[] {
-  const seenIds = new Set<string>();
-  const seenContent = new Set<string>();
-  const ranked: RankedEvidenceItem[] = [];
-
-  for (const item of items) {
-    const id = item.id ?? (
-      item.sessionId && typeof item.turnIndex === "number"
-        ? `${item.sessionId}:${item.turnIndex}`
-        : undefined
-    );
-    if (id && seenIds.has(id)) continue;
-    const contentKey = item.content.toLowerCase().replace(/\s+/g, " ").trim();
-    if (seenContent.has(contentKey)) continue;
-    if (id) seenIds.add(id);
-    seenContent.add(contentKey);
-    ranked.push({
-      ...item,
-      content: appendNormalizedNumericCues(item.content),
-      rank: scoreTargetedFactEvidence(item, query),
-    });
-  }
-
-  return ranked.sort((left, right) => {
-    if (right.rank !== left.rank) return right.rank - left.rank;
-    const leftTurn = typeof left.turnIndex === "number" ? left.turnIndex : -1;
-    const rightTurn = typeof right.turnIndex === "number" ? right.turnIndex : -1;
-    if (rightTurn !== leftTurn) return rightTurn - leftTurn;
-    return (right.score ?? 0) - (left.score ?? 0);
+  return unifiedDedupeAndRank(items, {
+    query,
+    intents: [],
+    scoreEvidence: (item, q) => scoreTargetedFactEvidence(item, q),
+    transformContent: (content) => appendNormalizedNumericCues(content),
   });
 }
 

package/src/types.ts

@@ -2852,6 +2852,30 @@ export interface ExtractedFact {
    * "no verified span" and skip gracefully.
    */
   sources?: ProvenanceSource[];
+  /**
+   * Coarse provenance strength tag (issue #1575 PR 2). Set by the extraction
+   * validator: "verified" when the quote was located in source turns,
+   * "unverified" when the quote survived but no offsets were recoverable.
+   * Absent when `provenance.enabled` is false or no quote was provided —
+   * readers treat absent as "none" (rule 34 spirit).
+   */
+  provenance?: "verified" | "unverified" | "none";
+  /**
+   * Transient LLM-provided supporting quote (issue #1575 PR 2). Consumed by
+   * the post-parse validator to build `sources`, then stripped before the
+   * fact reaches persistence. NEVER included in content-hash dedup (rule 23 /
+   * checklist §13). Absent on facts that have already passed the validator.
+   */
+  quote?: string;
+  /**
+   * Transient requireSpans signal (issue #1575 PR 2). Set by the provenance
+   * validator (`ProvenanceBuildResult.requireSpansPending`) when
+   * `provenance.requireSpans` is enabled and the quote could not be located
+   * in any source turn. Read by the persist path to route the fact to
+   * `pending_review` instead of `active`. Stripped before persistence —
+   * it never reaches frontmatter (chatgpt-codex-connector thread 4xB).
+   */
+  requireSpansPending?: boolean;
   promptedByQuestion?: string;
   /**
    * Whether this fact is project-scoped or globally applicable.

package/src/utils/serialize-mutations.ts

@@ -670,10 +670,14 @@ async function lockHeldBySelf(held: HeldLock): Promise<boolean> {
 }
 
 function sleep(ms: number): Promise<void> {
-  const { promise, resolve } = Promise.withResolvers<void>();
-  // NOT unref'd: this polls inside an awaited acquire loop, so the caller's
-  // await chain keeps the loop alive; unref would let Node exit mid-poll when
-  // nothing else is pending (the heartbeat interval IS unref'd separately).
-  setTimeout(resolve, ms);
-  return promise;
+  // Manual deferred instead of Promise.withResolvers (ES2024) — plugin-openclaw's
+  // standalone tsconfig targets ES2022 lib and this module is reachable from its
+  // type graph, so withResolvers would TS2550 there (same fix as
+  // extraction-faithfulness.ts:467).
+  return new Promise<void>((resolve) => {
+    // NOT unref'd: this polls inside an awaited acquire loop, so the caller's
+    // await chain keeps the loop alive; unref would let Node exit mid-poll when
+    // nothing else is pending (the heartbeat interval IS unref'd separately).
+    setTimeout(resolve, ms);
+  });
 }