@remnic/core 9.69.88 → 9.69.90

package/src/rerank-outcome.test.ts

@@ -0,0 +1,218 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { parseConfig } from "./config.js";
+import type { MemoryFile, QmdSearchResult } from "./types.js";
+import {
+  OUTCOME_STATE_OBSERVED,
+  OUTCOME_STATE_UNOBSERVED,
+  applyOutcomePriorScores,
+  blendOutcomeScore,
+  outcomePriorBoostActive,
+  parseOutcomeBoostWeight,
+  rerankWithOutcomePrior,
+  resolveOutcomeObservation,
+  scaleTextScore,
+  type OutcomeObservationState,
+} from "./rerank-outcome.js";
+
+// @ts-expect-error "bogus" is not an OutcomeObservationState
+const badOutcomeState: OutcomeObservationState = "bogus";
+void badOutcomeState;
+
+function memory(counters: { mw_success?: number; mw_fail?: number }): MemoryFile {
+  return {
+    path: "facts/a.md",
+    content: "synthetic fact",
+    frontmatter: counters as MemoryFile["frontmatter"],
+  };
+}
+
+function result(path: string, score: number, namespace?: string): QmdSearchResult {
+  return { docid: path, path, snippet: "", score, namespace } as QmdSearchResult;
+}
+
+test("scaleTextScore clamps into [0, 1] without a batch min-max", () => {
+  assert.equal(scaleTextScore(0.4), 0.4);
+  assert.equal(scaleTextScore(2.5), 1);
+  assert.equal(scaleTextScore(-3), 0);
+  assert.equal(scaleTextScore(Number.NaN), 0);
+  assert.equal(scaleTextScore(Number.POSITIVE_INFINITY), 0);
+});
+
+test("UNOBSERVED is not the Memory Worth Beta prior", () => {
+  const unobserved = resolveOutcomeObservation(undefined);
+  assert.equal(unobserved.state, OUTCOME_STATE_UNOBSERVED);
+  assert.equal(blendOutcomeScore(1, unobserved, 0.5), 1);
+  assert.notEqual(blendOutcomeScore(1, unobserved, 0.5), 0.75);
+
+  const zeros = resolveOutcomeObservation({ success: 0, fail: 0 });
+  assert.equal(zeros.state, OUTCOME_STATE_UNOBSERVED);
+
+  const missing = resolveOutcomeObservation({});
+  assert.equal(missing.state, OUTCOME_STATE_UNOBSERVED);
+
+  const invalid = resolveOutcomeObservation({ success: Number.NaN, fail: 1 });
+  assert.equal(invalid.state, OUTCOME_STATE_UNOBSERVED);
+  const negative = resolveOutcomeObservation({ success: -1, fail: 2 });
+  assert.equal(negative.state, OUTCOME_STATE_UNOBSERVED);
+
+  const observed = resolveOutcomeObservation({ success: 0, fail: 1 });
+  assert.equal(observed.state, OUTCOME_STATE_OBSERVED);
+  assert.equal(observed.outcomeScore, 0);
+  assert.equal(blendOutcomeScore(1, observed, 0.5), 0.5);
+  assert.notEqual(blendOutcomeScore(1, observed, 0.5), 2 / 3);
+
+  const oneSided = resolveOutcomeObservation({ success: 2 });
+  assert.equal(oneSided.state, OUTCOME_STATE_OBSERVED);
+  assert.equal(oneSided.outcomeScore, 1);
+});
+
+test("pre-main unit gates cover ties, equal scores, and w=0 identity", () => {
+  const unobserved = resolveOutcomeObservation(undefined);
+  const observedZero = resolveOutcomeObservation({ success: 0, fail: 4 });
+  assert.equal(unobserved.state, OUTCOME_STATE_UNOBSERVED);
+  assert.equal(observedZero.state, OUTCOME_STATE_OBSERVED);
+  assert.equal(blendOutcomeScore(1.7, unobserved, 0), 1.7);
+  assert.equal(blendOutcomeScore(1.7, observedZero, 0), 1.7);
+  assert.equal(blendOutcomeScore(1.7, observedZero, 0.5), scaleTextScore(1.7) * 0.5);
+
+  const tied = rerankWithOutcomePrior(
+    [
+      { item: "left", textScore: 0.4, success: 3, fail: 1 },
+      { item: "right", textScore: 0.4, success: 0, fail: 9 },
+    ],
+    0,
+  );
+  assert.deepEqual(
+    tied.map((row) => [row.item, row.score, row.state]),
+    [
+      ["left", 0.4, OUTCOME_STATE_OBSERVED],
+      ["right", 0.4, OUTCOME_STATE_OBSERVED],
+    ],
+  );
+
+  const equalBlend = rerankWithOutcomePrior(
+    [
+      { item: "first", textScore: 0.2, success: 1, fail: 1 },
+      { item: "second", textScore: 0.2, success: 1, fail: 1 },
+    ],
+    0.5,
+  );
+  assert.deepEqual(
+    equalBlend.map((row) => row.item),
+    ["first", "second"],
+  );
+  assert.equal(equalBlend[0]?.score, equalBlend[1]?.score);
+});
+
+test("weight 0 keeps raw scores and original order", () => {
+  const ranked = rerankWithOutcomePrior(
+    [
+      { item: "a", textScore: 1.4, success: 0, fail: 5 },
+      { item: "b", textScore: 0.2 },
+      { item: "c", textScore: -1, success: 9, fail: 0 },
+    ],
+    0,
+  );
+  assert.deepEqual(
+    ranked.map((row) => [row.item, row.score]),
+    [
+      ["a", 1.4],
+      ["b", 0.2],
+      ["c", -1],
+    ],
+  );
+});
+
+test("equal blended scores keep the original index order", () => {
+  const ranked = rerankWithOutcomePrior(
+    [
+      { item: "first", textScore: 1 },
+      { item: "second", textScore: 0, success: 1, fail: 0 },
+    ],
+    1,
+  );
+  assert.deepEqual(
+    ranked.map((row) => row.item),
+    ["first", "second"],
+  );
+  assert.equal(ranked[0]?.score, ranked[1]?.score);
+});
+
+test("observed outcomes reorder above a weaker text score", () => {
+  const ranked = rerankWithOutcomePrior(
+    [
+      { item: "text", textScore: 0.9 },
+      { item: "worked", textScore: 0.2, success: 4, fail: 0 },
+    ],
+    1,
+  );
+  assert.deepEqual(
+    ranked.map((row) => row.item),
+    ["worked", "text"],
+  );
+});
+
+test("parseOutcomeBoostWeight rejects invalid values and keeps zero", () => {
+  assert.equal(parseOutcomeBoostWeight(undefined), 0);
+  assert.equal(parseOutcomeBoostWeight(""), 0);
+  assert.equal(parseOutcomeBoostWeight("0"), 0);
+  assert.equal(parseOutcomeBoostWeight("0.3"), 0.3);
+  assert.throws(() => parseOutcomeBoostWeight(false), /outcomeBoostWeight/);
+  assert.throws(() => parseOutcomeBoostWeight("false"), /outcomeBoostWeight/);
+  assert.throws(() => parseOutcomeBoostWeight("1.5"), /outcomeBoostWeight/);
+  assert.throws(() => parseOutcomeBoostWeight(Number.NaN), /outcomeBoostWeight/);
+  assert.throws(() => rerankWithOutcomePrior([{ item: "a", textScore: 1 }], 1.2), /outcomeBoostWeight/);
+});
+
+test("parseConfig leaves the outcome boost off", () => {
+  const defaults = parseConfig({});
+  assert.equal(defaults.outcomeBoostEnabled, false);
+  assert.equal(defaults.outcomeBoostWeight, 0);
+  assert.equal(outcomePriorBoostActive(defaults), false);
+  assert.equal(parseConfig({ outcomeBoostEnabled: "false" }).outcomeBoostEnabled, false);
+  assert.equal(parseConfig({ outcomeBoostEnabled: "0" }).outcomeBoostEnabled, false);
+  assert.equal(parseConfig({ outcomeBoostWeight: "0" }).outcomeBoostWeight, 0);
+  assert.equal(outcomePriorBoostActive(parseConfig({ outcomeBoostEnabled: true, outcomeBoostWeight: 0 })), false);
+  assert.equal(outcomePriorBoostActive(parseConfig({ outcomeBoostEnabled: false, outcomeBoostWeight: 0.3 })), false);
+  assert.equal(outcomePriorBoostActive(parseConfig({ outcomeBoostEnabled: true, outcomeBoostWeight: 0.3 })), true);
+  assert.throws(() => parseConfig({ outcomeBoostWeight: "no" }), /outcomeBoostWeight/);
+});
+
+test("applyOutcomePriorScores is an identity at weight 0 and uses the namespace key", () => {
+  const untouched = [result("facts/a.md", 1.4, "ns-a"), result("facts/a.md", 0.2)];
+  const before = untouched.map((row) => ({ path: row.path, score: row.score, namespace: row.namespace }));
+  applyOutcomePriorScores(untouched, new Map(), { outcomeBoostWeight: 0 });
+  assert.deepEqual(
+    untouched.map((row) => ({ path: row.path, score: row.score, namespace: row.namespace })),
+    before,
+  );
+  const negative = [result("facts/a.md", 1.4)];
+  applyOutcomePriorScores(negative, new Map(), { outcomeBoostWeight: -1 });
+  assert.equal(negative[0]?.score, 1.4);
+  assert.throws(
+    () => applyOutcomePriorScores([result("facts/a.md", 1)], new Map(), { outcomeBoostWeight: 1.2 }),
+    /outcomeBoostWeight/,
+  );
+
+  const namespaced = resolveOutcomeObservation({ success: 0, fail: 1 });
+  assert.equal(namespaced.state, OUTCOME_STATE_OBSERVED);
+  assert.equal(namespaced.outcomeScore, 0);
+  const bare = resolveOutcomeObservation({ success: 8, fail: 0 });
+  assert.equal(bare.state, OUTCOME_STATE_OBSERVED);
+  assert.equal(bare.outcomeScore, 1);
+
+  const results = [result("facts/a.md", 1, "ns-a"), result("facts/a.md", 0.2)];
+  const memoryByPath = new Map<string, MemoryFile>([
+    ["ns-a\0facts/a.md", memory({ mw_success: 0, mw_fail: 1 })],
+    ["facts/a.md", memory({ mw_success: 8, mw_fail: 0 })],
+  ]);
+  applyOutcomePriorScores(results, memoryByPath, { outcomeBoostWeight: 1 });
+  // 0/1 is an observed rate of 0, not UNOBSERVED. The namespaced key must not
+  // read the bare-path 8/0 counters. At weight 1 the bare path (rate 1) sorts
+  // ahead of ns-a (rate 0).
+  assert.equal(results[0]?.namespace, undefined);
+  assert.equal(results[0]?.score, 1);
+  assert.equal(results[1]?.namespace, "ns-a");
+  assert.equal(results[1]?.score, 0);
+});

package/src/rerank-outcome.ts

@@ -0,0 +1,219 @@
+/**
+ * H1 outcome-prior blend (issue #1958). Scaffolding only — no experiment runner.
+ *
+ * Text scores map onto [0, 1] with a fixed clamp. The clamp is not a batch
+ * min-max, so one candidate cannot move another's scale.
+ *
+ * UNOBSERVED (no past result) keeps that unit text score when the weight is
+ * positive. It does not use the Memory Worth Laplace prior (s+1)/(s+f+2).
+ * An observed result uses the maximum-likelihood rate success/(success+fail).
+ *
+ * Turning the blend on (enabled and weight > 0) clamps every candidate's text
+ * score into [0, 1] before mixing, including UNOBSERVED items. Raw scores
+ * outside that range can change order relative to the weight-0 path. Weight 0
+ * does not clamp.
+ *
+ * Weight 0 is an identity on the raw input score: same values, same order.
+ * The recall pipeline calls `applyOutcomePriorScores` only when
+ * `outcomeBoostEnabled === true` and the weight is greater than 0, so the
+ * production default stays on the existing sort.
+ */
+
+import {
+  type RecallEnhancementConfigProjection,
+  resolveRecallEnhancementCapabilities,
+} from "./capabilities.js";
+import { coerceNumber } from "./connectors/coerce.js";
+import { memoryForResult } from "./recall-memory-map.js";
+import type { MemoryFile, QmdSearchResult } from "./types.js";
+
+export const OUTCOME_STATE_UNOBSERVED = "UNOBSERVED" as const;
+export const OUTCOME_STATE_OBSERVED = "OBSERVED" as const;
+
+export type OutcomeObservationState =
+  | typeof OUTCOME_STATE_UNOBSERVED
+  | typeof OUTCOME_STATE_OBSERVED;
+
+export interface OutcomeCounters {
+  success?: number;
+  fail?: number;
+}
+
+export interface OutcomeObservation {
+  state: OutcomeObservationState;
+  /** Present only for OBSERVED. Maximum-likelihood rate in [0, 1]. */
+  outcomeScore?: number;
+}
+
+export function scaleTextScore(score: number): number {
+  if (!Number.isFinite(score) || score <= 0) return 0;
+  if (score >= 1) return 1;
+  return score;
+}
+
+function finiteNonNegative(value: number | undefined): number | undefined {
+  if (value === undefined) return undefined;
+  if (typeof value !== "number" || !Number.isFinite(value) || value < 0) return undefined;
+  return value;
+}
+
+/**
+ * Both counters missing, explicit 0+0, or any present-but-invalid counter
+ * (non-finite or negative) is UNOBSERVED. One counter present and positive
+ * with the other missing is OBSERVED; the missing counter counts as 0.
+ */
+export function resolveOutcomeObservation(
+  counters: OutcomeCounters | undefined,
+): OutcomeObservation {
+  if (!counters) return { state: OUTCOME_STATE_UNOBSERVED };
+  const successPresent = counters.success !== undefined;
+  const failPresent = counters.fail !== undefined;
+  if (!successPresent && !failPresent) return { state: OUTCOME_STATE_UNOBSERVED };
+
+  const success = finiteNonNegative(counters.success);
+  const fail = finiteNonNegative(counters.fail);
+  if (successPresent && success === undefined) return { state: OUTCOME_STATE_UNOBSERVED };
+  if (failPresent && fail === undefined) return { state: OUTCOME_STATE_UNOBSERVED };
+
+  const successes = success ?? 0;
+  const failures = fail ?? 0;
+  if (successes === 0 && failures === 0) return { state: OUTCOME_STATE_UNOBSERVED };
+  return {
+    state: OUTCOME_STATE_OBSERVED,
+    outcomeScore: successes / (successes + failures),
+  };
+}
+
+/**
+ * Weight 0 returns `textScore` unchanged. A positive weight clamps `textScore`
+ * into [0, 1] first. UNOBSERVED stays on that clamped value. OBSERVED mixes it
+ * with the success rate.
+ */
+export function blendOutcomeScore(
+  textScore: number,
+  observation: OutcomeObservation,
+  weight: number,
+): number {
+  if (weight === 0) return textScore;
+  const textUnit = scaleTextScore(textScore);
+  if (observation.state === OUTCOME_STATE_UNOBSERVED || observation.outcomeScore === undefined) {
+    return textUnit;
+  }
+  return (1 - weight) * textUnit + weight * observation.outcomeScore;
+}
+
+export interface OutcomePriorCandidate<T> {
+  item: T;
+  textScore: number;
+  success?: number;
+  fail?: number;
+}
+
+export interface OutcomePriorRanked<T> {
+  item: T;
+  textScore: number;
+  score: number;
+  state: OutcomeObservationState;
+  originalIndex: number;
+}
+
+function assertOutcomeWeight(weight: number): void {
+  if (!Number.isFinite(weight) || weight < 0 || weight > 1) {
+    throw new Error(
+      `outcomeBoostWeight must be a finite number in [0, 1] (got ${JSON.stringify(weight)}).`,
+    );
+  }
+}
+
+function compareRanked(
+  left: { score: number; originalIndex: number },
+  right: { score: number; originalIndex: number },
+): number {
+  const leftScore = Number.isFinite(left.score) ? left.score : Number.NEGATIVE_INFINITY;
+  const rightScore = Number.isFinite(right.score) ? right.score : Number.NEGATIVE_INFINITY;
+  if (leftScore !== rightScore) return leftScore > rightScore ? -1 : 1;
+  if (left.originalIndex === right.originalIndex) return 0;
+  return left.originalIndex < right.originalIndex ? -1 : 1;
+}
+
+export function rerankWithOutcomePrior<T>(
+  candidates: readonly OutcomePriorCandidate<T>[],
+  weight: number,
+): OutcomePriorRanked<T>[] {
+  assertOutcomeWeight(weight);
+  const ranked = candidates.map((candidate, originalIndex) => {
+    const observation = resolveOutcomeObservation({
+      success: candidate.success,
+      fail: candidate.fail,
+    });
+    return {
+      item: candidate.item,
+      textScore: candidate.textScore,
+      score: blendOutcomeScore(candidate.textScore, observation, weight),
+      state: observation.state,
+      originalIndex,
+    };
+  });
+  if (weight === 0) return ranked;
+  ranked.sort(compareRanked);
+  return ranked;
+}
+
+export function outcomePriorBoostActive(
+  config: RecallEnhancementConfigProjection & { outcomeBoostWeight?: number },
+): boolean {
+  const weight = config.outcomeBoostWeight;
+  return (
+    resolveRecallEnhancementCapabilities(config).outcomeBoost === true &&
+    typeof weight === "number" &&
+    Number.isFinite(weight) &&
+    weight > 0
+  );
+}
+
+export function applyOutcomePriorScores(
+  results: QmdSearchResult[],
+  memoryByPath: ReadonlyMap<string, MemoryFile>,
+  config: { outcomeBoostWeight?: number },
+): void {
+  const weight = config.outcomeBoostWeight;
+  // Non-numbers, non-finite values, and negatives are a no-op. Zero is an
+  // identity. assertOutcomeWeight is the upper bound (> 1 throws); it is not
+  // a second copy of the positivity check.
+  if (typeof weight !== "number" || !Number.isFinite(weight) || weight < 0) return;
+  assertOutcomeWeight(weight);
+  if (weight === 0) return;
+  const ranked = results.map((result, originalIndex) => {
+    const memory = memoryForResult(memoryByPath, result);
+    const observation = resolveOutcomeObservation(
+      memory
+        ? { success: memory.frontmatter.mw_success, fail: memory.frontmatter.mw_fail }
+        : undefined,
+    );
+    return {
+      result,
+      originalIndex,
+      score: blendOutcomeScore(result.score, observation, weight),
+    };
+  });
+  ranked.sort(compareRanked);
+  const next = ranked.map((row) =>
+    row.score === row.result.score ? row.result : { ...row.result, score: row.score },
+  );
+  results.splice(0, results.length, ...next);
+}
+
+/**
+ * Absent / null / exact empty string → 0. A present value that is not a
+ * finite number in [0, 1] throws. Boolean false is not a zero weight.
+ */
+export function parseOutcomeBoostWeight(raw: unknown): number {
+  if (raw === undefined || raw === null || raw === "") return 0;
+  const parsed = coerceNumber(raw, "outcomeBoostWeight");
+  if (parsed === undefined || parsed < 0 || parsed > 1) {
+    throw new Error(
+      `outcomeBoostWeight must be a number in [0, 1] (got ${JSON.stringify(raw)}).`,
+    );
+  }
+  return parsed;
+}

package/src/support-passport/offline-sync-guard.ts

@@ -26,10 +26,14 @@ type FrontmatterDecision = "allow" | "wait";
 interface SupportPassportOfflineSyncStorage {
   dir: string;
   readMemoryByPath(filePath: string): Promise<MemoryFile | null>;
-  readOfflineSyncFile(filePath: string): Promise<Buffer>;
+  readOfflineSyncFile(filePath: string, opts?: { aadFilePath?: string }): Promise<Buffer>;
   digestOfflineSyncFile(filePath: string): Promise<{ sha256: string; bytes: number }>;
   writeOfflineSyncFile(filePath: string, content: Buffer): Promise<void>;
-  writeOfflineSyncStagingFile(filePath: string, content: Buffer): Promise<void>;
+  writeOfflineSyncStagingFile(
+    filePath: string,
+    content: Buffer,
+    opts?: { aadFilePath?: string },
+  ): Promise<void>;
   writeOfflineSyncFileChunks(filePath: string, chunks: AsyncIterable<Buffer>): Promise<void>;
   deleteOfflineSyncFile(filePath: string, deletionMtimeMs?: number | null): Promise<void>;
   recordReplicatedDeletionRevision(filePath: string, mtimeMs: number): Promise<void>;
@@ -219,6 +223,16 @@ export async function applySupportPassportOfflineSyncChangeset(
       guard.assertContentAllowed(target.path, target.content);
       await storage.writeOfflineSyncFile(target.filePath, target.content);
     },
+    // Secure staging for embedding-generation replacement (#3150): staged
+    // ciphertext binds the FINAL published path, matching the snapshot apply.
+    writeStagingFile: async ({ filePath, content, finalAadFilePath }) =>
+      storage.writeOfflineSyncStagingFile(
+        filePath,
+        content,
+        finalAadFilePath === undefined ? undefined : { aadFilePath: finalAadFilePath },
+      ),
+    readStagingFile: async ({ filePath, finalAadFilePath }) =>
+      storage.readOfflineSyncFile(filePath, finalAadFilePath === undefined ? undefined : { aadFilePath: finalAadFilePath }),
     deleteFile: async (target) => {
       await guard.assertTargetAllowed(target);
       await storage.deleteOfflineSyncFile(target.filePath, target.mtimeMs ?? null);

package/src/types.ts

@@ -10,9 +10,9 @@ import type { ReplicaPeersConfig } from "./replica-peers-config.js";
 import type { ExternalWikiRoot } from "./external-wiki-config.js";
 import type { StateViewResult } from "./recall-state-view.js";
 import type { ContradictionLocalizationConfig, ContradictionScanConfig } from "./contradiction-config.js";
-import type { GraphPathScoringConfig } from "./graph-path-scoring-config.js";
+import type { GraphPathScoringConfig } from "./graph-path-scoring-config.js"; export type { GraphPathScoringConfig };
+import type { OutcomeBoostConfig } from "./outcome-boost-config.js"; export type { OutcomeBoostConfig };
 export type { ContradictionLocalizationConfig, ContradictionScanConfig } from "./contradiction-config.js";
-export type { GraphPathScoringConfig } from "./graph-path-scoring-config.js";
 export type { BackgroundGenerationConfig } from "./background-generation-config.js";
 import type { BackgroundGenerationConfig } from "./background-generation-config.js";
 import type {
@@ -700,7 +700,7 @@ export interface PluginConfig
     RecallNavigationSettings,
     RecognitionTierSettings,
     SeedGraduationSettings,
-    LocalLlmConfig {
+    LocalLlmConfig, OutcomeBoostConfig {
   openaiApiKey: string | undefined;
   openaiBaseUrl: string | undefined;
   backgroundGeneration?: BackgroundGenerationConfig;