@remnic/core 1.0.3 → 1.1.1

package/dist/{chunk-POMSFKTB.js → chunk-F5VP6YCB.js}

@@ -1,15 +1,12 @@
 import {
   SPECULATIVE_TTL_DAYS,
   confidenceTier
-} from "./chunk-U2IQTSBY.js";
+} from "./chunk-LTCGGW2D.js";
 import {
   DEFAULT_CITATION_FORMAT,
   hasCitation,
   stripCitationForTemplate
 } from "./chunk-4KAN3GZ3.js";
-import {
-  createVersion
-} from "./chunk-6ZH4TU6I.js";
 import {
   getCachedEntities,
   setCachedEntities
@@ -30,6 +27,9 @@ import {
   readProjectedMemoryState,
   readProjectedMemoryTimeline
 } from "./chunk-BOUYNNYD.js";
+import {
+  rotateMarkdownFileToArchive
+} from "./chunk-DM2T26WE.js";
 import {
   closeContinuityIncidentRecord,
   createContinuityIncidentRecord,
@@ -39,9 +39,6 @@ import {
   serializeContinuityIncident,
   upsertContinuityLoopInMarkdown
 } from "./chunk-QSVPYQPG.js";
-import {
-  rotateMarkdownFileToArchive
-} from "./chunk-DM2T26WE.js";
 import {
   sanitizeMemoryContent
 } from "./chunk-M62O4P4T.js";
@@ -50,6 +47,13 @@ import {
   normalizeEntityStructuredSection,
   sortStructuredSectionsBySchema
 } from "./chunk-4DJQYKMN.js";
+import {
+  isConsolidationOperator,
+  isValidDerivedFromEntry
+} from "./chunk-X6GF3FX2.js";
+import {
+  createVersion
+} from "./chunk-FAAFWE4G.js";
 import {
   log
 } from "./chunk-2ODBA7MQ.js";
@@ -91,6 +95,17 @@ var ARTIFACT_SEARCH_STOPWORDS = /* @__PURE__ */ new Set([
 function tokenizeArtifactSearchText(input) {
   return input.toLowerCase().split(/[^a-z0-9]+/i).map((t) => t.trim()).filter((t) => t.length >= 2).filter((t) => !ARTIFACT_SEARCH_STOPWORDS.has(t));
 }
+function assertMemoryWorthCounter(field, value) {
+  if (typeof value !== "number" || !Number.isFinite(value)) {
+    throw new Error(`${field} must be a finite number, got ${String(value)}`);
+  }
+  if (!Number.isInteger(value)) {
+    throw new Error(`${field} must be an integer, got ${value}`);
+  }
+  if (value < 0) {
+    throw new Error(`${field} must be >= 0, got ${value}`);
+  }
+}
 function serializeFrontmatter(fm) {
   const lines = [
     "---",
@@ -123,6 +138,14 @@ function serializeFrontmatter(fm) {
     lines.push(`accessCount: ${fm.accessCount}`);
   }
   if (fm.lastAccessed) lines.push(`lastAccessed: ${fm.lastAccessed}`);
+  if (fm.mw_success !== void 0) {
+    assertMemoryWorthCounter("mw_success", fm.mw_success);
+    lines.push(`mw_success: ${fm.mw_success}`);
+  }
+  if (fm.mw_fail !== void 0) {
+    assertMemoryWorthCounter("mw_fail", fm.mw_fail);
+    lines.push(`mw_fail: ${fm.mw_fail}`);
+  }
   if (fm.importance) {
     lines.push(`importanceScore: ${fm.importance.score}`);
     lines.push(`importanceLevel: ${fm.importance.level}`);
@@ -160,6 +183,33 @@ function serializeFrontmatter(fm) {
     lines.push(`structuredAttributes: ${JSON.stringify(fm.structuredAttributes)}`);
   }
   if (fm.contentHash) lines.push(`contentHash: ${fm.contentHash}`);
+  if (fm.derived_from !== void 0) {
+    if (!Array.isArray(fm.derived_from)) {
+      throw new Error(
+        `serializeFrontmatter: derived_from must be an array of "<path>:<version>" strings`
+      );
+    }
+    for (const entry of fm.derived_from) {
+      if (!isValidDerivedFromEntry(entry)) {
+        throw new Error(
+          `serializeFrontmatter: invalid derived_from entry ${JSON.stringify(entry)} \u2014 expected "<path>:<version>" with version >= 0`
+        );
+      }
+    }
+    if (fm.derived_from.length > 0) {
+      lines.push(
+        `derived_from: [${fm.derived_from.map((e) => `"${e.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`).join(", ")}]`
+      );
+    }
+  }
+  if (fm.derived_via !== void 0) {
+    if (!isConsolidationOperator(fm.derived_via)) {
+      throw new Error(
+        `serializeFrontmatter: invalid derived_via ${JSON.stringify(fm.derived_via)} \u2014 expected one of "split" | "merge" | "update"`
+      );
+    }
+    lines.push(`derived_via: ${fm.derived_via}`);
+  }
   lines.push("---");
   return lines.join("\n");
 }
@@ -192,13 +242,54 @@ function parseLinkReasonValue(rawValue) {
     return legacyValue;
   }
 }
+function parseMemoryWorthCounterField(raw) {
+  if (raw === void 0) return void 0;
+  const trimmed = raw.trim();
+  if (trimmed.length === 0) return void 0;
+  const n = Number(trimmed);
+  if (!Number.isFinite(n) || !Number.isInteger(n) || n < 0) return void 0;
+  return n;
+}
 function parseFrontmatter(raw) {
   const match = raw.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
   if (!match) return null;
   const fmBlock = match[1];
   const content = match[2].trim();
   const fm = {};
-  for (const line of fmBlock.split("\n")) {
+  const rawLines = fmBlock.split("\n");
+  const lines = [];
+  let i = 0;
+  while (i < rawLines.length) {
+    const line = rawLines[i];
+    const colonIdx = line.indexOf(":");
+    if (colonIdx !== -1 && line.slice(colonIdx + 1).trim() === "") {
+      const baseIndent = line.match(/^\s*/)[0].length;
+      const items = [];
+      let j = i + 1;
+      while (j < rawLines.length) {
+        const next = rawLines[j];
+        const m = next.match(/^(\s+)- (.*)$/);
+        if (!m || m[1].length <= baseIndent) break;
+        let item = m[2].trim();
+        if (item.startsWith('"') && item.endsWith('"') && item.length >= 2) {
+          item = item.slice(1, -1).replace(/\\"/g, '"').replace(/\\\\/g, "\\");
+        } else if (item.startsWith("'") && item.endsWith("'") && item.length >= 2) {
+          item = item.slice(1, -1).replace(/''/g, "'");
+        }
+        items.push(item);
+        j++;
+      }
+      if (items.length > 0) {
+        const inline = items.map((v) => `"${v.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`).join(", ");
+        lines.push(`${line.slice(0, colonIdx + 1)} [${inline}]`);
+        i = j;
+        continue;
+      }
+    }
+    lines.push(line);
+    i++;
+  }
+  for (const line of lines) {
     const colonIdx = line.indexOf(":");
     if (colonIdx === -1) continue;
     const key = line.slice(0, colonIdx).trim();
@@ -224,9 +315,84 @@ function parseFrontmatter(raw) {
   if (lineageMatch) {
     lineage = lineageMatch[1].split(",").map((l) => l.trim().replace(/^"|"$/g, "")).filter(Boolean);
   }
+  let derived_from;
+  const derivedFromStr = (fm.derived_from ?? "").trim();
+  if (derivedFromStr.startsWith("[") && derivedFromStr.endsWith("]")) {
+    const inner = derivedFromStr.slice(1, -1);
+    const entries = [];
+    let i2 = 0;
+    while (i2 < inner.length) {
+      const ch = inner[i2];
+      if (ch === '"') {
+        let buf = "";
+        i2++;
+        while (i2 < inner.length) {
+          const c = inner[i2];
+          if (c === "\\" && i2 + 1 < inner.length) {
+            const next = inner[i2 + 1];
+            if (next === '"') {
+              buf += '"';
+              i2 += 2;
+              continue;
+            }
+            if (next === "\\") {
+              buf += "\\";
+              i2 += 2;
+              continue;
+            }
+            buf += c;
+            i2++;
+            continue;
+          }
+          if (c === '"') {
+            i2++;
+            break;
+          }
+          buf += c;
+          i2++;
+        }
+        if (buf.length > 0) entries.push(buf);
+      } else if (ch === "'") {
+        let buf = "";
+        i2++;
+        while (i2 < inner.length) {
+          const c = inner[i2];
+          if (c === "'") {
+            if (i2 + 1 < inner.length && inner[i2 + 1] === "'") {
+              buf += "'";
+              i2 += 2;
+              continue;
+            }
+            i2++;
+            break;
+          }
+          buf += c;
+          i2++;
+        }
+        if (buf.length > 0) entries.push(buf);
+      } else if (ch === "," || /\s/.test(ch)) {
+        i2++;
+      } else {
+        let buf = "";
+        while (i2 < inner.length) {
+          const c = inner[i2];
+          if (c === "," || /\s/.test(c)) break;
+          buf += c;
+          i2++;
+        }
+        if (buf.length > 0) entries.push(buf);
+      }
+    }
+    if (entries.length > 0) derived_from = entries;
+  }
+  const derivedViaRaw = (fm.derived_via ?? "").trim();
+  const derivedViaUnquoted = derivedViaRaw.startsWith('"') && derivedViaRaw.endsWith('"') || derivedViaRaw.startsWith("'") && derivedViaRaw.endsWith("'") ? derivedViaRaw.slice(1, -1) : derivedViaRaw;
+  const derived_via = isConsolidationOperator(derivedViaUnquoted) ? derivedViaUnquoted : void 0;
   const accessCount = fm.accessCount ? parseInt(fm.accessCount, 10) : void 0;
   const decayScore = fm.decayScore !== void 0 ? parseFloat(fm.decayScore) : void 0;
   const heatScore = fm.heatScore !== void 0 ? parseFloat(fm.heatScore) : void 0;
+  const mw_success = parseMemoryWorthCounterField(fm.mw_success);
+  const mw_fail = parseMemoryWorthCounterField(fm.mw_fail);
   let importance;
   if (fm.importanceScore) {
     const score = parseFloat(fm.importanceScore);
@@ -278,6 +444,9 @@ function parseFrontmatter(raw) {
       // Access tracking
       accessCount: accessCount && accessCount > 0 ? accessCount : void 0,
       lastAccessed: fm.lastAccessed || void 0,
+      // Memory Worth counters (issue #560)
+      mw_success,
+      mw_fail,
       // Importance scoring
       importance,
       // Chunking
@@ -296,7 +465,11 @@ function parseFrontmatter(raw) {
       // Structured attributes (JSON on a single line)
       structuredAttributes: parseStructuredAttributes(fm.structuredAttributes),
       // Raw-content dedup hash (format-agnostic archive/consolidation cleanup)
-      contentHash: fm.contentHash || void 0
+      contentHash: fm.contentHash || void 0,
+      // Consolidation provenance (issue #561) — read-through only in this
+      // PR; no code produces these fields yet.
+      derived_from,
+      derived_via
     },
     content
   };
@@ -1271,6 +1444,27 @@ function buildEntitySchemaCacheKey(entitySchemas) {
   ]);
   return JSON.stringify(normalized);
 }
+function isValidBufferSurpriseEvent(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return false;
+  const v = value;
+  if (v.event !== "BUFFER_SURPRISE") return false;
+  if (typeof v.timestamp !== "string" || v.timestamp.length === 0) return false;
+  if (!Number.isFinite(Date.parse(v.timestamp))) return false;
+  if (typeof v.bufferKey !== "string" || v.bufferKey.length === 0) return false;
+  if (v.sessionKey !== null && typeof v.sessionKey !== "string") return false;
+  if (v.turnRole !== "user" && v.turnRole !== "assistant") return false;
+  if (typeof v.surpriseScore !== "number" || !Number.isFinite(v.surpriseScore)) {
+    return false;
+  }
+  if (v.surpriseScore < 0 || v.surpriseScore > 1) return false;
+  if (typeof v.threshold !== "number" || !Number.isFinite(v.threshold)) return false;
+  if (v.threshold < 0 || v.threshold > 1) return false;
+  if (typeof v.triggeredFlush !== "boolean") return false;
+  if (typeof v.turnCountInWindow !== "number" || !Number.isFinite(v.turnCountInWindow)) {
+    return false;
+  }
+  return true;
+}
 var StorageManager = class _StorageManager {
   constructor(baseDir, entitySchemas) {
     this.baseDir = baseDir;
@@ -1353,6 +1547,47 @@ var StorageManager = class _StorageManager {
     } catch {
     }
   }
+  /**
+   * Consolidation provenance helper (issue #561 PR 2).
+   *
+   * Captures the current on-disk content of a source memory as a
+   * page-version snapshot so the downstream consolidated write can record a
+   * `derived_from` pointer that actually resolves.  Returns the
+   * `"<relative-path>:<versionId>"` entry expected by the `derived_from`
+   * frontmatter field.
+   *
+   * Returns `null` when versioning is disabled (snapshots would not be
+   * created), when the file does not exist (nothing to snapshot), or when
+   * the snapshot write itself fails (best-effort — callers skip the entry
+   * rather than block the consolidation).
+   */
+  async snapshotForProvenance(filePath) {
+    if (!this._versioningConfig || !this._versioningConfig.enabled) return null;
+    let existing;
+    try {
+      existing = await readFile(filePath, "utf-8");
+    } catch {
+      return null;
+    }
+    try {
+      const version = await createVersion(
+        filePath,
+        existing,
+        "consolidation",
+        this._versioningConfig,
+        log,
+        void 0,
+        this.baseDir
+      );
+      const rel = path.relative(this.baseDir, filePath).split(path.sep).join("/");
+      return `${rel}:${version.versionId}`;
+    } catch (err) {
+      log.warn(
+        `storage.snapshotForProvenance: failed to snapshot ${filePath}: ${err instanceof Error ? err.message : String(err)}`
+      );
+      return null;
+    }
+  }
   /** The root directory of this storage instance. */
   get dir() {
     return this.baseDir;
@@ -1410,6 +1645,9 @@ var StorageManager = class _StorageManager {
   get proceduresDir() {
     return path.join(this.baseDir, "procedures");
   }
+  get reasoningTracesDir() {
+    return path.join(this.baseDir, "reasoning-traces");
+  }
   get entitiesDir() {
     return path.join(this.baseDir, "entities");
   }
@@ -1545,6 +1783,19 @@ var StorageManager = class _StorageManager {
   get behaviorSignalsPath() {
     return path.join(this.stateDir, "behavior-signals.jsonl");
   }
+  /**
+   * Buffer surprise telemetry ledger (issue #563 PR 3).
+   *
+   * Append-only JSONL of per-turn `BUFFER_SURPRISE` events emitted by
+   * `SmartBuffer` when `bufferSurpriseTriggerEnabled` is on. Each row
+   * captures the score, the threshold in force at the time, whether the
+   * turn caused an extract_now upgrade, and the buffer size. Kept in
+   * `state/` alongside the other append-only ledgers so cleanup and
+   * governance sweeps can treat it uniformly.
+   */
+  get bufferSurpriseLedgerPath() {
+    return path.join(this.stateDir, "buffer-surprise-ledger.jsonl");
+  }
   /**
    * Load user-defined entity aliases from config/aliases.json in the memory store.
    * File format: { "variant": "canonical", "variant2": "canonical", ... }
@@ -1567,6 +1818,7 @@ var StorageManager = class _StorageManager {
     const today = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
     await mkdir(path.join(this.factsDir, today), { recursive: true });
     await mkdir(path.join(this.proceduresDir, today), { recursive: true });
+    await mkdir(path.join(this.reasoningTracesDir, today), { recursive: true });
     await mkdir(this.correctionsDir, { recursive: true });
     await mkdir(this.entitiesDir, { recursive: true });
     await mkdir(this.stateDir, { recursive: true });
@@ -1619,6 +1871,12 @@ var StorageManager = class _StorageManager {
     if (options.status !== void 0) {
       fm.status = options.status;
     }
+    if (options.derivedFrom !== void 0 && options.derivedFrom.length > 0) {
+      fm.derived_from = options.derivedFrom;
+    }
+    if (options.derivedVia !== void 0) {
+      fm.derived_via = options.derivedVia;
+    }
     let enrichedContent = content;
     if (options.structuredAttributes && Object.keys(options.structuredAttributes).length > 0) {
       enrichedContent = `${content}
@@ -1642,6 +1900,9 @@ ${sanitized.text}
     } else if (category === "procedure") {
       await mkdir(path.join(this.proceduresDir, today), { recursive: true });
       filePath = path.join(this.proceduresDir, today, `${id}.md`);
+    } else if (category === "reasoning_trace") {
+      await mkdir(path.join(this.reasoningTracesDir, today), { recursive: true });
+      filePath = path.join(this.reasoningTracesDir, today, `${id}.md`);
     } else {
       filePath = path.join(this.factsDir, today, `${id}.md`);
     }
@@ -2062,6 +2323,7 @@ ${sanitized.text}
     };
     await collectPaths(this.factsDir);
     await collectPaths(this.proceduresDir);
+    await collectPaths(this.reasoningTracesDir);
     await collectPaths(this.correctionsDir);
     return filePaths;
   }
@@ -2372,6 +2634,9 @@ ${sanitized.text}
     if (memory.frontmatter.category === "procedure") {
       return path.join(root, "procedures", this.resolveMemoryDateDir(memory), `${memory.frontmatter.id}.md`);
     }
+    if (memory.frontmatter.category === "reasoning_trace") {
+      return path.join(root, "reasoning-traces", this.resolveMemoryDateDir(memory), `${memory.frontmatter.id}.md`);
+    }
     return path.join(root, "facts", this.resolveMemoryDateDir(memory), `${memory.frontmatter.id}.md`);
   }
   async writeMemoryFileAtomic(targetPath, memory) {
@@ -2757,6 +3022,96 @@ ${memory.content}
     await appendFile(this.memoryLifecycleLedgerPath, payload, "utf-8");
     return events.length;
   }
+  /**
+   * Append a batch of `BUFFER_SURPRISE` telemetry events (issue #563 PR 3).
+   *
+   * Each event records a single buffer flush decision driven by the
+   * surprise gate. The ledger is consumed by
+   * `reportBufferSurpriseDistribution` (Doctor report) and by downstream
+   * benchmark analysis. This method is fire-and-forget by contract:
+   * callers log but do not fail the hot path if the append throws.
+   */
+  async appendBufferSurpriseEvents(events) {
+    if (events.length === 0) return 0;
+    await this.ensureDirectories();
+    const nowIso = (/* @__PURE__ */ new Date()).toISOString();
+    const payload = events.map((event) => {
+      const normalized = {
+        ...event,
+        event: "BUFFER_SURPRISE",
+        timestamp: event.timestamp && event.timestamp.length > 0 ? event.timestamp : nowIso
+      };
+      return `${JSON.stringify(normalized)}
+`;
+    }).join("");
+    await appendFile(this.bufferSurpriseLedgerPath, payload, "utf-8");
+    return events.length;
+  }
+  /**
+   * Read the buffer-surprise ledger, most recent rows last.
+   *
+   * `limit` bounds the number of **valid rows** returned (not the
+   * number of raw lines parsed). We parse every row, discard malformed
+   * ones, then take the tail — so a partial/truncated trailing line
+   * (the common failure mode after an interrupted append) cannot hide
+   * otherwise-valid recent data above it.
+   *
+   * Non-positive / non-integer / non-finite limits return `[]` rather
+   * than the entire file, matching the other ledger readers in this
+   * class and protecting against `slice(-0.5)` → `slice(-0)` silently
+   * devolving into an unbounded parse.
+   *
+   * # Performance note
+   *
+   * For very large ledgers (issue #563 follow-up), a tail-first reader
+   * would avoid parsing the full file when only a recent window is
+   * needed. We keep the full-scan implementation here because:
+   *
+   *   - the ledger is opt-in (flag off by default), so early deployments
+   *     accumulate rows slowly;
+   *   - telemetry rows are small (~200 bytes), so even 100k rows parse
+   *     in well under a second;
+   *   - the governance archive/cleanup flow can trim the ledger when
+   *     size becomes a concern, reusing the existing maintenance hooks.
+   *
+   * Swap to a chunked tail-reader if production logs show this is a
+   * hot path — leaving that work for a follow-up keeps this PR scoped
+   * to correctness, not optimization.
+   */
+  async readBufferSurpriseEvents(options = {}) {
+    let raw;
+    try {
+      raw = await readFile(this.bufferSurpriseLedgerPath, "utf-8");
+    } catch (err) {
+      const code = err.code;
+      if (code === "ENOENT") return [];
+      throw err;
+    }
+    let effectiveLimit = null;
+    if (options.limit !== void 0) {
+      if (typeof options.limit !== "number" || !Number.isFinite(options.limit) || options.limit <= 0) {
+        return [];
+      }
+      const floored = Math.floor(options.limit);
+      if (floored <= 0) return [];
+      effectiveLimit = floored;
+    }
+    const lines = raw.split("\n");
+    const events = [];
+    for (const line of lines) {
+      const trimmed = line.trim();
+      if (trimmed.length === 0) continue;
+      try {
+        const parsed = JSON.parse(trimmed);
+        if (isValidBufferSurpriseEvent(parsed)) {
+          events.push(parsed);
+        }
+      } catch {
+      }
+    }
+    if (effectiveLimit === null) return events;
+    return events.slice(-effectiveLimit);
+  }
   async appendBehaviorSignals(events) {
     if (events.length === 0) return 0;
     await this.ensureDirectories();
@@ -4073,6 +4428,9 @@ ${sanitized.text}
     } else if (category === "procedure") {
       await mkdir(path.join(this.proceduresDir, today), { recursive: true });
       filePath = path.join(this.proceduresDir, today, `${id}.md`);
+    } else if (category === "reasoning_trace") {
+      await mkdir(path.join(this.reasoningTracesDir, today), { recursive: true });
+      filePath = path.join(this.reasoningTracesDir, today, `${id}.md`);
     } else {
       filePath = path.join(this.factsDir, today, `${id}.md`);
     }
@@ -4317,4 +4675,4 @@ export {
   serializeEntityFile,
   StorageManager
 };
-//# sourceMappingURL=chunk-POMSFKTB.js.map
+//# sourceMappingURL=chunk-F5VP6YCB.js.map