@remnic/core 9.3.645 → 9.3.646

package/src/cli.ts

@@ -202,6 +202,10 @@ import {
   type SessionRepairApplyResult,
   type SessionRepairPlan,
 } from "./session-integrity.js";
+import {
+  migrateSessionTranscripts,
+  planSessionTranscriptMigration,
+} from "./session-transcript-migration.js";
 import type { TierMigrationCycleSummary, TierMigrationStatusSnapshot } from "./recall-state.js";
 import {
   readRuntimePolicySnapshot as readPolicyRuntimeSnapshot,
@@ -8650,6 +8654,81 @@ export function registerCli(
           }
         });
 
+      // ── Sessions subcommand (issue #1496) ───────────────────────────────
+      const sessionsCmd = cmd
+        .command("sessions")
+        .description("Inspect and migrate session transcript storage");
+
+      sessionsCmd
+        .command("migrate-transcripts")
+        .description(
+          "Split conflated other/default transcripts into first-class session/<hash> dirs",
+        )
+        .option("--dry-run", "Report the migration plan without moving files (default)")
+        .option("--apply", "Apply the migration (move files; default is dry-run)")
+        .action(async (...args: unknown[]) => {
+          const options = (args[0] ?? {}) as Record<string, unknown>;
+          const dryRunFlag = options.dryRun === true;
+          const applyFlag = options.apply === true;
+
+          // Rule #51: reject contradictory flags rather than silently picking one.
+          if (dryRunFlag && applyFlag) {
+            console.error("Cannot pass both --dry-run and --apply. Choose one.");
+            process.exit(1);
+          }
+
+          // Safe default: dry-run when neither flag is given.
+          const apply = applyFlag;
+          const memoryDir = orchestrator.config.memoryDir;
+
+          if (!apply) {
+            const plan = await planSessionTranscriptMigration({ memoryDir });
+            console.log("=== Session Transcript Migration (DRY RUN) ===\n");
+            if (!dryRunFlag && !applyFlag) {
+              console.log("No flag given — defaulting to --dry-run. Pass --apply to migrate.\n");
+            }
+            console.log(`transcripts dir: ${plan.transcriptsDir}`);
+            console.log(`files to migrate: ${plan.files.length}`);
+            console.log(`distinct sessions: ${plan.distinctSessions}`);
+            console.log(`entries to move: ${plan.movedEntries}`);
+            if (plan.files.length > 0) {
+              console.log("\nPlanned splits:");
+              for (const file of plan.files) {
+                console.log(`- ${file.sourceRelPath}`);
+                for (const group of file.groups) {
+                  console.log(
+                    `    ${group.entryCount} entr${group.entryCount === 1 ? "y" : "ies"} → ${group.destDir} (${group.legacy ? "legacy" : "session"})`,
+                  );
+                }
+                if (file.unmovableLines > 0) {
+                  console.log(`    ${file.unmovableLines} unmovable line(s) retained in source`);
+                }
+              }
+            }
+            console.log("\nDRY RUN — no files changed.");
+            return;
+          }
+
+          const result = await migrateSessionTranscripts({ memoryDir, apply: true });
+          console.log("=== Session Transcript Migration (APPLY) ===\n");
+          console.log(`files rewritten: ${result.filesRewritten}`);
+          console.log(`files removed: ${result.filesRemoved}`);
+          console.log(`distinct sessions: ${result.plan.distinctSessions}`);
+          console.log(`entries moved: ${result.plan.movedEntries}`);
+          if (result.manifestPath) {
+            console.log(`manifest: ${result.manifestPath}`);
+          }
+          if (result.errors.length > 0) {
+            console.log("\nErrors:");
+            for (const err of result.errors) {
+              console.log(`- ${err}`);
+            }
+            process.exitCode = 1;
+            return;
+          }
+          console.log("\nOK");
+        });
+
       // Checkpoint command
       cmd
         .command("checkpoint")

package/src/index.ts

@@ -324,6 +324,24 @@ export { MeilisearchBackend } from "./search/meilisearch-backend.js";
 export { buildEntityRecallSection } from "./entity-retrieval.js";
 export { resolvePrincipal } from "./namespaces/principal.js";
 
+// ---------------------------------------------------------------------------
+// Session identity / transcript pathing (issue #1496)
+// ---------------------------------------------------------------------------
+
+// Shared, deterministic session-key → storage-path resolution. Exported so
+// downstream consumers (the @remnic/bench adapter + its tests, host adapters)
+// resolve transcript directories through the SAME layer the core writer uses
+// instead of hard-coding `other/default` (rule #20, #22, #26).
+export {
+  SESSION_CHANNEL_TYPE,
+  parseSessionIdentity,
+  sessionStoragePaths,
+  legacyParserReadbackDir,
+  type SessionIdentity,
+  type SessionStoragePaths,
+} from "./session-identity.js";
+export { storagePathHash } from "./storage-paths.js";
+
 // ---------------------------------------------------------------------------
 // Trust zones
 // ---------------------------------------------------------------------------

package/src/session-identity.test.ts

@@ -0,0 +1,130 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  LEGACY_FALLBACK_CHANNEL_ID,
+  LEGACY_FALLBACK_CHANNEL_TYPE,
+  SESSION_CHANNEL_TYPE,
+  legacyParserReadbackDir,
+  parseSessionIdentity,
+  sessionStoragePaths,
+} from "./session-identity.js";
+
+test("legacy agent:<id>:main keeps main/default channel identity", () => {
+  const id = parseSessionIdentity("agent:generalist:main");
+  assert.equal(id.legacy, true);
+  assert.equal(id.channelType, "main");
+  assert.equal(id.channelId, "default");
+  assert.equal(id.canonicalSessionKey, "agent:generalist:main");
+});
+
+test("legacy discord channel keeps discord/<channelId> identity", () => {
+  const id = parseSessionIdentity("agent:generalist:discord:channel:998877");
+  assert.equal(id.legacy, true);
+  assert.equal(id.channelType, "discord");
+  assert.equal(id.channelId, "998877");
+});
+
+test("legacy slack channel keeps slack/<channelId> identity", () => {
+  const id = parseSessionIdentity("agent:generalist:slack:channel:C12345");
+  assert.equal(id.legacy, true);
+  assert.equal(id.channelType, "slack");
+  assert.equal(id.channelId, "C12345");
+});
+
+test("legacy cron job keeps cron/<jobId> identity", () => {
+  const id = parseSessionIdentity("agent:generalist:cron:nightly-sync");
+  assert.equal(id.legacy, true);
+  assert.equal(id.channelType, "cron");
+  assert.equal(id.channelId, "nightly-sync");
+});
+
+test("arbitrary key becomes a first-class session/<hash> identity, never other/default", () => {
+  const id = parseSessionIdentity("pi-geek:abc123");
+  assert.equal(id.legacy, false);
+  assert.equal(id.channelType, SESSION_CHANNEL_TYPE);
+  assert.notEqual(id.channelType, LEGACY_FALLBACK_CHANNEL_TYPE);
+  assert.notEqual(id.channelId, LEGACY_FALLBACK_CHANNEL_ID);
+  assert.match(id.channelId, /^[0-9a-f]{16}$/);
+});
+
+test("distinct arbitrary keys get distinct, collision-resistant hashes", () => {
+  const geek = parseSessionIdentity("pi-geek:abc123");
+  const friend = parseSessionIdentity("pi-friend:def456");
+  assert.notEqual(geek.channelId, friend.channelId);
+});
+
+test("identity is deterministic for the same key", () => {
+  const a = parseSessionIdentity("pi-geek:abc123");
+  const b = parseSessionIdentity("pi-geek:abc123");
+  assert.deepEqual(a, b);
+});
+
+test("a bare token without the agent prefix is treated as arbitrary, not legacy", () => {
+  // "foo:bar:baz" must NOT be misread as channelType="baz".
+  const id = parseSessionIdentity("foo:bar:baz");
+  assert.equal(id.legacy, false);
+  assert.equal(id.channelType, SESSION_CHANNEL_TYPE);
+});
+
+test("empty session key resolves without throwing", () => {
+  const id = parseSessionIdentity("");
+  assert.equal(id.legacy, false);
+  assert.equal(id.channelType, SESSION_CHANNEL_TYPE);
+  assert.equal(typeof id.channelId, "string");
+});
+
+test("sessionStoragePaths routes arbitrary keys to session/<hash>", () => {
+  const paths = sessionStoragePaths("pi-geek:abc123");
+  assert.equal(paths.channelType, SESSION_CHANNEL_TYPE);
+  assert.match(paths.dir, /^session\/[0-9a-f]{16}$/);
+});
+
+test("sessionStoragePaths keeps legacy main at main/default", () => {
+  const paths = sessionStoragePaths("agent:generalist:main");
+  assert.equal(paths.dir, "main/default");
+});
+
+test("two arbitrary keys never share a storage dir", () => {
+  const a = sessionStoragePaths("pi-geek:abc123");
+  const b = sessionStoragePaths("pi-friend:def456");
+  assert.notEqual(a.dir, b.dir);
+  assert.notEqual(a.dir, "other/default");
+  assert.notEqual(b.dir, "other/default");
+});
+
+test("display label strips control characters and separators", () => {
+  const id = parseSessionIdentity("pi-geek:abc/123");
+  assert.ok(!id.displayLabel.includes("/"));
+  assert.ok(id.displayLabel.length > 0);
+});
+
+test("legacyParserReadbackDir reconstructs the OLD parser dir for >=3-part keys", () => {
+  // foo:bar:baz → channelType=baz, channelId=default
+  assert.equal(legacyParserReadbackDir("foo:bar:baz"), "baz/default");
+  // foo:bar:baz:qux → channelType=baz, channelId=qux
+  assert.equal(legacyParserReadbackDir("foo:bar:baz:qux"), "baz/qux");
+});
+
+test("legacyParserReadbackDir returns undefined for <3-part keys (old build used other/default)", () => {
+  assert.equal(legacyParserReadbackDir("pi-geek:abc123"), undefined);
+  assert.equal(legacyParserReadbackDir("bare"), undefined);
+  assert.equal(legacyParserReadbackDir(""), undefined);
+});
+
+test("readbackDirs for a >=3-part arbitrary key include other/default AND the old parser dir", () => {
+  const paths = sessionStoragePaths("foo:bar:baz");
+  assert.match(paths.dir, /^session\/[0-9a-f]{16}$/);
+  assert.ok(paths.readbackDirs.includes("other/default"));
+  assert.ok(paths.readbackDirs.includes("baz/default"));
+});
+
+test("readbackDirs for a 2-part arbitrary key include only other/default", () => {
+  const paths = sessionStoragePaths("pi-geek:abc123");
+  assert.deepEqual(paths.readbackDirs, ["other/default"]);
+});
+
+test("legacy agent:<id>:... keys expose no read-back dirs (their location never moved)", () => {
+  assert.deepEqual(sessionStoragePaths("agent:generalist:main").readbackDirs, []);
+  assert.deepEqual(sessionStoragePaths("agent:generalist:discord:channel:998877").readbackDirs, []);
+});

package/src/session-identity.ts

@@ -0,0 +1,281 @@
+import path from "node:path";
+import {
+  encodeStoragePathSegment,
+  encodeStoragePathSegmentWithHash,
+  isSafeLegacyPathSegment,
+  storagePathHash,
+} from "./storage-paths.js";
+
+/**
+ * Shared session-identity / transcript-pathing layer (issue #1496).
+ *
+ * This module is the SINGLE source of truth for turning a `sessionKey` into a
+ * channel identity and a set of deterministic, collision-resistant storage
+ * paths. Every subsystem that previously re-parsed session keys (transcript
+ * pathing, tool-usage pathing, hourly summaries, session auditing, conversation
+ * indexing) MUST route through `parseSessionIdentity()` / `sessionStoragePaths()`
+ * so behavior stays identical across paths (rule #22, #39).
+ *
+ * Two shapes are recognized:
+ *
+ *   1. Legacy `agent:<agentId>:<channelType>:...` keys keep their existing
+ *      readable channel identity:
+ *        - agent:<id>:main                      → type="main",    id="default"
+ *        - agent:<id>:discord:channel:<chanId>  → type="discord", id="<chanId>"
+ *        - agent:<id>:slack:channel:<chanId>    → type="slack",   id="<chanId>"
+ *        - agent:<id>:cron:<jobId>              → type="cron",    id="<jobId>"
+ *        - agent:<id>:<other>[:<id>]            → type="<other>", id="<id?>"
+ *
+ *   2. Arbitrary / non-legacy keys (e.g. `pi-geek:abc123`) become first-class
+ *      from the FIRST write — they NEVER start life under `other/default`:
+ *        channelType = "session"
+ *        channelId   = storagePathHash(sessionKey)   (collision-resistant)
+ *      yielding `transcripts/session/<hash>/YYYY-MM-DD.jsonl`.
+ *
+ * The reserved `"session"` channel type is what makes arbitrary keys isolated
+ * and auditable. Legacy data under `other/default` (and any legacy channel
+ * directory) remains READABLE via `legacyDir` / `alternateDir` candidates.
+ */
+
+/** Channel type reserved for first-class arbitrary (non-legacy) session keys. */
+export const SESSION_CHANNEL_TYPE = "session";
+
+/** Legacy fallback channel identity for un-parseable keys (read-back only). */
+export const LEGACY_FALLBACK_CHANNEL_TYPE = "other";
+export const LEGACY_FALLBACK_CHANNEL_ID = "default";
+
+/** Max characters of the raw session key kept in a human-readable display label. */
+const DISPLAY_LABEL_MAX_LENGTH = 64;
+
+export interface SessionIdentity {
+  /** Channel type — legacy channel type for known shapes, else `"session"`. */
+  channelType: string;
+  /** Channel id — legacy channel id for known shapes, else the key hash. */
+  channelId: string;
+  /** Human-readable, non-authoritative label derived from the raw key. */
+  displayLabel: string;
+  /** The original session key, unchanged. */
+  canonicalSessionKey: string;
+  /** True when the key matched a known legacy `agent:<id>:...` shape. */
+  legacy: boolean;
+}
+
+export interface SessionStoragePaths {
+  /** Channel type (mirrors {@link SessionIdentity.channelType}). */
+  channelType: string;
+  /** Channel id (mirrors {@link SessionIdentity.channelId}). */
+  channelId: string;
+  /** Primary encoded storage subdirectory: `<encodedType>/<encodedId>`. */
+  dir: string;
+  /**
+   * Collision-resistant alternate subdirectory that always embeds the full
+   * session-key hash. Used both as a write-time collision escape hatch and as a
+   * read-back candidate for data written before this layer existed.
+   */
+  alternateDir: string;
+  /**
+   * Un-encoded `<type>/<id>` directory for reading data written by older
+   * builds that did not URL-encode path segments. Undefined when it would be
+   * identical to `dir` or when either segment is path-unsafe.
+   */
+  legacyDir?: string;
+  /**
+   * Ordered, de-duplicated list of READ-BACK-ONLY directories where an older
+   * build may have stranded this key's data. NEW writes never target these —
+   * they exist purely so pre-#1496 transcripts/tool-usage stay discoverable
+   * (and migratable). Populated only for non-legacy keys. Includes:
+   *   1. the shared `other/default` fallback (every arbitrary key landed here);
+   *   2. the directory the OLD `parts.length >= 3` parser would have chosen
+   *      (e.g. `foo:bar:baz` → `baz/default`, `foo:bar:baz:qux` → `baz/qux`).
+   * See {@link legacyParserReadbackDir}.
+   */
+  readbackDirs: string[];
+}
+
+/**
+ * Parse a session key into a stable channel identity.
+ *
+ * Pure and deterministic — no filesystem access. Same input always yields the
+ * same identity, which is what makes arbitrary keys collision-resistant from
+ * the first write.
+ */
+export function parseSessionIdentity(sessionKey: string): SessionIdentity {
+  const canonicalSessionKey = typeof sessionKey === "string" ? sessionKey : "";
+  const legacy = parseLegacyChannelIdentity(canonicalSessionKey);
+  if (legacy) {
+    return {
+      channelType: legacy.channelType,
+      channelId: legacy.channelId,
+      displayLabel: safeDisplayLabel(canonicalSessionKey),
+      canonicalSessionKey,
+      legacy: true,
+    };
+  }
+
+  return {
+    channelType: SESSION_CHANNEL_TYPE,
+    channelId: storagePathHash(canonicalSessionKey),
+    displayLabel: safeDisplayLabel(canonicalSessionKey),
+    canonicalSessionKey,
+    legacy: false,
+  };
+}
+
+/**
+ * Resolve the storage path pieces for a session key. Wraps
+ * {@link parseSessionIdentity} so transcript, tool-usage, and summary writers
+ * share one implementation (rule #22).
+ */
+export function sessionStoragePaths(sessionKey: string): SessionStoragePaths {
+  const identity = parseSessionIdentity(sessionKey);
+  const { channelType, channelId, canonicalSessionKey } = identity;
+
+  const dir = path.join(encodeStoragePathSegment(channelType), encodeStoragePathSegment(channelId));
+
+  const alternateDir = path.join(
+    encodeStoragePathSegmentWithHash(channelType),
+    `${encodeStoragePathSegmentWithHash(channelId)}--session-${storagePathHash(canonicalSessionKey)}`
+  );
+
+  let legacyDir: string | undefined;
+  if (isSafeLegacyPathSegment(channelType) && isSafeLegacyPathSegment(channelId)) {
+    const candidate = path.join(channelType, channelId);
+    if (candidate !== dir) {
+      legacyDir = candidate;
+    }
+  }
+
+  // Read-back-only candidates are only relevant for non-legacy keys: legacy
+  // `agent:<id>:...` keys still resolve to their original channel directory, so
+  // nothing about their on-disk location moved.
+  const readbackDirs: string[] = [];
+  if (!identity.legacy) {
+    const seen = new Set<string>([dir]);
+    if (alternateDir) seen.add(alternateDir);
+    if (legacyDir) seen.add(legacyDir);
+    for (const candidate of [OTHER_DEFAULT_READBACK_DIR, legacyParserReadbackDir(canonicalSessionKey)]) {
+      if (!candidate || seen.has(candidate)) continue;
+      seen.add(candidate);
+      readbackDirs.push(candidate);
+    }
+  }
+
+  return { channelType, channelId, dir, alternateDir, legacyDir, readbackDirs };
+}
+
+/**
+ * The shared `other/default` directory every arbitrary key was routed into by
+ * builds predating issue #1496. Exposed so transcript/tool-usage/summary read
+ * paths and the migration scanner all agree on the same fallback location.
+ */
+export const OTHER_DEFAULT_READBACK_DIR = path.join(
+  LEGACY_FALLBACK_CHANNEL_TYPE,
+  LEGACY_FALLBACK_CHANNEL_ID
+);
+
+/**
+ * Reconstruct the transcript/tool-usage directory the OLD `getTranscriptPath`
+ * parser (pre-#1496) would have produced for a key the NEW parser reclassifies
+ * as a first-class `session/<hash>` identity.
+ *
+ * The OLD parser treated ANY key with `parts.length >= 3` as legacy — it did
+ * NOT require a leading `agent` segment — so an arbitrary key like
+ * `foo:bar:baz` was stored under `baz/default` and `foo:bar:baz:qux` under
+ * `baz/qux`. Those directories must stay readable (and migratable) for existing
+ * installs even though the key now writes to `session/<hash>` (Thread B / codex
+ * review on PR #1504). Path segments are encoded exactly as the old `dir` was
+ * built (`encodeStoragePathSegment`) so the candidate matches the bytes on disk.
+ *
+ * Returns `undefined` when the key has fewer than three colon parts (the old
+ * parser would have used the `other/default` fallback, already covered), when
+ * the channel type is empty, or when the result is unsafe.
+ */
+export function legacyParserReadbackDir(sessionKey: string): string | undefined {
+  if (typeof sessionKey !== "string" || sessionKey.length === 0) return undefined;
+  const parts = sessionKey.split(":");
+  if (parts.length < 3) return undefined;
+
+  const channelType = parts[2];
+  if (!channelType || channelType.length === 0) return undefined;
+
+  // Mirror the OLD parser's channelId derivation for parts.length >= 3 keys.
+  let channelId = LEGACY_FALLBACK_CHANNEL_ID;
+  if (channelType === "main") {
+    channelId = "default";
+  } else if (channelType === "discord" && parts.length >= 5 && parts[3] === "channel") {
+    channelId = parts[4];
+  } else if (channelType === "slack" && parts.length >= 5 && parts[3] === "channel") {
+    channelId = parts[4];
+  } else if (channelType === "cron" && parts.length >= 4) {
+    channelId = parts[3];
+  } else if (parts.length >= 4) {
+    channelId = parts[3];
+  }
+  if (!channelId || channelId.length === 0) {
+    channelId = LEGACY_FALLBACK_CHANNEL_ID;
+  }
+
+  return path.join(encodeStoragePathSegment(channelType), encodeStoragePathSegment(channelId));
+}
+
+/**
+ * Parse the legacy `agent:<agentId>:<channelType>:...` shape. Returns
+ * `undefined` for anything that is not a recognized legacy key, so the caller
+ * can route it to a first-class `session/<hash>` identity instead.
+ *
+ * A legacy key must start with the literal `agent` segment and a non-empty
+ * agent id; otherwise an arbitrary key like `pi-geek:abc123` would be
+ * misread as `channelType = "abc123"`.
+ */
+function parseLegacyChannelIdentity(sessionKey: string): { channelType: string; channelId: string } | undefined {
+  if (sessionKey.length === 0) return undefined;
+  const parts = sessionKey.split(":");
+  if (parts.length < 3) return undefined;
+  if (parts[0] !== "agent") return undefined;
+  if (!parts[1] || parts[1].length === 0) return undefined;
+
+  const channelType = parts[2];
+  if (!channelType || channelType.length === 0) return undefined;
+
+  let channelId = LEGACY_FALLBACK_CHANNEL_ID;
+  if (channelType === "main") {
+    channelId = "default";
+  } else if (channelType === "discord" && parts.length >= 5 && parts[3] === "channel") {
+    channelId = parts[4];
+  } else if (channelType === "slack" && parts.length >= 5 && parts[3] === "channel") {
+    channelId = parts[4];
+  } else if (channelType === "cron" && parts.length >= 4) {
+    channelId = parts[3];
+  } else if (parts.length >= 4) {
+    channelId = parts[3];
+  }
+
+  if (!channelId || channelId.length === 0) {
+    channelId = LEGACY_FALLBACK_CHANNEL_ID;
+  }
+
+  return { channelType, channelId };
+}
+
+/**
+ * A short, human-readable label for dashboards/audits. Never used for storage
+ * routing — `channelId` (the hash) owns isolation. Control characters and path
+ * separators are stripped so the label is safe to render.
+ */
+function safeDisplayLabel(sessionKey: string): string {
+  if (sessionKey.length === 0) return "(empty)";
+  let out = "";
+  for (const char of sessionKey) {
+    const code = char.codePointAt(0) ?? 0;
+    // Drop ASCII control characters (0x00-0x1F) and DEL (0x7F).
+    if (code <= 0x1f || code === 0x7f) continue;
+    if (char === "/" || char === "\\") {
+      out += "_";
+      continue;
+    }
+    out += char;
+  }
+  const trimmed = out.trim();
+  if (trimmed.length === 0) return "(empty)";
+  return trimmed.length > DISPLAY_LABEL_MAX_LENGTH ? `${trimmed.slice(0, DISPLAY_LABEL_MAX_LENGTH - 1)}…` : trimmed;
+}