@hypercarrier/rarebit 0.1.0-alpha.5 → 0.1.0-alpha.6

package/scripts/verify-package.mjs

@@ -13,17 +13,19 @@ const npm = (args, cwd = root) => run("npm", args, cwd);
 const required = [
   "package/README.md", "package/RUNBOOK.md", "package/VISUAL-LANGUAGE.md",
   "package/AGENTS.md", "package/CHANGELOG.md", "package/LICENSE",
-  "package/SECURITY.md", "package/bin/rarebit.mjs",
-  "package/src/index.mjs", "package/src/types.d.ts",
+  "package/SECURITY.md", "package/bin/rarebit.mjs", "package/bin/piq.mjs",
+  "package/src/index.mjs", "package/src/rarebit-fork-lineage.mjs", "package/src/rarebit-fork.mjs", "package/src/types.d.ts",
   "package/schemas/rarebit.schema.json", "package/test/rarebit-cli.test.mjs",
   "package/scripts/e2e-recall-pi083.mjs", "package/scripts/e2e-recall-pi0842.mjs", "package/scripts/verify-package.mjs",
 ];
 const forbiddenPath = /(^|\/)(\.git|node_modules|\.github|hc-rarebit\.mjs)(\/|$)|HyperCarrier|timeline|pi-team/i;
-const allowedBare = new Set(["@earendil-works/pi-ai"]);
+const allowedBare = new Set(["@earendil-works/pi-ai", "@earendil-works/pi-coding-agent"]);
 const packageFor = (specifier) => specifier.startsWith("@") ? specifier.split("/").slice(0, 2).join("/") : specifier.split("/")[0];
 const packageJson = JSON.parse(await readFile(join(root, "package.json"), "utf8"));
-assert.equal(packageJson.version, "0.1.0-alpha.5", "release version mismatch");
+assert.equal(packageJson.version, "0.1.0-alpha.6", "release version mismatch");
 assert.equal(packageJson.peerDependencies?.["@earendil-works/pi-ai"], ">=0.83.0", "Pi AI peer range mismatch");
+for (const peer of ["@earendil-works/pi-ai", "@earendil-works/pi-coding-agent"])
+  assert.equal(packageJson.peerDependenciesMeta?.[peer]?.optional, true, `${peer} peer must remain optional`);
 
 const packed = JSON.parse(npm(["pack", "--json", "--ignore-scripts"]))[0];
 const tarball = join(root, packed.filename);
@@ -35,7 +37,7 @@ assert(!entries.includes("package/bin/hc-rarebit.mjs"), "legacy CLI is packed");
 const sourceFiles = entries.filter((path) => path.endsWith(".mjs") && !path.startsWith("package/test/"));
 for (const entry of sourceFiles) {
   const text = run("tar", ["-xOf", tarball, entry]);
-  for (const match of text.matchAll(/(?:from\s*|import\s*\()\s*["']([^"']+)["']/g)) {
+  for (const match of text.matchAll(/(?:\bfrom\s+|import\s*\()\s*["']([^"']+)["']/g)) {
     const specifier = match[1];
     if (!specifier.startsWith(".") && !specifier.startsWith("node:") && !allowedBare.has(packageFor(specifier))) {
       throw new Error(`undeclared runtime import ${specifier} in ${entry}`);
@@ -52,10 +54,10 @@ try {
   npm(["init", "-y"], temp);
   npm(["install", "--ignore-scripts", "--omit=dev", tarball], temp);
   const installed = "@hypercarrier/rarebit";
-  for (const subpath of ["", "/core", "/session", "/service", "/artifact-state", "/visual-language", "/extension"]) {
+  for (const subpath of ["", "/core", "/session", "/service", "/artifact-state", "/fork", "/fork-lineage", "/visual-language", "/extension"]) {
     run(process.execPath, ["--input-type=module", "-e", `await import(${JSON.stringify(installed + subpath)})`], temp);
   }
-  const imports = ["", "/core", "/session", "/service", "/artifact-state", "/visual-language", "/extension"]
+  const imports = ["", "/core", "/session", "/service", "/artifact-state", "/fork", "/fork-lineage", "/visual-language", "/extension"]
     .map((subpath) => `import ${JSON.stringify(installed + subpath)};`).join("\n");
   await writeFile(join(temp, "imports.mts"), imports);
   run(join(root, "node_modules/.bin/tsc"), ["--noEmit", "--module", "NodeNext", "--moduleResolution", "NodeNext", "--target", "ES2022", "--skipLibCheck", "imports.mts"], temp);

package/src/extension.mjs

@@ -1,8 +1,6 @@
-import { readFile } from "node:fs/promises";
+import { readFile, rm } from "node:fs/promises";
 import { homedir } from "node:os";
 import { join, resolve } from "node:path";
-import { complete } from "@earendil-works/pi-ai/compat";
-
 export * from "./rarebit-core.mjs";
 export * from "./rarebit-model.mjs";
 export * from "./automatic-summary-policy.mjs";
@@ -26,6 +24,12 @@ import {
   parseRarebitCommand,
   rarebitCommandDescription,
 } from "./rarebit-command.mjs";
+import {
+  buildRarebitForkPlan,
+  createRarebitForkEntries,
+  sessionFilenameFor,
+  writeRarebitForkFile,
+} from "./rarebit-fork.mjs";
 import {
   RAREBIT_CONVERSATION_SCHEMA_VERSION,
   RAREBIT_RECALL_SCHEMA_VERSION,
@@ -49,6 +53,17 @@ async function readSettings(path) {
   }
 }
 
+function resolveReserveTokens(globalSettings, projectSettings, model) {
+  const globalCompaction = globalSettings?.compaction ?? {};
+  const projectCompaction = projectSettings?.compaction ?? {};
+  const compaction = { ...globalCompaction, ...projectCompaction };
+  const modelOverrides = { ...(globalCompaction.modelOverrides ?? {}), ...(projectCompaction.modelOverrides ?? {}) };
+  const key = model?.provider && model?.id ? `${model.provider}/${model.id}` : null;
+  const override = key ? modelOverrides[key] : undefined;
+  const value = override?.reserveTokens ?? (Object.hasOwn(projectCompaction, "reserveTokens") ? projectCompaction.reserveTokens : compaction.reserveTokens) ?? 16_384;
+  return Number.isSafeInteger(value) && value >= 0 ? value : 16_384;
+}
+
 function configuredAgentDir(env = process.env) {
   return env.PI_CODING_AGENT_DIR ?? join(homedir(), ".pi", "agent");
 }
@@ -56,6 +71,7 @@ function configuredAgentDir(env = process.env) {
 export async function readConfiguredRarebitSettings({
   cwd,
   projectTrusted,
+  model,
   agentDir = configuredAgentDir(),
 } = {}) {
   const global = await readSettings(join(agentDir, "settings.json"));
@@ -88,6 +104,7 @@ export async function readConfiguredRarebitSettings({
     },
     summaryPolicy: resolved.summaryPolicy,
     autoTitle: resolved.autoTitle,
+    reserveTokens: resolveReserveTokens(global.value, project.value, model),
   };
 }
 
@@ -104,10 +121,13 @@ export default function registerPiRarebit(pi, config = {}) {
     });
   const activityReporter =
     explicit.activityReporter ?? createHerdrActivityReporter();
-  const projectActivity = (ctx) =>
-    projectRarebitSessionActivity(
-      selectRarebits(ctx?.sessionManager?.getBranch?.() ?? []),
-    );
+  const projectActivity = (ctx) => {
+    const entries = ctx?.sessionManager?.getBranch?.() ?? [];
+    return projectRarebitSessionActivity({
+      ...selectRarebits(entries),
+      entries,
+    });
+  };
 
   const notify = (ctx, text, level = "info") => {
     if (!ctx?.hasUI || typeof ctx?.ui?.notify !== "function") return;
@@ -179,6 +199,7 @@ export default function registerPiRarebit(pi, config = {}) {
         injected = await settingsLoader({
           cwd: ctx?.cwd,
           projectTrusted: ctx?.isProjectTrusted?.() === true,
+          model: ctx?.model,
         });
       } catch (error) {
         injected = {
@@ -200,7 +221,9 @@ export default function registerPiRarebit(pi, config = {}) {
       ...injected,
       ...explicit,
       summaryPolicy,
-      piAi: explicit.piAi ?? { complete },
+      piAi: explicit.piAi ?? {
+        complete: async (...args) => (await import("@earendil-works/pi-ai/compat")).complete(...args),
+      },
     };
   };
 
@@ -434,6 +457,72 @@ ${fence(requestText)}`;
         return;
       }
       const { subcommand, arguments: rest } = command;
+      if (subcommand === "fork") {
+        if (typeof ctx?.switchSession !== "function") {
+          notify(ctx, "Rarebit fork is unavailable in this Pi runtime", "error");
+          return;
+        }
+        let destination;
+        let switchStarted = false;
+        try {
+          await ctx.waitForIdle?.();
+          const sourceFile = ctx.sessionManager?.getSessionFile?.();
+          const branch = ctx.sessionManager?.getBranch?.();
+          const sourceHeader = ctx.sessionManager?.getHeader?.();
+          if (typeof sourceFile !== "string" || !Array.isArray(branch))
+            throw new Error("the current Pi Session must be persisted before forking");
+          const maxTokenLength = Number(rest[1] ?? 64_000);
+          const model = ctx.model ?? null;
+          if (!model || typeof model.provider !== "string" || !model.provider || typeof model.id !== "string" || !model.id || !Number.isFinite(model.contextWindow) || model.contextWindow <= 0)
+            throw new Error("destination model/context window is unavailable; select a target Pi model before forking");
+          const effective = await loadEffective(ctx);
+          const promptOverheadTokens = Math.ceil(String(ctx.getSystemPrompt?.() ?? "").length / 4);
+          const activeToolNames = new Set(pi.getActiveTools?.() ?? []);
+          const activeToolSchema = (pi.getAllTools?.() ?? [])
+            .filter((tool) => activeToolNames.has(tool.name))
+            .map(({ name, description, parameters }) => ({ name, description, parameters }));
+          const toolOverheadTokens = Math.ceil(JSON.stringify(activeToolSchema).length / 4);
+          const plan = buildRarebitForkPlan({
+            header: sourceHeader, sessionFile: sourceFile, cwd: ctx.cwd, branch,
+            targetCwd: ctx.cwd, maxTokenLength, targetModel: model,
+            reserveTokens: effective.reserveTokens ?? 16_384,
+            promptOverheadTokens,
+            toolOverheadTokens,
+          });
+          const currentLeaf = ctx.sessionManager.getLeafId?.() ?? branch.at(-1)?.id ?? null;
+          if (currentLeaf !== (plan.branchLeafId ?? null))
+            throw new Error("the active source branch changed during fork preparation; retry without switching branches");
+          const session = createRarebitForkEntries(plan);
+          const sessionDir = ctx.sessionManager.getSessionDir?.();
+          if (typeof sessionDir !== "string" || !sessionDir)
+            throw new Error("the current Pi Session store is unavailable");
+          destination = sessionFilenameFor({ sessionDir, forkCreatedAt: plan.forkCreatedAt, sessionId: session.sessionId });
+          await writeRarebitForkFile(destination, session);
+          const currentSourceFile = ctx.sessionManager.getSessionFile?.();
+          const currentSourceId = ctx.sessionManager.getHeader?.()?.id;
+          const currentSourceLeaf = ctx.sessionManager.getLeafId?.() ?? ctx.sessionManager.getBranch?.()?.at(-1)?.id ?? null;
+          if (resolve(currentSourceFile ?? "") !== resolve(sourceFile) || currentSourceId !== sourceHeader?.id || currentSourceLeaf !== (plan.branchLeafId ?? null))
+            throw new Error("the active source Session or branch changed during fork preparation; retry without switching branches");
+          switchStarted = true;
+          const switched = await ctx.switchSession(destination, {
+            withSession: async (next) => {
+              notify(next, `Rarebit fork opened (${plan.importedTokens} imported tokens; ${plan.omittedOccurrences} omitted occurrences)`, "info");
+            },
+          });
+          if (switched?.cancelled) {
+            await rm(destination, { force: true });
+            destination = undefined;
+            notify(ctx, "Rarebit fork cancelled before switching", "warning");
+          }
+        } catch (error) {
+          if (destination && !switchStarted) await rm(destination, { force: true }).catch(() => {});
+          const retained = destination && switchStarted
+            ? ` Destination retained at ${destination}; resume it if the replacement is active.`
+            : "";
+          notify(ctx, `Rarebit fork failed: ${error?.message ?? error}.${retained}`, "error");
+        }
+        return;
+      }
       if (subcommand === "recall") {
         const prompt = rest[0];
         if (typeof pi.sendUserMessage !== "function") {

package/src/index.mjs

@@ -8,6 +8,7 @@ export * from "./rarebit-service.mjs";
 export * from "./rarebit-session.mjs";
 export * from "./rarebit-store.mjs";
 export * from "./rarebit-artifact-state.mjs";
+export * from "./rarebit-fork-lineage.mjs";
 export * from "./rarebit-visual-language.mjs";
 export * from "./rarebit-settings.mjs";
 export { default } from "./extension.mjs";

package/src/pi-session.mjs

@@ -0,0 +1,237 @@
+import { open, readFile, readdir, stat } from "node:fs/promises";
+import { homedir } from "node:os";
+import { join, resolve, sep } from "node:path";
+
+export const DEFAULT_PI_SESSION_ROOT = join(
+  homedir(),
+  ".pi",
+  "agent",
+  "sessions",
+);
+
+function queryError(message) {
+  const error = new Error(message);
+  error.name = "PiSessionQueryError";
+  return error;
+}
+
+async function isRegularFile(path) {
+  try {
+    return (await stat(path)).isFile();
+  } catch (error) {
+    if (error?.code === "ENOENT") return false;
+    throw error;
+  }
+}
+
+async function* jsonlFiles(root) {
+  let entries;
+  try {
+    entries = await readdir(root, { withFileTypes: true });
+  } catch (error) {
+    if (error?.code === "ENOENT") return;
+    throw error;
+  }
+  for (const entry of entries) {
+    const path = join(root, entry.name);
+    if (entry.isDirectory()) yield* jsonlFiles(path);
+    else if (entry.isFile() && entry.name.endsWith(".jsonl")) yield path;
+  }
+}
+
+async function readHeader(path) {
+  const handle = await open(path, "r");
+  try {
+    const buffer = Buffer.alloc(16384);
+    const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
+    const firstLine = buffer
+      .subarray(0, bytesRead)
+      .toString("utf8")
+      .split("\n", 1)[0]
+      ?.trim();
+    if (!firstLine) return null;
+    try {
+      const header = JSON.parse(firstLine);
+      return header?.type === "session" ? header : null;
+    } catch {
+      return null;
+    }
+  } finally {
+    await handle.close();
+  }
+}
+
+function looksLikePath(reference) {
+  return (
+    reference.includes(sep) ||
+    reference.startsWith(".") ||
+    reference.endsWith(".jsonl")
+  );
+}
+
+/** Resolve one exact path or one unambiguous persisted Session ID. */
+export async function resolvePiSessionFile(
+  reference,
+  { sessionRoot = DEFAULT_PI_SESSION_ROOT } = {},
+) {
+  if (typeof reference !== "string" || !reference.trim())
+    throw queryError("--session requires an exact Pi Session path or ID");
+  const input = reference.trim();
+  const path = resolve(input);
+  if (await isRegularFile(path)) return path;
+  if (looksLikePath(input))
+    throw queryError(`Pi Session file not found: ${input}`);
+  const matches = [];
+  for await (const candidate of jsonlFiles(resolve(sessionRoot))) {
+    const header = await readHeader(candidate);
+    if (header?.id === input) matches.push(candidate);
+  }
+  if (matches.length === 0)
+    throw queryError(`No persisted Pi Session has exact ID ${input}`);
+  if (matches.length > 1)
+    throw queryError(
+      `Pi Session ID ${input} is ambiguous across ${matches.length} files; pass an exact path`,
+    );
+  return resolve(matches[0]);
+}
+
+/** Parse native JSONL without normalizing away unknown evidence fields. */
+export function parseNativeSession(content, source = "Pi Session") {
+  const records = [];
+  for (const [index, line] of String(content).split("\n").entries()) {
+    if (!line.trim()) continue;
+    try {
+      records.push(JSON.parse(line));
+    } catch {
+      throw queryError(`${source} contains malformed JSON at line ${index + 1}`);
+    }
+  }
+  const header = records[0];
+  if (
+    !header ||
+    header.type !== "session" ||
+    typeof header.id !== "string" ||
+    !header.id
+  )
+    throw queryError(`${source} does not begin with a valid Pi Session header`);
+  const entries = records.slice(1);
+  const version = header.version ?? 1;
+  if (typeof version !== "number" || !Number.isInteger(version) || version < 1)
+    throw queryError(`${source} has an invalid Session version`);
+  const byId = new Map();
+  const linear = version < 2;
+  if (!linear) {
+    for (const entry of entries) {
+      if (typeof entry?.id !== "string" || !entry.id)
+        throw queryError(`${source} contains a Session entry without an ID`);
+      if (byId.has(entry.id))
+        throw queryError(
+          `${source} contains duplicate Session entry ID ${entry.id}`,
+        );
+      byId.set(entry.id, entry);
+    }
+  }
+  return { header, entries, byId, linear };
+}
+
+function ancestryFromLeaf(parsed, leaf, source) {
+  const reversed = [];
+  const visited = new Set();
+  let current = leaf;
+  while (current) {
+    if (visited.has(current.id))
+      throw queryError(`${source} contains a cycle at Session entry ${current.id}`);
+    visited.add(current.id);
+    reversed.push(current);
+    if (current.parentId === null || current.parentId === undefined) break;
+    if (typeof current.parentId !== "string" || !current.parentId)
+      throw queryError(`${source} contains an invalid parent ID at ${current.id}`);
+    current = parsed.byId.get(current.parentId);
+    if (!current)
+      throw queryError(
+        `${source} is missing parent Session entry ${reversed.at(-1).parentId}`,
+      );
+  }
+  return reversed.reverse();
+}
+
+/** Resolve the persisted active branch. The final entry is Pi's default leaf. */
+export function resolveActiveBranch(parsed, source = "Pi Session") {
+  if (parsed.linear) return parsed.entries.slice();
+  const leaf = parsed.entries.at(-1);
+  return leaf ? ancestryFromLeaf(parsed, leaf, source) : [];
+}
+
+/** Resolve one explicit leaf, or the native default leaf when omitted. */
+export function resolveSessionBranch(
+  parsed,
+  { leafId = null } = {},
+  source = "Pi Session",
+) {
+  if (parsed.linear) {
+    if (leafId !== null && leafId !== undefined)
+      throw queryError(
+        `${source} uses a linear format and has no explicit branch leaf`,
+      );
+    return parsed.entries.slice();
+  }
+  const leaf =
+    leafId === null || leafId === undefined
+      ? parsed.entries.at(-1)
+      : parsed.byId.get(leafId);
+  if (!leaf) {
+    if (leafId === null || leafId === undefined) return [];
+    throw queryError(`${source} has no Session entry with leaf ID ${leafId}`);
+  }
+  return ancestryFromLeaf(parsed, leaf, source);
+}
+
+function childIds(parsed) {
+  const children = new Set();
+  if (parsed.linear) return children;
+  for (const entry of parsed.entries) {
+    if (entry.parentId !== null && entry.parentId !== undefined) {
+      if (!parsed.byId.has(entry.parentId)) continue;
+      children.add(entry.parentId);
+    }
+  }
+  return children;
+}
+
+/** Return every persisted leaf in file order; no timestamp-based inference. */
+export function listSessionBranches(parsed, source = "Pi Session") {
+  if (parsed.linear) {
+    return parsed.entries.length
+      ? [{ leafId: null, entryCount: parsed.entries.length }]
+      : [];
+  }
+  // Validate every connected component before looking for leaves. A cycle has
+  // no leaf and must remain an explicit error instead of an empty branch list.
+  for (const entry of parsed.entries) ancestryFromLeaf(parsed, entry, source);
+  const children = childIds(parsed);
+  return parsed.entries
+    .filter((entry) => !children.has(entry.id))
+    .map((entry) => ({
+      leafId: entry.id,
+      entryCount: ancestryFromLeaf(parsed, entry, source).length,
+    }));
+}
+
+export async function readPiSession(reference, options = {}) {
+  const sessionFile = await resolvePiSessionFile(reference, options);
+  const parsed = parseNativeSession(
+    await readFile(sessionFile, "utf8"),
+    sessionFile,
+  );
+  const branch = resolveSessionBranch(
+    parsed,
+    { leafId: options.leafId ?? null },
+    sessionFile,
+  );
+  return {
+    sessionFile,
+    parsed,
+    branch,
+    leafId: parsed.linear ? null : (branch.at(-1)?.id ?? null),
+  };
+}

package/src/piq-cli.mjs

@@ -0,0 +1,96 @@
+import { queryPiQ } from "./piq.mjs";
+
+const FROM_FLAG = "--from";
+
+export const PIQ_CLI_USAGE = `Usage:
+  piq <entries|branches|tools|tool-calls|tool-results|compactions> --session <exact-path-or-id> [options]
+
+Options:
+  --leaf <entry-id>                 Read one explicit branch leaf (default: native active leaf)
+  --entry-id <entry-id>             Restrict entries or tools to one native entry ID
+  --role <role>                     Restrict entries to a native message role
+  --from <ISO-8601>                 Include evidence at or after this timestamp
+  --to <ISO-8601>                   Include evidence at or before this timestamp
+  --through-compaction <entry-id>  Stop the branch at this compaction boundary (or latest)
+
+PiQ writes one JSON object per result line. It never writes the source Session.`;
+
+function value(argv, index, flag) {
+  const next = argv[index + 1];
+  if (!next || next.startsWith("--")) throw new Error(`${flag} requires a value`);
+  return next;
+}
+
+export function parsePiQCliArgs(argv) {
+  if (argv.includes("--help") || argv.includes("-h")) return { help: true };
+  const [first, ...rest] = argv;
+  const commands = new Set([
+    "entries",
+    "branches",
+    "tools",
+    "tool-calls",
+    "tool-results",
+    "compactions",
+  ]);
+  const kind = commands.has(first) ? first : "entries";
+  const args = commands.has(first) ? rest : argv;
+  const options = { kind };
+  for (let index = 0; index < args.length; index += 1) {
+    const argument = args[index];
+    switch (argument) {
+      case "--session":
+        options.session = value(args, index, argument);
+        index += 1;
+        break;
+      case "--leaf":
+        options.leafId = value(args, index, argument);
+        index += 1;
+        break;
+      case "--entry-id":
+        options.entryId = value(args, index, argument);
+        index += 1;
+        break;
+      case "--role":
+        options.role = value(args, index, argument);
+        index += 1;
+        break;
+      case FROM_FLAG:
+        options.from = value(args, index, argument);
+        index += 1;
+        break;
+      case "--to":
+        options.to = value(args, index, argument);
+        index += 1;
+        break;
+      case "--through-compaction":
+        options.throughCompaction = value(args, index, argument);
+        index += 1;
+        break;
+      case "--session-root":
+        options.sessionRoot = value(args, index, argument);
+        index += 1;
+        break;
+      case "--jsonl":
+      case "--json":
+        break;
+      default:
+        throw new Error(`Unknown argument: ${argument}`);
+    }
+  }
+  if (!options.session) throw new Error("--session is required");
+  return options;
+}
+
+export async function runPiQCli(options, dependencies = {}) {
+  const query = dependencies.query ?? queryPiQ;
+  return query(options.session, {
+    kind: options.kind,
+    leafId: options.leafId,
+    sessionRoot: options.sessionRoot,
+    entryId: options.entryId,
+    role: options.role,
+    from: options.from,
+    to: options.to,
+    throughCompaction: options.throughCompaction,
+  });
+}