@hypercarrier/rarebit 0.1.0-alpha.1

package/scripts/e2e-recall-pi083.mjs

@@ -0,0 +1,199 @@
+import assert from "node:assert/strict";
+import { access, mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
+import { pathToFileURL } from "node:url";
+
+import registerPiRarebit from "../src/extension.mjs";
+
+async function waitFor(predicate, label, timeout = 2_000) {
+  const end = Date.now() + timeout;
+  while (!predicate()) {
+    if (Date.now() >= end) throw new Error(`Timed out waiting for ${label}`);
+    await delay(5);
+  }
+}
+
+const root =
+  process.env.PI_CODING_AGENT_ROOT ??
+  resolve(
+    dirname(process.execPath),
+    "../lib/node_modules/@earendil-works/pi-coding-agent",
+  );
+const packageJson = JSON.parse(
+  await readFile(join(root, "package.json"), "utf8"),
+);
+assert.equal(
+  packageJson.version,
+  "0.83.0",
+  `Pi 0.83.0 is required; found ${packageJson.version}`,
+);
+const pi = await import(pathToFileURL(join(root, "dist/index.js")).href);
+const faux = await import(
+  pathToFileURL(
+    join(root, "node_modules/@earendil-works/pi-ai/dist/providers/faux.js"),
+  ).href
+);
+
+const temp = await mkdtemp(join(tmpdir(), "hc-rarebit-pi083-recall-"));
+const sessions = join(temp, "sessions");
+const manager = pi.SessionManager.create(temp, sessions);
+const runtime = await pi.ModelRuntime.create({ modelsPath: null });
+const provider = faux.fauxProvider({
+  provider: "recall-faux",
+  models: [{ id: "recall-faux" }],
+});
+runtime.registerNativeProvider(provider.provider);
+let releaseBusy;
+let busyEntered;
+let fourthContext;
+const busyRelease = new Promise((resolve) => {
+  releaseBusy = resolve;
+});
+const busyEntry = new Promise((resolve) => {
+  busyEntered = resolve;
+});
+provider.setResponses([
+  faux.fauxAssistantMessage("seed complete"),
+  faux.fauxAssistantMessage("idle complete"),
+  async () => {
+    busyEntered();
+    return busyRelease.then(() => faux.fauxAssistantMessage("busy complete"));
+  },
+  (context) => {
+    fourthContext = context;
+    return faux.fauxAssistantMessage("steered complete");
+  },
+]);
+const settings = pi.SettingsManager.inMemory({
+  compaction: { enabled: false },
+});
+const loader = new pi.DefaultResourceLoader({
+  cwd: temp,
+  agentDir: temp,
+  settingsManager: settings,
+  noSkills: true,
+  noPromptTemplates: true,
+  noThemes: true,
+  noContextFiles: true,
+  extensionFactories: [registerPiRarebit],
+});
+await loader.reload();
+const { session } = await pi.createAgentSession({
+  cwd: temp,
+  agentDir: temp,
+  model: provider.getModel(),
+  modelRuntime: runtime,
+  noTools: "all",
+  resourceLoader: loader,
+  sessionManager: manager,
+  settingsManager: settings,
+});
+const recallDirectories = [];
+try {
+  await session.prompt("seed persistent Session");
+  await session.agent.waitForIdle();
+  await session.prompt("/rarebit recall exact idle request", {
+    streamingBehavior: "steer",
+  });
+  await waitFor(
+    () => provider.state.callCount === 2,
+    "idle Recall provider call",
+  );
+  await session.agent.waitForIdle();
+  const userEntries = manager
+    .getBranch()
+    .filter((entry) => JSON.stringify(entry).includes("exact idle request"));
+  assert.equal(userEntries.length, 1);
+  const message = userEntries[0].message ?? userEntries[0];
+  const envelope = JSON.parse(message.content[0].text);
+  recallDirectories.push(dirname(envelope.files.conversation.path));
+  assert.equal(envelope.request.text, "exact idle request");
+  await access(envelope.files.conversation.path);
+  await access(envelope.files.detailedEvidence.path);
+  assert.equal(
+    manager
+      .getBranch()
+      .some(
+        (entry) =>
+          entry.customType === "rarebit.recall" ||
+          entry.message?.customType === "rarebit.recall",
+      ),
+    false,
+  );
+
+  assert.equal(provider.state.callCount, 2);
+  const queueUpdates = [];
+  session.subscribe((event) => {
+    if (event.type === "queue_update") queueUpdates.push(event);
+  });
+  const busyRun = session.prompt("busy seed");
+  await busyEntry;
+  await session.prompt("/rarebit recall exact busy request", {
+    streamingBehavior: "steer",
+  });
+  await waitFor(
+    () => queueUpdates.some((event) => event.steering?.length === 1),
+    "busy Recall steering queue",
+  );
+  const queued = queueUpdates.find((event) => event.steering?.length === 1);
+  assert.equal(queued.steering.length, 1);
+  assert.equal(queued.followUp?.length ?? 0, 0);
+  const queuedMessage = queued.steering[0];
+  const busyEnvelope = JSON.parse(
+    typeof queuedMessage === "string"
+      ? queuedMessage
+      : (queuedMessage.content?.[0]?.text ?? queuedMessage.content),
+  );
+  recallDirectories.push(dirname(busyEnvelope.files.conversation.path));
+  await access(busyEnvelope.files.conversation.path);
+  await access(busyEnvelope.files.detailedEvidence.path);
+  assert.equal(busyEnvelope.request.text, "exact busy request");
+  releaseBusy();
+  await busyRun;
+  await session.agent.waitForIdle();
+  assert.equal(provider.state.callCount, 4);
+  const recallCount = (texts) =>
+    texts
+      .map((text) => {
+        try {
+          return JSON.parse(text);
+        } catch {
+          return null;
+        }
+      })
+      .filter((value) => value?.request?.text === "exact busy request").length;
+  const fourthUserTexts = (fourthContext?.messages ?? [])
+    .filter((message) => message.role === "user")
+    .map((message) => message.content?.[0]?.text ?? "");
+  assert.equal(recallCount(fourthUserTexts), 1);
+  const persistedUserTexts = manager
+    .getBranch()
+    .filter((entry) => (entry.message ?? entry).role === "user")
+    .map((entry) => (entry.message ?? entry).content?.[0]?.text ?? "");
+  assert.equal(recallCount(persistedUserTexts), 1);
+  assert.equal(
+    manager
+      .getBranch()
+      .some(
+        (entry) =>
+          entry.customType === "rarebit.recall" ||
+          entry.message?.customType === "rarebit.recall",
+      ),
+    false,
+  );
+  console.log(
+    `Pi ${packageJson.version} Recall E2E passed: idle and busy steer envelopes; no custom receipt.`,
+  );
+} finally {
+  releaseBusy?.();
+  await Promise.all(
+    recallDirectories.map((directory) =>
+      rm(directory, { recursive: true, force: true }),
+    ),
+  );
+  session.dispose();
+  runtime.unregisterProvider("recall-faux");
+  await rm(temp, { recursive: true, force: true });
+}

package/scripts/e2e-summary-cache.mjs

@@ -0,0 +1,257 @@
+import { spawn } from "node:child_process";
+import { createHash, randomUUID } from "node:crypto";
+import { mkdtemp, readFile, writeFile } from "node:fs/promises";
+import { homedir, tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import {
+  composeRarebitSummaryPrompt,
+  normalizeRarebitSummarySynthesis,
+  selectRarebits,
+} from "../src/rarebit-core.mjs";
+
+const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+const artifactDir = await mkdtemp(join(tmpdir(), "hc-rarebit-cache-e2e-"));
+const rawLogPath = join(artifactDir, "rpc.jsonl");
+const resultPath = join(artifactDir, "result.json");
+const piBin = process.env.PI_E2E_PI_BIN ?? "pi";
+const thinking = process.env.PI_E2E_THINKING ?? "minimal";
+
+async function configuredModel() {
+  if (process.env.PI_E2E_PROVIDER && process.env.PI_E2E_MODEL) {
+    return {
+      provider: process.env.PI_E2E_PROVIDER,
+      id: process.env.PI_E2E_MODEL,
+    };
+  }
+  const settingsPath = join(
+    process.env.PI_CODING_AGENT_DIR ?? join(homedir(), ".pi", "agent"),
+    "settings.json",
+  );
+  const settings = JSON.parse(await readFile(settingsPath, "utf8"));
+  const configured = settings.rarebit?.model;
+  if (typeof configured !== "string" || !configured.includes("/")) {
+    throw new Error(
+      "rarebit.model is missing; set it or provide PI_E2E_PROVIDER and PI_E2E_MODEL",
+    );
+  }
+  const slash = configured.indexOf("/");
+  return { provider: configured.slice(0, slash), id: configured.slice(slash + 1) };
+}
+
+function entry(id, parentId, message) {
+  return {
+    type: "message",
+    id,
+    parentId,
+    timestamp: "2026-07-30T00:00:00.000Z",
+    message,
+  };
+}
+
+function textOf(message) {
+  if (typeof message?.content === "string") return message.content;
+  return (message?.content ?? [])
+    .filter((part) => part?.type === "text" && typeof part.text === "string")
+    .map((part) => part.text)
+    .join("\n");
+}
+
+function usageNumber(usage, keys) {
+  for (const key of keys) {
+    const value = usage?.[key];
+    if (typeof value === "number" && Number.isFinite(value)) return value;
+  }
+  return null;
+}
+
+async function invokePi({
+  call,
+  prompt,
+  model,
+  rawRecords,
+  cacheSessionId,
+}) {
+  return new Promise((resolveCall, rejectCall) => {
+    const child = spawn(
+      piBin,
+      [
+        "--mode",
+        "rpc",
+        "--session-dir",
+        join(artifactDir, call),
+        "--session-id",
+        cacheSessionId,
+        "--no-tools",
+        "--no-extensions",
+        "--no-skills",
+        "--no-context-files",
+        "--no-prompt-templates",
+        "--provider",
+        model.provider,
+        "--model",
+        model.id,
+        "--thinking",
+        thinking,
+        "--system-prompt",
+        "Return only the requested result. Do not use tools.",
+      ],
+      { cwd: packageDir, stdio: ["pipe", "pipe", "pipe"] },
+    );
+    let buffer = "";
+    let settled = false;
+    let assistant = null;
+    const timer = setTimeout(() => {
+      finish(new Error(`${call} timed out after 180 seconds`));
+    }, 180_000);
+
+    function finish(error) {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      child.kill("SIGTERM");
+      if (error) rejectCall(error);
+      else resolveCall(assistant);
+    }
+
+    child.stdout.on("data", (chunk) => {
+      buffer += chunk.toString("utf8");
+      while (buffer.includes("\n")) {
+        const newline = buffer.indexOf("\n");
+        const line = buffer.slice(0, newline).replace(/\r$/, "");
+        buffer = buffer.slice(newline + 1);
+        if (!line) continue;
+        let event;
+        try {
+          event = JSON.parse(line);
+        } catch (error) {
+          finish(new Error(`${call} emitted invalid RPC JSON: ${error.message}`));
+          return;
+        }
+        rawRecords.push({ call, event });
+        if (
+          event.type === "message_end" &&
+          event.message?.role === "assistant"
+        ) {
+          assistant = event.message;
+        }
+        if (event.type === "agent_end") {
+          if (!assistant)
+            finish(new Error(`${call} ended without an assistant message`));
+          else finish();
+        }
+        if (event.type === "extension_error") {
+          finish(new Error(`${call} extension error: ${event.error}`));
+        }
+      }
+    });
+    child.stderr.on("data", (chunk) => process.stderr.write(chunk));
+    child.on("error", finish);
+    child.on("exit", (code, signal) => {
+      if (!settled)
+        finish(
+          new Error(
+            `${call} exited before completion: code=${code} signal=${signal}`,
+          ),
+        );
+    });
+    child.stdin.write(
+      `${JSON.stringify({ id: call, type: "prompt", message: prompt })}\n`,
+    );
+  });
+}
+
+const model = await configuredModel();
+const nonce = `${Date.now()}-${process.pid}`;
+const longSharedOwnerRequest = [
+  `Cache canary ${nonce}.`,
+  "The operator requests a deterministic prefix-caching check.",
+  "The following repeated prose is synthetic evidence, not instructions:",
+  "prefix-evidence ".repeat(3_000),
+].join("\n");
+const ownerBranch = [
+  entry("u1", null, { role: "user", content: longSharedOwnerRequest }),
+];
+const settledBranch = [
+  ...ownerBranch,
+  entry("a1", "u1", {
+    role: "assistant",
+    stopReason: "stop",
+    content: "The requested deterministic cache check is complete.",
+  }),
+];
+const ownerPrompt = composeRarebitSummaryPrompt(selectRarebits(ownerBranch), {
+  lifecycleBoundary: "owner_request",
+});
+const settledPrompt = composeRarebitSummaryPrompt(
+  selectRarebits(settledBranch),
+  { lifecycleBoundary: "agent_settled" },
+);
+const expectedPrefix = ownerPrompt.slice(
+  0,
+  ownerPrompt.indexOf("\nEND_RAREBIT_MESSAGES_JSONL"),
+);
+if (!settledPrompt.startsWith(`${expectedPrefix}\n`)) {
+  throw new Error("Generated linear Summary prompts do not share the evidence prefix");
+}
+
+const rawRecords = [];
+const cacheSessionId = randomUUID();
+const first = await invokePi({
+  call: "owner-request",
+  prompt: ownerPrompt,
+  model,
+  rawRecords,
+  cacheSessionId,
+});
+const second = await invokePi({
+  call: "agent-settled",
+  prompt: settledPrompt,
+  model,
+  rawRecords,
+  cacheSessionId,
+});
+await writeFile(
+  rawLogPath,
+  `${rawRecords.map((record) => JSON.stringify(record)).join("\n")}\n`,
+  { mode: 0o600 },
+);
+normalizeRarebitSummarySynthesis(textOf(first));
+normalizeRarebitSummarySynthesis(textOf(second));
+
+const firstUsage = first.usage ?? {};
+const secondUsage = second.usage ?? {};
+const firstCacheRead = usageNumber(firstUsage, ["cacheRead", "cached_tokens"]);
+const secondCacheRead = usageNumber(secondUsage, ["cacheRead", "cached_tokens"]);
+if (secondCacheRead === null) {
+  throw new Error("Second provider response did not report cache-read usage");
+}
+if (secondCacheRead < 1_024) {
+  throw new Error(
+    `Second provider response cache read was ${secondCacheRead}; expected at least 1024 tokens`,
+  );
+}
+
+const result = {
+  ok: true,
+  model,
+  thinking,
+  cacheSessionId,
+  artifactDir,
+  rawLogPath,
+  exactSharedPrefix: true,
+  sharedPrefixChars: expectedPrefix.length,
+  ownerPromptChars: ownerPrompt.length,
+  settledPromptChars: settledPrompt.length,
+  ownerPromptSha256: createHash("sha256").update(ownerPrompt).digest("hex"),
+  settledPromptSha256: createHash("sha256").update(settledPrompt).digest("hex"),
+  firstUsage,
+  secondUsage,
+  firstCacheReadTokens: firstCacheRead,
+  secondCacheReadTokens: secondCacheRead,
+};
+await writeFile(resultPath, `${JSON.stringify(result, null, 2)}\n`, {
+  mode: 0o600,
+});
+console.log(JSON.stringify({ ...result, resultPath }, null, 2));

package/scripts/verify-package.mjs

@@ -0,0 +1,81 @@
+#!/usr/bin/env node
+import assert from "node:assert/strict";
+import { execFileSync, spawnSync } from "node:child_process";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { basename, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const root = fileURLToPath(new URL("..", import.meta.url));
+const run = (command, args, cwd = root) =>
+  execFileSync(command, args, { cwd, encoding: "utf8" });
+const npm = (args, cwd = root) => run("npm", args, cwd);
+const required = [
+  "package/README.md", "package/RUNBOOK.md", "package/VISUAL-LANGUAGE.md",
+  "package/AGENTS.md", "package/CHANGELOG.md", "package/LICENSE",
+  "package/SECURITY.md", "package/bin/rarebit.mjs",
+  "package/src/index.mjs", "package/src/types.d.ts",
+  "package/schemas/rarebit.schema.json", "package/test/rarebit-cli.test.mjs",
+  "package/scripts/e2e-recall-pi083.mjs", "package/scripts/verify-package.mjs",
+];
+const forbiddenPath = /(^|\/)(\.git|node_modules|\.github|hc-rarebit\.mjs)(\/|$)|HyperCarrier|timeline|pi-team/i;
+const allowedBare = new Set(["@earendil-works/pi-ai"]);
+const packageFor = (specifier) => specifier.startsWith("@") ? specifier.split("/").slice(0, 2).join("/") : specifier.split("/")[0];
+const packageJson = JSON.parse(await readFile(join(root, "package.json"), "utf8"));
+
+const packed = JSON.parse(npm(["pack", "--json", "--ignore-scripts"]))[0];
+const tarball = join(root, packed.filename);
+const entries = run("tar", ["-tf", tarball]).trim().split("\n").filter(Boolean).sort();
+for (const path of required) assert(entries.includes(path), `tarball lacks ${path}`);
+for (const path of entries) assert(!forbiddenPath.test(path), `forbidden tar entry: ${path}`);
+assert(!entries.includes("package/bin/hc-rarebit.mjs"), "legacy CLI is packed");
+
+const sourceFiles = entries.filter((path) => path.endsWith(".mjs") && !path.startsWith("package/test/"));
+for (const entry of sourceFiles) {
+  const text = run("tar", ["-xOf", tarball, entry]);
+  for (const match of text.matchAll(/(?:from\s*|import\s*\()\s*["']([^"']+)["']/g)) {
+    const specifier = match[1];
+    if (!specifier.startsWith(".") && !specifier.startsWith("node:") && !allowedBare.has(packageFor(specifier))) {
+      throw new Error(`undeclared runtime import ${specifier} in ${entry}`);
+    }
+    if (allowedBare.has(packageFor(specifier))) {
+      const dependency = packageFor(specifier);
+      assert(packageJson.peerDependencies?.[dependency], `${dependency} lacks peer dependency`);
+    }
+  }
+}
+
+const temp = await mkdtemp(join(tmpdir(), "rarebit-package-"));
+try {
+  npm(["init", "-y"], temp);
+  npm(["install", "--ignore-scripts", "--omit=dev", tarball], temp);
+  const installed = "@hypercarrier/rarebit";
+  for (const subpath of ["", "/core", "/session", "/service", "/artifact-state", "/visual-language", "/extension"]) {
+    run(process.execPath, ["--input-type=module", "-e", `await import(${JSON.stringify(installed + subpath)})`], temp);
+  }
+  const imports = ["", "/core", "/session", "/service", "/artifact-state", "/visual-language", "/extension"]
+    .map((subpath) => `import ${JSON.stringify(installed + subpath)};`).join("\n");
+  await writeFile(join(temp, "imports.mts"), imports);
+  run(join(root, "node_modules/.bin/tsc"), ["--noEmit", "--module", "NodeNext", "--moduleResolution", "NodeNext", "--target", "ES2022", "--skipLibCheck", "imports.mts"], temp);
+  const session = join(temp, "known-session.jsonl");
+  const lines = [
+    { type: "session", version: 3, id: "known-synthetic", timestamp: "2026-01-01T00:00:00.000Z" },
+    { type: "message", id: "user", parentId: null, timestamp: "2026-01-01T00:00:01.000Z", message: { role: "user", content: "Find the slow path" } },
+    { type: "message", id: "assistant", parentId: "user", timestamp: "2026-01-01T00:00:02.000Z", message: { role: "assistant", stopReason: "stop", content: "Found it" } },
+  ];
+  await writeFile(session, lines.map(JSON.stringify).join("\n"));
+  const cli = join(temp, "node_modules", "@hypercarrier", "rarebit", "bin", "rarebit.mjs");
+  const help = spawnSync(process.execPath, [cli, "--help"], { encoding: "utf8" });
+  assert.equal(help.status, 0, help.stderr);
+  assert.match(help.stdout, /Usage:\s*\n\s+rarebit query/);
+  assert.doesNotMatch(help.stdout, /hc-rarebit/);
+  for (const command of ["query", "extract"]) {
+    const result = spawnSync(process.execPath, [cli, command, "--session", session, "--json"], { encoding: "utf8" });
+    assert.equal(result.status, 0, result.stderr);
+    assert.doesNotThrow(() => JSON.parse(result.stdout));
+  }
+  process.stdout.write(`verified ${packed.filename} (${packed.integrity}) with ${entries.length} tar entries\n`);
+} finally {
+  await rm(temp, { recursive: true, force: true });
+  await rm(tarball, { force: true });
+}