@hypercarrier/rarebit 0.3.0 → 0.4.0

package/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.4.0 — 2026-10-07
+
+- Replace Recall's hour-bucketed `rarebit_conversation/v1` with a flat
+  `rarebit_conversation/v2` message list. Each message has `seq`, `round`,
+  `role`, `kind`, second-precision UTC `time`, and `text`. Detailed evidence
+  occurrences gain the same `seq` join key.
+- Add the conversation schema and `jq` tail queries to the Recall message.
+
 ## 0.3.0 — 2026-10-04
 
 - Add `/rarebit view [context|all|peek|off]` and a View action in the palette.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hypercarrier/rarebit",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Sparse, provenance-bearing conversational evidence recovery for Pi sessions.",
   "license": "MIT",
   "author": "HyperCarrier contributors",

package/scripts/e2e-recall-pi0842.mjs

@@ -18,29 +18,47 @@ async function waitFor(predicate, label, timeout = 2_000) {
 function expectedRecallMarkdown(conversationPath, detailedPath, request) {
   return `# Rarebit Recall
 
-*Use these local private evidence files to recover historical context for this turn.*
+*Answer the **Current request** below. Use these local private files to recover earlier context from this Session.*
 
-## How to use this bundle
+## Files
 
-1. Treat **Current request** below as the exact current user request.
-2. Read **Conversation** first for meaning.
-3. Use **Detailed evidence** only for source, Session, branch, or lineage facts.
-4. Answer the current request using this evidence.
-
-## Local private evidence files
-
-**Conversation** — \`rarebit_conversation/v1\`
+**Conversation** — \`rarebit_conversation/v2\`. Read this first.
 
 \`\`\`text
 ${conversationPath}
 \`\`\`
 
-**Detailed evidence** — \`rarebit_message_recall/v1\`
+**Detailed evidence** — \`rarebit_message_recall/v1\`. Use it only for source, Session, branch, or lineage facts. Join on \`seq\`.
 
 \`\`\`text
 ${detailedPath}
 \`\`\`
 
+## Conversation schema
+
+\`\`\`text
+{ messageCount, lastRound, messages: [{ seq, round, role, kind, time, text }] }
+seq    1-based, oldest first
+round  increments at each user message; agent messages before the first user message have round 0
+role   "user" | "agent"
+kind   "request" (user) | "progress" (agent text before a tool call) | "reply" (agent final answer)
+time   UTC "YYYY-MM-DDTHH:MM:SSZ", or null
+\`\`\`
+
+Messages contain user and agent text only. Tool calls and tool results are excluded.
+
+## Example queries
+
+Set \`F\` to the Conversation path. Read the smallest slice that answers the request.
+
+\`\`\`sh
+jq '.messages[-20:]' "$F"                                              # last 20 messages
+jq '[.messages[] | select(.role == "user")][-3:]' "$F"                 # last 3 user messages
+jq '[.messages[] | select(.kind == "reply")][-10:]' "$F"               # last 10 agent replies
+jq '.lastRound as $r | [.messages[] | select(.round > $r - 5)]' "$F"  # last 5 rounds
+jq '(now - 8*3600 | todate) as $t | [.messages[] | select(.time >= $t)]' "$F"  # last 8 hours
+\`\`\`
+
 ## Current request
 
 \`\`\`text
@@ -50,7 +68,7 @@ ${request}
 
 function conversationPathFrom(message) {
   const match = message.match(
-    /\*\*Conversation\*\* — `rarebit_conversation\/v1`\n\n```text\n([^\n]*rarebit-conversation\.json)\n```/,
+    /\*\*Conversation\*\* — `rarebit_conversation\/v2`\. Read this first\.\n\n```text\n([^\n]*rarebit-conversation\.json)\n```/,
   );
   assert.ok(match, "Recall Markdown must contain the fenced conversation path");
   return match[1];
@@ -58,7 +76,7 @@ function conversationPathFrom(message) {
 
 function detailedPathFrom(message) {
   const match = message.match(
-    /\*\*Detailed evidence\*\* — `rarebit_message_recall\/v1`\n\n```text\n([^\n]*rarebit-evidence\.json)\n```/,
+    /\*\*Detailed evidence\*\* — `rarebit_message_recall\/v1`\. Use it only for source, Session, branch, or lineage facts\. Join on `seq`\.\n\n```text\n([^\n]*rarebit-evidence\.json)\n```/,
   );
   assert.ok(match, "Recall Markdown must contain the fenced detailed path");
   return match[1];

package/src/extension.mjs

@@ -543,25 +543,43 @@ export default function registerPiRarebit(pi, config = {}) {
   };
   const recallMessage = (recall, requestText) => `# Rarebit Recall
 
-*Use these local private evidence files to recover historical context for this turn.*
+*Answer the **Current request** below. Use these local private files to recover earlier context from this Session.*
 
-## How to use this bundle
+## Files
 
-1. Treat **Current request** below as the exact current user request.
-2. Read **Conversation** first for meaning.
-3. Use **Detailed evidence** only for source, Session, branch, or lineage facts.
-4. Answer the current request using this evidence.
-
-## Local private evidence files
-
-**Conversation** — \`rarebit_conversation/v${RAREBIT_CONVERSATION_SCHEMA_VERSION}\`
+**Conversation** — \`rarebit_conversation/v${RAREBIT_CONVERSATION_SCHEMA_VERSION}\`. Read this first.
 
 ${fence(recall.conversationPath)}
 
-**Detailed evidence** — \`rarebit_message_recall/v${RAREBIT_RECALL_SCHEMA_VERSION}\`
+**Detailed evidence** — \`rarebit_message_recall/v${RAREBIT_RECALL_SCHEMA_VERSION}\`. Use it only for source, Session, branch, or lineage facts. Join on \`seq\`.
 
 ${fence(recall.detailedPath)}
 
+## Conversation schema
+
+\`\`\`text
+{ messageCount, lastRound, messages: [{ seq, round, role, kind, time, text }] }
+seq    1-based, oldest first
+round  increments at each user message; agent messages before the first user message have round 0
+role   "user" | "agent"
+kind   "request" (user) | "progress" (agent text before a tool call) | "reply" (agent final answer)
+time   UTC "YYYY-MM-DDTHH:MM:SSZ", or null
+\`\`\`
+
+Messages contain user and agent text only. Tool calls and tool results are excluded.
+
+## Example queries
+
+Set \`F\` to the Conversation path. Read the smallest slice that answers the request.
+
+\`\`\`sh
+jq '.messages[-20:]' "$F"                                              # last 20 messages
+jq '[.messages[] | select(.role == "user")][-3:]' "$F"                 # last 3 user messages
+jq '[.messages[] | select(.kind == "reply")][-10:]' "$F"               # last 10 agent replies
+jq '.lastRound as $r | [.messages[] | select(.round > $r - 5)]' "$F"  # last 5 rounds
+jq '(now - 8*3600 | todate) as $t | [.messages[] | select(.time >= $t)]' "$F"  # last 8 hours
+\`\`\`
+
 ## Current request
 
 ${fence(requestText)}`;

package/src/rarebit-recall.mjs

@@ -6,7 +6,7 @@ import { isAbsolute, join, resolve } from "node:path";
 import { selectRarebits } from "./rarebit-core.mjs";
 
 export const RAREBIT_RECALL_SCHEMA_VERSION = 1;
-export const RAREBIT_CONVERSATION_SCHEMA_VERSION = 1;
+export const RAREBIT_CONVERSATION_SCHEMA_VERSION = 2;
 export const RAREBIT_RECALL_DETAILED_FILENAME = "rarebit-evidence.json";
 export const RAREBIT_RECALL_CONVERSATION_FILENAME = "rarebit-conversation.json";
 
@@ -52,16 +52,20 @@ function recallDocument({ sessionId, sessionFile, branch, selection, now }) {
       occurrenceCount: selection.occurrences.length,
       uniquePayloadCount: selection.payloads.length,
       occurrences: selection.occurrences.map(
-        ({
-          occurrenceId,
+        (
+          {
+            occurrenceId,
           sourceEntryId,
           order,
           timestamp,
           role,
           outcome,
-          contentHash,
-          text,
-        }) => ({
+            contentHash,
+            text,
+          },
+          index,
+        ) => ({
+          seq: index + 1,
           occurrenceId,
           sourceEntryId,
           order,
@@ -76,37 +80,45 @@ function recallDocument({ sessionId, sessionFile, branch, selection, now }) {
   };
 }
 
-function conversationHour(timestamp) {
+// Second precision matches jq `todate`, so `.time >= (now - N | todate)` and
+// `fromdate` work without parsing fractional seconds.
+function conversationTime(timestamp) {
   if (typeof timestamp !== "string") return null;
   const value = new Date(timestamp);
   if (Number.isNaN(value.getTime())) return null;
-  return `${value.toISOString().slice(0, 13)}:00Z`;
+  return `${value.toISOString().slice(0, 19)}Z`;
 }
 
+const CONVERSATION_KIND = Object.freeze({
+  user: "request",
+  continuation: "progress",
+  stop: "reply",
+});
+
+/**
+ * One flat, branch-ordered message list. `seq` matches the detailed
+ * occurrence `seq`. A round starts at each user message, so tail slices by
+ * message, role, kind, round, or time are single jq filters.
+ */
 function conversationDocument(selection) {
-  const byHour = new Map();
-  for (const occurrence of selection.occurrences) {
-    const hour = conversationHour(occurrence.timestamp);
-    let bucket = byHour.get(hour);
-    if (!bucket) {
-      bucket = { hour, messages: [] };
-      byHour.set(hour, bucket);
-    }
-    bucket.messages.push({
-      role: occurrence.role === "user" ? "user" : "agent",
-      content: occurrence.text,
-    });
-  }
-  const hours = [...byHour.values()].sort((left, right) => {
-    if (left.hour === null) return right.hour === null ? 0 : 1;
-    if (right.hour === null) return -1;
-    return left.hour.localeCompare(right.hour);
+  let round = 0;
+  const messages = selection.occurrences.map((occurrence, index) => {
+    if (occurrence.outcome === "user") round += 1;
+    return {
+      seq: index + 1,
+      round,
+      role: occurrence.outcome === "user" ? "user" : "agent",
+      kind: CONVERSATION_KIND[occurrence.outcome],
+      time: conversationTime(occurrence.timestamp),
+      text: occurrence.text,
+    };
   });
   return {
     schemaVersion: RAREBIT_CONVERSATION_SCHEMA_VERSION,
     type: "rarebit_conversation",
-    timeZone: "UTC",
-    hours,
+    messageCount: messages.length,
+    lastRound: round,
+    messages,
   };
 }
 

package/test/rarebit-recall.test.mjs

@@ -140,27 +140,52 @@ test("recall materialization writes private detailed evidence and a chronologica
       await readFile(result.conversationPath, "utf8"),
     );
     assert.deepEqual(conversation, {
-      schemaVersion: 1,
+      schemaVersion: 2,
       type: "rarebit_conversation",
-      timeZone: "UTC",
-      hours: [
+      messageCount: 3,
+      lastRound: 1,
+      messages: [
         {
-          hour: "2026-07-24T01:00Z",
-          messages: [{ role: "user", content: "Exact owner evidence." }],
+          seq: 1,
+          round: 1,
+          role: "user",
+          kind: "request",
+          time: "2026-07-24T01:02:03Z",
+          text: "Exact owner evidence.",
         },
         {
-          hour: "2026-07-24T02:00Z",
-          messages: [
-            { role: "agent", content: "Exact continuation evidence." },
-            { role: "agent", content: "Exact stop evidence." },
-          ],
+          seq: 2,
+          round: 1,
+          role: "agent",
+          kind: "progress",
+          time: "2026-07-24T02:00:00Z",
+          text: "Exact continuation evidence.",
+        },
+        {
+          seq: 3,
+          round: 1,
+          role: "agent",
+          kind: "reply",
+          time: "2026-07-24T02:45:00Z",
+          text: "Exact stop evidence.",
         },
       ],
     });
+    assert.deepEqual(
+      detailed.selection.occurrences.map(({ seq, sourceEntryId }) => ({
+        seq,
+        sourceEntryId,
+      })),
+      [
+        { seq: 1, sourceEntryId: "owner" },
+        { seq: 2, sourceEntryId: "continue" },
+        { seq: 3, sourceEntryId: "stop" },
+      ],
+    );
     const conversationText = await readFile(result.conversationPath, "utf8");
     assert.doesNotMatch(
       conversationText,
-      /sourceEntryId|occurrenceId|manifestHash|contentHash|timestamp|session/i,
+      /sourceEntryId|occurrenceId|manifestHash|contentHash|session/i,
     );
   } finally {
     await result.discard();
@@ -168,7 +193,7 @@ test("recall materialization writes private detailed evidence and a chronologica
   }
 });
 
-test("conversation buckets sort chronologically and treat producer-tagged input as a normal user message", async () => {
+test("conversation keeps branch order, numbers rounds, and treats producer-tagged input as a normal user message", async () => {
   const root = await fixtureRoot("bucket-order");
   const sessionFile = join(root, "session.jsonl");
   await writeFile(sessionFile, "{}\n");
@@ -218,23 +243,23 @@ test("conversation buckets sort chronologically and treat producer-tagged input
     const conversation = JSON.parse(
       await readFile(result.conversationPath, "utf8"),
     );
-    assert.deepEqual(conversation.hours, [
-      {
-        hour: "2026-07-24T01:00Z",
-        messages: [{ role: "agent", content: "Earlier agent evidence." }],
-      },
-      {
-        hour: "2026-07-24T02:00Z",
-        messages: [
-          { role: "user", content: "Unknown-origin user evidence." },
-          { role: "agent", content: "Later agent evidence." },
-        ],
-      },
-      {
-        hour: null,
-        messages: [{ role: "agent", content: "Unknown-time agent evidence." }],
-      },
-    ]);
+    assert.equal(conversation.lastRound, 1);
+    assert.deepEqual(
+      conversation.messages.map(({ seq, round, role, kind, time, text }) => [
+        seq,
+        round,
+        role,
+        kind,
+        time,
+        text,
+      ]),
+      [
+        [1, 1, "user", "request", "2026-07-24T02:10:00Z", "Unknown-origin user evidence."],
+        [2, 1, "agent", "reply", "2026-07-24T01:10:00Z", "Earlier agent evidence."],
+        [3, 1, "agent", "reply", null, "Unknown-time agent evidence."],
+        [4, 1, "agent", "reply", "2026-07-24T02:20:00Z", "Later agent evidence."],
+      ],
+    );
     assert.doesNotMatch(
       await readFile(result.conversationPath, "utf8"),
       /human|producer|rpc/,
@@ -318,29 +343,47 @@ test("recall sends one exact fenced Markdown steer message in idle and busy cont
     "\n# Heading-looking request\n- list-looking item\n```text\ninside backticks\n```\n~~~~\ninside tildes\n~~~~\nTrailing request\n";
   const expectedMessage = `# Rarebit Recall
 
-*Use these local private evidence files to recover historical context for this turn.*
-
-## How to use this bundle
+*Answer the **Current request** below. Use these local private files to recover earlier context from this Session.*
 
-1. Treat **Current request** below as the exact current user request.
-2. Read **Conversation** first for meaning.
-3. Use **Detailed evidence** only for source, Session, branch, or lineage facts.
-4. Answer the current request using this evidence.
+## Files
 
-## Local private evidence files
-
-**Conversation** — \`rarebit_conversation/v1\`
+**Conversation** — \`rarebit_conversation/v2\`. Read this first.
 
 \`\`\`text
 ${recall.conversationPath}
 \`\`\`
 
-**Detailed evidence** — \`rarebit_message_recall/v1\`
+**Detailed evidence** — \`rarebit_message_recall/v1\`. Use it only for source, Session, branch, or lineage facts. Join on \`seq\`.
 
 ~~~~text
 ${recall.detailedPath}
 ~~~~
 
+## Conversation schema
+
+\`\`\`text
+{ messageCount, lastRound, messages: [{ seq, round, role, kind, time, text }] }
+seq    1-based, oldest first
+round  increments at each user message; agent messages before the first user message have round 0
+role   "user" | "agent"
+kind   "request" (user) | "progress" (agent text before a tool call) | "reply" (agent final answer)
+time   UTC "YYYY-MM-DDTHH:MM:SSZ", or null
+\`\`\`
+
+Messages contain user and agent text only. Tool calls and tool results are excluded.
+
+## Example queries
+
+Set \`F\` to the Conversation path. Read the smallest slice that answers the request.
+
+\`\`\`sh
+jq '.messages[-20:]' "$F"                                              # last 20 messages
+jq '[.messages[] | select(.role == "user")][-3:]' "$F"                 # last 3 user messages
+jq '[.messages[] | select(.kind == "reply")][-10:]' "$F"               # last 10 agent replies
+jq '.lastRound as $r | [.messages[] | select(.round > $r - 5)]' "$F"  # last 5 rounds
+jq '(now - 8*3600 | todate) as $t | [.messages[] | select(.time >= $t)]' "$F"  # last 8 hours
+\`\`\`
+
 ## Current request
 
 \`\`\`\`text
@@ -390,7 +433,7 @@ test("long delimiter runs ending at value boundaries use the shortest safe fence
   assert.equal(harness.sent.length, 1);
   const message = harness.sent[0].args[0];
   const conversationBlock = message.match(
-    /\*\*Conversation\*\* — `rarebit_conversation\/v1`\n\n([`~]+)text\n([^\n]*)\n\1\n\n\*\*Detailed evidence\*\*/,
+    /\*\*Conversation\*\* — `rarebit_conversation\/v2`\. Read this first\.\n\n([`~]+)text\n([^\n]*)\n\1\n\n\*\*Detailed evidence\*\*/,
   );
   assert.ok(conversationBlock);
   assert.equal(conversationBlock[2], conversationPath);