@circuitwall/jarela 2.2.1 → 2.4.0

package/lib/agents/citation-checker.test.ts

@@ -27,11 +27,27 @@ afterAll(() => {
 });
 
 describe("normalizeSource", () => {
-  it("lowercases + strips query/fragment + trailing slash on URLs", () => {
-    expect(normalizeSource("HTTPS://Example.com/Docs/?x=1#a")).toBe("https://example.com/docs");
+  it("lowercases + strips tracking params, page anchors and trailing slash on URLs", () => {
+    expect(normalizeSource("HTTPS://Example.com/Docs/?utm_source=x&fbclid=y#a")).toBe("https://example.com/docs");
     expect(normalizeSource("https://a.io/p/")).toBe("https://a.io/p");
   });
 
+  it("keeps the query and hash route that identify a message permalink", () => {
+    const a = normalizeSource("https://outlook.office365.com/owa/?ItemID=AAMk%2BA%3D&exvsurl=1&viewmodel=ReadMessageItem");
+    const b = normalizeSource("https://outlook.office365.com/owa/?ItemID=AAMk%2BB%3D&exvsurl=1&viewmodel=ReadMessageItem");
+    expect(a).not.toBe(b);
+    // Base64 item ids are case-sensitive; folding case would merge distinct messages.
+    expect(a).toContain("ItemID=AAMk%2BA%3D");
+    expect(normalizeSource("https://mail.google.com/mail/u/0/#all/18c2f0a1b2c3d4e5"))
+      .toBe("https://mail.google.com/mail/u/0#all/18c2f0a1b2c3d4e5");
+    expect(normalizeSource("https://www.youtube.com/watch?v=dQw4w9WgXcQ&utm_medium=share"))
+      .toBe("https://www.youtube.com/watch?v=dQw4w9WgXcQ");
+  });
+
+  it("treats the same permalink with reordered params as one source", () => {
+    expect(normalizeSource("https://x.test/r?b=2&a=1")).toBe(normalizeSource("https://x.test/r?a=1&b=2&utm_campaign=z"));
+  });
+
   it("normalises backslashes and trailing slashes on paths", () => {
     expect(normalizeSource("docs\\Adr\\README.md")).toBe("docs/adr/readme.md");
     expect(normalizeSource("lib/agents/")).toBe("lib/agents");
@@ -137,6 +153,18 @@ describe("parseCitationVerdict", () => {
     expect(out).toEqual([{ text: "a", marker: null, link: null, verified: false, reason: "no link", impact: "med" }]);
   });
 
+  it("finds the verdict when the model wraps it in prose that also contains braces", () => {
+    const verdict = '{"claims":[{"text":"rate is 5%","marker":1,"verified":true,"reason":"in [1]","impact":"high"}]}';
+    const expected = [{ text: "rate is 5%", marker: 1, link: null, verified: true, reason: "in [1]", impact: "high" }];
+    expect(parseCitationVerdict(`Checked the {draft} reply. ${verdict}`)).toEqual(expected);
+    expect(parseCitationVerdict(`${verdict}\nNote: ignored the {code} block.`)).toEqual(expected);
+  });
+
+  it("keeps braces that belong to a claim string", () => {
+    const out = parseCitationVerdict('{"claims":[{"text":"set {a: 1} in config","verified":false,"reason":"} unmatched"}]}');
+    expect(out?.[0].text).toBe("set {a: 1} in config");
+    expect(out?.[0].reason).toBe("} unmatched");
+  });
   it("returns null on malformed JSON", () => {
     expect(parseCitationVerdict("{not json")).toBeNull();
   });
@@ -299,6 +327,16 @@ describe("mergeDeclaredReferences", () => {
     ]);
   });
 
+  it("keeps two declared emails that differ only by their permalink id", () => {
+    const out = mergeDeclaredReferences([], [
+      { label: "Invoice mail", href: "https://mail.google.com/mail/u/0/#all/18c2f0a1b2c3d4e5" },
+      { label: "Contract mail", href: "https://mail.google.com/mail/u/0/#all/18c2f0a1b2c3ffff" },
+      { label: "Board invite", href: "https://outlook.office365.com/owa/?ItemID=AAMk%2BA%3D&exvsurl=1" },
+      { label: "Offsite invite", href: "https://outlook.office365.com/owa/?ItemID=AAMk%2BB%3D&exvsurl=1" },
+    ]);
+    expect(out.map((e) => e.label)).toEqual(["Invoice mail", "Contract mail", "Board invite", "Offsite invite"]);
+  });
+
   it("returns a copy of the manifest unchanged when there are no declared refs", () => {
     const manifest = [{ n: 1, label: "x", href: "x" }];
     expect(mergeDeclaredReferences(manifest, [])).toEqual(manifest);

package/lib/agents/citation-checker.ts

@@ -85,17 +85,28 @@ const SOURCE_PRODUCING_TOOLS_RESULT = new Set([
   "web_search", "fetch_webpage", "web_fetch",
 ]);
 
-// Strip query/fragment for URLs and normalize backslashes for paths so a
-// citation that drops `?utm=...` or uses forward slashes on Windows still
-// matches the visited entry. Also fold `~/...` to the resolved home dir
-// so `~/Downloads/x` and `C:\Users\andre\Downloads\x` de-dupe.
+const TRACKING_PARAM_RE = /^(?:utm(?:_.*)?|fbclid|gclid|dclid|msclkid|mc_cid|mc_eid|igshid|ref_src)$/i;
+
+// Drop tracking params and in-page anchors for URLs and normalize
+// backslashes for paths so a citation that adds `?utm_source=...` or uses
+// forward slashes on Windows still matches the visited entry. Identifying
+// query params (Outlook `?ItemID=`, `watch?v=`) and hash routes (Gmail
+// `#all/<id>`) are kept verbatim: they are case-sensitive and are the only
+// thing that distinguishes two message permalinks. Also fold `~/...` to the
+// resolved home dir so `~/Downloads/x` and `C:\Users\andre\Downloads\x` de-dupe.
 export function normalizeSource(raw: string): string {
   const trimmed = raw.trim();
   if (!trimmed) return "";
   if (/^https?:\/\//i.test(trimmed)) {
     try {
       const u = new URL(trimmed);
-      return `${u.protocol}//${u.host}${u.pathname}`.replace(/\/+$/, "").toLowerCase();
+      const base = `${u.protocol}//${u.host}${u.pathname}`.replace(/\/+$/, "").toLowerCase();
+      const params = [...u.searchParams]
+        .filter(([key]) => !TRACKING_PARAM_RE.test(key))
+        .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+      const query = params.length > 0 ? `?${new URLSearchParams(params).toString()}` : "";
+      const hashRoute = u.hash.includes("/") ? u.hash : "";
+      return base + query + hashRoute;
     } catch {
       return trimmed.toLowerCase();
     }
@@ -436,12 +447,9 @@ ${sent}`;
 export function parseCitationVerdict(text: string): CitationClaim[] | null {
   if (!text) return null;
   const stripped = text.replace(/```(?:json)?\s*/gi, "").replace(/```/g, "").trim();
-  const m = /\{[^]*"claims"[^]*\}/.exec(stripped);
-  if (!m) return null;
-  let parsed: unknown;
-  try { parsed = JSON.parse(m[0]); } catch { return null; }
-  if (!parsed || typeof parsed !== "object") return null;
-  const claimsRaw = (parsed as { claims?: unknown }).claims;
+  const parsed = findClaimsObject(stripped);
+  if (!parsed) return null;
+  const claimsRaw = parsed.claims;
   if (!Array.isArray(claimsRaw)) return null;
   const claims: CitationClaim[] = [];
   for (const item of claimsRaw) {
@@ -464,6 +472,43 @@ export function parseCitationVerdict(text: string): CitationClaim[] | null {
   return claims;
 }
 
+// Models often wrap the verdict in prose that itself contains braces, so a
+// single greedy `{...}` match can start or end in the wrong place. Try each
+// `{` as a start and parse the balanced object it opens (string-aware).
+function findClaimsObject(text: string): { claims?: unknown } | null {
+  for (let start = text.indexOf("{"); start !== -1; start = text.indexOf("{", start + 1)) {
+    const end = balancedObjectEnd(text, start);
+    if (end === -1) continue;
+    const candidate = text.slice(start, end + 1);
+    if (!candidate.includes('"claims"')) continue;
+    try {
+      const parsed: unknown = JSON.parse(candidate);
+      if (parsed && typeof parsed === "object" && "claims" in parsed) return parsed as { claims?: unknown };
+    } catch { /* try the next opening brace */ }
+  }
+  return null;
+}
+
+function balancedObjectEnd(text: string, start: number): number {
+  let depth = 0;
+  let inString = false;
+  for (let i = start; i < text.length; i++) {
+    const ch = text[i];
+    if (inString) {
+      if (ch === "\\") i++;
+      else if (ch === '"') inString = false;
+    } else if (ch === '"') {
+      inString = true;
+    } else if (ch === "{") {
+      depth++;
+    } else if (ch === "}") {
+      depth--;
+      if (depth === 0) return i;
+    }
+  }
+  return -1;
+}
+
 function collectStringsFromPayload(payload: unknown, out: Set<string>): void {
   if (!payload) return;
   if (typeof payload === "string") {

package/lib/agents/compaction-pipeline.test.ts

@@ -0,0 +1,202 @@
+import { describe, expect, it } from "vitest";
+import {
+  assembleFreshTopics,
+  buildTopicGroupingMessages,
+  buildTopicRecapMessages,
+  runCompactionPipeline,
+  validateTopicGroups,
+  validateTopicRecap,
+  type PipelineSourceRow,
+} from "./compaction-pipeline";
+import type { SummaryTopicSegment } from "./conversation-summary";
+import type { ProviderMessage } from "@/lib/providers/types";
+
+const row = (seq: number, role = seq % 2 === 0 ? "user" : "assistant", tool_events: string | null = null): PipelineSourceRow => ({
+  seq,
+  role,
+  content: `message ${seq}`,
+  created_at: `2026-10-09T10:00:${String(seq).padStart(2, "0")}.000Z`,
+  tool_events,
+});
+
+const prior: SummaryTopicSegment[] = [{
+  title: "Release plan",
+  start_seq: 1,
+  end_seq: 4,
+  start_at: "2026-10-09T09:00:00.000Z",
+  end_at: "2026-10-09T09:00:04.000Z",
+  recap: "Planned the 2.2.2 patch.",
+  facts: [{ subject: "Release cadence", content: "Patch releases ship weekly.", tags: [], confidence: "explicit", key: "facts/release-cadence", origin: "compaction" }],
+  message_count: 4,
+  source_chars: 40,
+}];
+
+const groups = (value: unknown) => JSON.stringify({ groups: value });
+
+function scriptedProvider(script: (messages: ProviderMessage[], call: number) => string) {
+  let call = 0;
+  const calls: ProviderMessage[][] = [];
+  return {
+    calls,
+    provider: {
+      chat: async (_model: string, messages: ProviderMessage[]) => {
+        calls.push(messages);
+        const text = script(messages, call++);
+        return { stream: (async function* () { yield text; })() };
+      },
+    } as never,
+  };
+}
+
+describe("source-aware prompts", () => {
+  const group = { title: "T", continues: null, seqs: [1, 2] };
+  const sources = ["chat", "scheduled_task", "watcher", "bridge"] as const;
+
+  it("describes each message stream differently in both steps", () => {
+    const firstLines = sources.map((source) => [
+      String(buildTopicGroupingMessages("x", [], source)[0].content).split("\n")[0],
+      String(buildTopicRecapMessages(group, null, "x", source)[0].content).split("\n")[0],
+    ]);
+    expect(new Set(firstLines.map(([grouping]) => grouping)).size).toBe(sources.length);
+    expect(new Set(firstLines.map(([, recap]) => recap)).size).toBe(sources.length);
+  });
+
+  it("asks only chat to extract facts; automation channels must return an empty list", () => {
+    for (const source of sources) {
+      const system = String(buildTopicRecapMessages(group, null, "x", source)[0].content);
+      expect(system.includes('"confidence"')).toBe(source === "chat");
+      expect(system.includes('Leave "facts" as []')).toBe(source !== "chat");
+      expect(system).toMatch(/\n1\. [\s\S]*\n2\. [\s\S]*\n3\. /);
+    }
+  });
+});
+
+describe("step 1 — validateTopicGroups", () => {
+  const rows = [10, 11, 12, 13, 14, 15].map((seq) => row(seq));
+
+  it("rejects seqs that were not supplied, overlaps and unknown topic ids, so the model retries", () => {
+    expect(validateTopicGroups("not json", rows, prior)).toMatchObject({ ok: false });
+    const bad = validateTopicGroups(groups([
+      { start_seq: 9, end_seq: 12, title: "a" },
+      { start_seq: 12, end_seq: 15, continues: "t7", title: "b" },
+    ]), rows, prior);
+    expect(bad.ok).toBe(false);
+    if (!bad.ok) {
+      expect(bad.errors.join(" ")).toContain("start_seq 9 is not a supplied message seq");
+      expect(bad.errors.join(" ")).toContain('continues "t7" is not an existing topic id');
+    }
+    const overlap = validateTopicGroups(groups([
+      { start_seq: 10, end_seq: 12, title: "a" },
+      { start_seq: 12, end_seq: 15, title: "b" },
+    ]), rows, prior);
+    expect(overlap.ok).toBe(false);
+  });
+
+  it("repairs gaps, single-message groups and duplicate continuations in code", () => {
+    const result = validateTopicGroups(groups([
+      { start_seq: 10, end_seq: 11, continues: "t1", title: "" },
+      { start_seq: 13, end_seq: 13, title: "stray" },
+      { start_seq: 14, end_seq: 15, continues: "t1", title: "Release plan again" },
+    ]), rows, prior);
+    expect(result.ok).toBe(true);
+    if (!result.ok) return;
+    // seq 12 was ungrouped and joined the first group; the stray single folds
+    // in; the second continuation of t1 coalesces into the first.
+    expect(result.value).toEqual([{ title: "Release plan", continues: "t1", seqs: [10, 11, 12, 13, 14, 15] }]);
+    expect(result.repairs?.length).toBeGreaterThan(0);
+  });
+});
+
+describe("step 2 — validateTopicRecap", () => {
+  const group = { title: "Topic", seqs: [10, 11] };
+
+  it("requires a non-empty recap and rejects a bare sentinel", () => {
+    expect(validateTopicRecap("NO_REPLY", group).ok).toBe(false);
+    expect(validateTopicRecap(JSON.stringify({ recap: "" }), group).ok).toBe(false);
+    expect(validateTopicRecap(JSON.stringify({ recap: "x".repeat(4_001) }), group).ok).toBe(false);
+  });
+
+  it("accepts fenced JSON, defaults the title and drops malformed facts", () => {
+    const result = validateTopicRecap("```json\n" + JSON.stringify({
+      recap: "Newest first.",
+      facts: [
+        { subject: "Pref", content: "Uses pnpm.", tags: ["tooling"], confidence: "explicit" },
+        { subject: "Bad", content: "no confidence" },
+      ],
+    }) + "\n```", group);
+    expect(result).toMatchObject({ ok: true, value: { title: "Topic", recap: "Newest first.", facts: [{ subject: "Pref" }] } });
+  });
+});
+
+describe("step 3 — assembleFreshTopics", () => {
+  it("fails when a supplied message lands in no topic", () => {
+    const rows = [row(10), row(11), row(12)];
+    const result = assembleFreshTopics([{ title: "a", seqs: [10, 11] }], [{ title: "a", recap: "r", facts: [] }], rows);
+    expect(result).toMatchObject({ ok: false, errors: ["seq 12 is in no topic."] });
+  });
+});
+
+describe("runCompactionPipeline", () => {
+  const saved = JSON.stringify([
+    { id: "c1", phase: "call", name: "memory_upsert", payload: { namespace: "facts", key: "deploy-target", record: { subject: "Deploy target", content: "Deploys go to staging first.", confidence: "explicit" } } },
+    { id: "c1", phase: "result", name: "memory_upsert", payload: { kwargs: { content: JSON.stringify({ ok: true, namespace: "facts", key: "deploy-target" }) } } },
+  ]);
+  const rows = [row(10), row(11, "assistant", saved), row(12), row(13)];
+
+  it("groups, recaps each topic, and merges into the stored topics newest last", async () => {
+    const { provider, calls } = scriptedProvider((messages) => {
+      const system = String(messages[0].content);
+      if (system.includes("step 1 of 3")) {
+        return groups([
+          { start_seq: 10, end_seq: 11, continues: "t1", title: "Release plan" },
+          { start_seq: 12, end_seq: 13, continues: null, title: "Deploy" },
+        ]);
+      }
+      return String(messages[1].content).includes("Topic: Release plan")
+        ? JSON.stringify({ title: "Release plan", recap: "Tagged 2.2.2. Earlier: planned the patch.", facts: [] })
+        : JSON.stringify({ title: "Deploy", recap: "Discussed deploy.", facts: [{ subject: "Region", content: "Deploy to eu-west.", tags: [], confidence: "explicit" }] });
+    });
+
+    const result = await runCompactionPipeline({ provider, modelId: "m", params: {}, rows, priorTopics: prior, messageLimit: 50 });
+
+    expect(calls).toHaveLength(3);
+    expect(result?.topics.map((topic) => [topic.title, topic.message_count])).toEqual([["Release plan", 6], ["Deploy", 2]]);
+    const continued = result!.topics[0];
+    expect(continued.start_seq).toBe(1);
+    expect(continued.end_seq).toBe(11);
+    expect(continued.recap).toBe("Tagged 2.2.2. Earlier: planned the patch.");
+    // Prior memorized items stay; agent-saved memory is attributed by code.
+    expect(continued.facts.map((fact) => fact.key)).toEqual(["facts/release-cadence", "facts/deploy-target"]);
+    expect(result!.topics[1].facts[0]).toMatchObject({ key: "facts/region", origin: "compaction" });
+    expect(result!.freshTopics.every((topic) => !("continues" in topic))).toBe(true);
+    expect(result!.sourceMessages).toBe(8);
+  });
+
+  it("retries a step with the validation errors, then succeeds", async () => {
+    const { provider, calls } = scriptedProvider((messages, call) => {
+      if (call === 0) return groups([{ start_seq: 99, end_seq: 13, title: "x" }]);
+      if (String(messages[0].content).includes("step 1 of 3")) return groups([{ start_seq: 10, end_seq: 13, title: "All" }]);
+      return JSON.stringify({ recap: "Everything.", facts: [] });
+    });
+    const result = await runCompactionPipeline({ provider, modelId: "m", params: {}, rows, priorTopics: [], messageLimit: 50 });
+    expect(result?.topics).toHaveLength(1);
+    const retry = calls[1];
+    expect(String(retry[retry.length - 1].content)).toContain("start_seq 99 is not a supplied message seq");
+  });
+
+  it("returns null after a step fails twice, so the caller falls back", async () => {
+    const { provider, calls } = scriptedProvider(() => "no json here");
+    const result = await runCompactionPipeline({ provider, modelId: "m", params: {}, rows, priorTopics: [], messageLimit: 50 });
+    expect(result).toBeNull();
+    expect(calls).toHaveLength(2);
+  });
+
+  it("evicts the oldest topics once the merged topics exceed the message limit", async () => {
+    const { provider } = scriptedProvider((messages) => String(messages[0].content).includes("step 1 of 3")
+      ? groups([{ start_seq: 10, end_seq: 13, title: "New" }])
+      : JSON.stringify({ recap: "New work.", facts: [] }));
+    const result = await runCompactionPipeline({ provider, modelId: "m", params: {}, rows, priorTopics: prior, messageLimit: 5 });
+    expect(result?.topics.map((topic) => topic.title)).toEqual(["New"]);
+    expect(result?.evicted).toBe(1);
+  });
+});