@hassangameryt/freeflow 0.7.3 → 0.8.0

package/pi-extension/dist/{cognitive-routing-v2 → cognitive-routing}/projection.js

@@ -1,8 +1,9 @@
-import { annotateSources } from "../session-sources/provenance.js";
+import { annotateSources } from "./provenance.js";
 import { randomUUID } from "node:crypto";
-import { textRef } from "../session-sources/sources.js";
-import { canonical, emptySelection, isWorkerProfile, requireCondition as check } from "./types.js";
+import { bodyHash, deliveredSelection } from "./sources.js";
+import { canonical, emptySelection, isWorkerProfile, PROFILES, requireCondition as check } from "./types.js";
 import { estimateRequest } from "./budget.js";
+import { occurrenceIdentity, tagDerived } from "../host/projection-tags.js";
 export function changeSelection(prior, input, sources, state) {
   check(
     Array.isArray(input.refs) &&
@@ -38,9 +39,41 @@ export function changeSelection(prior, input, sources, state) {
   next.revision++;
   return next;
 }
-export function representationProblems(source, model, structural = false) {
-  const message = source.original?.message ?? source.message,
-    content = Array.isArray(source.message.content) ? source.message.content : [];
+/**
+ * Profiles share history, not private reasoning, except where they would otherwise send the same request: the
+ * same model under the same view (projection off, or Helper and Executor on worker history). There shared
+ * reasoning keeps one prompt cache across effort switches and stays valid for providers that bind reasoning to
+ * its conversation. Unknown and pre-routing history keeps its native form.
+ */
+function privateReasoning(source, view, projection, model) {
+  const producer = source.producer;
+  if (!PROFILES.includes(producer) || producer === view) return false;
+  const message = source.message;
+  const sameModel = message.provider === model?.provider && message.api === model?.api && message.model === model?.id;
+  const viewOf = (profile) => (!projection ? "shared" : isWorkerProfile(profile) ? "worker" : profile);
+  return !(sameModel && viewOf(producer) === viewOf(view));
+}
+/**
+ * Remove reasoning from another profile's turn. A provider may pair a call with the reasoning that produced
+ * it (OpenAI Responses `fc_` ids pair with `rs_` items), so such calls keep only their call id, as the host
+ * already does for another model's turns; `unpaired` carries the rename to the matching results.
+ */
+function withoutReasoning(message, unpaired) {
+  if (!message.content.some((b) => b.type === "thinking")) return message;
+  return {
+    ...message,
+    content: message.content.flatMap((b) => {
+      if (b.type === "thinking") return [];
+      if (b.type !== "toolCall" || typeof b.id !== "string" || !b.id.includes("|")) return [b];
+      const id = b.id.split("|")[0];
+      unpaired.set(b.id, id);
+      return [{ ...b, id }];
+    }),
+  };
+}
+export function representationProblems(source, model) {
+  const message = source.message,
+    content = Array.isArray(message.content) ? message.content : [];
   if (message.role === "assistant" && ["error", "aborted"].includes(message.stopReason))
     return [
       {
@@ -54,22 +87,49 @@ export function representationProblems(source, model, structural = false) {
     return [
       { ref: source.ref, code: "target_representation", detail: "The receiving model does not support image input." },
     ];
-  if (
-    !structural &&
-    !source.original &&
-    content.some((b) => b.type === "thinking" && (b.redacted || b.thinkingSignature)) &&
-    (message.provider !== model?.provider || message.model !== model?.id || message.api !== model?.api)
-  )
-    return [
-      {
-        ref: source.ref,
-        code: "target_representation",
-        detail: `Whole native signed content is not qualified across this model boundary. Inspect the explicit assistant-text source ${textRef(source.ref)} when visible text is the intended evidence.`,
-      },
-    ];
   return [];
 }
+/**
+ * Renderings reused across requests, so each view message keeps one identity (request history then fingerprints it
+ * once) and nothing is cloned per request. Entries are immutable, so a rendering keyed by its entry object and variant
+ * never goes stale; a replaced entry is a different object. Cached messages are shared and must never be mutated.
+ */
+export class ViewCache {
+  renderings = new WeakMap();
+  notes = new Map();
+  coordinatorVersions = new WeakMap();
+}
+/** Tests set FREEFLOW_FREEZE_VIEWS to make any later mutation of a shared rendering throw. */
+const FREEZE = !!process.env.FREEFLOW_FREEZE_VIEWS;
+function deepFreeze(value) {
+  if (value && typeof value === "object" && !Object.isFrozen(value)) {
+    Object.freeze(value);
+    for (const child of Object.values(value)) deepFreeze(child);
+  }
+  return value;
+}
+/** Messages normalized() produced, so normalizing a shared rendering again keeps it (and its identity) as it is. */
+const NORMALIZED = new WeakSet();
+/** Request metadata as Pi must see it in a routed view: a finite timestamp, and no recorded usage on assistants. */
+function normalized(message) {
+  const finite = Number.isFinite(message?.timestamp);
+  if ((finite && message?.role !== "assistant") || NORMALIZED.has(message)) return message;
+  const result = {
+    ...message,
+    timestamp: finite ? message.timestamp : 0,
+    ...(message.role === "assistant"
+      ? { usage: { ...message.usage, input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0 } }
+      : {}),
+  };
+  NORMALIZED.add(result);
+  return result;
+}
+function shareRendering(message, identity) {
+  tagDerived(message, identity);
+  return FREEZE ? deepFreeze(message) : message;
+}
 export function prepareView(options) {
+  const cache = options.cache ?? new ViewCache();
   const { sources, state } = options;
   const associated = sources.associate(options.messages);
   const selective = options.projection && options.view === "coordinator";
@@ -79,16 +139,19 @@ export function prepareView(options) {
   const handoff = handoffId ? state.handoffs.get(handoffId) : undefined;
   const assignmentId = handoff?.assignmentId ?? state.assignmentId;
   const current = assignmentId ? (state.selections.get(assignmentId) ?? emptySelection()) : emptySelection();
-  const required = new Set(selective && !attention && assignmentId ? current.selected : []);
+  // Assistant messages (whole or `#text`) that older sessions selected are not delivered.
+  const delivered = (ref) => deliveredSelection(sources, ref);
+  const required = new Set(selective && !attention && assignmentId ? current.selected.filter(delivered) : []);
   const promised = handoff && state.reservations.get(handoff.id);
   const admitted = new Set();
   if (selective)
-    for (const selection of state.selections.values()) for (const ref of selection.selected) admitted.add(ref);
+    for (const selection of state.selections.values())
+      for (const ref of selection.selected) if (delivered(ref)) admitted.add(ref);
   // Accepted return receipts are communication, independent of selected task evidence.
   // Their ordinary active occurrence survives closure; compaction still owns its lifetime.
   const full = new Map();
   const structural = new Map();
-  const problems = selective && !attention && assignmentId ? [...current.unresolved] : [];
+  const problems = selective && !attention && assignmentId ? current.unresolved.filter((p) => delivered(p.ref)) : [];
   for (const item of associated) {
     if (!item.source) continue;
     if (
@@ -98,8 +161,6 @@ export function prepareView(options) {
       (item.source.reportHandoff && state.handoffs.has(item.source.reportHandoff))
     )
       full.set(item.source.ref, item.source);
-    const visible = sources.byRef.get(textRef(item.source.ref));
-    if (selective && visible && admitted.has(visible.ref)) full.set(visible.ref, visible);
   }
   for (const ref of required) {
     if (sources.ambiguous.has(ref)) {
@@ -130,6 +191,15 @@ export function prepareView(options) {
     }
     full.set(ref, source);
   }
+  // After a compaction, selected evidence is rebuilt from history only while it is required. Evidence the
+  // Coordinator already received this cycle stays once that ends, so its later requests extend the earlier ones.
+  if (selective)
+    for (const id of options.assessedSinceCompaction ?? [])
+      for (const ref of state.selections.get(id)?.selected ?? []) {
+        const source = sources.byRef.get(ref);
+        if (!full.has(ref) && source && delivered(ref) && !sources.ambiguous.has(ref) && !sources.eligible(ref, state))
+          full.set(ref, source);
+      }
   // A selected result carries its complete native call group; siblings may be structural only.
   for (const source of full.values()) {
     const exchange = sources.exchange(source);
@@ -138,30 +208,74 @@ export function prepareView(options) {
   }
   for (const source of structural.values())
     if (source.message.role === "assistant" && required.size)
-      problems.push(...representationProblems(source, options.model, true));
+      problems.push(...representationProblems(source, options.model));
+  const unpaired = new Map();
+  // Rendering depends only on the entry and these facts, so a rendering is reused for the same variant.
   const render = (source) => {
-    const message = structuredClone(source.message);
-    if (source.message.role === "assistant" && !full.has(source.ref) && full.has(textRef(source.ref)))
-      message.content = message.content.filter((b) => b.type !== "text");
-    if (full.has(source.ref) || source.message.role !== "toolResult") return message;
-    return {
-      ...structuredClone(source.message),
-      content: [{ type: "text", text: "[Worker result omitted from this view]" }],
-      details: undefined,
-    };
+    const role = source.message.role;
+    const withheld = role === "assistant" && privateReasoning(source, options.view, options.projection, options.model);
+    // A structural worker envelope keeps only its calls for the selected results; worker narration is never evidence.
+    const structuralWorker =
+      role === "assistant" && selective && isWorkerProfile(source.producer) && !full.has(source.ref);
+    const omitted = role === "toolResult" && !full.has(source.ref);
+    const renamed = role === "toolResult" ? unpaired.get(source.message.toolCallId) : undefined;
+    const variant = `${withheld ? 1 : 0}${structuralWorker ? 1 : 0}${omitted ? 1 : 0}${renamed === undefined ? "" : `:${renamed}`}`;
+    let variants = cache.renderings.get(source.entry);
+    if (!variants) cache.renderings.set(source.entry, (variants = new Map()));
+    let rendering = variants.get(variant);
+    if (!rendering) {
+      const renames = new Map();
+      let message = structuredClone(source.message);
+      if (withheld) message = withoutReasoning(message, renames);
+      if (structuralWorker) message.content = message.content.filter((b) => b.type === "toolCall");
+      if (omitted)
+        message = {
+          ...message,
+          content: [{ type: "text", text: "[Worker result omitted from this view]" }],
+          details: undefined,
+        };
+      if (renamed !== undefined) message = { ...message, toolCallId: renamed };
+      rendering = {
+        message: shareRendering(normalized(message), `${source.entry.id}#${variant}`),
+        ...(renames.size ? { renames: [...renames] } : {}),
+      };
+      variants.set(variant, rendering);
+    }
+    // A reused assistant rendering still tells its results which call ids it unpaired.
+    for (const [from, to] of rendering.renames ?? []) unpaired.set(from, to);
+    return rendering.message;
   };
   // Insert resolved historical occurrences before the first later native occurrence, retaining
   // opaque/common messages in their original order rather than moving user input wholesale.
   const ordered = sources.ordered(new Map([...structural, ...full]).values());
-  const rank = new Map(options.sources.entries.map((entry, index) => [entry.id, index]));
+  const rank = sources.rank;
+  // Worker evidence admitted after the Coordinator has seen later content is delivered where it was admitted,
+  // with its whole call group, so the Coordinator's already-sent prefix is extended rather than rewritten.
+  const shared = sources.nonWorkerBefore;
   const activeRefs = new Set(associated.flatMap((i) => (i.source ? [i.source.ref] : [])));
+  const deferredAt = new Map();
+  if (selective && (options.admissions || options.resumedAt !== undefined))
+    for (const source of full.values()) {
+      // An attention view omitted required evidence that is no longer in the active context, so after a
+      // resume that evidence is new to the Coordinator: deliver it where the assessment resumed.
+      const resumed =
+        options.resumedAt !== undefined && required.has(source.ref) && !activeRefs.has(source.ref)
+          ? options.resumedAt
+          : undefined;
+      const admitted = options.admissions?.get(source.ref);
+      const at = admitted === undefined ? resumed : Math.max(admitted, resumed ?? 0),
+        own = rank.get(source.entry.id) ?? 0;
+      if (at === undefined || at <= own || !isWorkerProfile(source.producer) || shared[at] <= shared[own + 1]) continue;
+      const group = sources.exchange(source).sources;
+      for (const member of group.length ? group : [source])
+        deferredAt.set(member.ref, Math.max(deferredAt.get(member.ref) ?? 0, at));
+    }
   const historical = ordered.filter((s) => !activeRefs.has(s.ref));
   const emitted = new Set();
   let messages = [];
   const renderedSources = new Map();
   let cursor = 0;
   const emit = (source) => {
-    if (source.original && full.has(source.original.ref)) return;
     if (!emitted.has(source.ref)) {
       const message = render(source);
       messages.push(message);
@@ -169,21 +283,51 @@ export function prepareView(options) {
       emitted.add(source.ref);
     }
   };
+  const pending = [];
+  const place = (source) => (deferredAt.has(source.ref) ? pending.push(source) : emit(source));
+  const deliver = (limit) => {
+    pending.sort(
+      (a, b) =>
+        deferredAt.get(a.ref) - deferredAt.get(b.ref) || (rank.get(a.entry.id) ?? 0) - (rank.get(b.entry.id) ?? 0),
+    );
+    while (pending.length && deferredAt.get(pending[0].ref) <= limit) emit(pending.shift());
+  };
   for (const item of associated) {
     if (item.source) {
-      while (
-        cursor < historical.length &&
-        (rank.get(historical[cursor].entry.id) ?? 0) < (rank.get(item.source.entry.id) ?? 0)
-      )
-        emit(historical[cursor++]);
-      const visible = full.get(textRef(item.source.ref));
-      if (visible) emit(visible);
-      if (full.has(item.source.ref) || structural.has(item.source.ref)) emit(item.source);
+      const at = rank.get(item.source.entry.id) ?? 0;
+      deliver(at);
+      while (cursor < historical.length && (rank.get(historical[cursor].entry.id) ?? 0) < at)
+        place(historical[cursor++]);
+      if (full.has(item.source.ref) || structural.has(item.source.ref)) place(item.source);
     } else messages.push(item.message);
   }
-  while (cursor < historical.length) emit(historical[cursor++]);
+  while (cursor < historical.length) place(historical[cursor++]);
+  deliver(Infinity);
   const fullSources = [...full.values()];
-  messages = annotateSources(messages, renderedSources, new Set(full.keys()), sources, options.instance);
+  // The Coordinator only sees completed worker runs, so one note per run keeps its prefix stable.
+  messages = annotateSources(
+    messages,
+    renderedSources,
+    new Set(full.keys()),
+    sources,
+    options.instance,
+    selective,
+    cache.notes,
+  );
+  // A message may hold its own version for the Coordinator's selective view: Freeflow's carried context names the
+  // worker's copies there instead of repeating them, since the Coordinator receives evidence by selection.
+  if (selective)
+    messages = messages.map((message) => {
+      if (message?.role !== "custom" || typeof message.details?.coordinatorContent !== "string") return message;
+      let version = cache.coordinatorVersions.get(message);
+      if (!version) {
+        version = { ...message, content: message.details.coordinatorContent };
+        const identity = occurrenceIdentity(message);
+        if (identity) shareRendering(version, `${identity}#coordinator`);
+        cache.coordinatorVersions.set(message, version);
+      }
+      return version;
+    });
   // Restore exact current communication only when its accepted occurrence is absent.
   const a = state.assignmentId ? state.assignments.get(state.assignmentId) : undefined;
   const baseReport = assessment
@@ -222,6 +366,30 @@ export function prepareView(options) {
         }
       });
     });
+  // Current receipts carry metadata plus a text hash, while the text itself stays in the call arguments.
+  const hashedCommunication = (h, field, value, metadata) =>
+    messages.some(
+      (m) =>
+        m.role === "assistant" &&
+        m.content?.some((b) => b.type === "toolCall" && b.id === h.toolCallId && b.arguments?.[field] === value),
+    ) &&
+    messages.some(
+      (m) =>
+        m.role === "toolResult" &&
+        m.toolCallId === h.toolCallId &&
+        m.content?.some((b) => {
+          if (b.type !== "text") return false;
+          try {
+            const payload = JSON.parse(b.text);
+            return (
+              payload?.[`${field}Sha256`] === bodyHash(value) &&
+              Object.entries(metadata).every(([key, expected]) => canonical(payload?.[key]) === canonical(expected))
+            );
+          } catch {
+            return false;
+          }
+        }),
+    );
   const restore = (kind, id, body) =>
     messages.push({
       role: "custom",
@@ -234,19 +402,29 @@ export function prepareView(options) {
   if (a) {
     const accepted = state.handoffs.get(a.delegateHandoffId);
     if (accepted && !hasCommunication(accepted, "contract", a.contract))
-      restore("Current exact assignment", a.id, a.contract);
+      // Directly after the contract, so a contract asking for compaction does not read as still undone.
+      restore(
+        "Current exact assignment",
+        a.id,
+        options.compactedInAssignment === undefined
+          ? a.contract
+          : `${a.contract}\n\nCompaction: you compacted during this assignment, and cycle ${options.compactedInAssignment} began then. A compaction this contract asks for is done: continue with the rest of the contract, or return it with freeflow_return if compacting was all it asked.`,
+      );
   }
   if (baseReport) {
     const metadata = { outcome: baseReport.outcome, limitations: baseReport.limitations };
     const hasBody = hasCommunication(baseReport, "report", baseReport.text);
     // Tool arguments establish accepted content but do not carry harness-assigned
     // revision/lineage. Old receipts and partially retained calls need metadata too.
-    const complete = hasCommunication(baseReport, "report", baseReport.text, {
+    const lineage = {
       ...metadata,
       reportRevision: baseReport.reportRevision,
       assignmentRef: `assignment:${baseReport.assignmentId}`,
       reportRef: `report:${baseReport.id}:${baseReport.reportRevision}`,
-    });
+    };
+    const complete =
+      hasCommunication(baseReport, "report", baseReport.text, lineage) ||
+      hashedCommunication(baseReport, "report", baseReport.text, lineage);
     if (!complete)
       restore(
         "Saved report",
@@ -314,13 +492,8 @@ export function prepareView(options) {
     // Recorded usage describes a prior provider view, which may belong to the
     // other profile. Pi must estimate this assembled view when allocating output.
     // Normalize request metadata only; canonical usage/billing records stay intact.
-    messages: messages.map((message) => ({
-      ...message,
-      timestamp: Number.isFinite(message.timestamp) ? message.timestamp : 0,
-      ...(message.role === "assistant"
-        ? { usage: { ...message.usage, input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0 } }
-        : {}),
-    })),
+    // Renderings are already normalized; other messages are copied only when they need it.
+    messages: messages.map(normalized),
     problems,
     fullSources,
     reservation,

package/pi-extension/dist/cognitive-routing/provenance.js

@@ -0,0 +1,81 @@
+import { isWorkerProfile } from "./types.js";
+import { tagDerived } from "../host/projection-tags.js";
+import { isTaskEvidence } from "./sources.js";
+/**
+ * Follow each exchange with a note naming its attributed sources. With `mergeWorkerRuns`, consecutive
+ * exchanges from one worker assignment share a single note after the run. Use it only for a view that
+ * never sees a run still growing, or the note would move on every appended exchange.
+ */
+export function annotateSources(
+  messages,
+  renderedSources,
+  full,
+  sources,
+  instance,
+  mergeWorkerRuns = false,
+  /** Notes kept across requests by content, so an unchanged note keeps one identity and is fingerprinted once. */
+  notes,
+) {
+  const annotated = [];
+  let pending;
+  const flush = () => {
+    if (pending?.rows.length) {
+      const content = `Source provenance for the preceding ${pending.exchanges > 1 ? "worker run" : "message/exchange"}:\n${pending.rows.join("\n")}`;
+      const key = `${instance}\u0000${content}`;
+      let note = notes?.get(key);
+      if (!note) {
+        note = {
+          role: "custom",
+          customType: "freeflow-routing-v2-refs",
+          display: false,
+          content,
+          details: { routingInstance: instance },
+          timestamp: 0,
+        };
+        if (notes) {
+          tagDerived(note, `refs:${key}`);
+          notes.set(key, note);
+        }
+      }
+      annotated.push(note);
+    }
+    pending = undefined;
+  };
+  for (let i = 0; i < messages.length;) {
+    const group = [messages[i++]];
+    while (i < messages.length && messages[i].role === "toolResult") group.push(messages[i++]);
+    const attributed = [];
+    const rows = group.flatMap((message) => {
+      const s = renderedSources.get(message);
+      // Only observed routing attribution is informative; unattributed rows would repeat "unknown" on
+      // every exchange, and omitting them keeps the prefix identical whether routing is on or off.
+      if (!s || s.producer === "common") return [];
+      attributed.push(s);
+      const representation = full.has(s.ref) ? "full" : "structural only; omitted result bodies are not evidence";
+      const selection =
+        isWorkerProfile(s.producer) && isTaskEvidence(s) && full.has(s.ref)
+          ? "task evidence; selection checks apply"
+          : "not offered for new evidence selection";
+      return [
+        `${s.ref} | producer: ${s.producer} | ${s.message.role}${s.message.toolName ? ` | ${s.message.toolName}` : ""}${s.assignmentId ? ` | assignment: ${s.assignmentId}` : ""} | ${representation} | ${selection}`,
+      ];
+    });
+    const keys = new Set(attributed.map((s) => `${s.producer}\u0000${s.assignmentId ?? ""}`));
+    const key =
+      mergeWorkerRuns && attributed.length && keys.size === 1 && attributed.every((s) => isWorkerProfile(s.producer))
+        ? [...keys][0]
+        : undefined;
+    if (key !== undefined && pending?.key === key) {
+      annotated.push(...group);
+      pending.rows.push(...rows);
+      pending.exchanges++;
+      continue;
+    }
+    flush();
+    annotated.push(...group);
+    pending = { key, rows, exchanges: 1 };
+    if (key === undefined) flush();
+  }
+  flush();
+  return annotated;
+}