@pi-unipi/unipi 2.10.2 → 2.12.0

package/packages/trajectory/src/prefix-integrity.ts

@@ -0,0 +1,213 @@
+import { createHash } from "node:crypto";
+
+export type PrefixVerdict = "first_request" | "identical_retry" | "prefix_extended" | "boundary" | "violation";
+
+export interface PrefixDifference {
+  surface: "messages" | "system" | "tools" | "envelope" | "shape";
+  kind: "changed" | "inserted" | "removed" | "reordered";
+  path: string;
+  index?: number;
+  beforeFingerprint?: string;
+  afterFingerprint?: string;
+}
+
+export interface PrefixIntegrityResult {
+  epoch: number;
+  verdict: PrefixVerdict;
+  messageField: string;
+  previousItems: number;
+  currentItems: number;
+  extendedBy: number;
+  requestFingerprint: string;
+  messagesFingerprint: string;
+  systemFingerprint: string;
+  toolsFingerprint: string;
+  envelopeFingerprint: string;
+  boundaryReason?: string;
+  differences: PrefixDifference[];
+}
+
+interface RequestState {
+  requestFingerprint: string;
+  messageField: string;
+  messages: unknown[];
+  messageFingerprints: string[];
+  systemFingerprint: string;
+  toolsFingerprint: string;
+  envelopeFingerprint: string;
+}
+
+function canonicalize(value: unknown, seen = new WeakSet<object>()): unknown {
+  if (value === undefined) return { $type: "undefined" };
+  if (typeof value === "bigint") return { $type: "bigint", value: value.toString() };
+  if (typeof value === "number" && !Number.isFinite(value)) return { $type: "number", value: String(value) };
+  if (typeof value !== "object" || value === null) return value;
+  if (seen.has(value)) return { $type: "cycle" };
+  seen.add(value);
+  if (Array.isArray(value)) {
+    const result = value.map(item => canonicalize(item, seen));
+    seen.delete(value);
+    return result;
+  }
+  if (value instanceof Uint8Array) {
+    seen.delete(value);
+    return { $type: "bytes", length: value.byteLength };
+  }
+  const result: Record<string, unknown> = {};
+  for (const key of Object.keys(value as Record<string, unknown>).sort()) {
+    result[key] = canonicalize((value as Record<string, unknown>)[key], seen);
+  }
+  seen.delete(value);
+  return result;
+}
+
+function fingerprint(value: unknown): string {
+  return createHash("sha256").update(JSON.stringify(canonicalize(value))).digest("hex").slice(0, 16);
+}
+
+function payloadObject(payload: unknown): Record<string, unknown> {
+  return payload && typeof payload === "object" && !Array.isArray(payload)
+    ? payload as Record<string, unknown>
+    : { payload };
+}
+
+function messageSequence(payload: Record<string, unknown>): { field: string; items: unknown[] } {
+  for (const field of ["messages", "input", "contents"] as const) {
+    if (Array.isArray(payload[field])) return { field, items: payload[field] as unknown[] };
+  }
+  return { field: "none", items: [] };
+}
+
+function systemSurface(payload: Record<string, unknown>, field: string, messages: unknown[]): unknown {
+  const direct = Object.fromEntries(Object.entries(payload).filter(([key]) => /^(system|instructions|developer)/i.test(key)));
+  const leading = field === "messages"
+    ? messages.filter((message) => {
+      if (!message || typeof message !== "object") return false;
+      const role = (message as Record<string, unknown>).role;
+      return role === "system" || role === "developer";
+    })
+    : [];
+  return { direct, leading };
+}
+
+function toolsSurface(payload: Record<string, unknown>): unknown {
+  return { tools: payload.tools, toolChoice: payload.tool_choice ?? payload.toolChoice };
+}
+
+function envelopeSurface(payload: Record<string, unknown>, sequenceField: string): unknown {
+  const omitted = new Set([sequenceField, "tools", "tool_choice", "toolChoice"]);
+  return Object.fromEntries(Object.entries(payload).filter(([key]) => !omitted.has(key) && !/^(system|instructions|developer)/i.test(key)));
+}
+
+function state(payload: unknown, route?: unknown): RequestState {
+  const object = payloadObject(payload);
+  const sequence = messageSequence(object);
+  const system = systemSurface(object, sequence.field, sequence.items);
+  const tools = toolsSurface(object);
+  const envelope = { route, payload: envelopeSurface(object, sequence.field) };
+  return {
+    requestFingerprint: fingerprint({ route, payload: object }),
+    messageField: sequence.field,
+    messages: sequence.items,
+    messageFingerprints: sequence.items.map(fingerprint),
+    systemFingerprint: fingerprint(system),
+    toolsFingerprint: fingerprint(tools),
+    envelopeFingerprint: fingerprint(envelope),
+  };
+}
+
+function messageDifference(previous: RequestState, current: RequestState): PrefixDifference | undefined {
+  if (previous.messageField !== current.messageField) {
+    return { surface: "shape", kind: "changed", path: "$.messageField", beforeFingerprint: fingerprint(previous.messageField), afterFingerprint: fingerprint(current.messageField) };
+  }
+  const limit = Math.min(previous.messageFingerprints.length, current.messageFingerprints.length);
+  for (let index = 0; index < limit; index++) {
+    if (previous.messageFingerprints[index] === current.messageFingerprints[index]) continue;
+    const laterIndex = current.messageFingerprints.indexOf(previous.messageFingerprints[index]!, index + 1);
+    const earlierIndex = previous.messageFingerprints.indexOf(current.messageFingerprints[index]!, index + 1);
+    return {
+      surface: "messages",
+      kind: laterIndex >= 0 || earlierIndex >= 0 ? "reordered" : "changed",
+      path: `$.${current.messageField}[${index}]`, index,
+      beforeFingerprint: previous.messageFingerprints[index], afterFingerprint: current.messageFingerprints[index],
+    };
+  }
+  if (current.messageFingerprints.length < previous.messageFingerprints.length) {
+    const index = current.messageFingerprints.length;
+    return { surface: "messages", kind: "removed", path: `$.${current.messageField}[${index}]`, index, beforeFingerprint: previous.messageFingerprints[index] };
+  }
+  return undefined;
+}
+
+export class PrefixIntegrityTracker {
+  private epoch = 0;
+  private previous: RequestState | null = null;
+  private boundaryReason: string | undefined;
+
+  reset(): void {
+    this.epoch = 0;
+    this.previous = null;
+    this.boundaryReason = undefined;
+  }
+
+  markBoundary(reason: string): void {
+    if (this.previous) this.epoch++;
+    this.previous = null;
+    this.boundaryReason = reason;
+  }
+
+  observe(payload: unknown, route?: unknown): PrefixIntegrityResult {
+    const current = state(payload, route);
+    const boundaryReason = this.boundaryReason;
+    if (!this.previous) {
+      if (this.epoch === 0) this.epoch = 1;
+      this.previous = current;
+      this.boundaryReason = undefined;
+      return {
+        epoch: this.epoch,
+        verdict: boundaryReason ? "boundary" : "first_request",
+        messageField: current.messageField,
+        previousItems: 0,
+        currentItems: current.messages.length,
+        extendedBy: current.messages.length,
+        requestFingerprint: current.requestFingerprint,
+        messagesFingerprint: fingerprint(current.messageFingerprints),
+        systemFingerprint: current.systemFingerprint,
+        toolsFingerprint: current.toolsFingerprint,
+        envelopeFingerprint: current.envelopeFingerprint,
+        ...(boundaryReason ? { boundaryReason } : {}),
+        differences: [],
+      };
+    }
+
+    const previous = this.previous;
+    const differences: PrefixDifference[] = [];
+    const messageDiff = messageDifference(previous, current);
+    if (messageDiff) differences.push(messageDiff);
+    if (previous.systemFingerprint !== current.systemFingerprint) differences.push({ surface: "system", kind: "changed", path: "$.system", beforeFingerprint: previous.systemFingerprint, afterFingerprint: current.systemFingerprint });
+    if (previous.toolsFingerprint !== current.toolsFingerprint) differences.push({ surface: "tools", kind: "changed", path: "$.tools", beforeFingerprint: previous.toolsFingerprint, afterFingerprint: current.toolsFingerprint });
+    if (previous.envelopeFingerprint !== current.envelopeFingerprint) differences.push({ surface: "envelope", kind: "changed", path: "$.envelope", beforeFingerprint: previous.envelopeFingerprint, afterFingerprint: current.envelopeFingerprint });
+
+    const verdict: PrefixVerdict = current.requestFingerprint === previous.requestFingerprint
+      ? "identical_retry"
+      : differences.length === 0
+        ? "prefix_extended"
+        : "violation";
+    const result: PrefixIntegrityResult = {
+      epoch: this.epoch,
+      verdict,
+      messageField: current.messageField,
+      previousItems: previous.messages.length,
+      currentItems: current.messages.length,
+      extendedBy: Math.max(0, current.messages.length - previous.messages.length),
+      requestFingerprint: current.requestFingerprint,
+      messagesFingerprint: fingerprint(current.messageFingerprints),
+      systemFingerprint: current.systemFingerprint,
+      toolsFingerprint: current.toolsFingerprint,
+      envelopeFingerprint: current.envelopeFingerprint,
+      differences,
+    };
+    this.previous = current;
+    return result;
+  }
+}

package/packages/trajectory/src/project.ts

@@ -1,7 +1,7 @@
 import type { SessionEntry } from "@earendil-works/pi-coding-agent";
 import type { TelemetryEvent } from "./telemetry.js";
 
-export type TrajectoryKind = "system" | "user" | "assistant" | "tool" | "compaction" | "branch";
+export type TrajectoryKind = "system" | "unipi" | "prefix" | "user" | "assistant" | "tool" | "compaction" | "branch";
 
 export interface TrajectoryUsage {
   input?: number;
@@ -38,6 +38,13 @@ export interface TrajectoryRecord {
   request?: unknown;
   response?: unknown;
   tools?: unknown;
+  data?: unknown;
+  package?: string;
+  surface?: string;
+  phase?: string;
+  verdict?: string;
+  epoch?: number;
+  attribution?: unknown;
 }
 
 export interface TrajectorySnapshot {
@@ -165,12 +172,56 @@ export function projectTrajectory(
     }
   }
 
+  const firstEntryAt = records[0]?.timestamp ?? Date.now();
+  let syntheticSeq = -telemetry.length;
+  for (const event of telemetry) {
+    if (!["system-prompt", "unipi-trace", "prefix-integrity"].includes(event.type)) continue;
+    const data = event.data as AnyRecord | undefined;
+    // Package traces are attribution evidence, not a second activity log. Only
+    // operations that changed a provider-visible surface belong in trajectory.
+    if (event.type === "unipi-trace" && data?.affectsContext !== true) continue;
+    const traceAction = String(data?.hook ?? data?.action ?? data?.channel ?? data?.surface ?? "operation");
+    const kind: TrajectoryKind = event.type === "system-prompt" ? "system" : event.type === "unipi-trace" ? "unipi" : "prefix";
+    const title = event.type === "system-prompt"
+      ? "SYSTEM PROMPT"
+      : event.type === "unipi-trace"
+        ? `${String(data?.package ?? "unipi")} · ${traceAction}`
+        : `PREFIX · ${String(data?.verdict ?? "unknown")}`;
+    const value = event.type === "system-prompt" ? data?.systemPrompt : data;
+    records.push({
+      id: `telemetry:${event.type}:${event.at}:${syntheticSeq}`,
+      seq: syntheticSeq++,
+      turn: event.turnIndex === undefined ? null : event.turnIndex + 1,
+      step: null,
+      kind,
+      timestamp: event.at || firstEntryAt,
+      durationMs: typeof data?.durationMs === "number" ? data.durationMs : null,
+      title,
+      preview: event.type === "prefix-integrity"
+        ? `${String(data?.verdict ?? "unknown")} · epoch ${String(data?.epoch ?? "?")}${Array.isArray(data?.differences) && data.differences[0] ? ` · ${String(data.differences[0].path ?? data.differences[0].surface ?? "changed")}` : ""}`
+        : preview(event.type === "system-prompt" ? value : JSON.stringify(value ?? {})),
+      output: event.type === "system-prompt" ? value : undefined,
+      data,
+      package: typeof data?.package === "string" ? data.package : undefined,
+      surface: typeof data?.surface === "string" ? data.surface : undefined,
+      phase: typeof data?.phase === "string" ? data.phase : undefined,
+      verdict: typeof data?.verdict === "string" ? data.verdict : undefined,
+      epoch: typeof data?.epoch === "number" ? data.epoch : undefined,
+      attribution: data?.attribution,
+      isError: data?.phase === "error" || data?.verdict === "violation",
+      request: event.requestId === undefined ? undefined : { requestId: event.requestId },
+    });
+  }
+  records.sort((a, b) => a.timestamp - b.timestamp || a.seq - b.seq);
+
   const assistantRecords = records.filter(record => record.kind === "assistant");
-  const requestIds = [...new Set(telemetry.flatMap(event => event.requestId === undefined ? [] : [event.requestId]))];
-  for (const [index, requestId] of requestIds.entries()) {
-    const record = assistantRecords[index];
-    if (!record) break;
+  const requestIds = [...new Set(telemetry.flatMap(event => event.type === "request" && event.requestId !== undefined ? [event.requestId] : []))];
+  for (const requestId of requestIds) {
     const events = telemetry.filter(event => event.requestId === requestId);
+    const requestAt = events.find(event => event.type === "request")?.at ?? 0;
+    const record = assistantRecords.find(candidate => candidate.request === undefined && candidate.timestamp >= requestAt)
+      ?? assistantRecords.find(candidate => candidate.request === undefined);
+    if (!record) break;
     const request = events.find(event => event.type === "request")?.data as AnyRecord | undefined;
     const response = events.find(event => event.type === "response")?.data;
     const end = events.find(event => event.type === "message-end")?.data as AnyRecord | undefined;
@@ -188,6 +239,7 @@ export function projectTrajectory(
     const end = telemetry.find(event => event.type === "tool-end" && event.toolCallId === record.toolCallId)?.data as AnyRecord | undefined;
     if (typeof end?.durationMs === "number") record.durationMs = end.durationMs;
   }
+  records.forEach((record, index) => { record.seq = index; });
 
   return { ...meta, generatedAt: Date.now(), records };
 }

package/packages/trajectory/src/server.ts

@@ -11,10 +11,17 @@ export interface TrajectoryServerOptions {
 
 const HOST = "127.0.0.1";
 
-function json(res: ServerResponse, value: unknown): void {
+function json(res: ServerResponse, value: TrajectorySnapshot, requestEtag?: string): void {
+  const etag = `W/\"${value.sessionId}:${value.generatedAt}\"`;
+  if (requestEtag === etag) {
+    res.writeHead(304, { ETag: etag });
+    res.end();
+    return;
+  }
   res.writeHead(200, {
-    "Cache-Control": "no-store",
+    "Cache-Control": "no-cache",
     "Content-Type": "application/json; charset=utf-8",
+    ETag: etag,
     "X-Content-Type-Options": "nosniff",
   });
   res.end(JSON.stringify(value));
@@ -35,7 +42,7 @@ export class TrajectoryServer {
       try {
         const url = new URL(req.url ?? "/", `http://${HOST}`);
         if (req.method !== "GET") { res.writeHead(405).end("Method Not Allowed"); return; }
-        if (url.pathname === "/api/snapshot") { json(res, this.options.snapshot()); return; }
+        if (url.pathname === "/api/snapshot") { json(res, this.options.snapshot(), req.headers["if-none-match"]); return; }
         if (url.pathname === "/favicon.ico") { res.writeHead(204).end(); return; }
         if (url.pathname === "/app.js") {
         res.writeHead(200, { "Cache-Control": "no-store", "Content-Type": "application/javascript; charset=utf-8", "X-Content-Type-Options": "nosniff" });

package/packages/trajectory/src/telemetry.ts

@@ -1,8 +1,12 @@
-import { appendFileSync, mkdirSync, readFileSync } from "node:fs";
+import { appendFileSync, closeSync, mkdirSync, openSync, readSync, statSync } from "node:fs";
 import { homedir } from "node:os";
 import { join } from "node:path";
 
 export type TelemetryEventType =
+  | "hook"
+  | "unipi-trace"
+  | "prefix-integrity"
+  | "system-prompt"
   | "request"
   | "response"
   | "first-token"
@@ -15,6 +19,8 @@ export interface TelemetryEvent {
   type: TelemetryEventType;
   at: number;
   requestId?: number | string;
+  runId?: number | string;
+  turnIndex?: number;
   toolCallId?: string;
   data?: unknown;
 }
@@ -24,8 +30,58 @@ const SECRET_VALUE = /^(?:bearer\s+|sk-[a-z0-9_-]{12,}|gh[pousr]_[a-z0-9]{12,}|A
 const MAX_STRING = 200_000;
 /** Hard ceiling on a serialized event; guards against pathological payloads. */
 const MAX_EVENT_BYTES = 2_000_000;
-/** Sidecar reads are capped so projection cannot allocate unbounded memory. */
-const MAX_READ_BYTES = 20_000_000;
+/** Only a small recent tail is needed by the live context inspector. */
+const MAX_READ_BYTES = 5_000_000;
+
+interface SharedSidecarState {
+  events: TelemetryEvent[];
+  sizes: number[];
+  retainedBytes: number;
+  revision: number;
+}
+
+const sharedStates = new Map<string, SharedSidecarState>();
+
+function readTail(file: string): { events: TelemetryEvent[]; sizes: number[] } {
+  let fd: number | undefined;
+  try {
+    const size = statSync(file).size;
+    const length = Math.min(size, MAX_READ_BYTES);
+    const buffer = Buffer.allocUnsafe(length);
+    fd = openSync(file, "r");
+    readSync(fd, buffer, 0, length, size - length);
+    let text = buffer.toString("utf8");
+    if (size > length) text = text.slice(text.indexOf("\n") + 1);
+    const events: TelemetryEvent[] = [];
+    const sizes: number[] = [];
+    for (const line of text.split("\n")) {
+      if (!line) continue;
+      try {
+        events.push(JSON.parse(line) as TelemetryEvent);
+        sizes.push(Buffer.byteLength(line) + 1);
+      } catch { /* Ignore an incomplete or malformed line. */ }
+    }
+    return { events, sizes };
+  } catch {
+    return { events: [], sizes: [] };
+  } finally {
+    if (fd !== undefined) closeSync(fd);
+  }
+}
+
+function stateFor(file: string): SharedSidecarState {
+  const existing = sharedStates.get(file);
+  if (existing) return existing;
+  const loaded = readTail(file);
+  const state = {
+    events: loaded.events,
+    sizes: loaded.sizes,
+    retainedBytes: loaded.sizes.reduce((total, size) => total + size, 0),
+    revision: 0,
+  };
+  sharedStates.set(file, state);
+  return state;
+}
 
 export function redactTelemetry(value: unknown, seen = new WeakSet<object>()): unknown {
   if (typeof value === "string") {
@@ -44,32 +100,37 @@ export function redactTelemetry(value: unknown, seen = new WeakSet<object>()): u
 
 export class TelemetrySidecar {
   readonly file: string;
+  private readonly state: SharedSidecarState;
 
   constructor(sessionId: string, root = join(homedir(), ".unipi", "trajectory")) {
     mkdirSync(root, { recursive: true, mode: 0o700 });
     this.file = join(root, `${sessionId.replace(/[^a-zA-Z0-9._-]/g, "_")}.jsonl`);
+    this.state = stateFor(this.file);
   }
 
   append(event: Omit<TelemetryEvent, "v">): void {
-    const serialized = JSON.stringify(redactTelemetry({ v: 1, ...event }));
-    if (serialized.length > MAX_EVENT_BYTES) return;
+    const safe = redactTelemetry({ v: 1, ...event }) as TelemetryEvent;
+    const serialized = JSON.stringify(safe);
+    const bytes = Buffer.byteLength(serialized) + 1;
+    if (bytes > MAX_EVENT_BYTES) return;
     appendFileSync(this.file, `${serialized}\n`, { encoding: "utf8", mode: 0o600 });
+    this.state.events.push(safe);
+    this.state.sizes.push(bytes);
+    this.state.retainedBytes += bytes;
+    this.state.revision++;
+    while (this.state.retainedBytes > MAX_READ_BYTES && this.state.events.length > 1) {
+      this.state.events.shift();
+      this.state.retainedBytes -= this.state.sizes.shift() ?? 0;
+    }
   }
 
+  /** Return the cached recent tail; this never rereads the whole JSONL file. */
   read(): TelemetryEvent[] {
-    try {
-      let lines = readFileSync(this.file, "utf8");
-      if (lines.length > MAX_READ_BYTES) {
-        // Keep the most recent events; skip complete earlier lines.
-        const keep = lines.slice(lines.length - MAX_READ_BYTES);
-        lines = keep.slice(keep.indexOf("\n") + 1);
-      }
-      return lines.split("\n").filter(Boolean).flatMap(line => {
-        try { return [JSON.parse(line) as TelemetryEvent]; } catch { return []; }
-      });
-    } catch {
-      return [];
-    }
+    return this.state.events.slice();
+  }
+
+  revision(): number {
+    return this.state.revision;
   }
 }