memhouse 0.18.12 → 0.20.1

package/editors/normalise/copilot-cli.js

@@ -0,0 +1,321 @@
+'use strict';
+// v4 wave 1: GitHub Copilot CLI → normalised session log records (docs/design/sessions-log.md).
+//
+// Store: <COPILOT_HOME|~/.copilot>/session-state/<sid>/events.jsonl, plus workspace.yaml. VS Code's
+// Copilot engine writes into the same store (workspace.yaml client_name: vscode-agent-host), so those
+// sessions are captured here as harness copilot-cli with client 'vscode'.
+// Spec: docs/design/adapter-specs/copilot-cli.md (repository), sessions-model v0.
+//
+// Rules from the spec:
+// - order is line order; parentId is only a hint (it dangles after rewinds and side calls);
+// - message ids are event ids (uuid v4, stable across resume and compaction);
+// - a tool call is matched to tool.execution_complete by toolCallId; one part holds both;
+// - usage: session.usage_record attaches to the assistant.message with the same apiCallId;
+//   basis unknown (the vendor does not define it). The record's `cost` is Copilot's own credit
+//   unit (0 under BYOK), so it goes to credits (0 → null), never to cost. Usage with no
+//   assistant.message is a `usage` record of its own (sessions-model §2.4: "Copilot leaves a
+//   usage_record when a reply is capped and no assistant.message follows"):
+//   - a usage_record whose apiCallId no assistant.message has: usage_id = its apiCallId (one longer
+//     than 64 characters, VS Code's opaque base64, is keyed `sha256:<32 hex>` of it), at its own
+//     place, with its finishReason (`length` for a capped reply) as the reason. It waits, with all
+//     after it, until its reply can no longer come (its agent replied again, or its turn ended), so
+//     a reply written just after it still takes it, and its release only appends;
+//   - a usage_record with no apiCallId (optional in Copilot's schema): no reply can take it, so it
+//     is written at once, at its own place, keyed by its own event id (`usage_record:<event id>`).
+//     apiCallId is needed only to match a reply;
+//   - a compaction's own model call, which has no usage_record: `compactionTokensUsed`, usage_id
+//     `compaction:<event id>#1` (§2.4's form for a compaction anchor), reason compaction, after the
+//     compaction's summary;
+//   - neither is written for an event a fork copied from its origin (below): the origin bills it;
+// - a sub-agent's events carry the envelope agentId: same session, agent_id set;
+// - incarnation: the session.start event id (a reused --session-id gets a new one);
+// - a fork child copies its origin's events with their ids and timestamps, so every copy is older
+//   than the child's own session.start (the first one in the file): older than the earlier of its
+//   `startTime` and the event's own timestamp, the session's birth. A copied `session.info` marker
+//   (fork or rewind) is the origin's history: it states nothing about this session. A child's copy
+//   of "Forked this session into \"…\" (<id>)" names its origin's earlier fork, a sibling, never
+//   its own child;
+// - an assistant message with a tool call still running is held back until the call completes or
+//   the turn ends, and everything after it waits too (order is the native event index), so its
+//   release only appends;
+// - a turn ends with its agent's own user message, abort or turn end; a sub-agent's also with its
+//   subagent.completed or subagent.failed; and every agent's with the root agent's user message or
+//   abort (no agentId), or the session's shutdown. A sub-agent writes no turn end of its own;
+// - system.message, model.*, hook.*, permission.*, binary assets' bytes and other bookkeeping are
+//   not conversation: they are left out of the log.
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const harnessPaths = require('../harness-paths');
+
+const HARNESS = 'copilot-cli';
+// Bumped when the records change for an unchanged store, so the runner normalises every session again
+// (normalise-run.js) and the engine re-derives its rewritten log at a higher rev. 2: usage with no
+// assistant.message (a capped reply, a compaction's own tokens); a fork's copied markers ignored.
+const VERSION = 2;
+const SID_RE = /\(([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\)/i;
+
+function storeRootDefault() { return harnessPaths.defaultFolder(HARNESS); }
+
+/** Sessions in a store: one per session-state/<sid>/ that has an events.jsonl. */
+function list(storeRoot = storeRootDefault()) {
+  const base = path.join(storeRoot, 'session-state');
+  // The data folder's real path: absolute, symlinks resolved, in the filesystem's own case.
+  const harnessPath = harnessPaths.resolveNative(storeRoot);
+  let names = [];
+  try { names = fs.readdirSync(base); } catch { return []; }
+  const out = [];
+  for (const sid of names) {
+    const dir = path.join(base, sid);
+    const events = path.join(dir, 'events.jsonl');
+    try { if (!fs.statSync(events).isFile()) continue; } catch { continue; }
+    out.push({ harness: HARNESS, harness_path: harnessPath, session_id: sid, dir, events, workspace: path.join(dir, 'workspace.yaml') });
+  }
+  return out;
+}
+
+/** Complete lines only: a partial last line is held back. */
+function parseEvents(text) {
+  const end = text.lastIndexOf('\n');
+  if (end < 0) return [];
+  const out = [];
+  for (const line of text.slice(0, end).split('\n')) {
+    if (!line.trim()) continue;
+    try { out.push(JSON.parse(line)); } catch { /* malformed line: skipped, the archive keeps it */ }
+  }
+  return out;
+}
+
+/** workspace.yaml is flat `key: value`; only the top-level scalars are read. */
+function parseWorkspace(text) {
+  const ws = {};
+  for (const line of String(text || '').split('\n')) {
+    const m = /^([A-Za-z_][\w-]*):\s*(.*)$/.exec(line);
+    if (!m) continue;
+    let v = m[2].trim();
+    if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) v = v.slice(1, -1);
+    ws[m[1]] = v;
+  }
+  return ws;
+}
+
+const ms = (iso) => { const t = Date.parse(iso); return Number.isFinite(t) ? t : null; };
+const sha256 = (buf) => crypto.createHash('sha256').update(buf).digest('hex');
+function argsOf(a) {
+  if (a == null) return {};
+  if (typeof a === 'string') { try { return JSON.parse(a); } catch { return { raw: a }; } }
+  return a;
+}
+/** Copilot's usage counters (a usage_record's `usage`, or `compactionTokensUsed`) as the log's usage. */
+function usageOf(u) {
+  return { basis: 'unknown', input: u.inputTokens ?? null, output: u.outputTokens ?? null, cache_read: u.cacheReadTokens ?? null, cache_write: u.cacheWriteTokens ?? null, reasoning: u.reasoningTokens ?? null, cost: null, credits: u.cost || null };
+}
+const modelOf = (u) => (u.modelDisplayName || u.model ? { model: String(u.modelDisplayName || u.model) } : {});
+/**
+ * A usage record's key from its call's apiCallId: the id itself when it is short, else a hash of
+ * it (stable, 39 characters). VS Code's Copilot engine writes an opaque base64 apiCallId of
+ * hundreds of characters, which would otherwise fill a key column. A reply is still matched on the
+ * raw value.
+ */
+const KEY_MAX = 64;
+const keyOfApiCall = (api) => { const s = String(api); return s.length <= KEY_MAX ? s : `sha256:${sha256(s).slice(0, 32)}`; };
+function resultText(d) {
+  if (!d) return null;
+  const r = d.result;
+  if (r && typeof r.content === 'string') return r.content;
+  if (r && Array.isArray(r.content)) return r.content.map((c) => (typeof c === 'string' ? c : c.text || '')).join('');
+  if (d.error) return typeof d.error === 'string' ? d.error : (d.error.message || d.error.code || 'error');
+  return '';
+}
+
+/**
+ * Normalise one session. `session` is a list() entry, or `{ session_id, events: [...], workspace: {...} }`
+ * already parsed (tests). Returns the log records: the session record first, then messages.
+ */
+function normalise(session) {
+  const events = Array.isArray(session.events)
+    ? session.events
+    : parseEvents(fs.readFileSync(session.events, 'utf-8'));
+  let ws = session.workspace;
+  if (!ws || typeof ws === 'string') {
+    let txt = '';
+    try { txt = fs.readFileSync(ws || '', 'utf-8'); } catch { txt = ''; }
+    ws = parseWorkspace(txt);
+  }
+  const start = events.find((e) => e.type === 'session.start');
+  if (!start) return null;
+  // An event a fork copied from its origin: older than this session's birth, the earlier of its
+  // start's data.startTime and the start event's own timestamp (the fixtures write the event a
+  // fraction of a second after startTime, so an event of its own may fall between the two).
+  const born = [ms((start.data || {}).startTime), ms(start.timestamp)].filter((t) => t !== null);
+  const birth = born.length ? Math.min(...born) : null;
+  const copied = (e) => { const t = ms(e.timestamp); return birth !== null && t !== null && t < birth; };
+
+  // Index tool results, usage and assets.
+  const results = new Map();     // toolCallId → completion event
+  const usage = new Map();       // apiCallId → usage
+  const replied = new Set();     // apiCallIds an assistant.message has
+  const assets = new Map();      // assetId → { mime, bytes }
+  const servers = new Map();     // toolCallId → MCP server name (tool.execution_start)
+  for (const e of events) {
+    const d = e.data || {};
+    if (e.type === 'tool.execution_complete' && d.toolCallId) results.set(d.toolCallId, e);
+    else if (e.type === 'tool.execution_start' && d.toolCallId && d.mcpServerName) servers.set(d.toolCallId, d.mcpServerName);
+    else if (e.type === 'session.usage_record' && d.usage && d.usage.apiCallId) usage.set(d.usage.apiCallId, d.usage);
+    else if (e.type === 'session.binary_asset' && d.assetId) assets.set(d.assetId, d);
+    else if (e.type === 'assistant.message' && d.apiCallId) replied.add(d.apiCallId);
+  }
+
+  // For each event, whether its agent's turn ends later in the file: scanned once from the end.
+  // - Its own agent ends it: a user message, an abort or the turn's end of that agent, or, for a
+  //   sub-agent, its subagent.completed / subagent.failed (they carry the sub-agent's agentId).
+  // - The root agent ends every agent: a shutdown, or a user message or abort with no agentId. A
+  //   sub-agent writes no turn end of its own, so without this a sub-agent's unanswered call (or
+  //   its running tool) would hold back the rest of the session for good.
+  const ENDS = new Set(['user.message', 'abort', 'session.shutdown', 'assistant.turn_end', 'subagent.completed', 'subagent.failed']);
+  const ROOT_ENDS = new Set(['user.message', 'abort']);
+  const endsAfter = new Array(events.length).fill(false);
+  const seenEnd = new Set();
+  let seenRootEnd = false;
+  // And whether its agent writes another assistant.message later (a reply after it).
+  const repliesAfter = new Array(events.length).fill(false);
+  const seenReply = new Set();
+  for (let i = events.length - 1; i >= 0; i--) {
+    const a = events[i].agentId || '';
+    endsAfter[i] = seenRootEnd || seenEnd.has(a);
+    repliesAfter[i] = seenReply.has(a);
+    if (ENDS.has(events[i].type)) seenEnd.add(a);
+    if (events[i].type === 'session.shutdown' || (!a && ROOT_ENDS.has(events[i].type))) seenRootEnd = true;
+    if (events[i].type === 'assistant.message') seenReply.add(a);
+  }
+
+  const links = [];
+  const messages = [];
+  for (let i = 0; i < events.length; i++) {
+    const e = events[i];
+    const d = e.data || {};
+    const agent = e.agentId || '';
+    const ts = ms(e.timestamp);
+    if (e.type === 'user.message') {
+      const parts = [];
+      if (d.content) parts.push({ part_id: '0', kind: 'text', text: String(d.content) });
+      (d.attachments || []).forEach((a, j) => {
+        const asset = a && a.assetId ? assets.get(a.assetId) : null;
+        const bytes = asset && asset.data ? Buffer.from(asset.data, 'base64') : null;
+        parts.push({
+          part_id: `att:${j}`, kind: 'file',
+          media_type: (asset && asset.mimeType) || a.mimeType || '',
+          att_sha: bytes ? sha256(bytes) : null, att_len: bytes ? bytes.length : (asset && asset.byteLength) || null,
+          ref: a.path || a.displayName || a.assetId || '',
+        });
+      });
+      // A sub-agent's prompt is written by its parent agent (data.source 'agent-<sid>…'): automated.
+      const auto = typeof d.source === 'string' && d.source.startsWith('agent-');
+      messages.push({ k: 'message', message_id: e.id, role: 'user', ts, order: i, agent_id: agent, state: 'current', status: 'complete', ...(auto ? { source: 'automated' } : {}), parts });
+      continue;
+    }
+    if (e.type === 'assistant.message') {
+      const reqs = d.toolRequests || [];
+      const pending = reqs.some((r) => !results.has(r.toolCallId));
+      if (pending && !endsAfter[i]) break;   // a tool still running: held back, and all after it
+      const parts = [];
+      if (d.reasoningText) parts.push({ part_id: 'r', kind: 'reasoning', text: String(d.reasoningText) });
+      if (d.content) parts.push({ part_id: 't', kind: 'text', text: String(d.content) });
+      for (const r of reqs) {
+        const done = results.get(r.toolCallId);
+        const dd = done ? done.data || {} : null;
+        parts.push({
+          part_id: `tool:${r.toolCallId}`, kind: 'tool', call_id: r.toolCallId, tool: r.name, args: argsOf(r.arguments),
+          output: dd ? resultText(dd) : null, is_error: dd ? dd.success === false : false,
+          status: dd ? (dd.success === false ? (dd.error && dd.error.code === 'denied' ? 'denied' : 'error') : 'completed') : 'interrupted',
+          ...(dd && dd.shellExecution && Number.isFinite(dd.shellExecution.exitCode) ? { exit_code: dd.shellExecution.exitCode } : {}),
+          ...(servers.has(r.toolCallId) ? { server: servers.get(r.toolCallId) } : {}),
+          ts: done ? ms(done.timestamp) : ts,
+        });
+      }
+      const u = d.apiCallId ? usage.get(d.apiCallId) : null;
+      messages.push({
+        k: 'message', message_id: e.id, role: 'assistant', ts, order: i, agent_id: agent, state: 'current',
+        status: pending ? 'interrupted' : 'complete',
+        model: (u && u.modelDisplayName) || d.model || '', model_requested: d.model || (u && u.model) || '', effort: '',
+        usage: u ? usageOf(u) : null,
+        parts,
+      });
+      continue;
+    }
+    if (e.type === 'system.notification') {
+      // The text, or a type/status line built from plain strings only: never the native record
+      // itself, whose other fields are not conversation (and could carry anything).
+      const kind = d.kind && typeof d.kind === 'object' ? d.kind : {};
+      const word = (v) => (typeof v === 'string' && v ? v : '');
+      const text = typeof d.content === 'string' ? d.content
+        : [word(typeof d.kind === 'string' ? d.kind : kind.type), word(kind.status || d.status)].filter(Boolean).join(' ');
+      if (!text) continue;
+      messages.push({ k: 'message', message_id: e.id, role: 'system', ts, order: i, agent_id: agent, state: 'current', status: 'complete', source: 'injected', parts: [{ part_id: '0', kind: 'notice', text }] });
+      continue;
+    }
+    if (e.type === 'session.usage_record') {
+      // Usage with no assistant.message (the header's usage rule): a capped reply, or a record with
+      // no apiCallId (the schema makes it optional), which no reply can take.
+      const u = d.usage;
+      if (!u || typeof u !== 'object' || copied(e)) continue;
+      const api = u.apiCallId != null && u.apiCallId !== '' ? u.apiCallId : null;
+      if (api !== null && replied.has(api)) continue;   // on its assistant.message
+      const id = api !== null ? keyOfApiCall(api) : (e.id ? `usage_record:${e.id}` : null);
+      if (id === null) continue;   // no stable key of any kind
+      if (api !== null && !endsAfter[i] && !repliesAfter[i]) break;   // its reply may still come: held back, and all after it
+      messages.push({ k: 'usage', usage_id: id, ts, ...modelOf(u), ...(agent ? { agent_id: agent } : {}), ...(typeof u.finishReason === 'string' && u.finishReason ? { reason: u.finishReason } : {}), usage: usageOf(u) });
+      continue;
+    }
+    if (e.type === 'session.compaction_complete') {
+      if (d.summaryContent) {
+        messages.push({ k: 'lifecycle', event: 'compaction', at: ts, ...(Number.isFinite(d.messagesRemoved) ? { count: d.messagesRemoved } : {}) });
+        messages.push({
+          k: 'message', message_id: e.id, role: 'system', ts, order: i, agent_id: agent, state: 'current', status: 'complete',
+          parts: [{ part_id: '0', kind: 'summary', text: String(d.summaryContent), replaces: { count: d.messagesRemoved ?? null } }],
+        });
+      }
+      // The compaction's own tokens (the header's usage rule), whether or not a summary was kept.
+      const t = d.compactionTokensUsed;
+      const cu = t && typeof t === 'object' && e.id && !copied(e) ? usageOf(t) : null;
+      if (cu && [cu.input, cu.output, cu.cache_read, cu.cache_write, cu.reasoning].some((x) => Number(x) > 0)) {
+        messages.push({ k: 'usage', usage_id: `compaction:${e.id}#1`, ts, ...modelOf(t), ...(agent ? { agent_id: agent } : {}), reason: 'compaction', usage: cu });
+      }
+      continue;
+    }
+    if (e.type === 'session.info' && copied(e)) continue;   // the origin's marker, copied with its events
+    if (e.type === 'session.info' && d.infoType === 'fork') {
+      // Stated on both sides, naming the other session in text. The child's statement gives its
+      // origin (links); the origin's ("Forked this session into …") names the child (a child record).
+      const m = SID_RE.exec(String(d.message || ''));
+      if (m && m[1] !== session.session_id) {
+        // The child at the fork event's own place, not after the message before it: two forks in a
+        // row would otherwise put the second child before the first fork's lifecycle record.
+        if (/\binto\b/i.test(d.message)) messages.push({ k: 'child', kind: 'fork', session_id: m[1], evidence: 'explicit' });
+        else links.push({ kind: 'fork', origin: m[1], evidence: 'explicit', at: '' });
+        messages.push({ k: 'lifecycle', event: 'fork', at: ts });
+      }
+      continue;
+    }
+    if (e.type === 'session.info' && d.infoType === 'rewind') {
+      // A rewind rewrites events.jsonl (the rewound events vanish) and appends this marker.
+      messages.push({ k: 'lifecycle', event: 'rewind', at: ts });
+    }
+  }
+
+  const head = {
+    k: 'session',
+    title: ws.name || '',
+    cwd: ((start.data || {}).context || {}).cwd || '',
+    created: ms((start.data || {}).startTime) || ms(start.timestamp),
+    updated: null, archived: null,
+    version: (start.data || {}).copilotVersion || '',
+    incarnation: start.id,
+    client: ws.client_name === 'vscode-agent-host' ? 'vscode' : 'cli',
+    links,
+  };
+  return [head, ...messages];
+}
+
+module.exports = { HARNESS, VERSION, list, normalise, parseEvents, parseWorkspace };