@elench/testkit 0.1.101 → 0.1.103

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -281,6 +281,9 @@ function colorWelcomeValue(label, value) {
281
281
  function colorMarker(block) {
282
282
  if (block.kind === "user") return cyan(block.marker);
283
283
  if (block.kind === "system") return red(block.marker);
284
+ if (block.kind === "provider-error") return red(block.marker);
285
+ if (block.kind === "provider-activity") return dim(block.marker);
286
+ if (block.kind === "provider-tool") return yellow(block.marker);
284
287
  if (block.kind === "tool-running") return yellow(block.marker);
285
288
  if (block.kind === "testkit-run") return green(block.marker);
286
289
  return block.marker;
@@ -289,6 +292,9 @@ function colorMarker(block) {
289
292
  function colorBlockText(block, text) {
290
293
  if (block.kind === "user") return text;
291
294
  if (block.kind === "system") return red(text);
295
+ if (block.kind === "provider-error") return red(text);
296
+ if (block.kind === "provider-activity") return dim(text);
297
+ if (block.kind === "provider-tool") return yellow(text);
292
298
  if (block.kind === "tool-running") return yellow(text);
293
299
  return text;
294
300
  }
@@ -4,14 +4,15 @@ import {
4
4
  buildStatusEvent,
5
5
  buildToolEvent,
6
6
  createHostedSessionRunner,
7
- extractTextFragments,
8
7
  } from "./shared.mjs";
8
+ import { providerAssistantDelta, providerAssistantFinal, providerToolStart } from "./events.mjs";
9
9
 
10
10
  export function startClaudeHostedSession({
11
11
  command = "claude",
12
12
  cwd,
13
13
  prompt,
14
14
  onEvent,
15
+ onRawLine,
15
16
  purpose = "assistant",
16
17
  model = null,
17
18
  effort = null,
@@ -49,6 +50,7 @@ export function startClaudeHostedSession({
49
50
  provider: "claude",
50
51
  child,
51
52
  onEvent,
53
+ onRawLine,
52
54
  parsePayload: parseClaudePayload,
53
55
  readFinalText(result) {
54
56
  return readClaudeFinalText(result?.stdout || "") || null;
@@ -77,7 +79,20 @@ export function parseClaudePayload(payload) {
77
79
  const streamEvent = payload.event || {};
78
80
  if (streamEvent.type === "content_block_delta" && streamEvent.delta?.type === "text_delta") {
79
81
  const text = String(streamEvent.delta.text || "");
80
- if (text) events.push({ type: "delta", text });
82
+ const event = providerAssistantDelta(text);
83
+ if (event) events.push(event);
84
+ return events;
85
+ }
86
+ if (streamEvent.type === "content_block_start" && streamEvent.content_block?.type === "tool_use") {
87
+ const tool = streamEvent.content_block;
88
+ const event = providerToolStart(
89
+ tool.name || tool.tool_name || "tool_use",
90
+ {
91
+ id: tool.id || streamEvent.index,
92
+ input: tool.input || null,
93
+ }
94
+ );
95
+ if (event) events.push(event);
81
96
  return events;
82
97
  }
83
98
  if (streamEvent.type === "tool_use" || streamEvent.content_block?.type === "tool_use") {
@@ -102,10 +117,9 @@ export function parseClaudePayload(payload) {
102
117
  }
103
118
 
104
119
  if (type === "assistant") {
105
- const fragments = [...new Set(extractTextFragments(payload.message?.content || payload.content || [], []))];
106
- for (const fragment of fragments) {
107
- events.push({ type: "delta", text: fragment });
108
- }
120
+ const fragments = extractClaudeTextFragments(payload.message?.content || payload.content || []);
121
+ const event = providerAssistantFinal(fragments.join(""));
122
+ if (event) events.push(event);
109
123
  return events;
110
124
  }
111
125
 
@@ -113,6 +127,9 @@ export function parseClaudePayload(payload) {
113
127
  if (payload.is_error || payload.subtype === "error") {
114
128
  const event = buildErrorEvent(payload.result || payload.error || "Claude command failed");
115
129
  if (event) events.push(event);
130
+ } else if (typeof payload.result === "string") {
131
+ const event = providerAssistantFinal(payload.result);
132
+ if (event) events.push(event);
116
133
  }
117
134
  return events;
118
135
  }
@@ -147,10 +164,27 @@ export function readClaudeFinalText(stdout) {
147
164
  }
148
165
 
149
166
  if (payload.type === "assistant") {
150
- const fragments = [...new Set(extractTextFragments(payload.message?.content || payload.content || [], []))];
167
+ const fragments = extractClaudeTextFragments(payload.message?.content || payload.content || []);
151
168
  if (fragments.length > 0) fallback = fragments.join("");
152
169
  }
153
170
  }
154
171
 
155
172
  return fallback;
156
173
  }
174
+
175
+ function extractClaudeTextFragments(content) {
176
+ const entries = Array.isArray(content) ? content : [content];
177
+ const fragments = [];
178
+ for (const entry of entries) {
179
+ if (typeof entry === "string") {
180
+ const text = entry.trim();
181
+ if (text) fragments.push(text);
182
+ continue;
183
+ }
184
+ if (!entry || typeof entry !== "object") continue;
185
+ if (entry.type !== "text") continue;
186
+ const text = String(entry.text || "").trim();
187
+ if (text) fragments.push(text);
188
+ }
189
+ return [...new Set(fragments)];
190
+ }
@@ -4,18 +4,25 @@ import path from "path";
4
4
  import { execa } from "execa";
5
5
  import {
6
6
  buildErrorEvent,
7
- buildStatusEvent,
8
7
  buildToolEvent,
9
8
  createHostedSessionRunner,
10
- extractTextFragments,
11
9
  readTextFileIfPresent,
12
10
  } from "./shared.mjs";
11
+ import {
12
+ providerAssistantDelta,
13
+ providerAssistantFinal,
14
+ providerStatus,
15
+ providerToolEnd,
16
+ providerToolStart,
17
+ providerToolUpdate,
18
+ } from "./events.mjs";
13
19
 
14
20
  export function startCodexHostedSession({
15
21
  command = "codex",
16
22
  cwd,
17
23
  prompt,
18
24
  onEvent,
25
+ onRawLine,
19
26
  purpose = "assistant",
20
27
  model = null,
21
28
  providerArgs = [],
@@ -44,6 +51,7 @@ export function startCodexHostedSession({
44
51
  provider: "codex",
45
52
  child,
46
53
  onEvent,
54
+ onRawLine,
47
55
  parsePayload: parseCodexPayload,
48
56
  shouldIgnoreStatus(message) {
49
57
  return String(message || "").trim() === "Reading additional input from stdin...";
@@ -96,6 +104,48 @@ export function parseCodexPayload(payload) {
96
104
  return events;
97
105
  }
98
106
 
107
+ if (type === "thread.started") {
108
+ const event = providerStatus(
109
+ payload.thread_id ? `Codex thread ${payload.thread_id} started` : "Codex thread started",
110
+ { transient: true }
111
+ );
112
+ if (event) events.push(event);
113
+ return events;
114
+ }
115
+
116
+ if (type === "turn.started") {
117
+ const event = providerStatus("Codex turn started", { transient: true });
118
+ if (event) events.push(event);
119
+ return events;
120
+ }
121
+
122
+ if (type === "turn.completed") {
123
+ const event = providerStatus("Codex turn completed", { transient: true, usage: payload.usage || null });
124
+ if (event) events.push(event);
125
+ return events;
126
+ }
127
+
128
+ if (type === "item.started") {
129
+ const item = payload.item || {};
130
+ const event = codexItemStartedEvent(item);
131
+ if (event) events.push(event);
132
+ return events;
133
+ }
134
+
135
+ if (type === "item.updated") {
136
+ const item = payload.item || {};
137
+ const event = codexItemUpdatedEvent(item);
138
+ if (event) events.push(event);
139
+ return events;
140
+ }
141
+
142
+ if (type === "item.completed") {
143
+ const item = payload.item || {};
144
+ const event = codexItemCompletedEvent(item);
145
+ if (event) events.push(event);
146
+ return events;
147
+ }
148
+
99
149
  if (type && /(tool|command|patch|exec)/i.test(type)) {
100
150
  const event = buildToolEvent(
101
151
  payload.name || payload.tool_name || payload.command || type,
@@ -105,15 +155,73 @@ export function parseCodexPayload(payload) {
105
155
  return events;
106
156
  }
107
157
 
108
- const fragments = [...new Set(extractTextFragments(payload, []))];
109
- if (fragments.length > 0) {
110
- for (const fragment of fragments) {
111
- events.push({ type: "delta", text: fragment });
112
- }
113
- return events;
114
- }
115
-
116
- const statusEvent = buildStatusEvent(type ? `Codex event: ${type}` : JSON.stringify(payload));
158
+ const statusEvent = providerStatus(type ? `Codex event: ${type}` : JSON.stringify(payload));
117
159
  if (statusEvent) events.push(statusEvent);
118
160
  return events;
119
161
  }
162
+
163
+ function codexItemStartedEvent(item) {
164
+ if (!item || typeof item !== "object") return null;
165
+ if (item.type === "command_execution") {
166
+ return providerToolStart("command", {
167
+ id: item.id || null,
168
+ input: item.command || item.input || null,
169
+ });
170
+ }
171
+ if (item.type === "tool_call") {
172
+ return providerToolStart(item.name || item.type, {
173
+ id: item.id || null,
174
+ input: item.arguments || item.input || null,
175
+ });
176
+ }
177
+ if (item.type === "agent_message") return providerStatus("Codex started response", { transient: true });
178
+ if (item.type) return providerStatus(`Codex started ${item.type}`);
179
+ return null;
180
+ }
181
+
182
+ function codexItemUpdatedEvent(item) {
183
+ if (!item || typeof item !== "object") return null;
184
+ if (item.type === "agent_message" && typeof item.text === "string") {
185
+ return providerAssistantDelta(item.text);
186
+ }
187
+ if (item.type === "command_execution") {
188
+ return providerToolUpdate("command", {
189
+ id: item.id || null,
190
+ text: item.output || item.status || null,
191
+ data: item,
192
+ });
193
+ }
194
+ if (item.type === "tool_call") {
195
+ return providerToolUpdate(item.name || item.type, {
196
+ id: item.id || null,
197
+ text: item.output || item.status || null,
198
+ data: item,
199
+ });
200
+ }
201
+ return null;
202
+ }
203
+
204
+ function codexItemCompletedEvent(item) {
205
+ if (!item || typeof item !== "object") return null;
206
+ if (item.type === "agent_message" && typeof item.text === "string") {
207
+ return providerAssistantFinal(item.text);
208
+ }
209
+ if (item.type === "command_execution") {
210
+ return providerToolEnd("command", {
211
+ id: item.id || null,
212
+ status: item.status === "failed" ? "error" : "ok",
213
+ output: item.output || null,
214
+ data: item,
215
+ });
216
+ }
217
+ if (item.type === "tool_call") {
218
+ return providerToolEnd(item.name || item.type, {
219
+ id: item.id || null,
220
+ status: item.status === "failed" ? "error" : "ok",
221
+ output: item.output || null,
222
+ data: item,
223
+ });
224
+ }
225
+ if (item.type) return providerStatus(`Codex completed ${item.type}`);
226
+ return null;
227
+ }
@@ -0,0 +1,71 @@
1
+ export const PROVIDER_EVENT_TYPES = new Set([
2
+ "session-start",
3
+ "status",
4
+ "assistant-delta",
5
+ "assistant-final",
6
+ "tool-start",
7
+ "tool-update",
8
+ "tool-end",
9
+ "error",
10
+ "session-end",
11
+ ]);
12
+
13
+ export function providerEvent(type, fields = {}) {
14
+ if (!PROVIDER_EVENT_TYPES.has(type)) {
15
+ throw new Error(`Unknown provider event type: ${type}`);
16
+ }
17
+ return {
18
+ type,
19
+ ...fields,
20
+ };
21
+ }
22
+
23
+ export function providerStatus(text, fields = {}) {
24
+ const normalized = normalizeText(text);
25
+ if (!normalized) return null;
26
+ return providerEvent("status", { text: normalized, ...fields });
27
+ }
28
+
29
+ export function providerAssistantDelta(text, fields = {}) {
30
+ if (!text) return null;
31
+ return providerEvent("assistant-delta", { text: String(text), ...fields });
32
+ }
33
+
34
+ export function providerAssistantFinal(text, fields = {}) {
35
+ const normalized = normalizeText(text);
36
+ if (!normalized) return null;
37
+ return providerEvent("assistant-final", { text: normalized, ...fields });
38
+ }
39
+
40
+ export function providerToolStart(name, fields = {}) {
41
+ const normalized = normalizeText(name);
42
+ if (!normalized) return null;
43
+ return providerEvent("tool-start", { name: normalized, ...fields });
44
+ }
45
+
46
+ export function providerToolUpdate(name, fields = {}) {
47
+ const normalized = normalizeText(name);
48
+ if (!normalized && !fields.text && !fields.data) return null;
49
+ return providerEvent("tool-update", { ...(normalized ? { name: normalized } : {}), ...fields });
50
+ }
51
+
52
+ export function providerToolEnd(name, fields = {}) {
53
+ const normalized = normalizeText(name);
54
+ if (!normalized) return null;
55
+ return providerEvent("tool-end", { name: normalized, status: fields.status || "ok", ...fields });
56
+ }
57
+
58
+ export function providerError(text, fields = {}) {
59
+ const normalized = normalizeText(text);
60
+ if (!normalized) return null;
61
+ return providerEvent("error", { text: normalized, ...fields });
62
+ }
63
+
64
+ export function providerSessionEnd(fields = {}) {
65
+ return providerEvent("session-end", fields);
66
+ }
67
+
68
+ export function normalizeText(value) {
69
+ const text = String(value || "").trim();
70
+ return text || null;
71
+ }
@@ -3,7 +3,7 @@ import path from "path";
3
3
  import { startClaudeHostedSession } from "./claude.mjs";
4
4
  import { startCodexHostedSession } from "./codex.mjs";
5
5
 
6
- const PROVIDERS = ["codex", "claude"];
6
+ export const HOSTED_ASSISTANT_PROVIDERS = Object.freeze(["codex", "claude"]);
7
7
 
8
8
  export function resolvePreferredProvider(preferred = null, env = process.env) {
9
9
  if (preferred && preferred !== "auto") {
@@ -13,7 +13,7 @@ export function resolvePreferredProvider(preferred = null, env = process.env) {
13
13
  return preferred;
14
14
  }
15
15
 
16
- for (const provider of PROVIDERS) {
16
+ for (const provider of HOSTED_ASSISTANT_PROVIDERS) {
17
17
  if (isProviderInstalled(provider, env)) return provider;
18
18
  }
19
19
  throw new Error("Neither codex nor claude was found on PATH");
@@ -63,13 +63,14 @@ export function startProviderSession({
63
63
  cwd,
64
64
  prompt,
65
65
  onEvent,
66
+ onRawLine,
66
67
  purpose = "assistant",
67
68
  env = process.env,
68
69
  } = {}) {
69
70
  const resolvedProvider = resolvePreferredProvider(provider, env);
70
71
  const command = resolveProviderBinary(resolvedProvider, env);
71
72
  if (resolvedProvider === "claude") {
72
- return startClaudeHostedSession({ command, cwd, prompt, onEvent, purpose, model, effort, providerArgs, env });
73
+ return startClaudeHostedSession({ command, cwd, prompt, onEvent, onRawLine, purpose, model, effort, providerArgs, env });
73
74
  }
74
- return startCodexHostedSession({ command, cwd, prompt, onEvent, purpose, model, providerArgs, env });
75
+ return startCodexHostedSession({ command, cwd, prompt, onEvent, onRawLine, purpose, model, providerArgs, env });
75
76
  }
@@ -1,29 +1,51 @@
1
1
  import fs from "fs";
2
2
  import readline from "readline";
3
+ import {
4
+ providerError,
5
+ providerEvent,
6
+ providerSessionEnd,
7
+ providerStatus,
8
+ providerToolStart,
9
+ } from "./events.mjs";
3
10
 
4
- export function createHostedSessionRunner({ provider, child, onEvent, parsePayload, readFinalText, shouldIgnoreStatus } = {}) {
11
+ export function createHostedSessionRunner({
12
+ provider,
13
+ child,
14
+ onEvent,
15
+ onRawLine,
16
+ parsePayload,
17
+ readFinalText,
18
+ shouldIgnoreStatus,
19
+ } = {}) {
5
20
  let cancelled = false;
6
21
  let settled = false;
7
22
  let assistantText = "";
23
+ let finalText = null;
8
24
  let lastErrorMessage = null;
9
25
 
10
26
  const emit = (event) => {
11
- if (event?.type === "delta" || event?.type === "final") {
27
+ if (!event) return;
28
+ if (event.type === "assistant-delta") {
12
29
  assistantText += event.text || "";
13
30
  }
14
- if (event?.type === "error") {
15
- lastErrorMessage = event.message || lastErrorMessage;
31
+ if (event.type === "assistant-final") {
32
+ finalText = event.text || finalText;
33
+ }
34
+ if (event.type === "error") {
35
+ lastErrorMessage = event.text || lastErrorMessage;
16
36
  }
17
37
  if (typeof onEvent === "function" && event) onEvent({ provider, ...event });
18
38
  };
19
39
 
20
- emit({ type: "start" });
40
+ emit(providerEvent("session-start"));
21
41
 
22
42
  const stdoutReader = readline.createInterface({ input: child.stdout });
43
+ const stdoutClosed = waitForReaderClose(stdoutReader);
23
44
  stdoutReader.on("line", (line) => {
45
+ onRawLine?.({ provider, stream: "stdout", line });
24
46
  const parsed = tryParseJson(line);
25
47
  if (parsed == null) {
26
- emit({ type: "status", message: line });
48
+ emit(providerStatus(line));
27
49
  return;
28
50
  }
29
51
  const events = parsePayload ? parsePayload(parsed) : [];
@@ -32,28 +54,34 @@ export function createHostedSessionRunner({ provider, child, onEvent, parsePaylo
32
54
  });
33
55
 
34
56
  const stderrReader = readline.createInterface({ input: child.stderr });
57
+ const stderrClosed = waitForReaderClose(stderrReader);
35
58
  stderrReader.on("line", (line) => {
59
+ onRawLine?.({ provider, stream: "stderr", line });
36
60
  if (shouldIgnoreStatus?.(line)) return;
37
- emit({ type: "status", message: line });
61
+ emit(providerStatus(line, { stream: "stderr" }));
38
62
  });
39
63
 
40
64
  const completion = (async () => {
41
65
  const result = await child;
42
- const finalText = (readFinalText ? readFinalText(result) : null) || assistantText.trim() || null;
66
+ await Promise.all([stdoutClosed, stderrClosed]);
67
+ const fileFinalText = readFinalText ? readFinalText(result) : null;
68
+ const resolvedFinalText = fileFinalText || finalText || assistantText.trim() || null;
43
69
  if ((result.exitCode ?? 0) !== 0) {
44
70
  const message = lastErrorMessage || result.stderr || `${provider} exited with code ${result.exitCode ?? 1}`;
45
- emit({ type: "error", message });
71
+ emit(providerError(message));
46
72
  throw new Error(message);
47
73
  }
48
- if (finalText) emit({ type: "final", text: finalText });
49
- emit({ type: "exit", code: result.exitCode ?? 0 });
74
+ if (resolvedFinalText && resolvedFinalText !== finalText) {
75
+ emit(providerEvent("assistant-final", { text: resolvedFinalText }));
76
+ }
77
+ emit(providerSessionEnd({ exitCode: result.exitCode ?? 0 }));
50
78
  settled = true;
51
79
  return {
52
80
  provider,
53
81
  exitCode: result.exitCode ?? 0,
54
82
  stdout: result.stdout || "",
55
83
  stderr: result.stderr || "",
56
- finalText: finalText || result.stdout || "",
84
+ finalText: resolvedFinalText || result.stdout || "",
57
85
  cancelled,
58
86
  };
59
87
  })();
@@ -72,6 +100,12 @@ export function createHostedSessionRunner({ provider, child, onEvent, parsePaylo
72
100
  };
73
101
  }
74
102
 
103
+ function waitForReaderClose(reader) {
104
+ return new Promise((resolve) => {
105
+ reader.once("close", resolve);
106
+ });
107
+ }
108
+
75
109
  export function tryParseJson(line) {
76
110
  const normalized = String(line || "").trim();
77
111
  if (!normalized) return null;
@@ -84,21 +118,15 @@ export function tryParseJson(line) {
84
118
 
85
119
  export function buildToolEvent(name, detail = null) {
86
120
  if (!name) return null;
87
- return {
88
- type: "tool",
89
- name: String(name),
90
- ...(detail ? { detail: String(detail) } : {}),
91
- };
121
+ return providerToolStart(name, detail ? { detail: String(detail) } : {});
92
122
  }
93
123
 
94
124
  export function buildStatusEvent(message) {
95
- if (!message) return null;
96
- return { type: "status", message: String(message) };
125
+ return providerStatus(message);
97
126
  }
98
127
 
99
128
  export function buildErrorEvent(message) {
100
- if (!message) return null;
101
- return { type: "error", message: String(message) };
129
+ return providerError(message);
102
130
  }
103
131
 
104
132
  export function extractTextFragments(payload, fragments = [], depth = 0) {
@@ -1,3 +1,5 @@
1
+ import fs from "fs";
2
+ import path from "path";
1
3
  import { startProviderSession, resolvePreferredProvider } from "./providers/index.mjs";
2
4
  import { buildAssistantPrompt } from "./prompt-builder.mjs";
3
5
  import { createAssistantCommandObserver } from "./command-observer.mjs";
@@ -14,6 +16,7 @@ export async function runAssistantConversationTurn({
14
16
  commandLog,
15
17
  onStatus,
16
18
  onToolEvent,
19
+ onProviderEvent,
17
20
  onResolvedProvider,
18
21
  onPrompt,
19
22
  } = {}) {
@@ -36,6 +39,12 @@ export async function runAssistantConversationTurn({
36
39
  const runtimeSettings = settings || { provider };
37
40
  const resolvedProvider = resolvePreferredProvider(runtimeSettings.provider || provider, env);
38
41
  const providerEnv = commandLog?.providerEnv?.(env) || env;
42
+ const tracePath = shouldTraceProviderEvents(env, providerEnv)
43
+ ? path.join(commandLog?.contextDir || path.join(productDir, ".testkit", "assistant"), "provider-events.jsonl")
44
+ : null;
45
+ const rawTracePath = shouldTraceProviderEvents(env, providerEnv)
46
+ ? path.join(commandLog?.contextDir || path.join(productDir, ".testkit", "assistant"), "provider-raw.jsonl")
47
+ : null;
39
48
  onResolvedProvider?.(resolvedProvider);
40
49
  onPrompt?.({
41
50
  prompt,
@@ -44,6 +53,12 @@ export async function runAssistantConversationTurn({
44
53
  effort: runtimeSettings.effort || null,
45
54
  });
46
55
  onStatus?.(`Thinking with ${resolvedProvider}...`);
56
+ onProviderEvent?.({
57
+ type: "status",
58
+ provider: resolvedProvider,
59
+ text: `Thinking with ${resolvedProvider}...`,
60
+ transient: true,
61
+ });
47
62
 
48
63
  observer.start();
49
64
  try {
@@ -56,25 +71,53 @@ export async function runAssistantConversationTurn({
56
71
  prompt,
57
72
  purpose: "assistant",
58
73
  env: providerEnv,
74
+ onRawLine(line) {
75
+ appendProviderTrace(rawTracePath, line);
76
+ },
59
77
  onEvent(event) {
60
- if (event.type === "status" || event.type === "tool") onStatus?.(formatProviderEvent(event));
78
+ appendProviderTrace(tracePath, event);
79
+ onProviderEvent?.(event);
80
+ if (event.type === "status" || event.type === "tool-start" || event.type === "tool-update" || event.type === "tool-end") {
81
+ onStatus?.(formatProviderEvent(event));
82
+ }
61
83
  },
62
84
  });
63
- const result = await session.completion;
85
+ await session.completion;
64
86
  observer.scan();
65
87
  commandLog?.refresh?.();
66
- return [{
67
- role: "assistant",
68
- text: result.finalText || "",
69
- }];
88
+ return { provider: resolvedProvider };
70
89
  } finally {
71
90
  observer.stop();
72
91
  }
73
92
  }
74
93
 
75
94
  function formatProviderEvent(event) {
76
- if (event.type === "tool") {
95
+ if (event.type === "tool-start") {
77
96
  return `${event.provider}: ${event.name}${event.detail ? ` (${event.detail})` : ""}`;
78
97
  }
79
- return `${event.provider}: ${event.message}`;
98
+ if (event.type === "tool-update") {
99
+ return `${event.provider}: ${event.name || "tool"}${event.text ? ` (${event.text})` : ""}`;
100
+ }
101
+ if (event.type === "tool-end") {
102
+ return `${event.provider}: ${event.name || "tool"} ${event.status || "done"}`;
103
+ }
104
+ return `${event.provider}: ${event.text || "working"}`;
105
+ }
106
+
107
+ function shouldTraceProviderEvents(...envs) {
108
+ return envs.some((entry) => String(entry?.TESTKIT_PROVIDER_TRACE || "").trim() === "1");
109
+ }
110
+
111
+ function appendProviderTrace(tracePath, event) {
112
+ if (!tracePath || !event) return;
113
+ try {
114
+ fs.mkdirSync(path.dirname(tracePath), { recursive: true });
115
+ fs.appendFileSync(
116
+ tracePath,
117
+ `${JSON.stringify({ timestamp: new Date().toISOString(), ...event })}\n`,
118
+ "utf8"
119
+ );
120
+ } catch {
121
+ // Provider tracing must never interfere with the assistant turn.
122
+ }
80
123
  }
@@ -1,7 +1,8 @@
1
1
  import fs from "fs";
2
2
  import path from "path";
3
+ import { HOSTED_ASSISTANT_PROVIDERS } from "./providers/index.mjs";
3
4
 
4
- export const ASSISTANT_PROVIDERS = ["auto", "codex", "claude"];
5
+ export const ASSISTANT_PROVIDERS = ["auto", ...HOSTED_ASSISTANT_PROVIDERS];
5
6
  export const ASSISTANT_EFFORTS = ["low", "medium", "high", "xhigh", "max"];
6
7
 
7
8
  export const DEFAULT_ASSISTANT_SETTINGS = Object.freeze({
@@ -99,11 +99,25 @@ export function createAssistantState({
99
99
  }
100
100
 
101
101
  function appendMessage(message) {
102
- messages.push({
102
+ const entry = {
103
103
  id: `msg-${messages.length + 1}`,
104
104
  ...message,
105
- });
105
+ };
106
+ messages.push(entry);
107
+ notify();
108
+ return entry.id;
109
+ }
110
+
111
+ function updateMessage(id, updater) {
112
+ const index = messages.findIndex((message) => message.id === id);
113
+ if (index < 0) return false;
114
+ const next = updater(messages[index]);
115
+ messages[index] = {
116
+ ...messages[index],
117
+ ...next,
118
+ };
106
119
  notify();
120
+ return true;
107
121
  }
108
122
 
109
123
  function setBusy(nextBusy, status = null) {
@@ -354,10 +368,11 @@ export function createAssistantState({
354
368
 
355
369
  try {
356
370
  setBusy(true, `Thinking with ${settings.provider === "auto" ? "provider" : settings.provider}...`);
357
- const emitted = await runAssistantConversationTurn({
371
+ const providerTurn = createProviderTurnState();
372
+ await runAssistantConversationTurn({
358
373
  productDir,
359
374
  runState,
360
- transcript: messages.map((entry) => ({ role: entry.role, text: entry.text })),
375
+ transcript: buildConversationTranscript(messages),
361
376
  userMessage: trimmed,
362
377
  settings,
363
378
  env,
@@ -379,11 +394,20 @@ export function createAssistantState({
379
394
  });
380
395
  notify();
381
396
  },
397
+ onProviderEvent(event) {
398
+ handleProviderEvent(providerTurn, event, {
399
+ appendMessage,
400
+ updateMessage,
401
+ setStatus(status) {
402
+ activeStatus = status;
403
+ notify();
404
+ },
405
+ });
406
+ },
382
407
  onToolEvent(event) {
383
408
  handleAssistantToolEvent(state, event, appendMessage);
384
409
  },
385
410
  });
386
- for (const message of emitted) appendMessage(message);
387
411
  } catch (error) {
388
412
  appendMessage({
389
413
  role: "system",
@@ -608,6 +632,139 @@ function handleAssistantToolEvent(state, event, appendMessage) {
608
632
  }
609
633
  }
610
634
 
635
+ function createProviderTurnState() {
636
+ return {
637
+ assistantMessageId: null,
638
+ lastActivityText: null,
639
+ };
640
+ }
641
+
642
+ function handleProviderEvent(turn, event, { appendMessage, updateMessage, setStatus } = {}) {
643
+ if (!event) return;
644
+ if (event.transient || event.display === false) {
645
+ if (event.type === "status" && event.text) setStatus?.(event.text);
646
+ return;
647
+ }
648
+ if (event.type === "assistant-delta") {
649
+ appendAssistantDelta(turn, event, { appendMessage, updateMessage });
650
+ setStatus?.(`${event.provider || "provider"} responding`);
651
+ return;
652
+ }
653
+ if (event.type === "assistant-final") {
654
+ finalizeAssistantMessage(turn, event, { appendMessage, updateMessage });
655
+ setStatus?.(`${event.provider || "provider"} complete`);
656
+ return;
657
+ }
658
+ if (event.type === "session-start") {
659
+ setStatus?.(`${event.provider || "provider"} session started`);
660
+ return;
661
+ }
662
+ if (event.type === "session-end") {
663
+ setStatus?.(`${event.provider || "provider"} session ended`);
664
+ return;
665
+ }
666
+ if (event.type === "status") {
667
+ appendProviderActivity(turn, {
668
+ role: "provider-activity",
669
+ title: formatProviderName(event.provider),
670
+ text: event.text || "Working",
671
+ data: event,
672
+ }, { appendMessage, setStatus });
673
+ return;
674
+ }
675
+ if (event.type === "tool-start" || event.type === "tool-update" || event.type === "tool-end") {
676
+ appendProviderActivity(turn, {
677
+ role: "provider-tool",
678
+ title: formatProviderToolTitle(event),
679
+ text: formatProviderToolText(event),
680
+ data: event,
681
+ }, { appendMessage, setStatus });
682
+ return;
683
+ }
684
+ if (event.type === "error") {
685
+ appendMessage({
686
+ role: "provider-error",
687
+ title: formatProviderName(event.provider),
688
+ text: event.text || "Provider error",
689
+ data: event,
690
+ });
691
+ setStatus?.(`${event.provider || "provider"} error`);
692
+ }
693
+ }
694
+
695
+ function appendAssistantDelta(turn, event, { appendMessage, updateMessage }) {
696
+ if (!turn.assistantMessageId) {
697
+ turn.assistantMessageId = appendMessage({
698
+ role: "assistant",
699
+ status: "streaming",
700
+ provider: event.provider || null,
701
+ text: "",
702
+ });
703
+ }
704
+ updateMessage(turn.assistantMessageId, (message) => ({
705
+ text: `${message.text || ""}${event.text || ""}`,
706
+ status: "streaming",
707
+ }));
708
+ }
709
+
710
+ function finalizeAssistantMessage(turn, event, { appendMessage, updateMessage }) {
711
+ const finalText = event.text || "";
712
+ if (!turn.assistantMessageId) {
713
+ turn.assistantMessageId = appendMessage({
714
+ role: "assistant",
715
+ provider: event.provider || null,
716
+ text: finalText,
717
+ });
718
+ return;
719
+ }
720
+ updateMessage(turn.assistantMessageId, (message) => ({
721
+ text: finalText || message.text || "",
722
+ status: null,
723
+ }));
724
+ }
725
+
726
+ function appendProviderActivity(turn, message, { appendMessage, setStatus }) {
727
+ const text = String(message.text || "").trim();
728
+ if (!text) return;
729
+ const signature = `${message.role}:${message.title || ""}:${text}`;
730
+ if (turn.lastActivityText === signature) return;
731
+ turn.lastActivityText = signature;
732
+ appendMessage(message);
733
+ setStatus?.(message.title ? `${message.title}: ${text}` : text);
734
+ }
735
+
736
+ function formatProviderName(provider) {
737
+ return provider ? String(provider) : "provider";
738
+ }
739
+
740
+ function formatProviderToolTitle(event) {
741
+ const provider = formatProviderName(event.provider);
742
+ const name = event.name || "tool";
743
+ if (event.type === "tool-end") return `${provider} finished ${name}`;
744
+ if (event.type === "tool-update") return `${provider} updated ${name}`;
745
+ return `${provider} started ${name}`;
746
+ }
747
+
748
+ function formatProviderToolText(event) {
749
+ const lines = [];
750
+ if (event.detail) lines.push(String(event.detail));
751
+ if (event.text) lines.push(String(event.text));
752
+ if (event.input) lines.push(formatProviderData("input", event.input));
753
+ if (event.output) lines.push(formatProviderData("output", event.output));
754
+ if (event.status) lines.push(`status: ${event.status}`);
755
+ return lines.filter(Boolean).join("\n") || event.name || "Provider tool activity";
756
+ }
757
+
758
+ function formatProviderData(label, value) {
759
+ if (value == null) return null;
760
+ if (typeof value === "string") return `${label}: ${value}`;
761
+ try {
762
+ return `${label}: ${JSON.stringify(value)}`;
763
+ } catch {
764
+ return `${label}: ${String(value)}`;
765
+ }
766
+ }
767
+
611
768
  function formatSettings(snapshot) {
612
769
  const rows = [
613
770
  ["Provider", snapshot.provider || "auto"],
@@ -654,6 +811,12 @@ function serializeRunSession(session) {
654
811
  };
655
812
  }
656
813
 
814
+ function buildConversationTranscript(messages) {
815
+ return (messages || [])
816
+ .filter((entry) => !["provider-activity", "provider-tool", "provider-error"].includes(entry.role))
817
+ .map((entry) => ({ role: entry.role, text: entry.text }));
818
+ }
819
+
657
820
  function formatObservedCommandTitle(command) {
658
821
  const kind = command?.kind || "command";
659
822
  return `testkit ${kind}`;
@@ -0,0 +1,35 @@
1
+ import stripAnsi from "strip-ansi";
2
+ import { buildAssistantViewModel } from "./view-model.mjs";
3
+ import { renderMarkdownToAnsi } from "./markdown-block.mjs";
4
+
5
+ export function renderAssistantSnapshotText(snapshot, { cwd = process.cwd(), ansi = false } = {}) {
6
+ const view = buildAssistantViewModel(snapshot || {}, { cwd });
7
+ const lines = [view.title, view.welcome.rows.find(([label]) => label === "Provider")?.[1] || ""]
8
+ .filter(Boolean);
9
+ for (const block of view.blocks || []) {
10
+ lines.push("");
11
+ lines.push(...renderBlockLines(block, { ansi }));
12
+ }
13
+ lines.push("");
14
+ lines.push(view.statusLine);
15
+ const text = `${lines.join("\n").trimEnd()}\n`;
16
+ return ansi ? text : stripAnsi(text);
17
+ }
18
+
19
+ function renderBlockLines(block, { ansi = false } = {}) {
20
+ const marker = block.marker || "";
21
+ const title = block.title ? ` ${block.title}` : "";
22
+ const text = String(block.text || "").trimEnd();
23
+ if (block.format === "markdown") {
24
+ const rendered = text ? renderMarkdownToAnsi(text) : "";
25
+ const normalized = ansi ? rendered : stripAnsi(rendered);
26
+ return prefixLines(`${marker}${title}`, normalized);
27
+ }
28
+ return prefixLines(`${marker}${title}`, text);
29
+ }
30
+
31
+ function prefixLines(prefix, text) {
32
+ const lines = String(text || "").split(/\r?\n/);
33
+ if (lines.length === 0 || (lines.length === 1 && lines[0] === "")) return [prefix];
34
+ return lines.map((line, index) => (index === 0 ? `${prefix}${prefix && line ? " " : ""}${line}` : ` ${line}`));
35
+ }
@@ -65,6 +65,17 @@ export function buildTranscriptBlocks(messages) {
65
65
  exitCode: message.data?.exitCode ?? null,
66
66
  };
67
67
  }
68
+ if (role === "provider-activity" || role === "provider-tool" || role === "provider-error") {
69
+ return {
70
+ id: message.id,
71
+ kind: role,
72
+ format: "plain",
73
+ marker: role === "provider-error" ? "!" : "◌",
74
+ title: message.title || null,
75
+ text: message.text || "",
76
+ status: message.status || null,
77
+ };
78
+ }
68
79
  if (role === "user") {
69
80
  return {
70
81
  id: message.id,
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@elench/next-analysis",
3
- "version": "0.1.101",
3
+ "version": "0.1.103",
4
4
  "description": "SWC-backed Next.js source analysis primitives for Erench tools",
5
5
  "type": "module",
6
6
  "exports": {
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@elench/testkit-bridge",
3
- "version": "0.1.101",
3
+ "version": "0.1.103",
4
4
  "description": "Browser bridge helpers for testkit",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
@@ -22,7 +22,7 @@
22
22
  "typecheck": "tsc -p tsconfig.json --noEmit"
23
23
  },
24
24
  "dependencies": {
25
- "@elench/testkit-protocol": "0.1.101"
25
+ "@elench/testkit-protocol": "0.1.103"
26
26
  },
27
27
  "private": false
28
28
  }
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@elench/testkit-protocol",
3
- "version": "0.1.101",
3
+ "version": "0.1.103",
4
4
  "description": "Shared browser protocol for testkit bridge and extension consumers",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@elench/ts-analysis",
3
- "version": "0.1.101",
3
+ "version": "0.1.103",
4
4
  "description": "TypeScript compiler-backed source analysis primitives for Erench tools",
5
5
  "type": "module",
6
6
  "exports": {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@elench/testkit",
3
- "version": "0.1.101",
3
+ "version": "0.1.103",
4
4
  "description": "Assistant-first CLI for running, inspecting, and debugging local testkit suites",
5
5
  "type": "module",
6
6
  "workspaces": [
@@ -58,11 +58,12 @@
58
58
  "scripts": {
59
59
  "build:packages": "npm --workspace packages/testkit-protocol run build && npm --workspace packages/ts-analysis run build && npm --workspace packages/next-analysis run build && npm --workspace packages/testkit-bridge run build",
60
60
  "typecheck:packages": "npm --workspace packages/testkit-protocol run typecheck && npm --workspace packages/ts-analysis run typecheck && npm --workspace packages/next-analysis run typecheck && npm --workspace packages/testkit-bridge run typecheck && npm --workspace packages/testkit-extension run compile",
61
- "test": "npm run build:packages && vitest run && node scripts/live-sandbox/harness.mjs",
61
+ "test": "npm run build:packages && vitest run && npm run test:live",
62
62
  "test:audit": "node scripts/test-boundary-audit.mjs",
63
+ "test:live": "node scripts/live-sandbox/harness.mjs",
63
64
  "test:unit": "npm run build:packages && npm run test:audit && vitest run --config vitest.unit.config.mjs",
64
65
  "test:integration": "npm run build:packages && vitest run test/integration",
65
- "test:system": "npm run build:packages && vitest run test/system --passWithNoTests"
66
+ "test:system": "npm run build:packages && vitest run test/system"
66
67
  },
67
68
  "files": [
68
69
  "bin/",
@@ -89,10 +90,10 @@
89
90
  },
90
91
  "dependencies": {
91
92
  "@babel/code-frame": "^7.29.0",
92
- "@elench/next-analysis": "0.1.101",
93
- "@elench/testkit-bridge": "0.1.101",
94
- "@elench/testkit-protocol": "0.1.101",
95
- "@elench/ts-analysis": "0.1.101",
93
+ "@elench/next-analysis": "0.1.103",
94
+ "@elench/testkit-bridge": "0.1.103",
95
+ "@elench/testkit-protocol": "0.1.103",
96
+ "@elench/ts-analysis": "0.1.103",
96
97
  "@oclif/core": "^4.10.6",
97
98
  "esbuild": "^0.25.11",
98
99
  "execa": "^9.5.0",
@@ -109,6 +110,11 @@
109
110
  "wrap-ansi": "^10.0.0"
110
111
  },
111
112
  "engines": {
112
- "node": ">=18"
113
+ "node": ">=24.14.1",
114
+ "npm": ">=11.11.0"
115
+ },
116
+ "volta": {
117
+ "node": "24.14.1",
118
+ "npm": "11.11.0"
113
119
  }
114
120
  }