@elench/testkit 0.1.111 → 0.1.113
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/lib/bundler/index.mjs +95 -1
- package/lib/cli/args.mjs +1 -1
- package/lib/cli/assistant/app.mjs +70 -20
- package/lib/cli/assistant/command-normalize.mjs +22 -0
- package/lib/cli/assistant/command-observer.mjs +49 -4
- package/lib/cli/assistant/command-results.mjs +10 -1
- package/lib/cli/assistant/context-pack.mjs +45 -15
- package/lib/cli/assistant/domain.d.mts +59 -0
- package/lib/cli/assistant/domain.d.mts.map +1 -0
- package/lib/cli/assistant/domain.mjs +2 -0
- package/lib/cli/assistant/domain.mjs.map +1 -0
- package/lib/cli/assistant/session.mjs +3 -1
- package/lib/cli/assistant/state.mjs +109 -2
- package/lib/cli/assistant/view-model.mjs +69 -9
- package/lib/cli/commands/run.mjs +1 -1
- package/lib/cli/components/blocks/run-tree.mjs +30 -64
- package/lib/cli/entrypoint.mjs +1 -1
- package/lib/cli/renderers/run/inline-detail.mjs +64 -0
- package/lib/cli/state/run/model.mjs +24 -95
- package/lib/cli/state/run/state.mjs +0 -22
- package/lib/config/discovery.mjs +0 -10
- package/lib/discovery/index.mjs +1 -1
- package/lib/domain/test-types.mjs +5 -14
- package/lib/runner/default-runtime-runner.mjs +3 -1
- package/lib/runner/failure-details.mjs +22 -0
- package/lib/runner/maintenance.mjs +1 -1
- package/lib/runner/provenance.mjs +4 -1
- package/lib/runner/results.mjs +31 -0
- package/lib/runner/status-model.mjs +15 -7
- package/lib/runner/suite-selection.mjs +2 -3
- package/node_modules/@elench/next-analysis/package.json +1 -1
- package/node_modules/@elench/testkit-bridge/package.json +2 -2
- package/node_modules/@elench/testkit-protocol/package.json +1 -1
- package/node_modules/@elench/ts-analysis/package.json +1 -1
- package/package.json +5 -5
- package/lib/cli/components/primitives/filter-bar.mjs +0 -12
- package/lib/cli/state/tree/fuzzy-match.mjs +0 -106
- package/packages/testkit-bridge/node_modules/@elench/testkit-protocol/dist/index.d.ts +0 -188
- package/packages/testkit-bridge/node_modules/@elench/testkit-protocol/dist/index.d.ts.map +0 -1
- package/packages/testkit-bridge/node_modules/@elench/testkit-protocol/dist/index.js +0 -293
- package/packages/testkit-bridge/node_modules/@elench/testkit-protocol/dist/index.js.map +0 -1
- package/packages/testkit-bridge/node_modules/@elench/testkit-protocol/package.json +0 -25
package/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
`@elench/testkit` discovers `*.testkit.ts` files, infers suite ownership from the
|
|
4
4
|
filesystem, starts local services, provisions Docker-managed local Postgres
|
|
5
|
-
databases, and runs
|
|
5
|
+
databases, and runs test suites.
|
|
6
6
|
|
|
7
7
|
The package is now driven by `testkit.config.ts`, not `testkit.config.json`.
|
|
8
8
|
|
package/lib/bundler/index.mjs
CHANGED
|
@@ -11,6 +11,7 @@ const ROOT_ENTRY = path.join(PACKAGE_ROOT, "lib", "index.mjs");
|
|
|
11
11
|
const CONFIG_ENTRY = path.join(PACKAGE_ROOT, "lib", "config-api", "index.mjs");
|
|
12
12
|
const RUNTIME_ENTRY = path.join(PACKAGE_ROOT, "lib", "runtime", "index.mjs");
|
|
13
13
|
const MANIFEST_FILE = "manifest.json";
|
|
14
|
+
const BUNDLE_WRAPPER_VERSION = "summary-check-artifacts-v2";
|
|
14
15
|
const bundleCache = new Map();
|
|
15
16
|
|
|
16
17
|
export async function bundleK6File({
|
|
@@ -105,7 +106,9 @@ async function buildCacheMetadata(sourceFile, configFile = null) {
|
|
|
105
106
|
.update("\0")
|
|
106
107
|
.update(source)
|
|
107
108
|
.update("\0")
|
|
108
|
-
.update(packageJsonText)
|
|
109
|
+
.update(packageJsonText)
|
|
110
|
+
.update("\0")
|
|
111
|
+
.update(BUNDLE_WRAPPER_VERSION);
|
|
109
112
|
let configHash = null;
|
|
110
113
|
|
|
111
114
|
if (configFile && fs.existsSync(configFile)) {
|
|
@@ -119,6 +122,7 @@ async function buildCacheMetadata(sourceFile, configFile = null) {
|
|
|
119
122
|
cacheKey: hash.digest("hex"),
|
|
120
123
|
sourceHash,
|
|
121
124
|
configHash,
|
|
125
|
+
wrapperVersion: BUNDLE_WRAPPER_VERSION,
|
|
122
126
|
testkitVersion: packageJson.version || null,
|
|
123
127
|
};
|
|
124
128
|
}
|
|
@@ -169,6 +173,96 @@ export function setup(...args) {
|
|
|
169
173
|
export default function exec(...args) {
|
|
170
174
|
return suite.exec(...args);
|
|
171
175
|
}
|
|
176
|
+
export function handleSummary(data) {
|
|
177
|
+
const checksData = extractChecksFromSummary(data);
|
|
178
|
+
emitTestkitArtifact("checks", checksData, {
|
|
179
|
+
kind: "testkit.checks",
|
|
180
|
+
summary: checksData.summary
|
|
181
|
+
? \`\${checksData.summary.passed}/\${checksData.summary.total} checks passed\`
|
|
182
|
+
: "no checks",
|
|
183
|
+
});
|
|
184
|
+
return {};
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
function extractChecksFromSummary(data) {
|
|
188
|
+
const checks = [];
|
|
189
|
+
collectChecksRecursive(data && data.root_group, [], checks);
|
|
190
|
+
const total = checks.length;
|
|
191
|
+
const passed = checks.filter((c) => c.passes > 0 && c.fails === 0).length;
|
|
192
|
+
|
|
193
|
+
const thresholds = {};
|
|
194
|
+
for (const [name, metric] of Object.entries((data && data.metrics) || {})) {
|
|
195
|
+
if (metric && metric.thresholds) {
|
|
196
|
+
thresholds[name] = Object.entries(metric.thresholds).map(([expr, result]) => ({
|
|
197
|
+
expression: expr,
|
|
198
|
+
ok: Boolean(result && result.ok),
|
|
199
|
+
}));
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
const http = {};
|
|
204
|
+
for (const key of ["http_reqs", "http_req_duration", "http_req_failed"]) {
|
|
205
|
+
if (data && data.metrics && data.metrics[key]) {
|
|
206
|
+
http[key] = data.metrics[key].values || {};
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
return { checks, summary: { total, passed, failed: total - passed }, thresholds, http };
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function collectChecksRecursive(group, path, out) {
|
|
214
|
+
if (!group) return;
|
|
215
|
+
const currentPath = normalizeNonEmptyString(group.name) ? [...path, normalizeNonEmptyString(group.name)] : path;
|
|
216
|
+
for (const check of Object.values(group.checks || {})) {
|
|
217
|
+
if (!check) continue;
|
|
218
|
+
out.push({
|
|
219
|
+
name: normalizeNonEmptyString(check.name) || "unnamed",
|
|
220
|
+
path: currentPath,
|
|
221
|
+
passes: normalizeCount(check.passes),
|
|
222
|
+
fails: normalizeCount(check.fails),
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
for (const subGroup of Object.values(group.groups || {})) {
|
|
226
|
+
collectChecksRecursive(subGroup, currentPath, out);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
const TESTKIT_ARTIFACT_MARKER = "TESTKIT_ARTIFACT:";
|
|
231
|
+
|
|
232
|
+
function emitTestkitArtifact(name, data, options = {}) {
|
|
233
|
+
const payload = encodeURIComponent(JSON.stringify({
|
|
234
|
+
name: normalizeArtifactName(name),
|
|
235
|
+
kind: normalizeOptionalString(options.kind),
|
|
236
|
+
summary: normalizeOptionalString(options.summary),
|
|
237
|
+
contentType: normalizeOptionalString(options.contentType) || "application/json",
|
|
238
|
+
data,
|
|
239
|
+
emittedAt: new Date().toISOString(),
|
|
240
|
+
}));
|
|
241
|
+
console.log(\`\${TESTKIT_ARTIFACT_MARKER}\${payload}\`);
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
function normalizeArtifactName(name) {
|
|
245
|
+
const normalized = normalizeNonEmptyString(name);
|
|
246
|
+
if (!normalized) {
|
|
247
|
+
throw new Error("emitArtifact(name, data) requires a non-empty artifact name");
|
|
248
|
+
}
|
|
249
|
+
return normalized;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
function normalizeOptionalString(value) {
|
|
253
|
+
if (value === undefined || value === null) return null;
|
|
254
|
+
const normalized = String(value).trim();
|
|
255
|
+
return normalized.length > 0 ? normalized : null;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function normalizeNonEmptyString(value) {
|
|
259
|
+
return normalizeOptionalString(value);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
function normalizeCount(value) {
|
|
263
|
+
const count = Number(value);
|
|
264
|
+
return Number.isFinite(count) && count > 0 ? count : 0;
|
|
265
|
+
}
|
|
172
266
|
|
|
173
267
|
function normalizeTestkitSuite(module) {
|
|
174
268
|
const candidate = module?.default;
|
package/lib/cli/args.mjs
CHANGED
|
@@ -6,7 +6,7 @@ import {
|
|
|
6
6
|
parseWorkersOption,
|
|
7
7
|
} from "../runner/execution-config.mjs";
|
|
8
8
|
|
|
9
|
-
export const POSITIONAL_TYPES = new Set(publicTestTypeList({ includeAll: true
|
|
9
|
+
export const POSITIONAL_TYPES = new Set(publicTestTypeList({ includeAll: true }));
|
|
10
10
|
export const LIFECYCLE = new Set(["status", "destroy", "cleanup"]);
|
|
11
11
|
export const RESERVED = new Set([...POSITIONAL_TYPES, ...LIFECYCLE]);
|
|
12
12
|
|
|
@@ -2,15 +2,16 @@ import React, { createElement, useEffect, useMemo, useRef, useState } from "reac
|
|
|
2
2
|
import { Box, Text, useApp, useBoxMetrics, useCursor, useInput, useStdout } from "ink";
|
|
3
3
|
import { bold, cyan, dim, green, red, yellow } from "../terminal/colors.mjs";
|
|
4
4
|
import { RunTreeView } from "../components/blocks/run-tree.mjs";
|
|
5
|
-
import {
|
|
5
|
+
import { colorCodeLine } from "./code-block.mjs";
|
|
6
6
|
import { getComposerDisplayModel } from "./composer.mjs";
|
|
7
7
|
import { MarkdownBlock } from "./markdown-block.mjs";
|
|
8
8
|
import { QualitySignalStrip } from "./quality-signal-strip.mjs";
|
|
9
9
|
import { buildAssistantViewModel } from "./view-model.mjs";
|
|
10
|
-
import { truncateText
|
|
10
|
+
import { truncateText } from "../terminal/layout.mjs";
|
|
11
11
|
|
|
12
12
|
const FALLBACK_COMMAND_BLOCK_WIDTH = 100;
|
|
13
13
|
const COMMAND_BLOCK_CHROME_WIDTH = 4;
|
|
14
|
+
const COMMAND_BLOCK_BODY_ROWS = 8;
|
|
14
15
|
|
|
15
16
|
export function AssistantApp({
|
|
16
17
|
assistantState,
|
|
@@ -58,6 +59,10 @@ export function AssistantApp({
|
|
|
58
59
|
const runSession = assistantState.getLiveRunSession?.() || assistantState.getLastRunSession?.() || null;
|
|
59
60
|
const serializedRunSession = snapshot.liveRunSession || snapshot.lastRunSession || null;
|
|
60
61
|
const runSessionProductDir = serializedRunSession?.productDir || runSession?.productDir || snapshot.productDir;
|
|
62
|
+
const visibleBlocks = useMemo(
|
|
63
|
+
() => runSession ? view.blocks.filter((block) => !isRunCommandBlock(block)) : view.blocks,
|
|
64
|
+
[runSession, view.blocks]
|
|
65
|
+
);
|
|
61
66
|
|
|
62
67
|
return createElement(
|
|
63
68
|
Box,
|
|
@@ -70,9 +75,9 @@ export function AssistantApp({
|
|
|
70
75
|
})
|
|
71
76
|
: null,
|
|
72
77
|
createElement(HeaderChrome, { view }),
|
|
73
|
-
|
|
78
|
+
visibleBlocks.length === 0
|
|
74
79
|
? createElement(WelcomePanel, { view })
|
|
75
|
-
: createElement(Transcript, { view }),
|
|
80
|
+
: createElement(Transcript, { view: { ...view, blocks: visibleBlocks } }),
|
|
76
81
|
runSession
|
|
77
82
|
? createElement(
|
|
78
83
|
Box,
|
|
@@ -105,6 +110,12 @@ export function AssistantApp({
|
|
|
105
110
|
);
|
|
106
111
|
}
|
|
107
112
|
|
|
113
|
+
function isRunCommandBlock(block) {
|
|
114
|
+
if (!block || block.kind !== "testkit-run") return false;
|
|
115
|
+
const command = String(block.command || "").trim();
|
|
116
|
+
return /^testkit\s+(run\s+)?(ui|e2e|scenario|int|dal|load|all|run)\b/.test(command);
|
|
117
|
+
}
|
|
118
|
+
|
|
108
119
|
function HeaderChrome({ view }) {
|
|
109
120
|
const provider = view.welcome.rows.find(([label]) => label === "Provider")?.[1] || "";
|
|
110
121
|
return createElement(
|
|
@@ -257,12 +268,16 @@ function renderCommandBlock(block, view = {}) {
|
|
|
257
268
|
: block.outputPreview?.omittedLineCount || block.omittedOutputLineCount || 0;
|
|
258
269
|
const blockWidth = Math.max(1, Number(view.terminalWidth) || FALLBACK_COMMAND_BLOCK_WIDTH);
|
|
259
270
|
const contentWidth = Math.max(1, blockWidth - COMMAND_BLOCK_CHROME_WIDTH);
|
|
260
|
-
const
|
|
271
|
+
const commandLine = command ? truncateText(`${dim("$")} ${command}`, contentWidth) : null;
|
|
261
272
|
const statusLine = status ? truncateText(status, contentWidth) : null;
|
|
262
|
-
const
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
273
|
+
const bodyRows = buildCommandBodyRows({
|
|
274
|
+
commandLine,
|
|
275
|
+
statusLine,
|
|
276
|
+
codeBlock,
|
|
277
|
+
outputLines,
|
|
278
|
+
omitted,
|
|
279
|
+
contentWidth,
|
|
280
|
+
});
|
|
266
281
|
|
|
267
282
|
return [
|
|
268
283
|
createElement(
|
|
@@ -277,21 +292,56 @@ function renderCommandBlock(block, view = {}) {
|
|
|
277
292
|
width: blockWidth,
|
|
278
293
|
},
|
|
279
294
|
createElement(Text, { key: "title" }, `${marker} ${title}`),
|
|
280
|
-
...
|
|
281
|
-
statusLine ? createElement(Text, { key: "status" }, colorCommandStatus(block, statusLine)) : null,
|
|
282
|
-
codeBlock ? createElement(Text, { key: "code-gap" }, "") : null,
|
|
283
|
-
...(codeBlock ? CodeBlock({ lines: codeBlock.lines, language: codeBlock.language, width: contentWidth }) : []),
|
|
284
|
-
...previewLines.map((line, index) => (
|
|
285
|
-
createElement(Text, { key: `output-${index}` }, dim(line))
|
|
286
|
-
)),
|
|
287
|
-
omittedLine ? createElement(Text, { key: "omitted" }, dim(omittedLine)) : null,
|
|
288
|
-
block.text && !command && outputLines.length === 0
|
|
289
|
-
? createElement(Text, { key: "text" }, colorBlockText(block, truncateText(block.text, contentWidth)))
|
|
290
|
-
: null
|
|
295
|
+
...bodyRows.map((row, index) => renderCommandBodyRow(block, row, index)),
|
|
291
296
|
),
|
|
292
297
|
];
|
|
293
298
|
}
|
|
294
299
|
|
|
300
|
+
function buildCommandBodyRows({ commandLine, statusLine, codeBlock, outputLines = [], omitted = 0, contentWidth }) {
|
|
301
|
+
const rows = [];
|
|
302
|
+
if (commandLine) rows.push({ tone: "command", text: commandLine });
|
|
303
|
+
if (statusLine) rows.push({ tone: "status", text: statusLine });
|
|
304
|
+
|
|
305
|
+
if (codeBlock) {
|
|
306
|
+
const available = Math.max(0, COMMAND_BLOCK_BODY_ROWS - rows.length - (omitted > 0 ? 1 : 0));
|
|
307
|
+
for (const line of (codeBlock.lines || []).slice(0, available)) {
|
|
308
|
+
rows.push({ tone: "code", language: codeBlock.language, text: truncateText(line, contentWidth) });
|
|
309
|
+
}
|
|
310
|
+
const omittedCount = Math.max(omitted, (codeBlock.lines || []).length - available);
|
|
311
|
+
if (omittedCount > 0 && rows.length < COMMAND_BLOCK_BODY_ROWS) {
|
|
312
|
+
rows.push({ tone: "muted", text: truncateText(`… ${omittedCount} more line${omittedCount === 1 ? "" : "s"} omitted`, contentWidth) });
|
|
313
|
+
}
|
|
314
|
+
} else {
|
|
315
|
+
const available = Math.max(0, COMMAND_BLOCK_BODY_ROWS - rows.length - (omitted > 0 ? 1 : 0));
|
|
316
|
+
for (const line of outputLines.slice(0, available)) {
|
|
317
|
+
rows.push({ tone: "muted", text: truncateText(line, contentWidth) });
|
|
318
|
+
}
|
|
319
|
+
const omittedCount = Math.max(omitted, outputLines.length - available);
|
|
320
|
+
if (omittedCount > 0 && rows.length < COMMAND_BLOCK_BODY_ROWS) {
|
|
321
|
+
rows.push({ tone: "muted", text: truncateText(`… ${omittedCount} more line${omittedCount === 1 ? "" : "s"} omitted`, contentWidth) });
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
while (rows.length < COMMAND_BLOCK_BODY_ROWS) rows.push({ tone: "blank", text: "" });
|
|
326
|
+
return rows.slice(0, COMMAND_BLOCK_BODY_ROWS);
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
function renderCommandBodyRow(block, row, index) {
|
|
330
|
+
if (row.tone === "status") {
|
|
331
|
+
return createElement(Text, { key: `body-${index}` }, colorCommandStatus(block, row.text));
|
|
332
|
+
}
|
|
333
|
+
if (row.tone === "code") {
|
|
334
|
+
return createElement(Text, { key: `body-${index}` }, colorCodeLine(row.text, row.language));
|
|
335
|
+
}
|
|
336
|
+
if (row.tone === "muted") {
|
|
337
|
+
return createElement(Text, { key: `body-${index}` }, dim(row.text));
|
|
338
|
+
}
|
|
339
|
+
if (row.tone === "blank") {
|
|
340
|
+
return createElement(Text, { key: `body-${index}` }, "");
|
|
341
|
+
}
|
|
342
|
+
return createElement(Text, { key: `body-${index}` }, row.text);
|
|
343
|
+
}
|
|
344
|
+
|
|
295
345
|
function formatCommandLine(block) {
|
|
296
346
|
if (!block.command) return null;
|
|
297
347
|
if (typeof block.command === "string") return block.command;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export function normalizeCommandLine(value) {
|
|
2
|
+
const text = unwrapShellCommand(value)
|
|
3
|
+
.replace(/\s+/g, " ")
|
|
4
|
+
.trim();
|
|
5
|
+
if (!text) return null;
|
|
6
|
+
return canonicalizeTestkitInvocation(text);
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export function unwrapShellCommand(value) {
|
|
10
|
+
const text = String(value || "").trim();
|
|
11
|
+
const match = text.match(/(?:^|\s)(?:[^\s'"]*\/)?(?:bash|sh|zsh)\s+-lc\s+(['"])([\s\S]*)\1\s*$/);
|
|
12
|
+
if (!match) return text;
|
|
13
|
+
return match[2].replace(/\\(["'\\$`])/g, "$1").trim();
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function canonicalizeTestkitInvocation(command) {
|
|
17
|
+
return command
|
|
18
|
+
.replace(/^npx\s+(?:--yes\s+|-y\s+)?testkit\b/, "testkit")
|
|
19
|
+
.replace(/^node(?:\s+--[^\s]+)*\s+(?:(?:\.?\.?\/)?[^\s]+\/)?(?:bin\/)?testkit(?:\.(?:mjs|js))?\b/, "testkit")
|
|
20
|
+
.replace(/^(?:(?:\.?\.?\/)?[^\s]+\/)?(?:bin\/)?testkit(?:\.(?:mjs|js))?\b/, "testkit")
|
|
21
|
+
.trim();
|
|
22
|
+
}
|
|
@@ -8,12 +8,14 @@ const OBSERVED_KINDS = new Set(["run", "discover", "status", "doctor", "typechec
|
|
|
8
8
|
export function createAssistantCommandObserver({
|
|
9
9
|
productDir,
|
|
10
10
|
commandLog,
|
|
11
|
+
turnId = null,
|
|
11
12
|
runState,
|
|
12
13
|
onEvent,
|
|
13
14
|
intervalMs = POLL_INTERVAL_MS,
|
|
14
15
|
} = {}) {
|
|
15
16
|
const seenResultFiles = new Set();
|
|
16
17
|
const observedRunCommandIds = new Set();
|
|
18
|
+
const hydratedArtifactKeys = new Set();
|
|
17
19
|
let timer = null;
|
|
18
20
|
let running = false;
|
|
19
21
|
let lastArtifactSignatures = new Map();
|
|
@@ -22,8 +24,9 @@ export function createAssistantCommandObserver({
|
|
|
22
24
|
function start() {
|
|
23
25
|
if (running) return;
|
|
24
26
|
running = true;
|
|
27
|
+
commandLogOffset = currentCommandLogSize();
|
|
28
|
+
markExistingResultFilesSeen();
|
|
25
29
|
lastArtifactSignatures = readArtifactSignatures();
|
|
26
|
-
scan();
|
|
27
30
|
timer = setInterval(scan, intervalMs);
|
|
28
31
|
}
|
|
29
32
|
|
|
@@ -66,7 +69,7 @@ export function createAssistantCommandObserver({
|
|
|
66
69
|
} catch {
|
|
67
70
|
continue;
|
|
68
71
|
}
|
|
69
|
-
if (event
|
|
72
|
+
if (!isCurrentObservation(event)) continue;
|
|
70
73
|
if (event.type === "command_start" && isRunCommand(event)) {
|
|
71
74
|
observedRunCommandIds.add(event.commandId);
|
|
72
75
|
}
|
|
@@ -89,9 +92,12 @@ export function createAssistantCommandObserver({
|
|
|
89
92
|
} catch {
|
|
90
93
|
continue;
|
|
91
94
|
}
|
|
92
|
-
if (document
|
|
93
|
-
|
|
95
|
+
if (!isCurrentObservation(document)) {
|
|
96
|
+
seenResultFiles.add(filePath);
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
94
99
|
seenResultFiles.add(filePath);
|
|
100
|
+
if (!OBSERVED_KINDS.has(document.kind)) continue;
|
|
95
101
|
if (document.kind === "run") {
|
|
96
102
|
observedRunCommandIds.add(document.commandId);
|
|
97
103
|
hydrateRunArtifact("command-result", document);
|
|
@@ -137,6 +143,9 @@ export function createAssistantCommandObserver({
|
|
|
137
143
|
function hydrateRunArtifact(source, command = null) {
|
|
138
144
|
const artifact = loadObservedRunArtifact(command);
|
|
139
145
|
if (!artifact) return;
|
|
146
|
+
const key = artifactKey(source, command, artifact);
|
|
147
|
+
if (hydratedArtifactKeys.has(key)) return;
|
|
148
|
+
hydratedArtifactKeys.add(key);
|
|
140
149
|
runState?.hydrateFromArtifact?.(artifact);
|
|
141
150
|
onEvent?.({
|
|
142
151
|
type: "observed-run-artifact",
|
|
@@ -157,6 +166,7 @@ export function createAssistantCommandObserver({
|
|
|
157
166
|
const assistant = artifact?.provenance?.assistant || {};
|
|
158
167
|
if (!assistant.sessionId || !assistant.commandId) return false;
|
|
159
168
|
if (commandLog?.sessionId && assistant.sessionId !== commandLog.sessionId) return false;
|
|
169
|
+
if (turnId && assistant.turnId !== turnId) return false;
|
|
160
170
|
return observedRunCommandIds.has(assistant.commandId);
|
|
161
171
|
}
|
|
162
172
|
|
|
@@ -174,6 +184,29 @@ export function createAssistantCommandObserver({
|
|
|
174
184
|
}
|
|
175
185
|
}
|
|
176
186
|
|
|
187
|
+
function isCurrentObservation(entry) {
|
|
188
|
+
if (entry?.sessionId && commandLog?.sessionId && entry.sessionId !== commandLog.sessionId) return false;
|
|
189
|
+
if (turnId && entry?.turnId !== turnId) return false;
|
|
190
|
+
return true;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function markExistingResultFilesSeen() {
|
|
194
|
+
const resultDir = commandLog?.resultDir;
|
|
195
|
+
if (!resultDir || !fs.existsSync(resultDir)) return;
|
|
196
|
+
for (const entry of fs.readdirSync(resultDir, { withFileTypes: true })) {
|
|
197
|
+
if (entry.isFile() && entry.name.endsWith(".json")) {
|
|
198
|
+
seenResultFiles.add(path.join(resultDir, entry.name));
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
function currentCommandLogSize() {
|
|
204
|
+
const commandLogPath = commandLog?.commandLogPath;
|
|
205
|
+
if (!commandLogPath) return 0;
|
|
206
|
+
const stat = safeStat(commandLogPath);
|
|
207
|
+
return stat?.size || 0;
|
|
208
|
+
}
|
|
209
|
+
|
|
177
210
|
function safeStat(filePath) {
|
|
178
211
|
try {
|
|
179
212
|
return fs.statSync(filePath);
|
|
@@ -182,6 +215,18 @@ export function createAssistantCommandObserver({
|
|
|
182
215
|
}
|
|
183
216
|
}
|
|
184
217
|
|
|
218
|
+
function artifactKey(source, command, artifact) {
|
|
219
|
+
const assistant = artifact?.provenance?.assistant || {};
|
|
220
|
+
const runId = artifact?.run?.id || command?.result?.runArtifact?.run?.id || command?.artifact?.run?.id || "unknown-run";
|
|
221
|
+
return [
|
|
222
|
+
source,
|
|
223
|
+
assistant.sessionId || commandLog?.sessionId || "",
|
|
224
|
+
assistant.turnId || turnId || "",
|
|
225
|
+
assistant.commandId || command?.commandId || "",
|
|
226
|
+
runId,
|
|
227
|
+
].join("|");
|
|
228
|
+
}
|
|
229
|
+
|
|
185
230
|
return {
|
|
186
231
|
start,
|
|
187
232
|
stop,
|
|
@@ -6,6 +6,7 @@ export const ASSISTANT_SESSION_ENV = "TESTKIT_ASSISTANT_SESSION_ID";
|
|
|
6
6
|
export const ASSISTANT_RESULT_DIR_ENV = "TESTKIT_ASSISTANT_RESULT_DIR";
|
|
7
7
|
export const ASSISTANT_COMMAND_LOG_ENV = "TESTKIT_ASSISTANT_COMMAND_LOG";
|
|
8
8
|
export const ASSISTANT_COMMAND_ID_ENV = "TESTKIT_ASSISTANT_COMMAND_ID";
|
|
9
|
+
export const ASSISTANT_TURN_ENV = "TESTKIT_ASSISTANT_TURN_ID";
|
|
9
10
|
export const ASSISTANT_WRAPPER_LOGGED_ENV = "TESTKIT_ASSISTANT_WRAPPER_LOGGED";
|
|
10
11
|
|
|
11
12
|
export function createAssistantCommandContext({
|
|
@@ -15,6 +16,7 @@ export function createAssistantCommandContext({
|
|
|
15
16
|
env = process.env,
|
|
16
17
|
} = {}) {
|
|
17
18
|
const sessionId = env[ASSISTANT_SESSION_ENV] || null;
|
|
19
|
+
const turnId = env[ASSISTANT_TURN_ENV] || null;
|
|
18
20
|
const resultDir = env[ASSISTANT_RESULT_DIR_ENV] || null;
|
|
19
21
|
const commandLogPath = env[ASSISTANT_COMMAND_LOG_ENV] || null;
|
|
20
22
|
if (!sessionId && !resultDir && !commandLogPath) return null;
|
|
@@ -23,6 +25,7 @@ export function createAssistantCommandContext({
|
|
|
23
25
|
const startedAt = new Date().toISOString();
|
|
24
26
|
return {
|
|
25
27
|
sessionId,
|
|
28
|
+
turnId,
|
|
26
29
|
resultDir,
|
|
27
30
|
commandLogPath,
|
|
28
31
|
commandId,
|
|
@@ -103,6 +106,7 @@ export function writeAssistantCommandResult(context, payload = {}) {
|
|
|
103
106
|
schemaVersion: 1,
|
|
104
107
|
source: "testkit-command-result",
|
|
105
108
|
sessionId: context.sessionId,
|
|
109
|
+
turnId: context.turnId,
|
|
106
110
|
commandId: context.commandId,
|
|
107
111
|
kind: context.kind,
|
|
108
112
|
argv: context.argv,
|
|
@@ -128,7 +132,12 @@ export function appendAssistantCommandLog(context, event) {
|
|
|
128
132
|
fs.mkdirSync(path.dirname(commandLogPath), { recursive: true });
|
|
129
133
|
fs.appendFileSync(
|
|
130
134
|
commandLogPath,
|
|
131
|
-
`${JSON.stringify({
|
|
135
|
+
`${JSON.stringify({
|
|
136
|
+
timestamp: new Date().toISOString(),
|
|
137
|
+
sessionId: context?.sessionId || null,
|
|
138
|
+
turnId: context?.turnId || null,
|
|
139
|
+
...event,
|
|
140
|
+
})}\n`,
|
|
132
141
|
"utf8"
|
|
133
142
|
);
|
|
134
143
|
} catch {
|
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
ASSISTANT_COMMAND_LOG_ENV,
|
|
9
9
|
ASSISTANT_RESULT_DIR_ENV,
|
|
10
10
|
ASSISTANT_SESSION_ENV,
|
|
11
|
+
ASSISTANT_TURN_ENV,
|
|
11
12
|
ASSISTANT_WRAPPER_LOGGED_ENV,
|
|
12
13
|
} from "./command-results.mjs";
|
|
13
14
|
|
|
@@ -38,9 +39,11 @@ export function prepareAssistantContextPack({
|
|
|
38
39
|
} = paths;
|
|
39
40
|
fs.mkdirSync(binDir, { recursive: true });
|
|
40
41
|
fs.mkdirSync(resultDir, { recursive: true });
|
|
41
|
-
|
|
42
|
+
let activeTurnId = null;
|
|
43
|
+
const currentDocument = {
|
|
42
44
|
schemaVersion: 1,
|
|
43
45
|
sessionId,
|
|
46
|
+
activeTurnId,
|
|
44
47
|
contextDir,
|
|
45
48
|
contextPath,
|
|
46
49
|
commandLogPath,
|
|
@@ -48,7 +51,8 @@ export function prepareAssistantContextPack({
|
|
|
48
51
|
providerEventsPath,
|
|
49
52
|
providerRawPath,
|
|
50
53
|
createdAt: new Date().toISOString(),
|
|
51
|
-
}
|
|
54
|
+
};
|
|
55
|
+
writeCurrent();
|
|
52
56
|
|
|
53
57
|
function refresh() {
|
|
54
58
|
const snapshot = runState?.getSnapshot?.() || {};
|
|
@@ -63,12 +67,12 @@ export function prepareAssistantContextPack({
|
|
|
63
67
|
artifactPath: path.join(productDir, ".testkit", "results", "latest.json"),
|
|
64
68
|
});
|
|
65
69
|
writeJson(selectionPath, buildContextSelection(snapshot));
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
70
|
+
writeText(commandsPath, buildCommandsMarkdown());
|
|
71
|
+
writeText(focusedDetailPath, `${detailContent.lines.join("\n")}\n`);
|
|
72
|
+
writeText(focusedLogsPath, `${logsContent.lines.join("\n")}\n`);
|
|
73
|
+
writeText(focusedArtifactsPath, `${artifactsContent.lines.join("\n")}\n`);
|
|
74
|
+
writeText(focusedSetupPath, `${setupContent.lines.join("\n")}\n`);
|
|
75
|
+
writeText(
|
|
72
76
|
contextPath,
|
|
73
77
|
buildContextMarkdown(productDir, snapshot, {
|
|
74
78
|
contextPath,
|
|
@@ -81,9 +85,8 @@ export function prepareAssistantContextPack({
|
|
|
81
85
|
focusedArtifactsPath,
|
|
82
86
|
focusedSetupPath,
|
|
83
87
|
}),
|
|
84
|
-
"utf8"
|
|
85
88
|
);
|
|
86
|
-
fs.writeFileSync(wrapperPath, buildWrapperScript({
|
|
89
|
+
if (!fs.existsSync(wrapperPath)) fs.writeFileSync(wrapperPath, buildWrapperScript({
|
|
87
90
|
cliPath: resolveCliPath(),
|
|
88
91
|
classifierUrl: resolveClassifierUrl(),
|
|
89
92
|
sessionId,
|
|
@@ -93,7 +96,7 @@ export function prepareAssistantContextPack({
|
|
|
93
96
|
encoding: "utf8",
|
|
94
97
|
mode: 0o755,
|
|
95
98
|
});
|
|
96
|
-
fs.chmodSync(wrapperPath, 0o755);
|
|
99
|
+
if (fs.existsSync(wrapperPath)) fs.chmodSync(wrapperPath, 0o755);
|
|
97
100
|
}
|
|
98
101
|
|
|
99
102
|
refresh();
|
|
@@ -117,11 +120,18 @@ export function prepareAssistantContextPack({
|
|
|
117
120
|
focusedSetupPath,
|
|
118
121
|
binDir,
|
|
119
122
|
wrapperPath,
|
|
120
|
-
|
|
123
|
+
setActiveTurnId(turnId = null) {
|
|
124
|
+
activeTurnId = turnId ? String(turnId) : null;
|
|
125
|
+
currentDocument.activeTurnId = activeTurnId;
|
|
126
|
+
currentDocument.updatedAt = new Date().toISOString();
|
|
127
|
+
writeCurrent();
|
|
128
|
+
},
|
|
129
|
+
providerEnv(baseEnv = process.env, { turnId = activeTurnId } = {}) {
|
|
121
130
|
return {
|
|
122
131
|
...baseEnv,
|
|
123
132
|
PATH: [binDir, baseEnv?.PATH, process.env.PATH].filter(Boolean).join(path.delimiter),
|
|
124
133
|
[ASSISTANT_SESSION_ENV]: sessionId,
|
|
134
|
+
[ASSISTANT_TURN_ENV]: turnId || "",
|
|
125
135
|
[ASSISTANT_RESULT_DIR_ENV]: resultDir,
|
|
126
136
|
[ASSISTANT_COMMAND_LOG_ENV]: commandLogPath,
|
|
127
137
|
};
|
|
@@ -132,7 +142,12 @@ export function prepareAssistantContextPack({
|
|
|
132
142
|
try {
|
|
133
143
|
fs.appendFileSync(
|
|
134
144
|
commandLogPath,
|
|
135
|
-
`${JSON.stringify({
|
|
145
|
+
`${JSON.stringify({
|
|
146
|
+
timestamp: new Date().toISOString(),
|
|
147
|
+
sessionId,
|
|
148
|
+
turnId: activeTurnId,
|
|
149
|
+
...event,
|
|
150
|
+
})}\n`,
|
|
136
151
|
"utf8"
|
|
137
152
|
);
|
|
138
153
|
} catch {
|
|
@@ -140,6 +155,10 @@ export function prepareAssistantContextPack({
|
|
|
140
155
|
}
|
|
141
156
|
},
|
|
142
157
|
};
|
|
158
|
+
|
|
159
|
+
function writeCurrent() {
|
|
160
|
+
writeJson(currentPath, currentDocument);
|
|
161
|
+
}
|
|
143
162
|
}
|
|
144
163
|
|
|
145
164
|
function resolveCliPath() {
|
|
@@ -160,10 +179,12 @@ import { classifyAssistantCommandKind } from ${JSON.stringify(classifierUrl)};
|
|
|
160
179
|
const commandId = process.env.${ASSISTANT_COMMAND_ID_ENV} || \`cmd-\${Date.now()}-\${Math.random().toString(36).slice(2, 10)}\`;
|
|
161
180
|
const commandLogPath = process.env.${ASSISTANT_COMMAND_LOG_ENV} || ${JSON.stringify(commandLogPath)};
|
|
162
181
|
const sessionId = process.env.${ASSISTANT_SESSION_ENV} || ${JSON.stringify(sessionId)};
|
|
182
|
+
const turnId = process.env.${ASSISTANT_TURN_ENV} || null;
|
|
163
183
|
const argv = process.argv.slice(2);
|
|
164
184
|
|
|
165
185
|
appendCommandLog({
|
|
166
186
|
type: "command_start",
|
|
187
|
+
turnId,
|
|
167
188
|
commandId,
|
|
168
189
|
command: "testkit",
|
|
169
190
|
kind: classifyAssistantCommandKind(argv),
|
|
@@ -177,6 +198,7 @@ const result = spawnSync(process.execPath, [${JSON.stringify(cliPath)}, ...proce
|
|
|
177
198
|
...process.env,
|
|
178
199
|
TESTKIT_NO_ASSISTANT_DEFAULT: "1",
|
|
179
200
|
${ASSISTANT_SESSION_ENV}: sessionId,
|
|
201
|
+
${ASSISTANT_TURN_ENV}: turnId || "",
|
|
180
202
|
${ASSISTANT_RESULT_DIR_ENV}: process.env.${ASSISTANT_RESULT_DIR_ENV} || ${JSON.stringify(resultDir)},
|
|
181
203
|
${ASSISTANT_COMMAND_LOG_ENV}: commandLogPath,
|
|
182
204
|
${ASSISTANT_COMMAND_ID_ENV}: commandId,
|
|
@@ -190,6 +212,7 @@ if (result.error) {
|
|
|
190
212
|
}
|
|
191
213
|
appendCommandLog({
|
|
192
214
|
type: "command_exit",
|
|
215
|
+
turnId,
|
|
193
216
|
commandId,
|
|
194
217
|
command: "testkit",
|
|
195
218
|
kind: classifyAssistantCommandKind(argv),
|
|
@@ -203,7 +226,7 @@ process.exit(result.status ?? 0);
|
|
|
203
226
|
function appendCommandLog(event) {
|
|
204
227
|
try {
|
|
205
228
|
fs.mkdirSync(path.dirname(commandLogPath), { recursive: true });
|
|
206
|
-
fs.appendFileSync(commandLogPath, \`\${JSON.stringify({ timestamp: new Date().toISOString(), sessionId, ...event })}\\n\`, "utf8");
|
|
229
|
+
fs.appendFileSync(commandLogPath, \`\${JSON.stringify({ timestamp: new Date().toISOString(), sessionId, turnId, ...event })}\\n\`, "utf8");
|
|
207
230
|
} catch {
|
|
208
231
|
// Command observation must not affect command execution.
|
|
209
232
|
}
|
|
@@ -284,5 +307,12 @@ function buildCommandsMarkdown() {
|
|
|
284
307
|
}
|
|
285
308
|
|
|
286
309
|
function writeJson(filePath, value) {
|
|
287
|
-
|
|
310
|
+
writeText(filePath, `${JSON.stringify(value, null, 2)}\n`);
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
function writeText(filePath, value) {
|
|
314
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
315
|
+
const tempPath = `${filePath}.${process.pid}.${Date.now()}.tmp`;
|
|
316
|
+
fs.writeFileSync(tempPath, String(value), "utf8");
|
|
317
|
+
fs.renameSync(tempPath, filePath);
|
|
288
318
|
}
|