@showrun/core 0.1.0 → 0.1.1-b
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__tests__/config.test.d.ts +2 -0
- package/dist/__tests__/config.test.d.ts.map +1 -0
- package/dist/__tests__/config.test.js +164 -0
- package/dist/__tests__/httpReplay.test.d.ts +2 -0
- package/dist/__tests__/httpReplay.test.d.ts.map +1 -0
- package/dist/__tests__/httpReplay.test.js +306 -0
- package/dist/__tests__/requestSnapshot.test.d.ts +2 -0
- package/dist/__tests__/requestSnapshot.test.d.ts.map +1 -0
- package/dist/__tests__/requestSnapshot.test.js +323 -0
- package/dist/browserLauncher.d.ts.map +1 -1
- package/dist/browserLauncher.js +7 -1
- package/dist/config.d.ts +96 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +268 -0
- package/dist/dsl/interpreter.d.ts +9 -0
- package/dist/dsl/interpreter.d.ts.map +1 -1
- package/dist/dsl/interpreter.js +24 -5
- package/dist/dsl/stepHandlers.d.ts +7 -0
- package/dist/dsl/stepHandlers.d.ts.map +1 -1
- package/dist/dsl/stepHandlers.js +141 -5
- package/dist/dsl/templating.d.ts.map +1 -1
- package/dist/dsl/templating.js +11 -0
- package/dist/dsl/types.d.ts +16 -4
- package/dist/dsl/types.d.ts.map +1 -1
- package/dist/dsl/validation.d.ts.map +1 -1
- package/dist/dsl/validation.js +29 -14
- package/dist/httpReplay.d.ts +43 -0
- package/dist/httpReplay.d.ts.map +1 -0
- package/dist/httpReplay.js +102 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/jsonPackValidator.d.ts.map +1 -1
- package/dist/jsonPackValidator.js +12 -3
- package/dist/loader.d.ts.map +1 -1
- package/dist/loader.js +4 -0
- package/dist/networkCapture.d.ts +10 -4
- package/dist/networkCapture.d.ts.map +1 -1
- package/dist/networkCapture.js +20 -12
- package/dist/requestSnapshot.d.ts +91 -0
- package/dist/requestSnapshot.d.ts.map +1 -0
- package/dist/requestSnapshot.js +200 -0
- package/dist/runner.d.ts.map +1 -1
- package/dist/runner.js +209 -13
- package/dist/storage/index.d.ts +3 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/dist/storage/index.js +2 -0
- package/dist/storage/keys.d.ts +12 -0
- package/dist/storage/keys.d.ts.map +1 -0
- package/dist/storage/keys.js +39 -0
- package/dist/storage/types.d.ts +112 -0
- package/dist/storage/types.d.ts.map +1 -0
- package/dist/storage/types.js +7 -0
- package/dist/types.d.ts +5 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Request Snapshot types and utilities.
|
|
3
|
+
*
|
|
4
|
+
* Snapshots record the HTTP request/response details of `network_replay` steps
|
|
5
|
+
* so they can be replayed at runtime via direct HTTP calls — no browser needed.
|
|
6
|
+
*/
|
|
7
|
+
import { readFileSync, writeFileSync, existsSync } from 'fs';
|
|
8
|
+
import { join } from 'path';
|
|
9
|
+
import { resolveTemplate } from './dsl/templating.js';
|
|
10
|
+
// ---------------------------------------------------------------------------
|
|
11
|
+
// Staleness
|
|
12
|
+
// ---------------------------------------------------------------------------
|
|
13
|
+
/**
|
|
14
|
+
* Check if a snapshot is stale based on its TTL.
|
|
15
|
+
* Returns false when `ttl` is null (indefinite).
|
|
16
|
+
*/
|
|
17
|
+
export function isSnapshotStale(snapshot) {
|
|
18
|
+
if (snapshot.ttl === null)
|
|
19
|
+
return false;
|
|
20
|
+
return Date.now() - snapshot.capturedAt > snapshot.ttl;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Validate an HTTP response against a snapshot's expected shape.
|
|
24
|
+
*/
|
|
25
|
+
export function validateResponse(snapshot, response) {
|
|
26
|
+
const v = snapshot.responseValidation;
|
|
27
|
+
if (response.status !== v.expectedStatus) {
|
|
28
|
+
return {
|
|
29
|
+
valid: false,
|
|
30
|
+
reason: `Expected status ${v.expectedStatus}, got ${response.status}`,
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
if (v.expectedContentType &&
|
|
34
|
+
response.contentType &&
|
|
35
|
+
!response.contentType.toLowerCase().startsWith(v.expectedContentType.toLowerCase())) {
|
|
36
|
+
return {
|
|
37
|
+
valid: false,
|
|
38
|
+
reason: `Expected content-type starting with "${v.expectedContentType}", got "${response.contentType}"`,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
if (v.expectedKeys.length > 0) {
|
|
42
|
+
try {
|
|
43
|
+
const parsed = JSON.parse(response.body);
|
|
44
|
+
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
|
|
45
|
+
const bodyKeys = Object.keys(parsed);
|
|
46
|
+
const missing = v.expectedKeys.filter((k) => !bodyKeys.includes(k));
|
|
47
|
+
if (missing.length > 0) {
|
|
48
|
+
return {
|
|
49
|
+
valid: false,
|
|
50
|
+
reason: `Missing expected keys: ${missing.join(', ')}`,
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
// If we expected JSON keys but can't parse, that's a validation failure
|
|
57
|
+
return {
|
|
58
|
+
valid: false,
|
|
59
|
+
reason: 'Response body is not valid JSON but expectedKeys were specified',
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return { valid: true };
|
|
64
|
+
}
|
|
65
|
+
// ---------------------------------------------------------------------------
|
|
66
|
+
// Override application
|
|
67
|
+
// ---------------------------------------------------------------------------
|
|
68
|
+
/**
|
|
69
|
+
* Resolve a template string using Nunjucks (same engine as the DSL interpreter).
|
|
70
|
+
* Supports {{inputs.x}}, {{vars.x}}, {{secret.x}} and filters like `| urlencode`.
|
|
71
|
+
*/
|
|
72
|
+
function resolveTemplateValue(template, ctx) {
|
|
73
|
+
return resolveTemplate(template, ctx);
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Apply overrides from the snapshot to produce the final request parameters.
|
|
77
|
+
* Template expressions ({{inputs.x}}, {{vars.y}}, {{secret.z}}) are resolved
|
|
78
|
+
* using Nunjucks — the same engine as the DSL interpreter, including filters.
|
|
79
|
+
*/
|
|
80
|
+
export function applyOverrides(snapshot, inputs, vars, secrets) {
|
|
81
|
+
const ov = snapshot.overrides;
|
|
82
|
+
let url = snapshot.request.url;
|
|
83
|
+
let body = snapshot.request.body;
|
|
84
|
+
const method = snapshot.request.method;
|
|
85
|
+
const headers = { ...snapshot.request.headers };
|
|
86
|
+
if (!ov) {
|
|
87
|
+
return { url, method, headers, body };
|
|
88
|
+
}
|
|
89
|
+
const ctx = { inputs, vars, secrets: secrets ?? {} };
|
|
90
|
+
// urlReplace
|
|
91
|
+
if (ov.urlReplace) {
|
|
92
|
+
for (const r of ov.urlReplace) {
|
|
93
|
+
const find = resolveTemplateValue(r.find, ctx);
|
|
94
|
+
const replace = resolveTemplateValue(r.replace, ctx);
|
|
95
|
+
try {
|
|
96
|
+
url = url.replace(new RegExp(find, 'g'), replace);
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
// If regex is invalid, try literal replace
|
|
100
|
+
url = url.split(find).join(replace);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
// setQuery
|
|
105
|
+
if (ov.setQuery) {
|
|
106
|
+
const u = new URL(url);
|
|
107
|
+
for (const [k, v] of Object.entries(ov.setQuery)) {
|
|
108
|
+
const resolved = resolveTemplateValue(v, ctx);
|
|
109
|
+
u.searchParams.set(k, resolved);
|
|
110
|
+
}
|
|
111
|
+
url = u.toString();
|
|
112
|
+
}
|
|
113
|
+
// setHeaders
|
|
114
|
+
if (ov.setHeaders) {
|
|
115
|
+
for (const [k, v] of Object.entries(ov.setHeaders)) {
|
|
116
|
+
headers[k] = resolveTemplateValue(v, ctx);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
// Direct URL override (applied after urlReplace, matching networkCapture.ts)
|
|
120
|
+
if (ov.url != null) {
|
|
121
|
+
url = resolveTemplateValue(ov.url, ctx);
|
|
122
|
+
}
|
|
123
|
+
// bodyReplace
|
|
124
|
+
if (body && ov.bodyReplace) {
|
|
125
|
+
for (const r of ov.bodyReplace) {
|
|
126
|
+
const find = resolveTemplateValue(r.find, ctx);
|
|
127
|
+
const replace = resolveTemplateValue(r.replace, ctx);
|
|
128
|
+
try {
|
|
129
|
+
body = body.replace(new RegExp(find, 'g'), replace);
|
|
130
|
+
}
|
|
131
|
+
catch {
|
|
132
|
+
body = body.split(find).join(replace);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
// Direct body override (applied after bodyReplace, matching networkCapture.ts)
|
|
137
|
+
if (ov.body != null) {
|
|
138
|
+
body = resolveTemplateValue(ov.body, ctx);
|
|
139
|
+
}
|
|
140
|
+
return { url, method, headers, body };
|
|
141
|
+
}
|
|
142
|
+
// ---------------------------------------------------------------------------
|
|
143
|
+
// File I/O
|
|
144
|
+
// ---------------------------------------------------------------------------
|
|
145
|
+
const SNAPSHOTS_FILENAME = 'snapshots.json';
|
|
146
|
+
/**
|
|
147
|
+
* Load snapshots.json from a pack directory. Returns null if not found.
|
|
148
|
+
*/
|
|
149
|
+
export function loadSnapshots(packPath) {
|
|
150
|
+
const filePath = join(packPath, SNAPSHOTS_FILENAME);
|
|
151
|
+
if (!existsSync(filePath))
|
|
152
|
+
return null;
|
|
153
|
+
try {
|
|
154
|
+
const content = readFileSync(filePath, 'utf-8');
|
|
155
|
+
const data = JSON.parse(content);
|
|
156
|
+
if (data.version !== 1) {
|
|
157
|
+
console.warn(`[requestSnapshot] Unsupported snapshot version: ${data.version}`);
|
|
158
|
+
return null;
|
|
159
|
+
}
|
|
160
|
+
return data;
|
|
161
|
+
}
|
|
162
|
+
catch (err) {
|
|
163
|
+
console.warn(`[requestSnapshot] Failed to load snapshots.json: ${err instanceof Error ? err.message : String(err)}`);
|
|
164
|
+
return null;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Write snapshots.json to a pack directory.
|
|
169
|
+
*/
|
|
170
|
+
export function writeSnapshots(packPath, snapshots) {
|
|
171
|
+
const filePath = join(packPath, SNAPSHOTS_FILENAME);
|
|
172
|
+
writeFileSync(filePath, JSON.stringify(snapshots, null, 2), 'utf-8');
|
|
173
|
+
}
|
|
174
|
+
// ---------------------------------------------------------------------------
|
|
175
|
+
// Capture helpers
|
|
176
|
+
// ---------------------------------------------------------------------------
|
|
177
|
+
const SENSITIVE_HEADER_NAMES = ['authorization', 'cookie', 'set-cookie', 'x-api-key', 'proxy-authorization'];
|
|
178
|
+
/**
|
|
179
|
+
* Extract top-level keys from a JSON string. Returns empty array on parse failure.
|
|
180
|
+
*/
|
|
181
|
+
export function extractTopLevelKeys(jsonText) {
|
|
182
|
+
if (!jsonText)
|
|
183
|
+
return [];
|
|
184
|
+
try {
|
|
185
|
+
const parsed = JSON.parse(jsonText);
|
|
186
|
+
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
|
|
187
|
+
return Object.keys(parsed);
|
|
188
|
+
}
|
|
189
|
+
return [];
|
|
190
|
+
}
|
|
191
|
+
catch {
|
|
192
|
+
return [];
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Detect which headers in a record are sensitive (contain auth data).
|
|
197
|
+
*/
|
|
198
|
+
export function detectSensitiveHeaders(headers) {
|
|
199
|
+
return Object.keys(headers).filter((h) => SENSITIVE_HEADER_NAMES.includes(h.toLowerCase()));
|
|
200
|
+
}
|
package/dist/runner.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAc,MAAM,YAAY,CAAC;AAElE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAc,MAAM,YAAY,CAAC;AAElE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAWzC;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAkB,SAAQ,SAAS;IAClD;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;CACtB;AAED;;;GAGG;AACH,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC/B,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,iBAAiB,CAAC,CAmQ5B"}
|
package/dist/runner.js
CHANGED
|
@@ -2,6 +2,8 @@ import { mkdirSync, writeFileSync } from 'fs';
|
|
|
2
2
|
import { join } from 'path';
|
|
3
3
|
import { InputValidator, RunContextFactory, runFlow, attachNetworkCapture, TaskPackLoader } from './index.js';
|
|
4
4
|
import { launchBrowser } from './browserLauncher.js';
|
|
5
|
+
import { isFlowHttpCompatible } from './httpReplay.js';
|
|
6
|
+
import { writeSnapshots, extractTopLevelKeys, detectSensitiveHeaders, } from './requestSnapshot.js';
|
|
5
7
|
/**
|
|
6
8
|
* Runs a task pack with Playwright
|
|
7
9
|
* This is a reusable function that can be used by both CLI and MCP server
|
|
@@ -10,8 +12,6 @@ export async function runTaskPack(taskPack, inputs, options) {
|
|
|
10
12
|
const { runDir, logger, headless: requestedHeadless = true, packPath, secrets: providedSecrets } = options;
|
|
11
13
|
const artifactsDir = join(runDir, 'artifacts');
|
|
12
14
|
const eventsPath = join(runDir, 'events.jsonl');
|
|
13
|
-
// Load secrets from pack directory if not provided and packPath is given
|
|
14
|
-
const secrets = providedSecrets ?? (packPath ? TaskPackLoader.loadSecrets(packPath) : {});
|
|
15
15
|
// Ensure directories exist
|
|
16
16
|
mkdirSync(runDir, { recursive: true });
|
|
17
17
|
mkdirSync(artifactsDir, { recursive: true });
|
|
@@ -24,14 +24,14 @@ export async function runTaskPack(taskPack, inputs, options) {
|
|
|
24
24
|
'Falling back to headless mode. Set DISPLAY or use xvfb-run to enable headful mode.');
|
|
25
25
|
}
|
|
26
26
|
const startTime = Date.now();
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
27
|
+
// Apply defaults and validate inputs early (needed for both HTTP and browser modes)
|
|
28
|
+
const inputsWithDefaults = InputValidator.applyDefaults(inputs, taskPack.inputs);
|
|
29
|
+
InputValidator.validate(inputsWithDefaults, taskPack.inputs);
|
|
30
|
+
// Load secrets early (needed for both modes)
|
|
31
|
+
const secrets = providedSecrets ?? (packPath ? TaskPackLoader.loadSecrets(packPath) : {});
|
|
32
|
+
// ─── HTTP-first execution ───────────────────────────────────────────
|
|
33
|
+
const snapshots = taskPack.snapshots ?? null;
|
|
34
|
+
if (isFlowHttpCompatible(taskPack.flow, snapshots)) {
|
|
35
35
|
logger.log({
|
|
36
36
|
type: 'run_started',
|
|
37
37
|
data: {
|
|
@@ -40,6 +40,34 @@ export async function runTaskPack(taskPack, inputs, options) {
|
|
|
40
40
|
inputs: inputsWithDefaults,
|
|
41
41
|
},
|
|
42
42
|
});
|
|
43
|
+
try {
|
|
44
|
+
const httpResult = await runHttpOnly(taskPack, inputsWithDefaults, snapshots, secrets, logger, options);
|
|
45
|
+
const durationMs = Date.now() - startTime;
|
|
46
|
+
logger.log({ type: 'run_finished', data: { success: true, durationMs } });
|
|
47
|
+
return { ...httpResult, runDir, eventsPath, artifactsDir };
|
|
48
|
+
}
|
|
49
|
+
catch (httpError) {
|
|
50
|
+
// HTTP-only execution failed — fall through to browser mode
|
|
51
|
+
const reason = httpError instanceof Error ? httpError.message : String(httpError);
|
|
52
|
+
console.log(`[runner] HTTP-only mode failed (${reason}), falling back to browser mode`);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
// ─── Browser execution (fallback or primary) ───────────────────────
|
|
56
|
+
let browserSession = null;
|
|
57
|
+
let page = null;
|
|
58
|
+
let runContext = null;
|
|
59
|
+
try {
|
|
60
|
+
// Log run start (only if not already logged above)
|
|
61
|
+
if (!isFlowHttpCompatible(taskPack.flow, snapshots)) {
|
|
62
|
+
logger.log({
|
|
63
|
+
type: 'run_started',
|
|
64
|
+
data: {
|
|
65
|
+
packId: taskPack.metadata.id,
|
|
66
|
+
packVersion: taskPack.metadata.version,
|
|
67
|
+
inputs: inputsWithDefaults,
|
|
68
|
+
},
|
|
69
|
+
});
|
|
70
|
+
}
|
|
43
71
|
// Launch browser with unified launcher
|
|
44
72
|
browserSession = await launchBrowser({
|
|
45
73
|
browserSettings: taskPack.browser,
|
|
@@ -93,6 +121,15 @@ export async function runTaskPack(taskPack, inputs, options) {
|
|
|
93
121
|
if (flowResult._hints && flowResult._hints.length > 0) {
|
|
94
122
|
result._hints = flowResult._hints;
|
|
95
123
|
}
|
|
124
|
+
// Capture snapshots for network_replay steps after successful browser run
|
|
125
|
+
if (packPath && networkCapture) {
|
|
126
|
+
try {
|
|
127
|
+
captureSnapshots(taskPack, flowResult, networkCapture, packPath);
|
|
128
|
+
}
|
|
129
|
+
catch (snapErr) {
|
|
130
|
+
console.warn(`[runner] Failed to capture snapshots: ${snapErr instanceof Error ? snapErr.message : String(snapErr)}`);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
96
133
|
const durationMs = Date.now() - startTime;
|
|
97
134
|
// Log run finish
|
|
98
135
|
logger.log({
|
|
@@ -141,6 +178,18 @@ export async function runTaskPack(taskPack, inputs, options) {
|
|
|
141
178
|
console.error('Failed to save artifacts:', artifactError);
|
|
142
179
|
}
|
|
143
180
|
}
|
|
181
|
+
// Extract partial results from enriched error (set by interpreter)
|
|
182
|
+
const partialResult = error?.partialResult;
|
|
183
|
+
// Filter partial collectibles to only include declared ones
|
|
184
|
+
let partialCollectibles = {};
|
|
185
|
+
if (partialResult?.collectibles) {
|
|
186
|
+
const definedCollectibleNames = new Set((taskPack.collectibles || []).map(c => c.name));
|
|
187
|
+
for (const [key, value] of Object.entries(partialResult.collectibles)) {
|
|
188
|
+
if (definedCollectibleNames.has(key)) {
|
|
189
|
+
partialCollectibles[key] = value;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
}
|
|
144
193
|
// Log run finish with failure
|
|
145
194
|
logger.log({
|
|
146
195
|
type: 'run_finished',
|
|
@@ -150,16 +199,21 @@ export async function runTaskPack(taskPack, inputs, options) {
|
|
|
150
199
|
},
|
|
151
200
|
});
|
|
152
201
|
// Return partial result with paths even on error
|
|
153
|
-
|
|
154
|
-
collectibles:
|
|
202
|
+
const failResult = {
|
|
203
|
+
collectibles: partialCollectibles,
|
|
155
204
|
meta: {
|
|
156
205
|
durationMs,
|
|
157
|
-
notes: `Error: ${errorMessage}`,
|
|
206
|
+
notes: `Error at step "${partialResult?.failedStepId ?? 'unknown'}": ${errorMessage}`,
|
|
158
207
|
},
|
|
159
208
|
runDir,
|
|
160
209
|
eventsPath,
|
|
161
210
|
artifactsDir,
|
|
162
211
|
};
|
|
212
|
+
// Include failedStepId for AI agents
|
|
213
|
+
if (partialResult?.failedStepId) {
|
|
214
|
+
failResult.failedStepId = partialResult.failedStepId;
|
|
215
|
+
}
|
|
216
|
+
return failResult;
|
|
163
217
|
}
|
|
164
218
|
finally {
|
|
165
219
|
// Cleanup using unified browser session close
|
|
@@ -168,3 +222,145 @@ export async function runTaskPack(taskPack, inputs, options) {
|
|
|
168
222
|
}
|
|
169
223
|
}
|
|
170
224
|
}
|
|
225
|
+
// ---------------------------------------------------------------------------
|
|
226
|
+
// HTTP-only execution helper
|
|
227
|
+
// ---------------------------------------------------------------------------
|
|
228
|
+
/**
|
|
229
|
+
* Run a flow in HTTP-only mode using request snapshots.
|
|
230
|
+
* No browser is launched; network_replay steps use Node fetch().
|
|
231
|
+
* Throws if any snapshot response fails validation (caller should fall back to browser mode).
|
|
232
|
+
*/
|
|
233
|
+
async function runHttpOnly(taskPack, inputs, snapshots, secrets, logger, options) {
|
|
234
|
+
console.log(`[runner] Running in HTTP-only mode (${Object.keys(snapshots.snapshots).length} snapshots)`);
|
|
235
|
+
// Build a minimal RunContext that doesn't require a browser.
|
|
236
|
+
// In HTTP mode the interpreter skips all DOM steps, so page/browser are never accessed.
|
|
237
|
+
const noopPage = null;
|
|
238
|
+
const noopBrowser = null;
|
|
239
|
+
const noopArtifacts = {
|
|
240
|
+
saveScreenshot: async () => '',
|
|
241
|
+
saveHTML: async () => '',
|
|
242
|
+
};
|
|
243
|
+
const runContext = {
|
|
244
|
+
page: noopPage,
|
|
245
|
+
browser: noopBrowser,
|
|
246
|
+
logger,
|
|
247
|
+
artifacts: noopArtifacts,
|
|
248
|
+
};
|
|
249
|
+
const flowResult = await runFlow(runContext, taskPack.flow, {
|
|
250
|
+
inputs,
|
|
251
|
+
auth: taskPack.auth,
|
|
252
|
+
sessionId: options.sessionId,
|
|
253
|
+
profileId: options.profileId,
|
|
254
|
+
cacheDir: options.cacheDir,
|
|
255
|
+
secrets,
|
|
256
|
+
httpMode: true,
|
|
257
|
+
snapshots,
|
|
258
|
+
});
|
|
259
|
+
// Validate responses: re-check each network_replay step's snapshot validation.
|
|
260
|
+
// The actual validation happens inside the step handler via replayFromSnapshot +
|
|
261
|
+
// validateResponse. If validation fails, the step throws and runFlow propagates
|
|
262
|
+
// the error, which the caller catches and falls back to browser mode.
|
|
263
|
+
// Filter collectibles to only include those defined in the pack
|
|
264
|
+
const definedCollectibleNames = new Set((taskPack.collectibles || []).map((c) => c.name));
|
|
265
|
+
const filteredCollectibles = {};
|
|
266
|
+
for (const [key, value] of Object.entries(flowResult.collectibles)) {
|
|
267
|
+
if (definedCollectibleNames.has(key)) {
|
|
268
|
+
filteredCollectibles[key] = value;
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
const result = {
|
|
272
|
+
collectibles: filteredCollectibles,
|
|
273
|
+
meta: {
|
|
274
|
+
durationMs: flowResult.meta.durationMs,
|
|
275
|
+
notes: `HTTP-only: ${flowResult.meta.stepsExecuted}/${flowResult.meta.stepsTotal} steps`,
|
|
276
|
+
},
|
|
277
|
+
};
|
|
278
|
+
if (flowResult._hints && flowResult._hints.length > 0) {
|
|
279
|
+
result._hints = flowResult._hints;
|
|
280
|
+
}
|
|
281
|
+
return result;
|
|
282
|
+
}
|
|
283
|
+
// ---------------------------------------------------------------------------
|
|
284
|
+
// Snapshot capture helper
|
|
285
|
+
// ---------------------------------------------------------------------------
|
|
286
|
+
/**
|
|
287
|
+
* After a successful browser run, capture snapshots for all network_replay steps.
|
|
288
|
+
* Uses the resolved vars from the flow result to look up request IDs,
|
|
289
|
+
* then exports full entry data from the network capture buffer.
|
|
290
|
+
* Writes snapshots.json to the pack directory.
|
|
291
|
+
*/
|
|
292
|
+
function captureSnapshots(taskPack, flowResult, networkCapture, packPath) {
|
|
293
|
+
const replaySteps = taskPack.flow.filter((s) => s.type === 'network_replay');
|
|
294
|
+
if (replaySteps.length === 0)
|
|
295
|
+
return;
|
|
296
|
+
const vars = flowResult._vars ?? {};
|
|
297
|
+
const newSnapshots = {};
|
|
298
|
+
for (const step of replaySteps) {
|
|
299
|
+
if (step.type !== 'network_replay')
|
|
300
|
+
continue;
|
|
301
|
+
// Resolve the requestId template (e.g. "{{vars.reqId}}" → actual ID)
|
|
302
|
+
const rawRequestId = step.params.requestId;
|
|
303
|
+
let requestId;
|
|
304
|
+
// Check if it's a template reference
|
|
305
|
+
const varMatch = rawRequestId.match(/\{\{vars\.([^}]+)\}\}/);
|
|
306
|
+
if (varMatch) {
|
|
307
|
+
const varName = varMatch[1];
|
|
308
|
+
requestId = typeof vars[varName] === 'string' ? vars[varName] : undefined;
|
|
309
|
+
}
|
|
310
|
+
else {
|
|
311
|
+
// Literal request ID
|
|
312
|
+
requestId = rawRequestId;
|
|
313
|
+
}
|
|
314
|
+
if (!requestId)
|
|
315
|
+
continue;
|
|
316
|
+
// Export the full entry from the network capture buffer
|
|
317
|
+
const fullEntry = networkCapture.exportEntry(requestId);
|
|
318
|
+
if (!fullEntry || fullEntry.status === undefined)
|
|
319
|
+
continue;
|
|
320
|
+
// Build the snapshot
|
|
321
|
+
const snapshot = {
|
|
322
|
+
stepId: step.id,
|
|
323
|
+
capturedAt: Date.now(),
|
|
324
|
+
ttl: null, // indefinite by default
|
|
325
|
+
request: {
|
|
326
|
+
method: fullEntry.method,
|
|
327
|
+
url: fullEntry.url,
|
|
328
|
+
headers: fullEntry.requestHeadersFull,
|
|
329
|
+
body: fullEntry.postData ?? null,
|
|
330
|
+
},
|
|
331
|
+
overrides: step.params.overrides
|
|
332
|
+
? {
|
|
333
|
+
url: step.params.overrides.url,
|
|
334
|
+
body: step.params.overrides.body,
|
|
335
|
+
setQuery: step.params.overrides.setQuery
|
|
336
|
+
? Object.fromEntries(Object.entries(step.params.overrides.setQuery).map(([k, v]) => [k, String(v)]))
|
|
337
|
+
: undefined,
|
|
338
|
+
setHeaders: step.params.overrides.setHeaders,
|
|
339
|
+
urlReplace: step.params.overrides.urlReplace
|
|
340
|
+
? (Array.isArray(step.params.overrides.urlReplace) ? step.params.overrides.urlReplace : [step.params.overrides.urlReplace])
|
|
341
|
+
: undefined,
|
|
342
|
+
bodyReplace: step.params.overrides.bodyReplace
|
|
343
|
+
? (Array.isArray(step.params.overrides.bodyReplace) ? step.params.overrides.bodyReplace : [step.params.overrides.bodyReplace])
|
|
344
|
+
: undefined,
|
|
345
|
+
}
|
|
346
|
+
: undefined,
|
|
347
|
+
responseValidation: {
|
|
348
|
+
expectedStatus: fullEntry.status ?? 200,
|
|
349
|
+
expectedContentType: fullEntry.contentType ?? 'application/json',
|
|
350
|
+
expectedKeys: extractTopLevelKeys(fullEntry.responseBodyText),
|
|
351
|
+
},
|
|
352
|
+
sensitiveHeaders: detectSensitiveHeaders(fullEntry.requestHeadersFull),
|
|
353
|
+
};
|
|
354
|
+
newSnapshots[step.id] = snapshot;
|
|
355
|
+
}
|
|
356
|
+
if (Object.keys(newSnapshots).length > 0) {
|
|
357
|
+
// Merge with existing snapshots (update existing, add new)
|
|
358
|
+
const existing = taskPack.snapshots ?? { version: 1, snapshots: {} };
|
|
359
|
+
const merged = {
|
|
360
|
+
version: 1,
|
|
361
|
+
snapshots: { ...existing.snapshots, ...newSnapshots },
|
|
362
|
+
};
|
|
363
|
+
writeSnapshots(packPath, merged);
|
|
364
|
+
console.log(`[runner] Captured ${Object.keys(newSnapshots).length} snapshot(s) → snapshots.json`);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storage/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,WAAW,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Produce a canonical, stable JSON representation of inputs.
|
|
3
|
+
* - Sorts object keys recursively
|
|
4
|
+
* - Strips `undefined` values
|
|
5
|
+
*/
|
|
6
|
+
export declare function canonicalizeInputs(inputs: Record<string, unknown>): string;
|
|
7
|
+
/**
|
|
8
|
+
* Generate a deterministic result key from pack ID and inputs.
|
|
9
|
+
* Same packId + same inputs ⇒ same key (latest-wins cache).
|
|
10
|
+
*/
|
|
11
|
+
export declare function generateResultKey(packId: string, inputs: Record<string, unknown>): string;
|
|
12
|
+
//# sourceMappingURL=keys.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"keys.d.ts","sourceRoot":"","sources":["../../src/storage/keys.ts"],"names":[],"mappings":"AAOA;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAE1E;AAkBD;;;GAGG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC9B,MAAM,CAGR"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic key generation for stored results.
|
|
3
|
+
*
|
|
4
|
+
* key = sha256(packId + ":" + canonicalized_inputs)[0..16] (16-char hex)
|
|
5
|
+
*/
|
|
6
|
+
import { createHash } from 'crypto';
|
|
7
|
+
/**
|
|
8
|
+
* Produce a canonical, stable JSON representation of inputs.
|
|
9
|
+
* - Sorts object keys recursively
|
|
10
|
+
* - Strips `undefined` values
|
|
11
|
+
*/
|
|
12
|
+
export function canonicalizeInputs(inputs) {
|
|
13
|
+
return JSON.stringify(sortKeys(inputs));
|
|
14
|
+
}
|
|
15
|
+
function sortKeys(obj) {
|
|
16
|
+
if (obj === null || obj === undefined)
|
|
17
|
+
return obj;
|
|
18
|
+
if (Array.isArray(obj))
|
|
19
|
+
return obj.map(sortKeys);
|
|
20
|
+
if (typeof obj === 'object') {
|
|
21
|
+
const sorted = {};
|
|
22
|
+
for (const key of Object.keys(obj).sort()) {
|
|
23
|
+
const val = obj[key];
|
|
24
|
+
if (val !== undefined) {
|
|
25
|
+
sorted[key] = sortKeys(val);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
return sorted;
|
|
29
|
+
}
|
|
30
|
+
return obj;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Generate a deterministic result key from pack ID and inputs.
|
|
34
|
+
* Same packId + same inputs ⇒ same key (latest-wins cache).
|
|
35
|
+
*/
|
|
36
|
+
export function generateResultKey(packId, inputs) {
|
|
37
|
+
const payload = packId + ':' + canonicalizeInputs(inputs);
|
|
38
|
+
return createHash('sha256').update(payload).digest('hex').slice(0, 16);
|
|
39
|
+
}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pluggable Result Store — Pure interfaces (no external deps)
|
|
3
|
+
*
|
|
4
|
+
* Providers declare their capabilities so consumers (MCP tools)
|
|
5
|
+
* know which operations are available.
|
|
6
|
+
*/
|
|
7
|
+
export type StorageCapability = 'get' | 'store' | 'list' | 'delete' | 'filter' | 'search' | 'aggregate';
|
|
8
|
+
/**
|
|
9
|
+
* Schema field describing a collectible column (for AI-friendly descriptions).
|
|
10
|
+
*/
|
|
11
|
+
export interface CollectibleSchemaField {
|
|
12
|
+
name: string;
|
|
13
|
+
type: 'string' | 'number' | 'boolean';
|
|
14
|
+
description?: string;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* A single stored run result.
|
|
18
|
+
*/
|
|
19
|
+
export interface StoredResult {
|
|
20
|
+
/** Deterministic hash of packId + inputs */
|
|
21
|
+
key: string;
|
|
22
|
+
packId: string;
|
|
23
|
+
toolName: string;
|
|
24
|
+
inputs: Record<string, unknown>;
|
|
25
|
+
collectibles: Record<string, unknown>;
|
|
26
|
+
meta: {
|
|
27
|
+
url?: string;
|
|
28
|
+
durationMs: number;
|
|
29
|
+
notes?: string;
|
|
30
|
+
};
|
|
31
|
+
collectibleSchema: CollectibleSchemaField[];
|
|
32
|
+
/** ISO 8601 — when the result was (last) stored */
|
|
33
|
+
storedAt: string;
|
|
34
|
+
/** ISO 8601 — when the run actually executed */
|
|
35
|
+
ranAt: string;
|
|
36
|
+
/** Incremented on overwrite (same key) */
|
|
37
|
+
version: number;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Options for listing stored results.
|
|
41
|
+
*/
|
|
42
|
+
export interface ListOptions {
|
|
43
|
+
limit?: number;
|
|
44
|
+
offset?: number;
|
|
45
|
+
sortBy?: 'storedAt' | 'ranAt';
|
|
46
|
+
sortDir?: 'asc' | 'desc';
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Lightweight summary returned by list().
|
|
50
|
+
*/
|
|
51
|
+
export interface ResultSummary {
|
|
52
|
+
key: string;
|
|
53
|
+
packId: string;
|
|
54
|
+
toolName: string;
|
|
55
|
+
storedAt: string;
|
|
56
|
+
version: number;
|
|
57
|
+
fieldCount: number;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Filter/paginate within a single stored result's collectibles.
|
|
61
|
+
*/
|
|
62
|
+
export interface FilterOptions {
|
|
63
|
+
/** Which stored result to filter within */
|
|
64
|
+
key: string;
|
|
65
|
+
/** JMESPath expression for extraction/transform */
|
|
66
|
+
jmesPath?: string;
|
|
67
|
+
/** Limit items (for array results) */
|
|
68
|
+
limit?: number;
|
|
69
|
+
/** Pagination offset */
|
|
70
|
+
offset?: number;
|
|
71
|
+
/** Field to sort by within collectibles */
|
|
72
|
+
sortBy?: string;
|
|
73
|
+
sortDir?: 'asc' | 'desc';
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Full-text search options (for future vector/FTS providers).
|
|
77
|
+
*/
|
|
78
|
+
export interface SearchOptions {
|
|
79
|
+
query: string;
|
|
80
|
+
limit?: number;
|
|
81
|
+
}
|
|
82
|
+
export interface SearchHit {
|
|
83
|
+
key: string;
|
|
84
|
+
packId: string;
|
|
85
|
+
toolName: string;
|
|
86
|
+
score: number;
|
|
87
|
+
snippet?: string;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* The pluggable store interface.
|
|
91
|
+
* Only `capabilities()`, `store()`, and `get()` are required.
|
|
92
|
+
* Optional methods should only be called if the corresponding capability is declared.
|
|
93
|
+
*/
|
|
94
|
+
export interface ResultStoreProvider {
|
|
95
|
+
capabilities(): StorageCapability[];
|
|
96
|
+
store(result: StoredResult): Promise<void>;
|
|
97
|
+
get(key: string): Promise<StoredResult | null>;
|
|
98
|
+
list?(options?: ListOptions): Promise<{
|
|
99
|
+
results: ResultSummary[];
|
|
100
|
+
total: number;
|
|
101
|
+
}>;
|
|
102
|
+
delete?(key: string): Promise<boolean>;
|
|
103
|
+
filter?(options: FilterOptions): Promise<{
|
|
104
|
+
data: unknown;
|
|
105
|
+
total?: number;
|
|
106
|
+
}>;
|
|
107
|
+
search?(options: SearchOptions): Promise<{
|
|
108
|
+
results: SearchHit[];
|
|
109
|
+
}>;
|
|
110
|
+
close?(): Promise<void>;
|
|
111
|
+
}
|
|
112
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/storage/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,MAAM,iBAAiB,GACzB,KAAK,GACL,OAAO,GACP,MAAM,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,WAAW,CAAC;AAEhB;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;IACtC,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,4CAA4C;IAC5C,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACtC,IAAI,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3D,iBAAiB,EAAE,sBAAsB,EAAE,CAAC;IAC5C,mDAAmD;IACnD,QAAQ,EAAE,MAAM,CAAC;IACjB,gDAAgD;IAChD,KAAK,EAAE,MAAM,CAAC;IACd,0CAA0C;IAC1C,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC;IAC9B,OAAO,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,2CAA2C;IAC3C,GAAG,EAAE,MAAM,CAAC;IACZ,mDAAmD;IACnD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sCAAsC;IACtC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,2CAA2C;IAC3C,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;GAIG;AACH,MAAM,WAAW,mBAAmB;IAClC,YAAY,IAAI,iBAAiB,EAAE,CAAC;IACpC,KAAK,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3C,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAC/C,IAAI,CAAC,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,aAAa,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACnF,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,CAAC,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5E,MAAM,CAAC,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,SAAS,EAAE,CAAA;KAAE,CAAC,CAAC;IACnE,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB"}
|