@zenalexa/unicli 0.208.0 → 0.209.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/README.md +6 -6
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +6 -0
- package/dist/cli.js.map +1 -1
- package/dist/commands/agents.d.ts.map +1 -1
- package/dist/commands/agents.js +82 -2
- package/dist/commands/agents.js.map +1 -1
- package/dist/commands/generate.d.ts.map +1 -1
- package/dist/commands/generate.js +20 -1
- package/dist/commands/generate.js.map +1 -1
- package/dist/commands/hub.d.ts +13 -0
- package/dist/commands/hub.d.ts.map +1 -0
- package/dist/commands/hub.js +232 -0
- package/dist/commands/hub.js.map +1 -0
- package/dist/commands/research.d.ts +17 -0
- package/dist/commands/research.d.ts.map +1 -0
- package/dist/commands/research.js +257 -0
- package/dist/commands/research.js.map +1 -0
- package/dist/commands/test-gen.d.ts +10 -0
- package/dist/commands/test-gen.d.ts.map +1 -0
- package/dist/commands/test-gen.js +124 -0
- package/dist/commands/test-gen.js.map +1 -0
- package/dist/discovery/loader.d.ts.map +1 -1
- package/dist/discovery/loader.js +3 -0
- package/dist/discovery/loader.js.map +1 -1
- package/dist/engine/capability.d.ts +40 -0
- package/dist/engine/capability.d.ts.map +1 -0
- package/dist/engine/capability.js +191 -0
- package/dist/engine/capability.js.map +1 -0
- package/dist/engine/endpoint.d.ts +47 -0
- package/dist/engine/endpoint.d.ts.map +1 -0
- package/dist/engine/endpoint.js +295 -0
- package/dist/engine/endpoint.js.map +1 -0
- package/dist/engine/framework.d.ts +28 -0
- package/dist/engine/framework.d.ts.map +1 -0
- package/dist/engine/framework.js +66 -0
- package/dist/engine/framework.js.map +1 -0
- package/dist/engine/probe.d.ts +19 -0
- package/dist/engine/probe.d.ts.map +1 -0
- package/dist/engine/probe.js +85 -0
- package/dist/engine/probe.js.map +1 -0
- package/dist/engine/research.d.ts +38 -0
- package/dist/engine/research.d.ts.map +1 -0
- package/dist/engine/research.js +414 -0
- package/dist/engine/research.js.map +1 -0
- package/dist/engine/yaml-runner.d.ts.map +1 -1
- package/dist/engine/yaml-runner.js +80 -5
- package/dist/engine/yaml-runner.js.map +1 -1
- package/dist/manifest.json +403 -1
- package/dist/mcp/server.js +59 -1
- package/dist/mcp/server.js.map +1 -1
- package/package.json +1 -1
- package/src/adapters/cnn/top.yaml +21 -0
- package/src/adapters/cocoapods/search.yaml +16 -0
- package/src/adapters/crates-io/search.yaml +27 -0
- package/src/adapters/docker-hub/search.yaml +26 -0
- package/src/adapters/eastmoney/hot.yaml +23 -0
- package/src/adapters/eastmoney/search.yaml +25 -0
- package/src/adapters/exchangerate/convert.yaml +19 -0
- package/src/adapters/feishu/calendar.yaml +24 -0
- package/src/adapters/feishu/docs.yaml +17 -0
- package/src/adapters/feishu/send.yaml +29 -0
- package/src/adapters/feishu/tasks.yaml +24 -0
- package/src/adapters/gitee/search.yaml +25 -0
- package/src/adapters/gitee/trending.yaml +22 -0
- package/src/adapters/gitlab/search.yaml +24 -0
- package/src/adapters/gitlab/trending.yaml +22 -0
- package/src/adapters/homebrew/info.yaml +15 -0
- package/src/adapters/huggingface-papers/daily.yaml +21 -0
- package/src/adapters/infoq/articles.yaml +29 -0
- package/src/adapters/ip-info/lookup.yaml +15 -0
- package/src/adapters/itch-io/popular.yaml +22 -0
- package/src/adapters/ithome/news.yaml +21 -0
- package/src/adapters/mastodon/search.yaml +29 -0
- package/src/adapters/mastodon/trending.yaml +27 -0
- package/src/adapters/meituan/search.yaml +30 -0
- package/src/adapters/minimax/chat.yaml +33 -0
- package/src/adapters/minimax/models.yaml +18 -0
- package/src/adapters/minimax/tts.yaml +33 -0
- package/src/adapters/netease-music/hot.yaml +24 -0
- package/src/adapters/netease-music/search.yaml +29 -0
- package/src/adapters/npm-trends/compare.yaml +19 -0
- package/src/adapters/nytimes/top.yaml +26 -0
- package/src/adapters/openrouter/models.yaml +22 -0
- package/src/adapters/pexels/search.yaml +28 -0
- package/src/adapters/pinduoduo/hot.yaml +20 -0
- package/src/adapters/pypi/info.yaml +16 -0
- package/src/adapters/qweather/now.yaml +16 -0
- package/src/adapters/replicate/search.yaml +25 -0
- package/src/adapters/replicate/trending.yaml +22 -0
- package/src/adapters/sspai/hot.yaml +21 -0
- package/src/adapters/sspai/latest.yaml +22 -0
- package/src/adapters/techcrunch/latest.yaml +22 -0
- package/src/adapters/theverge/latest.yaml +21 -0
- package/src/adapters/twitch/top.yaml +26 -0
- package/src/adapters/unsplash/search.yaml +28 -0
- package/src/adapters/ycombinator/launches.yaml +20 -0
|
@@ -0,0 +1,414 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AutoResearch Engine — Karpathy-style self-improvement loop.
|
|
3
|
+
*
|
|
4
|
+
* 8-phase loop: precondition → review → modify (Claude Code) →
|
|
5
|
+
* commit → verify (eval) → guard → decide (keep/discard) → log.
|
|
6
|
+
*
|
|
7
|
+
* Adapted from Open-CLI's autoresearch concept, scoped to YAML
|
|
8
|
+
* adapter modification and integrated with Uni-CLI's eval harness.
|
|
9
|
+
*/
|
|
10
|
+
import { execFileSync } from "node:child_process";
|
|
11
|
+
import { existsSync, readFileSync, readdirSync, appendFileSync, mkdirSync, } from "node:fs";
|
|
12
|
+
import { join } from "node:path";
|
|
13
|
+
import { homedir } from "node:os";
|
|
14
|
+
/** Strict site name pattern — prevents shell injection in interpolated commands. */
|
|
15
|
+
const SAFE_SITE_NAME = /^[a-zA-Z0-9_-]+$/;
|
|
16
|
+
// ── Stuck Hints ──────────────────────────────────────────────────────────
|
|
17
|
+
const STUCK_HINTS = [
|
|
18
|
+
"Try a completely different approach — the current direction is not working.",
|
|
19
|
+
"Try the OPPOSITE of what has been failing.",
|
|
20
|
+
"Consider changing the pipeline pattern entirely (e.g. fetch→intercept).",
|
|
21
|
+
"Consider if the endpoint URL itself needs to change.",
|
|
22
|
+
"The adapter may need a fundamentally different strategy.",
|
|
23
|
+
];
|
|
24
|
+
// ── Engine ───────────────────────────────────────────────────────────────
|
|
25
|
+
const EXEC_OPTS = { encoding: "utf-8", timeout: 60_000 };
|
|
26
|
+
const LOG_DIR = join(homedir(), ".unicli");
|
|
27
|
+
const LOG_FILE = join(LOG_DIR, "research.tsv");
|
|
28
|
+
function extractMetric(output, metricRegex) {
|
|
29
|
+
const re = new RegExp(metricRegex);
|
|
30
|
+
// Reset lastIndex for safety with global/sticky regexes
|
|
31
|
+
re.lastIndex = 0;
|
|
32
|
+
const match = re.exec(output);
|
|
33
|
+
if (!match)
|
|
34
|
+
return 0;
|
|
35
|
+
// If capturing groups exist, use first group; otherwise use full match
|
|
36
|
+
const raw = match[1] ?? match[0];
|
|
37
|
+
const val = parseFloat(raw);
|
|
38
|
+
return Number.isFinite(val) ? val : 0;
|
|
39
|
+
}
|
|
40
|
+
function gitExec(args) {
|
|
41
|
+
return execFileSync("git", args, EXEC_OPTS);
|
|
42
|
+
}
|
|
43
|
+
function appendLog(result, site) {
|
|
44
|
+
mkdirSync(LOG_DIR, { recursive: true });
|
|
45
|
+
const line = [
|
|
46
|
+
new Date().toISOString(),
|
|
47
|
+
site,
|
|
48
|
+
result.iteration,
|
|
49
|
+
result.status,
|
|
50
|
+
result.metric,
|
|
51
|
+
result.description.slice(0, 120),
|
|
52
|
+
result.durationMs,
|
|
53
|
+
].join("\t");
|
|
54
|
+
appendFileSync(LOG_FILE, line + "\n");
|
|
55
|
+
}
|
|
56
|
+
// ── Safe glob resolution (no shell) ─────────────────────────────────────
|
|
57
|
+
/**
|
|
58
|
+
* Resolve scope patterns to actual files without shell execution.
|
|
59
|
+
* Supports simple patterns like `src/adapters/site/*.yaml`.
|
|
60
|
+
*/
|
|
61
|
+
function resolveScope(patterns) {
|
|
62
|
+
const files = [];
|
|
63
|
+
for (const pattern of patterns) {
|
|
64
|
+
// Split at last directory separator before the glob
|
|
65
|
+
const starIdx = pattern.indexOf("*");
|
|
66
|
+
if (starIdx === -1) {
|
|
67
|
+
// No glob — treat as literal file
|
|
68
|
+
if (existsSync(pattern))
|
|
69
|
+
files.push(pattern);
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
const dir = pattern.slice(0, pattern.lastIndexOf("/", starIdx));
|
|
73
|
+
const extFilter = pattern.slice(pattern.lastIndexOf("."));
|
|
74
|
+
if (!existsSync(dir))
|
|
75
|
+
continue;
|
|
76
|
+
try {
|
|
77
|
+
for (const entry of readdirSync(dir)) {
|
|
78
|
+
if (extFilter && !entry.endsWith(extFilter))
|
|
79
|
+
continue;
|
|
80
|
+
const fullPath = join(dir, entry);
|
|
81
|
+
files.push(fullPath);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
catch {
|
|
85
|
+
/* directory read failed */
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return files;
|
|
89
|
+
}
|
|
90
|
+
// ── Phase 0: Preconditions ──────────────────────────────────────────────
|
|
91
|
+
function checkPreconditions() {
|
|
92
|
+
// Must be in a git repo
|
|
93
|
+
try {
|
|
94
|
+
gitExec(["rev-parse", "--is-inside-work-tree"]);
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
throw new Error("Not inside a git repository");
|
|
98
|
+
}
|
|
99
|
+
// No index.lock
|
|
100
|
+
if (existsSync(".git/index.lock")) {
|
|
101
|
+
throw new Error(".git/index.lock exists — another git process may be running");
|
|
102
|
+
}
|
|
103
|
+
// Not detached HEAD
|
|
104
|
+
try {
|
|
105
|
+
gitExec(["symbolic-ref", "--short", "HEAD"]);
|
|
106
|
+
}
|
|
107
|
+
catch {
|
|
108
|
+
throw new Error("Detached HEAD — checkout a branch first");
|
|
109
|
+
}
|
|
110
|
+
// Clean working tree for scope files
|
|
111
|
+
const status = gitExec(["status", "--porcelain"]).trim();
|
|
112
|
+
if (status.length > 0) {
|
|
113
|
+
throw new Error("Working tree has uncommitted changes. Commit or stash first.");
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
// ── Phase 4: Verify ─────────────────────────────────────────────────────
|
|
117
|
+
function runVerify(config) {
|
|
118
|
+
// Validate site name to prevent injection when used in command construction
|
|
119
|
+
if (!SAFE_SITE_NAME.test(config.site)) {
|
|
120
|
+
return { output: "invalid site name", metric: 0 };
|
|
121
|
+
}
|
|
122
|
+
try {
|
|
123
|
+
// Use execFileSync with explicit args — no shell interpretation
|
|
124
|
+
const output = execFileSync("unicli", ["eval", "run", config.site, "--json"], { encoding: "utf-8", timeout: 120_000, stdio: ["pipe", "pipe", "pipe"] });
|
|
125
|
+
return { output, metric: extractMetric(output, config.metric) };
|
|
126
|
+
}
|
|
127
|
+
catch (err) {
|
|
128
|
+
const output = err instanceof Error && "stdout" in err
|
|
129
|
+
? String(err.stdout)
|
|
130
|
+
: "";
|
|
131
|
+
return {
|
|
132
|
+
output,
|
|
133
|
+
metric: config.direction === "lower"
|
|
134
|
+
? Infinity
|
|
135
|
+
: extractMetric(output, config.metric),
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
// ── Phase 5.5: Guard ────────────────────────────────────────────────────
|
|
140
|
+
function runGuard(config) {
|
|
141
|
+
if (!config.guard)
|
|
142
|
+
return "skip";
|
|
143
|
+
// Guard runs `unicli eval run --all` or similar — use execFile with args
|
|
144
|
+
try {
|
|
145
|
+
execFileSync("unicli", ["eval", "run", "--all"], {
|
|
146
|
+
encoding: "utf-8",
|
|
147
|
+
timeout: 120_000,
|
|
148
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
149
|
+
});
|
|
150
|
+
return "pass";
|
|
151
|
+
}
|
|
152
|
+
catch {
|
|
153
|
+
return "fail";
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
// ── Phase 6: Keep/Discard ───────────────────────────────────────────────
|
|
157
|
+
function safeRevert() {
|
|
158
|
+
try {
|
|
159
|
+
gitExec(["revert", "HEAD", "--no-edit"]);
|
|
160
|
+
}
|
|
161
|
+
catch {
|
|
162
|
+
try {
|
|
163
|
+
gitExec(["revert", "--abort"]);
|
|
164
|
+
}
|
|
165
|
+
catch {
|
|
166
|
+
/* ignore */
|
|
167
|
+
}
|
|
168
|
+
gitExec(["reset", "--hard", "HEAD~1"]);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
// ── Phase 2: Modify (Claude Code invocation) ────────────────────────────
|
|
172
|
+
function buildModifyPrompt(config, iteration, bestMetric, currentMetric, consecutiveDiscards, recentLog, scopeContents) {
|
|
173
|
+
const stuckHint = consecutiveDiscards >= 5
|
|
174
|
+
? STUCK_HINTS[Math.min(consecutiveDiscards - 5, STUCK_HINTS.length - 1)]
|
|
175
|
+
: null;
|
|
176
|
+
const logSummary = recentLog
|
|
177
|
+
.slice(-10)
|
|
178
|
+
.map((r) => ` ${String(r.status).padEnd(12)} metric=${r.metric} ${r.description}`)
|
|
179
|
+
.join("\n");
|
|
180
|
+
return `You are improving a Uni-CLI YAML adapter for "${config.site}".
|
|
181
|
+
|
|
182
|
+
## Goal
|
|
183
|
+
${config.goal}
|
|
184
|
+
|
|
185
|
+
## Current State
|
|
186
|
+
- Iteration: ${iteration}
|
|
187
|
+
- Current metric: ${currentMetric}
|
|
188
|
+
- Best metric: ${bestMetric}
|
|
189
|
+
- Consecutive discards: ${consecutiveDiscards}
|
|
190
|
+
${stuckHint ? `\n## STUCK HINT\n${stuckHint}\n` : ""}
|
|
191
|
+
## Recent History
|
|
192
|
+
${logSummary || "(none)"}
|
|
193
|
+
|
|
194
|
+
## Scope Files
|
|
195
|
+
${scopeContents}
|
|
196
|
+
|
|
197
|
+
## Rules
|
|
198
|
+
1. Make exactly ONE focused change per iteration
|
|
199
|
+
2. Only modify files matching the scope patterns: ${config.scope.join(", ")}
|
|
200
|
+
3. The verify command is: ${config.verify}
|
|
201
|
+
4. Metric direction: ${config.direction} is better
|
|
202
|
+
5. Do NOT add comments explaining your changes
|
|
203
|
+
6. Do NOT modify test files or configuration`;
|
|
204
|
+
}
|
|
205
|
+
function invokeClaudeCode(prompt) {
|
|
206
|
+
try {
|
|
207
|
+
// Use execFileSync (no shell) to avoid injection — safer than Open-CLI's
|
|
208
|
+
// execSync approach which requires manual quote escaping.
|
|
209
|
+
const result = execFileSync("claude", [
|
|
210
|
+
"-p",
|
|
211
|
+
"--dangerously-skip-permissions",
|
|
212
|
+
"--allowedTools",
|
|
213
|
+
"Read,Edit,Glob,Grep",
|
|
214
|
+
"--output-format",
|
|
215
|
+
"text",
|
|
216
|
+
"--no-session-persistence",
|
|
217
|
+
prompt,
|
|
218
|
+
], { encoding: "utf-8", timeout: 300_000, stdio: ["pipe", "pipe", "pipe"] });
|
|
219
|
+
// Extract description from last non-empty line
|
|
220
|
+
const lines = result.trim().split("\n").filter(Boolean);
|
|
221
|
+
const desc = lines.length > 0 ? lines[lines.length - 1].trim() : null;
|
|
222
|
+
return desc ? desc.slice(0, 120) : "change made by research loop";
|
|
223
|
+
}
|
|
224
|
+
catch {
|
|
225
|
+
return null;
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
// ── Main Loop ────────────────────────────────────────────────────────────
|
|
229
|
+
export async function runResearchLoop(config, callbacks) {
|
|
230
|
+
const results = [];
|
|
231
|
+
let bestMetric;
|
|
232
|
+
let consecutiveDiscards = 0;
|
|
233
|
+
// Phase 0: Preconditions
|
|
234
|
+
checkPreconditions();
|
|
235
|
+
// Baseline measurement
|
|
236
|
+
callbacks?.onStatus?.("Running baseline verification...");
|
|
237
|
+
const baseline = runVerify(config);
|
|
238
|
+
bestMetric = baseline.metric;
|
|
239
|
+
const baselineResult = {
|
|
240
|
+
iteration: 0,
|
|
241
|
+
status: "baseline",
|
|
242
|
+
metric: baseline.metric,
|
|
243
|
+
description: "Baseline measurement",
|
|
244
|
+
durationMs: 0,
|
|
245
|
+
};
|
|
246
|
+
results.push(baselineResult);
|
|
247
|
+
appendLog(baselineResult, config.site);
|
|
248
|
+
callbacks?.onIteration?.(baselineResult);
|
|
249
|
+
callbacks?.onStatus?.(`Baseline: metric=${baseline.metric}`);
|
|
250
|
+
// Main loop
|
|
251
|
+
for (let i = 1; i <= config.maxIterations; i++) {
|
|
252
|
+
const startMs = Date.now();
|
|
253
|
+
// Stuck detection: abort if too many consecutive discards
|
|
254
|
+
if (consecutiveDiscards >= 13) {
|
|
255
|
+
callbacks?.onStatus?.(`Stuck after ${consecutiveDiscards} consecutive discards. Stopping.`);
|
|
256
|
+
break;
|
|
257
|
+
}
|
|
258
|
+
callbacks?.onStatus?.(`Iteration ${i}/${config.maxIterations} (best=${bestMetric}, discards=${consecutiveDiscards})`);
|
|
259
|
+
// Phase 1: Review — read scope files (safe glob, no shell)
|
|
260
|
+
let scopeContents = "";
|
|
261
|
+
const scopeFiles = resolveScope(config.scope);
|
|
262
|
+
for (const f of scopeFiles) {
|
|
263
|
+
scopeContents += `\n--- ${f} ---\n${readFileSync(f, "utf-8")}\n`;
|
|
264
|
+
}
|
|
265
|
+
// Phase 2+3: Modify
|
|
266
|
+
const prompt = buildModifyPrompt(config, i, bestMetric, bestMetric, // pass current best, not stale baseline
|
|
267
|
+
consecutiveDiscards, results, scopeContents);
|
|
268
|
+
const description = invokeClaudeCode(prompt);
|
|
269
|
+
if (!description) {
|
|
270
|
+
const result = {
|
|
271
|
+
iteration: i,
|
|
272
|
+
status: "crash",
|
|
273
|
+
metric: bestMetric,
|
|
274
|
+
description: "Claude Code invocation failed",
|
|
275
|
+
durationMs: Date.now() - startMs,
|
|
276
|
+
};
|
|
277
|
+
results.push(result);
|
|
278
|
+
appendLog(result, config.site);
|
|
279
|
+
callbacks?.onIteration?.(result);
|
|
280
|
+
consecutiveDiscards++;
|
|
281
|
+
continue;
|
|
282
|
+
}
|
|
283
|
+
// Phase 4: Commit
|
|
284
|
+
try {
|
|
285
|
+
// Stage only scope files (safe — no shell, explicit file list)
|
|
286
|
+
const filesToStage = resolveScope(config.scope);
|
|
287
|
+
if (filesToStage.length > 0) {
|
|
288
|
+
gitExec(["add", "--", ...filesToStage]);
|
|
289
|
+
}
|
|
290
|
+
const diff = gitExec(["diff", "--cached", "--stat"]).trim();
|
|
291
|
+
if (!diff) {
|
|
292
|
+
const result = {
|
|
293
|
+
iteration: i,
|
|
294
|
+
status: "no-op",
|
|
295
|
+
metric: bestMetric,
|
|
296
|
+
description: "No changes made",
|
|
297
|
+
durationMs: Date.now() - startMs,
|
|
298
|
+
};
|
|
299
|
+
results.push(result);
|
|
300
|
+
appendLog(result, config.site);
|
|
301
|
+
callbacks?.onIteration?.(result);
|
|
302
|
+
consecutiveDiscards++;
|
|
303
|
+
continue;
|
|
304
|
+
}
|
|
305
|
+
try {
|
|
306
|
+
gitExec(["commit", "-m", `research(${config.site}): ${description}`]);
|
|
307
|
+
}
|
|
308
|
+
catch {
|
|
309
|
+
// Hook failure
|
|
310
|
+
gitExec(["reset", "HEAD"]);
|
|
311
|
+
const result = {
|
|
312
|
+
iteration: i,
|
|
313
|
+
status: "hook-blocked",
|
|
314
|
+
metric: bestMetric,
|
|
315
|
+
description: "Pre-commit hook blocked",
|
|
316
|
+
durationMs: Date.now() - startMs,
|
|
317
|
+
};
|
|
318
|
+
results.push(result);
|
|
319
|
+
appendLog(result, config.site);
|
|
320
|
+
callbacks?.onIteration?.(result);
|
|
321
|
+
consecutiveDiscards++;
|
|
322
|
+
continue;
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
catch {
|
|
326
|
+
const result = {
|
|
327
|
+
iteration: i,
|
|
328
|
+
status: "crash",
|
|
329
|
+
metric: bestMetric,
|
|
330
|
+
description: "Git commit failed",
|
|
331
|
+
durationMs: Date.now() - startMs,
|
|
332
|
+
};
|
|
333
|
+
results.push(result);
|
|
334
|
+
appendLog(result, config.site);
|
|
335
|
+
callbacks?.onIteration?.(result);
|
|
336
|
+
consecutiveDiscards++;
|
|
337
|
+
continue;
|
|
338
|
+
}
|
|
339
|
+
// Phase 5: Verify
|
|
340
|
+
const verification = runVerify(config);
|
|
341
|
+
// Phase 5.5: Guard
|
|
342
|
+
const guardResult = runGuard(config);
|
|
343
|
+
// Phase 6: Decide
|
|
344
|
+
const improved = config.direction === "higher"
|
|
345
|
+
? verification.metric > bestMetric
|
|
346
|
+
: verification.metric < bestMetric;
|
|
347
|
+
const delta = Math.abs(verification.metric - bestMetric);
|
|
348
|
+
if (improved && delta >= config.minDelta && guardResult !== "fail") {
|
|
349
|
+
// Keep
|
|
350
|
+
bestMetric = verification.metric;
|
|
351
|
+
consecutiveDiscards = 0;
|
|
352
|
+
const result = {
|
|
353
|
+
iteration: i,
|
|
354
|
+
status: "keep",
|
|
355
|
+
metric: verification.metric,
|
|
356
|
+
description,
|
|
357
|
+
durationMs: Date.now() - startMs,
|
|
358
|
+
};
|
|
359
|
+
results.push(result);
|
|
360
|
+
appendLog(result, config.site);
|
|
361
|
+
callbacks?.onIteration?.(result);
|
|
362
|
+
callbacks?.onStatus?.(`KEEP: metric ${bestMetric} (${description})`);
|
|
363
|
+
}
|
|
364
|
+
else {
|
|
365
|
+
// Discard
|
|
366
|
+
safeRevert();
|
|
367
|
+
consecutiveDiscards++;
|
|
368
|
+
const reason = guardResult === "fail"
|
|
369
|
+
? "guard blocked"
|
|
370
|
+
: !improved
|
|
371
|
+
? "no improvement"
|
|
372
|
+
: "below min delta";
|
|
373
|
+
const result = {
|
|
374
|
+
iteration: i,
|
|
375
|
+
status: "discard",
|
|
376
|
+
metric: verification.metric,
|
|
377
|
+
description: `${reason}: ${description}`,
|
|
378
|
+
durationMs: Date.now() - startMs,
|
|
379
|
+
};
|
|
380
|
+
results.push(result);
|
|
381
|
+
appendLog(result, config.site);
|
|
382
|
+
callbacks?.onIteration?.(result);
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
return results;
|
|
386
|
+
}
|
|
387
|
+
// ── Log Reading ──────────────────────────────────────────────────────────
|
|
388
|
+
export function readResearchLog(opts) {
|
|
389
|
+
if (!existsSync(LOG_FILE))
|
|
390
|
+
return [];
|
|
391
|
+
const lines = readFileSync(LOG_FILE, "utf-8").trim().split("\n");
|
|
392
|
+
const results = [];
|
|
393
|
+
for (const line of lines) {
|
|
394
|
+
if (!line.trim())
|
|
395
|
+
continue;
|
|
396
|
+
const parts = line.split("\t");
|
|
397
|
+
if (parts.length < 7)
|
|
398
|
+
continue;
|
|
399
|
+
const [ts, site, iteration, status, metric, description, durationMs] = parts;
|
|
400
|
+
if (opts?.site && site !== opts.site)
|
|
401
|
+
continue;
|
|
402
|
+
if (opts?.since && new Date(ts).getTime() < opts.since)
|
|
403
|
+
continue;
|
|
404
|
+
results.push({
|
|
405
|
+
iteration: parseInt(iteration, 10),
|
|
406
|
+
status: status,
|
|
407
|
+
metric: parseFloat(metric),
|
|
408
|
+
description,
|
|
409
|
+
durationMs: parseInt(durationMs, 10),
|
|
410
|
+
});
|
|
411
|
+
}
|
|
412
|
+
return results;
|
|
413
|
+
}
|
|
414
|
+
//# sourceMappingURL=research.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"research.js","sourceRoot":"","sources":["../../src/engine/research.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,YAAY,EAA4B,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EACL,UAAU,EACV,YAAY,EACZ,WAAW,EACX,cAAc,EACd,SAAS,GACV,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAElC,oFAAoF;AACpF,MAAM,cAAc,GAAG,kBAAkB,CAAC;AA8B1C,4EAA4E;AAE5E,MAAM,WAAW,GAAG;IAClB,6EAA6E;IAC7E,4CAA4C;IAC5C,yEAAyE;IACzE,sDAAsD;IACtD,0DAA0D;CAC3D,CAAC;AAEF,4EAA4E;AAE5E,MAAM,SAAS,GAAwB,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AAC9E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,CAAC,CAAC;AAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;AAE/C,SAAS,aAAa,CAAC,MAAc,EAAE,WAAmB;IACxD,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,WAAW,CAAC,CAAC;IACnC,wDAAwD;IACxD,EAAE,CAAC,SAAS,GAAG,CAAC,CAAC;IACjB,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9B,IAAI,CAAC,KAAK;QAAE,OAAO,CAAC,CAAC;IACrB,uEAAuE;IACvE,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC;IACjC,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAC5B,OAAO,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACxC,CAAC;AAED,SAAS,OAAO,CAAC,IAAc;IAC7B,OAAO,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,SAAS,CAAW,CAAC;AACxD,CAAC;AAED,SAAS,SAAS,CAAC,MAAuB,EAAE,IAAY;IACtD,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG;QACX,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACxB,IAAI;QACJ,MAAM,CAAC,SAAS;QAChB,MAAM,CAAC,MAAM;QACb,MAAM,CAAC,MAAM;QACb,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;QAChC,MAAM,CAAC,UAAU;KAClB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACb,cAAc,CAAC,QAAQ,EAAE,IAAI,GAAG,IAAI,CAAC,CAAC;AACxC,CAAC;AAED,2EAA2E;AAE3E;;;GAGG;AACH,SAAS,YAAY,CAAC,QAAkB;IACtC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,oDAAoD;QACpD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,OAAO,KAAK,CAAC,CAAC,EAAE,CAAC;YACnB,kCAAkC;YAClC,IAAI,UAAU,CAAC,OAAO,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC7C,SAAS;QACX,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;QAChE,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1D,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAC/B,IAAI,CAAC;YACH,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrC,IAAI,SAAS,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC;oBAAE,SAAS;gBACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gBAClC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,2BAA2B;QAC7B,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,2EAA2E;AAE3E,SAAS,kBAAkB;IACzB,wBAAwB;IACxB,IAAI,CAAC;QACH,OAAO,CAAC,CAAC,WAAW,EAAE,uBAAuB,CAAC,CAAC,CAAC;IAClD,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;IACjD,CAAC;IAED,gBAAgB;IAChB,IAAI,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CACb,6DAA6D,CAC9D,CAAC;IACJ,CAAC;IAED,oBAAoB;IACpB,IAAI,CAAC;QACH,OAAO,CAAC,CAAC,cAAc,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,CAAC;IAED,qCAAqC;IACrC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CACb,8DAA8D,CAC/D,CAAC;IACJ,CAAC;AACH,CAAC;AAED,2EAA2E;AAE3E,SAAS,SAAS,CAAC,MAAsB;IAIvC,4EAA4E;IAC5E,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QACtC,OAAO,EAAE,MAAM,EAAE,mBAAmB,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IACpD,CAAC;IACD,IAAI,CAAC;QACH,gEAAgE;QAChE,MAAM,MAAM,GAAG,YAAY,CACzB,QAAQ,EACR,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,EACtC,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAC/D,CAAC;QACZ,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;IAClE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,MAAM,GACV,GAAG,YAAY,KAAK,IAAI,QAAQ,IAAI,GAAG;YACrC,CAAC,CAAC,MAAM,CAAE,GAA2B,CAAC,MAAM,CAAC;YAC7C,CAAC,CAAC,EAAE,CAAC;QACT,OAAO;YACL,MAAM;YACN,MAAM,EACJ,MAAM,CAAC,SAAS,KAAK,OAAO;gBAC1B,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC;SAC3C,CAAC;IACJ,CAAC;AACH,CAAC;AAED,2EAA2E;AAE3E,SAAS,QAAQ,CAAC,MAAsB;IACtC,IAAI,CAAC,MAAM,CAAC,KAAK;QAAE,OAAO,MAAM,CAAC;IACjC,yEAAyE;IACzE,IAAI,CAAC;QACH,YAAY,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE;YAC/C,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,OAAO;YAChB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM,CAAC;IAChB,CAAC;AACH,CAAC;AAED,2EAA2E;AAE3E,SAAS,UAAU;IACjB,IAAI,CAAC;QACH,OAAO,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;IAC3C,CAAC;IAAC,MAAM,CAAC;QACP,IAAI,CAAC;YACH,OAAO,CAAC,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;QACjC,CAAC;QAAC,MAAM,CAAC;YACP,YAAY;QACd,CAAC;QACD,OAAO,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;IACzC,CAAC;AACH,CAAC;AAED,2EAA2E;AAE3E,SAAS,iBAAiB,CACxB,MAAsB,EACtB,SAAiB,EACjB,UAAkB,EAClB,aAAqB,EACrB,mBAA2B,EAC3B,SAA4B,EAC5B,aAAqB;IAErB,MAAM,SAAS,GACb,mBAAmB,IAAI,CAAC;QACtB,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,mBAAmB,GAAG,CAAC,EAAE,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACxE,CAAC,CAAC,IAAI,CAAC;IAEX,MAAM,UAAU,GAAG,SAAS;SACzB,KAAK,CAAC,CAAC,EAAE,CAAC;SACV,GAAG,CACF,CAAC,CAAC,EAAE,EAAE,CACJ,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,WAAW,EAAE,CACzE;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO,iDAAiD,MAAM,CAAC,IAAI;;;EAGnE,MAAM,CAAC,IAAI;;;eAGE,SAAS;oBACJ,aAAa;iBAChB,UAAU;0BACD,mBAAmB;EAC3C,SAAS,CAAC,CAAC,CAAC,oBAAoB,SAAS,IAAI,CAAC,CAAC,CAAC,EAAE;;EAElD,UAAU,IAAI,QAAQ;;;EAGtB,aAAa;;;;oDAIqC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;4BAC/C,MAAM,CAAC,MAAM;uBAClB,MAAM,CAAC,SAAS;;6CAEM,CAAC;AAC9C,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAc;IACtC,IAAI,CAAC;QACH,yEAAyE;QACzE,0DAA0D;QAC1D,MAAM,MAAM,GAAG,YAAY,CACzB,QAAQ,EACR;YACE,IAAI;YACJ,gCAAgC;YAChC,gBAAgB;YAChB,qBAAqB;YACrB,iBAAiB;YACjB,MAAM;YACN,0BAA0B;YAC1B,MAAM;SACP,EACD,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAC/D,CAAC;QAEZ,+CAA+C;QAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACxD,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACtE,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,8BAA8B,CAAC;IACpE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,4EAA4E;AAE5E,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAsB,EACtB,SAA6B;IAE7B,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,IAAI,UAAkB,CAAC;IACvB,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAE5B,yBAAyB;IACzB,kBAAkB,EAAE,CAAC;IAErB,uBAAuB;IACvB,SAAS,EAAE,QAAQ,EAAE,CAAC,kCAAkC,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IACnC,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC;IAE7B,MAAM,cAAc,GAAoB;QACtC,SAAS,EAAE,CAAC;QACZ,MAAM,EAAE,UAAU;QAClB,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,WAAW,EAAE,sBAAsB;QACnC,UAAU,EAAE,CAAC;KACd,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC7B,SAAS,CAAC,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;IACvC,SAAS,EAAE,WAAW,EAAE,CAAC,cAAc,CAAC,CAAC;IACzC,SAAS,EAAE,QAAQ,EAAE,CAAC,oBAAoB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAE7D,YAAY;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE3B,0DAA0D;QAC1D,IAAI,mBAAmB,IAAI,EAAE,EAAE,CAAC;YAC9B,SAAS,EAAE,QAAQ,EAAE,CACnB,eAAe,mBAAmB,kCAAkC,CACrE,CAAC;YACF,MAAM;QACR,CAAC;QAED,SAAS,EAAE,QAAQ,EAAE,CACnB,aAAa,CAAC,IAAI,MAAM,CAAC,aAAa,UAAU,UAAU,cAAc,mBAAmB,GAAG,CAC/F,CAAC;QAEF,2DAA2D;QAC3D,IAAI,aAAa,GAAG,EAAE,CAAC;QACvB,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC9C,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;YAC3B,aAAa,IAAI,SAAS,CAAC,SAAS,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC;QACnE,CAAC;QAED,oBAAoB;QACpB,MAAM,MAAM,GAAG,iBAAiB,CAC9B,MAAM,EACN,CAAC,EACD,UAAU,EACV,UAAU,EAAE,wCAAwC;QACpD,mBAAmB,EACnB,OAAO,EACP,aAAa,CACd,CAAC;QACF,MAAM,WAAW,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAE7C,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,MAAM,GAAoB;gBAC9B,SAAS,EAAE,CAAC;gBACZ,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,UAAU;gBAClB,WAAW,EAAE,+BAA+B;gBAC5C,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YAC/B,SAAS,EAAE,WAAW,EAAE,CAAC,MAAM,CAAC,CAAC;YACjC,mBAAmB,EAAE,CAAC;YACtB,SAAS;QACX,CAAC;QAED,kBAAkB;QAClB,IAAI,CAAC;YACH,+DAA+D;YAC/D,MAAM,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAChD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,OAAO,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC;YAC1C,CAAC;YAED,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC5D,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM,MAAM,GAAoB;oBAC9B,SAAS,EAAE,CAAC;oBACZ,MAAM,EAAE,OAAO;oBACf,MAAM,EAAE,UAAU;oBAClB,WAAW,EAAE,iBAAiB;oBAC9B,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;iBACjC,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrB,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC/B,SAAS,EAAE,WAAW,EAAE,CAAC,MAAM,CAAC,CAAC;gBACjC,mBAAmB,EAAE,CAAC;gBACtB,SAAS;YACX,CAAC;YAED,IAAI,CAAC;gBACH,OAAO,CAAC,CAAC,QAAQ,EAAE,IAAI,EAAE,YAAY,MAAM,CAAC,IAAI,MAAM,WAAW,EAAE,CAAC,CAAC,CAAC;YACxE,CAAC;YAAC,MAAM,CAAC;gBACP,eAAe;gBACf,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;gBAC3B,MAAM,MAAM,GAAoB;oBAC9B,SAAS,EAAE,CAAC;oBACZ,MAAM,EAAE,cAAc;oBACtB,MAAM,EAAE,UAAU;oBAClB,WAAW,EAAE,yBAAyB;oBACtC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;iBACjC,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrB,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC/B,SAAS,EAAE,WAAW,EAAE,CAAC,MAAM,CAAC,CAAC;gBACjC,mBAAmB,EAAE,CAAC;gBACtB,SAAS;YACX,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,MAAM,GAAoB;gBAC9B,SAAS,EAAE,CAAC;gBACZ,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,UAAU;gBAClB,WAAW,EAAE,mBAAmB;gBAChC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YAC/B,SAAS,EAAE,WAAW,EAAE,CAAC,MAAM,CAAC,CAAC;YACjC,mBAAmB,EAAE,CAAC;YACtB,SAAS;QACX,CAAC;QAED,kBAAkB;QAClB,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;QAEvC,mBAAmB;QACnB,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QAErC,kBAAkB;QAClB,MAAM,QAAQ,GACZ,MAAM,CAAC,SAAS,KAAK,QAAQ;YAC3B,CAAC,CAAC,YAAY,CAAC,MAAM,GAAG,UAAU;YAClC,CAAC,CAAC,YAAY,CAAC,MAAM,GAAG,UAAU,CAAC;QAEvC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC;QAEzD,IAAI,QAAQ,IAAI,KAAK,IAAI,MAAM,CAAC,QAAQ,IAAI,WAAW,KAAK,MAAM,EAAE,CAAC;YACnE,OAAO;YACP,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC;YACjC,mBAAmB,GAAG,CAAC,CAAC;YACxB,MAAM,MAAM,GAAoB;gBAC9B,SAAS,EAAE,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,MAAM,EAAE,YAAY,CAAC,MAAM;gBAC3B,WAAW;gBACX,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YAC/B,SAAS,EAAE,WAAW,EAAE,CAAC,MAAM,CAAC,CAAC;YACjC,SAAS,EAAE,QAAQ,EAAE,CAAC,gBAAgB,UAAU,KAAK,WAAW,GAAG,CAAC,CAAC;QACvE,CAAC;aAAM,CAAC;YACN,UAAU;YACV,UAAU,EAAE,CAAC;YACb,mBAAmB,EAAE,CAAC;YACtB,MAAM,MAAM,GACV,WAAW,KAAK,MAAM;gBACpB,CAAC,CAAC,eAAe;gBACjB,CAAC,CAAC,CAAC,QAAQ;oBACT,CAAC,CAAC,gBAAgB;oBAClB,CAAC,CAAC,iBAAiB,CAAC;YAC1B,MAAM,MAAM,GAAoB;gBAC9B,SAAS,EAAE,CAAC;gBACZ,MAAM,EAAE,SAAS;gBACjB,MAAM,EAAE,YAAY,CAAC,MAAM;gBAC3B,WAAW,EAAE,GAAG,MAAM,KAAK,WAAW,EAAE;gBACxC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YAC/B,SAAS,EAAE,WAAW,EAAE,CAAC,MAAM,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,4EAA4E;AAE5E,MAAM,UAAU,eAAe,CAAC,IAG/B;IACC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,EAAE,CAAC;IACrC,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACjE,MAAM,OAAO,GAAsB,EAAE,CAAC;IAEtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,SAAS;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC/B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAE/B,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,CAAC,GAClE,KAAK,CAAC;QAER,IAAI,IAAI,EAAE,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC,IAAI;YAAE,SAAS;QAC/C,IAAI,IAAI,EAAE,KAAK,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK;YAAE,SAAS;QAEjE,OAAO,CAAC,IAAI,CAAC;YACX,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC;YAClC,MAAM,EAAE,MAAmC;YAC3C,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC;YAC1B,WAAW;YACX,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,EAAE,CAAC;SACrC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"yaml-runner.d.ts","sourceRoot":"","sources":["../../src/engine/yaml-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;
|
|
1
|
+
{"version":3,"file":"yaml-runner.d.ts","sourceRoot":"","sources":["../../src/engine/yaml-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAkBH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAMhD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAmBtD,MAAM,WAAW,eAAe;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,KAAK,eAAe,GAAG;IACrB,IAAI,EAAE,OAAO,CAAC;IACd,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,WAAW,CAAC;CACpB,CAAC;AAEF;;;;GAIG;AACH,qBAAa,aAAc,SAAQ,KAAK;aAGpB,MAAM,EAAE;QACtB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,OAAO,CAAC;QAChB,SAAS,EACL,YAAY,GACZ,eAAe,GACf,cAAc,GACd,aAAa,GACb,SAAS,GACT,kBAAkB,GAClB,kBAAkB,CAAC;QACvB,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,UAAU,EAAE,MAAM,CAAC;KACpB;gBAjBD,OAAO,EAAE,MAAM,EACC,MAAM,EAAE;QACtB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,OAAO,CAAC;QAChB,SAAS,EACL,YAAY,GACZ,eAAe,GACf,cAAc,GACd,aAAa,GACb,SAAS,GACT,kBAAkB,GAClB,kBAAkB,CAAC;QACvB,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,UAAU,EAAE,MAAM,CAAC;KACpB;IAMH,4EAA4E;IAC5E,WAAW,CAAC,WAAW,CAAC,EAAE,MAAM;cAtBtB,MAAM;gBACJ,MAAM;gBACN,OAAO;mBAEX,YAAY,GACZ,eAAe,GACf,cAAc,GACd,aAAa,GACb,SAAS,GACT,kBAAkB,GAClB,kBAAkB;cAChB,MAAM;qBACC,MAAM;0BACD,MAAM;oBACZ,MAAM;;;;CAevB;AAmID,wBAAsB,WAAW,CAC/B,KAAK,EAAE,YAAY,EAAE,EACrB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC7B,IAAI,CAAC,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,OAAO,EAAE,CAAC,CAqPpB;AA+wBD,iBAAS,UAAU,CAAC,GAAG,EAAE,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAqBjE;AAwOD;;;;;GAKG;AACH,QAAA,MAAM,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,OAAO,CAyFjE,CAAC;AA0GF;;;;;;;GAOG;AACH,iBAAS,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CA6D7E;AAqwBD,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,UAAU,EAAE,CAAC"}
|
|
@@ -15,11 +15,11 @@
|
|
|
15
15
|
* Available variables: item, index, args, base, temp
|
|
16
16
|
*/
|
|
17
17
|
import { execFile } from "node:child_process";
|
|
18
|
-
import { existsSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
|
|
18
|
+
import { existsSync, mkdirSync, writeFileSync, rmSync, readFileSync, } from "node:fs";
|
|
19
19
|
import { stat } from "node:fs/promises";
|
|
20
20
|
import { join, resolve } from "node:path";
|
|
21
21
|
import { tmpdir, homedir } from "node:os";
|
|
22
|
-
import { randomBytes } from "node:crypto";
|
|
22
|
+
import { randomBytes, createHash } from "node:crypto";
|
|
23
23
|
import { promisify } from "node:util";
|
|
24
24
|
import { runInNewContext } from "node:vm";
|
|
25
25
|
import TurndownService from "turndown";
|
|
@@ -501,11 +501,83 @@ async function stepFetch(ctx, config) {
|
|
|
501
501
|
const resolvedConfig = config.body
|
|
502
502
|
? { ...config, body: resolveTemplateDeep(config.body, ctx) }
|
|
503
503
|
: config;
|
|
504
|
-
|
|
505
|
-
|
|
504
|
+
// Strategy fallback: if no cookie and fetch returns 401/403, try with cookies
|
|
505
|
+
try {
|
|
506
|
+
const data = await fetchJson(url, resolvedConfig, ctx.cookieHeader);
|
|
507
|
+
return { ...ctx, data };
|
|
508
|
+
}
|
|
509
|
+
catch (err) {
|
|
510
|
+
if (err instanceof PipelineError &&
|
|
511
|
+
(err.detail.statusCode === 401 || err.detail.statusCode === 403) &&
|
|
512
|
+
!ctx.cookieHeader) {
|
|
513
|
+
// Attempt cookie fallback — try loading cookies for the domain
|
|
514
|
+
try {
|
|
515
|
+
const hostname = new URL(url).hostname;
|
|
516
|
+
const siteName = hostname
|
|
517
|
+
.replace(/^www\./, "")
|
|
518
|
+
.split(".")
|
|
519
|
+
.slice(0, -1)
|
|
520
|
+
.join("-");
|
|
521
|
+
const cookies = await loadCookiesWithCDP(siteName);
|
|
522
|
+
if (cookies) {
|
|
523
|
+
const fallbackCookie = formatCookieHeader(cookies);
|
|
524
|
+
const data = await fetchJson(url, resolvedConfig, fallbackCookie);
|
|
525
|
+
return { ...ctx, data, cookieHeader: fallbackCookie };
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
catch {
|
|
529
|
+
// Cookie fallback also failed — throw original
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
throw err;
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
// --- Fetch response cache ---
|
|
536
|
+
const CACHE_DIR = join(homedir(), ".unicli", "cache");
|
|
537
|
+
function fetchCacheKey(url, method) {
|
|
538
|
+
return createHash("sha256")
|
|
539
|
+
.update(`${method}:${url}`)
|
|
540
|
+
.digest("hex")
|
|
541
|
+
.slice(0, 16);
|
|
542
|
+
}
|
|
543
|
+
function readFetchCache(url, method, ttlSeconds) {
|
|
544
|
+
const key = fetchCacheKey(url, method);
|
|
545
|
+
const filePath = join(CACHE_DIR, `${key}.json`);
|
|
546
|
+
if (!existsSync(filePath))
|
|
547
|
+
return null;
|
|
548
|
+
try {
|
|
549
|
+
const raw = readFileSync(filePath, "utf-8");
|
|
550
|
+
const entry = JSON.parse(raw);
|
|
551
|
+
if (Date.now() - entry.ts > ttlSeconds * 1000)
|
|
552
|
+
return null;
|
|
553
|
+
return entry.data;
|
|
554
|
+
}
|
|
555
|
+
catch {
|
|
556
|
+
return null;
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
const MAX_CACHE_ENTRY_BYTES = 10 * 1024 * 1024; // 10MB per entry
|
|
560
|
+
function writeFetchCache(url, method, data) {
|
|
561
|
+
try {
|
|
562
|
+
const payload = JSON.stringify({ ts: Date.now(), url, data });
|
|
563
|
+
if (payload.length > MAX_CACHE_ENTRY_BYTES)
|
|
564
|
+
return; // reject oversized responses
|
|
565
|
+
mkdirSync(CACHE_DIR, { recursive: true });
|
|
566
|
+
const key = fetchCacheKey(url, method);
|
|
567
|
+
writeFileSync(join(CACHE_DIR, `${key}.json`), payload);
|
|
568
|
+
}
|
|
569
|
+
catch {
|
|
570
|
+
/* cache write failure is non-fatal */
|
|
571
|
+
}
|
|
506
572
|
}
|
|
507
573
|
async function fetchJson(url, config, cookieHeader) {
|
|
508
574
|
const method = config.method ?? "GET";
|
|
575
|
+
// Check cache before making network request
|
|
576
|
+
if (config.cache && config.cache > 0) {
|
|
577
|
+
const cached = readFetchCache(url, method, config.cache);
|
|
578
|
+
if (cached !== null)
|
|
579
|
+
return cached;
|
|
580
|
+
}
|
|
509
581
|
const headers = {
|
|
510
582
|
Accept: "application/json",
|
|
511
583
|
"User-Agent": USER_AGENT,
|
|
@@ -528,7 +600,10 @@ async function fetchJson(url, config, cookieHeader) {
|
|
|
528
600
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
529
601
|
const resp = await fetch(url, init);
|
|
530
602
|
if (resp.ok) {
|
|
531
|
-
|
|
603
|
+
const data = await resp.json();
|
|
604
|
+
if (config.cache && config.cache > 0)
|
|
605
|
+
writeFetchCache(url, method, data);
|
|
606
|
+
return data;
|
|
532
607
|
}
|
|
533
608
|
const isRetryable = resp.status === 429 || resp.status >= 500;
|
|
534
609
|
const isLastAttempt = attempt === maxAttempts;
|