explorbot 0.3.5 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/explorbot-cli.ts +18 -13
- package/boat/doc-collector/src/cli.ts +3 -0
- package/boat/doc-collector/src/docbot.ts +3 -1
- package/boat/prima/src/cli.ts +21 -8
- package/boat/prima/src/envelope.ts +35 -9
- package/boat/prima/src/prima.ts +23 -10
- package/dist/bin/explorbot-cli.js +19 -13
- package/dist/boat/doc-collector/src/cli.js +3 -0
- package/dist/boat/doc-collector/src/docbot.js +3 -1
- package/dist/boat/prima/src/cli.js +19 -8
- package/dist/boat/prima/src/envelope.js +24 -6
- package/dist/boat/prima/src/prima.js +23 -11
- package/dist/package.json +2 -2
- package/dist/src/action-result.d.ts +9 -1
- package/dist/src/action-result.js +57 -18
- package/dist/src/action.d.ts +1 -1
- package/dist/src/action.js +87 -12
- package/dist/src/ai/driller.d.ts +0 -1
- package/dist/src/ai/driller.js +8 -20
- package/dist/src/ai/fisherman-tools.d.ts +9 -0
- package/dist/src/ai/fisherman-tools.js +52 -6
- package/dist/src/ai/fisherman.d.ts +4 -2
- package/dist/src/ai/fisherman.js +48 -27
- package/dist/src/ai/historian/codeceptjs.js +1 -1
- package/dist/src/ai/historian/playwright.js +1 -1
- package/dist/src/ai/pilot.d.ts +1 -0
- package/dist/src/ai/pilot.js +13 -1
- package/dist/src/ai/provider.js +20 -3
- package/dist/src/ai/researcher/deep-analysis.js +1 -3
- package/dist/src/ai/researcher.js +3 -3
- package/dist/src/ai/tester.d.ts +3 -0
- package/dist/src/ai/tester.js +40 -3
- package/dist/src/ai/tools.d.ts +1 -0
- package/dist/src/ai/tools.js +13 -6
- package/dist/src/api/request-result.d.ts +2 -0
- package/dist/src/api/request-result.js +8 -2
- package/dist/src/api/request-store.d.ts +3 -2
- package/dist/src/api/request-store.js +66 -14
- package/dist/src/commands/explore-command.d.ts +6 -0
- package/dist/src/commands/explore-command.js +27 -2
- package/dist/src/commands/freesail-command.js +10 -1
- package/dist/src/commands/plans-command.js +6 -6
- package/dist/src/config.js +1 -0
- package/dist/src/experience-tracker.js +5 -0
- package/dist/src/explorbot.d.ts +0 -1
- package/dist/src/explorbot.js +23 -36
- package/dist/src/state-manager.d.ts +5 -1
- package/dist/src/state-manager.js +10 -7
- package/dist/src/test-plan.d.ts +3 -0
- package/dist/src/test-plan.js +27 -0
- package/dist/src/utils/aria.d.ts +1 -1
- package/dist/src/utils/aria.js +6 -42
- package/dist/src/utils/html-diff.d.ts +4 -0
- package/dist/src/utils/html-diff.js +62 -7
- package/dist/src/utils/html.d.ts +5 -15
- package/dist/src/utils/html.js +14 -85
- package/dist/src/utils/overlay.d.ts +56 -11
- package/dist/src/utils/overlay.js +191 -21
- package/dist/src/utils/request-map.d.ts +7 -0
- package/dist/src/utils/request-map.js +16 -0
- package/dist/src/utils/url-matcher.js +4 -2
- package/docs/reference/commands.md +8 -1
- package/docs/reference/websocket.md +1 -0
- package/docs/superpowers/plans/2026-08-29-fisherman-reliability.md +953 -0
- package/docs/superpowers/plans/2026-08-29-region-states.md +1292 -0
- package/docs/superpowers/plans/2026-08-30-fisherman-live-session-auth.md +457 -0
- package/docs/superpowers/specs/2026-08-29-fisherman-reliability-design.md +45 -0
- package/docs/superpowers/specs/2026-08-29-region-states-design.md +262 -0
- package/docs/superpowers/specs/2026-08-29-region-states-fixes-design.md +269 -0
- package/docs/superpowers/specs/2026-08-30-fisherman-live-session-auth-design.md +37 -0
- package/docs/workflow/agentic-usage.md +1 -0
- package/docs/workflow/ci.md +1 -0
- package/package.json +2 -2
- package/src/action-result.ts +61 -22
- package/src/action.ts +87 -14
- package/src/ai/driller.ts +7 -39
- package/src/ai/fisherman-tools.ts +56 -7
- package/src/ai/fisherman.ts +48 -28
- package/src/ai/historian/codeceptjs.ts +1 -1
- package/src/ai/historian/playwright.ts +1 -1
- package/src/ai/pilot.ts +9 -1
- package/src/ai/provider.ts +21 -3
- package/src/ai/researcher/deep-analysis.ts +1 -2
- package/src/ai/researcher.ts +3 -3
- package/src/ai/tester.ts +40 -3
- package/src/ai/tools.ts +17 -9
- package/src/api/request-result.ts +10 -2
- package/src/api/request-store.ts +60 -13
- package/src/commands/explore-command.ts +25 -2
- package/src/commands/freesail-command.ts +7 -1
- package/src/commands/plans-command.ts +6 -6
- package/src/config.ts +1 -0
- package/src/experience-tracker.ts +5 -1
- package/src/explorbot.ts +20 -36
- package/src/state-manager.ts +13 -7
- package/src/test-plan.ts +29 -0
- package/src/utils/aria.ts +7 -44
- package/src/utils/html-diff.ts +62 -7
- package/src/utils/html.ts +14 -91
- package/src/utils/overlay.ts +226 -23
- package/src/utils/request-map.ts +19 -0
- package/src/utils/url-matcher.ts +3 -2
|
@@ -64,6 +64,9 @@ export class RequestResult {
|
|
|
64
64
|
return raw;
|
|
65
65
|
}
|
|
66
66
|
}
|
|
67
|
+
get isWrite() {
|
|
68
|
+
return ['POST', 'PUT', 'PATCH', 'DELETE'].includes(this.method);
|
|
69
|
+
}
|
|
67
70
|
save(outputDir) {
|
|
68
71
|
const requestsDir = path.join(outputDir, 'requests');
|
|
69
72
|
if (!existsSync(requestsDir)) {
|
|
@@ -137,14 +140,17 @@ export class RequestResult {
|
|
|
137
140
|
statusText: meta.statusText || '',
|
|
138
141
|
responseHeaders: meta.responseHeaders || {},
|
|
139
142
|
timing: Number.parseInt(meta.timing) || 0,
|
|
140
|
-
timestamp: new Date(meta.timestamp ||
|
|
143
|
+
timestamp: new Date(meta.timestamp || 0),
|
|
141
144
|
});
|
|
142
145
|
result.requestFile = requestFile;
|
|
143
146
|
result.responseFile = responseFile;
|
|
144
147
|
return result;
|
|
145
148
|
}
|
|
149
|
+
toEndpoint() {
|
|
150
|
+
return `${this.method} ${this.path}`;
|
|
151
|
+
}
|
|
146
152
|
toSummary() {
|
|
147
|
-
return `${this.
|
|
153
|
+
return `${this.toEndpoint()} → ${this.status} (${this.timing}ms)`;
|
|
148
154
|
}
|
|
149
155
|
extractIdAndTitle() {
|
|
150
156
|
const body = this.responseBody;
|
|
@@ -5,6 +5,7 @@ export declare class RequestStore {
|
|
|
5
5
|
failedRequests: RequestResult[];
|
|
6
6
|
onFailedListeners: Array<(r: RequestResult) => void>;
|
|
7
7
|
outputDir: string;
|
|
8
|
+
sessionStartedAt: Date;
|
|
8
9
|
constructor(outputDir: string);
|
|
9
10
|
addCapturedRequest(result: RequestResult): void;
|
|
10
11
|
addFailedRequest(result: RequestResult): void;
|
|
@@ -19,9 +20,9 @@ export declare class RequestStore {
|
|
|
19
20
|
getRequestsByEndpoint(pathPrefix: string): RequestResult[];
|
|
20
21
|
getRequestsByMethod(method: string): RequestResult[];
|
|
21
22
|
getRequestsByStatus(status: number): RequestResult[];
|
|
22
|
-
toEndpointList(): string;
|
|
23
|
+
toEndpointList(scopePath?: string): string;
|
|
23
24
|
extractAuthHeaders(): Record<string, string>;
|
|
24
|
-
findCapturedRequest(method: string,
|
|
25
|
+
findCapturedRequest(method: string, searchPath: string): RequestResult | undefined;
|
|
25
26
|
toLog(): string;
|
|
26
27
|
loadFromDisk(): void;
|
|
27
28
|
getWriteRequestsForScope(scopePath: string): RequestResult[];
|
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import { existsSync, readdirSync } from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
|
+
import { isDynamicSegment } from "../utils/url-matcher.js";
|
|
3
4
|
import { RequestResult } from "./request-result.js";
|
|
4
|
-
const AUTH_HEADERS = ['authorization', '
|
|
5
|
+
const AUTH_HEADERS = ['authorization', 'x-api-key', 'x-csrf-token'];
|
|
5
6
|
export class RequestStore {
|
|
6
7
|
capturedRequests = [];
|
|
7
8
|
madeRequests = [];
|
|
8
9
|
failedRequests = [];
|
|
9
10
|
onFailedListeners = [];
|
|
10
11
|
outputDir;
|
|
12
|
+
sessionStartedAt = new Date();
|
|
11
13
|
constructor(outputDir) {
|
|
12
14
|
this.outputDir = outputDir;
|
|
13
15
|
}
|
|
@@ -61,12 +63,14 @@ export class RequestStore {
|
|
|
61
63
|
getRequestsByStatus(status) {
|
|
62
64
|
return this.madeRequests.filter((r) => r.status === status);
|
|
63
65
|
}
|
|
64
|
-
toEndpointList() {
|
|
66
|
+
toEndpointList(scopePath) {
|
|
67
|
+
let requests = this.capturedRequests;
|
|
68
|
+
if (scopePath)
|
|
69
|
+
requests = this.getWriteRequestsForScope(scopePath);
|
|
65
70
|
const seen = new Set();
|
|
66
71
|
const lines = [];
|
|
67
|
-
for (const req of
|
|
68
|
-
const
|
|
69
|
-
const key = `${req.method} ${normalized}`;
|
|
72
|
+
for (const req of requests) {
|
|
73
|
+
const key = `${req.method} ${normalizePathPattern(req.path)}`;
|
|
70
74
|
if (seen.has(key))
|
|
71
75
|
continue;
|
|
72
76
|
seen.add(key);
|
|
@@ -76,21 +80,42 @@ export class RequestStore {
|
|
|
76
80
|
}
|
|
77
81
|
extractAuthHeaders() {
|
|
78
82
|
const headers = {};
|
|
79
|
-
|
|
80
|
-
|
|
83
|
+
const sessionCaptures = this.capturedRequests.filter((r) => r.timestamp >= this.sessionStartedAt).sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime());
|
|
84
|
+
for (const req of sessionCaptures) {
|
|
81
85
|
for (const [key, value] of Object.entries(req.requestHeaders)) {
|
|
82
86
|
if (AUTH_HEADERS.includes(key.toLowerCase()) && !headers[key]) {
|
|
83
87
|
headers[key] = value;
|
|
84
88
|
}
|
|
85
89
|
}
|
|
86
|
-
if (AUTH_HEADERS.every((h) => Object.keys(headers).some((k) => k.toLowerCase() === h)))
|
|
87
|
-
break;
|
|
88
90
|
}
|
|
89
91
|
return headers;
|
|
90
92
|
}
|
|
91
|
-
findCapturedRequest(method,
|
|
93
|
+
findCapturedRequest(method, searchPath) {
|
|
92
94
|
const upper = method.toUpperCase();
|
|
93
|
-
|
|
95
|
+
const search = normalizePathPattern(searchPath).split('/').filter(Boolean);
|
|
96
|
+
let best;
|
|
97
|
+
let bestScore = -1;
|
|
98
|
+
for (const req of this.capturedRequests) {
|
|
99
|
+
if (req.method !== upper)
|
|
100
|
+
continue;
|
|
101
|
+
const segments = normalizePathPattern(req.path).split('/').filter(Boolean);
|
|
102
|
+
if (segments.length < search.length)
|
|
103
|
+
continue;
|
|
104
|
+
if (!search.every((segment, i) => segment === segments[i]))
|
|
105
|
+
continue;
|
|
106
|
+
let score = 0;
|
|
107
|
+
if (segments.length === search.length)
|
|
108
|
+
score += 4;
|
|
109
|
+
if (req.status < 400)
|
|
110
|
+
score += 2;
|
|
111
|
+
if (score < bestScore)
|
|
112
|
+
continue;
|
|
113
|
+
if (score === bestScore && best && req.timestamp <= best.timestamp)
|
|
114
|
+
continue;
|
|
115
|
+
best = req;
|
|
116
|
+
bestScore = score;
|
|
117
|
+
}
|
|
118
|
+
return best;
|
|
94
119
|
}
|
|
95
120
|
toLog() {
|
|
96
121
|
return this.madeRequests.map((r, i) => `${i + 1}. ${r.toSummary()} [${r.id}]`).join('\n');
|
|
@@ -100,7 +125,7 @@ export class RequestStore {
|
|
|
100
125
|
if (!existsSync(requestsDir))
|
|
101
126
|
return;
|
|
102
127
|
const existingIds = new Set(this.capturedRequests.map((r) => r.id));
|
|
103
|
-
const files = readdirSync(requestsDir).filter((f) => f.endsWith('.request.yaml'));
|
|
128
|
+
const files = readdirSync(requestsDir).filter((f) => f.startsWith('xhr_') && f.endsWith('.request.yaml'));
|
|
104
129
|
for (const file of files) {
|
|
105
130
|
try {
|
|
106
131
|
const result = RequestResult.load(path.join(requestsDir, file));
|
|
@@ -115,7 +140,31 @@ export class RequestStore {
|
|
|
115
140
|
}
|
|
116
141
|
getWriteRequestsForScope(scopePath) {
|
|
117
142
|
const writeMethods = new Set(['POST', 'PUT', 'PATCH', 'DELETE']);
|
|
118
|
-
|
|
143
|
+
const writes = this.capturedRequests.filter((r) => writeMethods.has(r.method));
|
|
144
|
+
const scopeSegments = scopePath.split('/').filter(Boolean);
|
|
145
|
+
if (scopeSegments.length === 0)
|
|
146
|
+
return writes;
|
|
147
|
+
let scoped = [];
|
|
148
|
+
let fewest = Number.POSITIVE_INFINITY;
|
|
149
|
+
let ambiguous = false;
|
|
150
|
+
for (const segment of scopeSegments) {
|
|
151
|
+
if (isDynamicSegment(segment))
|
|
152
|
+
continue;
|
|
153
|
+
const matches = writes.filter((r) => r.path.split('/').includes(segment));
|
|
154
|
+
if (matches.length === 0 || matches.length > fewest)
|
|
155
|
+
continue;
|
|
156
|
+
if (matches.length === fewest) {
|
|
157
|
+
if (!scoped.every((r, i) => r.id === matches[i].id))
|
|
158
|
+
ambiguous = true;
|
|
159
|
+
continue;
|
|
160
|
+
}
|
|
161
|
+
scoped = matches;
|
|
162
|
+
fewest = matches.length;
|
|
163
|
+
ambiguous = false;
|
|
164
|
+
}
|
|
165
|
+
if (ambiguous)
|
|
166
|
+
return [];
|
|
167
|
+
return scoped;
|
|
119
168
|
}
|
|
120
169
|
clear() {
|
|
121
170
|
this.capturedRequests = [];
|
|
@@ -124,5 +173,8 @@ export class RequestStore {
|
|
|
124
173
|
}
|
|
125
174
|
}
|
|
126
175
|
function normalizePathPattern(urlPath) {
|
|
127
|
-
return urlPath
|
|
176
|
+
return urlPath
|
|
177
|
+
.split('/')
|
|
178
|
+
.map((segment) => (segment && isDynamicSegment(segment) ? '{id}' : segment))
|
|
179
|
+
.join('/');
|
|
128
180
|
}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { type Plan, type Test } from '../test-plan.js';
|
|
2
2
|
import { BaseCommand, type Suggestion } from './base-command.js';
|
|
3
|
+
export declare const DEADLINE_RESERVE_MS: number;
|
|
4
|
+
export declare const DEADLINE_TEST_ALLOWANCE_MS: number;
|
|
3
5
|
export declare class ExploreCommand extends BaseCommand {
|
|
4
6
|
name: string;
|
|
5
7
|
description: string;
|
|
@@ -9,8 +11,11 @@ export declare class ExploreCommand extends BaseCommand {
|
|
|
9
11
|
}[];
|
|
10
12
|
suggestions: Suggestion[];
|
|
11
13
|
maxTests?: number;
|
|
14
|
+
maxDurationMinutes?: number;
|
|
15
|
+
hardDeadlineAt?: number;
|
|
12
16
|
dryRun: boolean;
|
|
13
17
|
testsRun: number;
|
|
18
|
+
deadlineLogged: boolean;
|
|
14
19
|
completedPlans: Plan[];
|
|
15
20
|
failedSubPages: Set<string>;
|
|
16
21
|
oldTestRefs: Set<Test>;
|
|
@@ -36,6 +41,7 @@ export declare class ExploreCommand extends BaseCommand {
|
|
|
36
41
|
printResults(): void;
|
|
37
42
|
printNextSteps(savedPlanPath?: string | null): void;
|
|
38
43
|
isLimitReached(): boolean;
|
|
44
|
+
isDeadlineReached(): boolean;
|
|
39
45
|
runPendingTests(): Promise<void>;
|
|
40
46
|
runOneTest(test: Test): Promise<void>;
|
|
41
47
|
}
|
|
@@ -13,11 +13,14 @@ import { safeFilename } from "../utils/strings.js";
|
|
|
13
13
|
import { BaseCommand } from './base-command.js';
|
|
14
14
|
const MAX_SUB_PAGE_ATTEMPTS = 30;
|
|
15
15
|
const PRIORITY_ORDER = { critical: 0, important: 1, high: 2, normal: 3, low: 4 };
|
|
16
|
+
export const DEADLINE_RESERVE_MS = 3 * 60_000;
|
|
17
|
+
export const DEADLINE_TEST_ALLOWANCE_MS = 5 * 60_000;
|
|
16
18
|
export class ExploreCommand extends BaseCommand {
|
|
17
19
|
name = 'explore';
|
|
18
20
|
description = 'Start web exploration';
|
|
19
21
|
options = [
|
|
20
22
|
{ flags: '--max-tests <number>', description: 'Maximum number of tests to run' },
|
|
23
|
+
{ flags: '--max-duration <minutes>', description: 'Wall-clock budget in minutes; wraps up the session before the limit is hit' },
|
|
21
24
|
{ flags: '--focus <feature>', description: 'Focus area for exploration' },
|
|
22
25
|
{ flags: '--configure <spec>', description: 'Reuse spec: keys new|from|style|subpages|pick_by|priority, e.g. "new:25%;pick_by=random;priority=critical,high"' },
|
|
23
26
|
{ flags: '--dry-run', description: 'Mark picked tests as skipped without executing or generating new ones' },
|
|
@@ -28,8 +31,11 @@ export class ExploreCommand extends BaseCommand {
|
|
|
28
31
|
{ command: 'plan <feature>', hint: 'plan testing' },
|
|
29
32
|
];
|
|
30
33
|
maxTests;
|
|
34
|
+
maxDurationMinutes;
|
|
35
|
+
hardDeadlineAt;
|
|
31
36
|
dryRun = false;
|
|
32
37
|
testsRun = 0;
|
|
38
|
+
deadlineLogged = false;
|
|
33
39
|
completedPlans = [];
|
|
34
40
|
failedSubPages = new Set();
|
|
35
41
|
oldTestRefs = new Set();
|
|
@@ -43,6 +49,12 @@ export class ExploreCommand extends BaseCommand {
|
|
|
43
49
|
if (opts.maxTests) {
|
|
44
50
|
this.maxTests = Number.parseInt(opts.maxTests, 10);
|
|
45
51
|
}
|
|
52
|
+
if (opts.maxDuration) {
|
|
53
|
+
this.maxDurationMinutes = Number.parseInt(opts.maxDuration, 10);
|
|
54
|
+
}
|
|
55
|
+
if (this.hardDeadlineAt == null && this.maxDurationMinutes != null) {
|
|
56
|
+
this.hardDeadlineAt = Date.now() + this.maxDurationMinutes * 60_000 - DEADLINE_RESERVE_MS;
|
|
57
|
+
}
|
|
46
58
|
const feature = opts.focus || remaining.join(' ') || undefined;
|
|
47
59
|
const cfg = this.parseConfigure(opts.configure);
|
|
48
60
|
if (cfg.priorities)
|
|
@@ -515,7 +527,20 @@ export class ExploreCommand extends BaseCommand {
|
|
|
515
527
|
printNextSteps(sections);
|
|
516
528
|
}
|
|
517
529
|
isLimitReached() {
|
|
518
|
-
|
|
530
|
+
if (this.maxTests != null && this.testsRun >= this.maxTests)
|
|
531
|
+
return true;
|
|
532
|
+
return this.isDeadlineReached();
|
|
533
|
+
}
|
|
534
|
+
isDeadlineReached() {
|
|
535
|
+
if (this.hardDeadlineAt == null)
|
|
536
|
+
return false;
|
|
537
|
+
if (Date.now() < this.hardDeadlineAt - DEADLINE_TEST_ALLOWANCE_MS)
|
|
538
|
+
return false;
|
|
539
|
+
if (!this.deadlineLogged) {
|
|
540
|
+
this.deadlineLogged = true;
|
|
541
|
+
tag('info').log(`Time budget reached after ${this.testsRun} test(s): stopping new work and finishing the session`);
|
|
542
|
+
}
|
|
543
|
+
return true;
|
|
519
544
|
}
|
|
520
545
|
async runPendingTests() {
|
|
521
546
|
const plan = this.explorBot.getCurrentPlan();
|
|
@@ -539,7 +564,7 @@ export class ExploreCommand extends BaseCommand {
|
|
|
539
564
|
test.finish(TestResult.SKIPPED);
|
|
540
565
|
}
|
|
541
566
|
else {
|
|
542
|
-
await this.explorBot.agentTester().test(test);
|
|
567
|
+
await this.explorBot.agentTester().test(test, { deadline: this.hardDeadlineAt });
|
|
543
568
|
}
|
|
544
569
|
this.testsRun++;
|
|
545
570
|
}
|
|
@@ -4,7 +4,7 @@ import { Stats } from '../stats.js';
|
|
|
4
4
|
import { tag } from '../utils/logger.js';
|
|
5
5
|
import { loop } from '../utils/loop.js';
|
|
6
6
|
import { BaseCommand } from './base-command.js';
|
|
7
|
-
import { ExploreCommand } from './explore-command.js';
|
|
7
|
+
import { DEADLINE_RESERVE_MS, DEADLINE_TEST_ALLOWANCE_MS, ExploreCommand } from './explore-command.js';
|
|
8
8
|
export class FreesailCommand extends BaseCommand {
|
|
9
9
|
name = 'freesail';
|
|
10
10
|
description = 'Continuously explore and navigate to new pages autonomously';
|
|
@@ -15,6 +15,7 @@ export class FreesailCommand extends BaseCommand {
|
|
|
15
15
|
{ flags: '--shallow', description: 'Use shallow navigation strategy' },
|
|
16
16
|
{ flags: '--scope <url>', description: 'Limit navigation to URLs starting with this prefix' },
|
|
17
17
|
{ flags: '--max-tests <number>', description: 'Maximum number of tests to run' },
|
|
18
|
+
{ flags: '--max-duration <number>', description: 'Wall-clock budget in minutes for the whole run' },
|
|
18
19
|
];
|
|
19
20
|
async execute(args) {
|
|
20
21
|
Stats.mode = 'freesail';
|
|
@@ -26,11 +27,17 @@ export class FreesailCommand extends BaseCommand {
|
|
|
26
27
|
strategy = 'shallow';
|
|
27
28
|
const scope = opts.scope;
|
|
28
29
|
const maxTests = opts.maxTests ? Number.parseInt(opts.maxTests, 10) : undefined;
|
|
30
|
+
const maxDuration = opts.maxDuration ? Number.parseInt(opts.maxDuration, 10) : undefined;
|
|
31
|
+
let hardDeadlineAt;
|
|
32
|
+
if (maxDuration != null)
|
|
33
|
+
hardDeadlineAt = Date.now() + maxDuration * 60_000 - DEADLINE_RESERVE_MS;
|
|
29
34
|
await this.explorBot.visitInitialState();
|
|
30
35
|
let testsRun = 0;
|
|
31
36
|
await loop(async (ctx) => {
|
|
32
37
|
if (maxTests != null && testsRun >= maxTests)
|
|
33
38
|
ctx.stop();
|
|
39
|
+
if (hardDeadlineAt != null && Date.now() >= hardDeadlineAt - DEADLINE_TEST_ALLOWANCE_MS)
|
|
40
|
+
ctx.stop();
|
|
34
41
|
const stateManager = this.explorBot.stateManager();
|
|
35
42
|
const state = stateManager.getCurrentState();
|
|
36
43
|
if (state && !Researcher.getCachedResearch(state)) {
|
|
@@ -44,6 +51,8 @@ export class FreesailCommand extends BaseCommand {
|
|
|
44
51
|
const exploreCmd = new ExploreCommand(this.explorBot);
|
|
45
52
|
if (maxTests != null)
|
|
46
53
|
exploreCmd.maxTests = maxTests - testsRun;
|
|
54
|
+
if (hardDeadlineAt != null)
|
|
55
|
+
exploreCmd.hardDeadlineAt = hardDeadlineAt;
|
|
47
56
|
await exploreCmd.execute('');
|
|
48
57
|
const plan = this.explorBot.getCurrentPlan();
|
|
49
58
|
if (plan)
|
|
@@ -70,14 +70,14 @@ export class PlansCommand extends BaseCommand {
|
|
|
70
70
|
throw new Error(`Plan #${target} not found. Available: 1-${files.length}`);
|
|
71
71
|
return file;
|
|
72
72
|
}
|
|
73
|
-
const
|
|
74
|
-
if (!
|
|
75
|
-
throw new Error(`Plan file not found: ${
|
|
73
|
+
const plan = Plan.loadFromFile(target, this.explorBot.getPlansDir());
|
|
74
|
+
if (!plan?.filePath) {
|
|
75
|
+
throw new Error(`Plan file not found: ${target}`);
|
|
76
76
|
}
|
|
77
77
|
return {
|
|
78
|
-
name: path.basename(
|
|
79
|
-
path:
|
|
80
|
-
modifiedAt: statSync(
|
|
78
|
+
name: path.basename(plan.filePath),
|
|
79
|
+
path: plan.filePath,
|
|
80
|
+
modifiedAt: statSync(plan.filePath).mtimeMs,
|
|
81
81
|
};
|
|
82
82
|
}
|
|
83
83
|
}
|
package/dist/src/config.js
CHANGED
|
@@ -48,6 +48,7 @@ export const EXPLORBOT_ENV_VARS = [
|
|
|
48
48
|
{ name: 'EXPLORBOT_KNOWLEDGE_FILE', description: 'Path to a knowledge markdown file' },
|
|
49
49
|
{ name: 'EXPLORBOT_API_SPEC', description: 'OpenAPI spec path for the API boat' },
|
|
50
50
|
{ name: 'EXPLORBOT_NO_BANNER', description: 'Suppress the startup banner, for machine-readable output' },
|
|
51
|
+
{ name: 'EXPLORBOT_MAX_DURATION', description: 'Wall-clock budget in minutes for an explore run; same as --max-duration' },
|
|
51
52
|
];
|
|
52
53
|
export class ConfigParser {
|
|
53
54
|
static instance;
|
|
@@ -105,6 +105,11 @@ export class ExperienceTracker {
|
|
|
105
105
|
url: state.url ? extractStatePath(state.url) : '',
|
|
106
106
|
title: state.title,
|
|
107
107
|
};
|
|
108
|
+
if (state.overlay.present && state.overlay.name) {
|
|
109
|
+
frontmatter.region = state.overlay.name;
|
|
110
|
+
if (state.overlay.root)
|
|
111
|
+
frontmatter.root = state.overlay.root;
|
|
112
|
+
}
|
|
108
113
|
this.writeExperienceFile(stateHash, '', frontmatter);
|
|
109
114
|
}
|
|
110
115
|
return filePath;
|
package/dist/src/explorbot.d.ts
CHANGED
|
@@ -114,7 +114,6 @@ export declare class ExplorBot {
|
|
|
114
114
|
savePlan(filename?: string): string | null;
|
|
115
115
|
savePlans(plans: Plan[], filename?: string): string | null;
|
|
116
116
|
generatePlanFilename(feature?: string): string;
|
|
117
|
-
resolvePlanPath(filename: string): string;
|
|
118
117
|
loadPlan(filename: string): Plan;
|
|
119
118
|
loadPlans(filename: string): Plan[];
|
|
120
119
|
setCurrentPlan(plan?: Plan): void;
|
package/dist/src/explorbot.js
CHANGED
|
@@ -290,14 +290,22 @@ export class ExplorBot {
|
|
|
290
290
|
return null;
|
|
291
291
|
}
|
|
292
292
|
};
|
|
293
|
-
const
|
|
294
|
-
const
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
293
|
+
const browserHeaderProvider = async () => {
|
|
294
|
+
const session = await this.explorer
|
|
295
|
+
.withPage(async (page) => ({
|
|
296
|
+
cookies: await page.context().cookies(baseEndpoint),
|
|
297
|
+
csrf: await page.evaluate(() => document.querySelector('meta[name="csrf-token"]')?.getAttribute('content') || '').catch(() => ''),
|
|
298
|
+
}))
|
|
299
|
+
.catch(() => ({ cookies: [], csrf: '' }));
|
|
300
|
+
const headers = {};
|
|
301
|
+
if (session.cookies.length)
|
|
302
|
+
headers.Cookie = session.cookies.map((c) => `${c.name}=${c.value}`).join('; ');
|
|
303
|
+
if (session.csrf)
|
|
304
|
+
headers['x-csrf-token'] = session.csrf;
|
|
305
|
+
return headers;
|
|
298
306
|
};
|
|
299
307
|
this.agents.fisherman = this.createAgent(({ ai }) => {
|
|
300
|
-
return new Fisherman(ai, apiClient, requestStore, specLoader, baseEndpoint,
|
|
308
|
+
return new Fisherman(ai, apiClient, requestStore, specLoader, baseEndpoint, browserHeaderProvider, configHeaders, hasApiConfig);
|
|
301
309
|
});
|
|
302
310
|
}
|
|
303
311
|
return this.agents.fisherman;
|
|
@@ -383,39 +391,18 @@ export class ExplorBot {
|
|
|
383
391
|
return urlPart.slice(0, 256 - suffix.length) + suffix;
|
|
384
392
|
return urlPart + featurePart.slice(0, maxFeatureLen) + suffix;
|
|
385
393
|
}
|
|
386
|
-
resolvePlanPath(filename) {
|
|
387
|
-
let planPath = filename;
|
|
388
|
-
if (path.isAbsolute(filename)) {
|
|
389
|
-
if (!existsSync(planPath) && !filename.endsWith('.md')) {
|
|
390
|
-
planPath = `${filename}.md`;
|
|
391
|
-
}
|
|
392
|
-
}
|
|
393
|
-
else if (existsSync(filename) || existsSync(`${filename}.md`)) {
|
|
394
|
-
planPath = existsSync(filename) ? filename : `${filename}.md`;
|
|
395
|
-
}
|
|
396
|
-
else {
|
|
397
|
-
const plansDir = this.getPlansDir();
|
|
398
|
-
planPath = path.join(plansDir, filename);
|
|
399
|
-
if (!existsSync(planPath) && !filename.endsWith('.md')) {
|
|
400
|
-
planPath = path.join(plansDir, `${filename}.md`);
|
|
401
|
-
}
|
|
402
|
-
}
|
|
403
|
-
return planPath;
|
|
404
|
-
}
|
|
405
394
|
loadPlan(filename) {
|
|
406
|
-
const
|
|
407
|
-
if (!
|
|
408
|
-
throw new Error(`Plan file not found: ${
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
return this.currentPlan;
|
|
395
|
+
const plan = Plan.loadFromFile(filename, this.getPlansDir());
|
|
396
|
+
if (!plan)
|
|
397
|
+
throw new Error(`Plan file not found: ${filename}`);
|
|
398
|
+
this.setCurrentPlan(plan);
|
|
399
|
+
return plan;
|
|
412
400
|
}
|
|
413
401
|
loadPlans(filename) {
|
|
414
|
-
const
|
|
415
|
-
if (!
|
|
416
|
-
throw new Error(`Plan file not found: ${
|
|
417
|
-
|
|
418
|
-
return parsePlansFromMarkdown(planPath);
|
|
402
|
+
const plan = Plan.loadFromFile(filename, this.getPlansDir());
|
|
403
|
+
if (!plan?.filePath)
|
|
404
|
+
throw new Error(`Plan file not found: ${filename}`);
|
|
405
|
+
return parsePlansFromMarkdown(plan.filePath);
|
|
419
406
|
}
|
|
420
407
|
setCurrentPlan(plan) {
|
|
421
408
|
this.currentPlan = plan;
|
|
@@ -42,6 +42,10 @@ export interface WebPageState {
|
|
|
42
42
|
links?: Link[];
|
|
43
43
|
verifications?: Record<string, boolean>;
|
|
44
44
|
overlay?: Overlay;
|
|
45
|
+
/** Region name, the persisted scalar form of overlay.name used in experience frontmatter */
|
|
46
|
+
region?: string;
|
|
47
|
+
/** Region root selector, the persisted scalar form of overlay.root used in experience frontmatter */
|
|
48
|
+
root?: string;
|
|
45
49
|
}
|
|
46
50
|
export interface StateTransition {
|
|
47
51
|
/** Previous state (null if this is the first state) */
|
|
@@ -91,7 +95,7 @@ export declare class StateManager {
|
|
|
91
95
|
* Update state from basic data (for navigation events)
|
|
92
96
|
*/
|
|
93
97
|
updateStateFromBasic(url: string, title?: string, trigger?: 'manual' | 'navigation' | 'automatic'): WebPageState;
|
|
94
|
-
|
|
98
|
+
hasRegionAppeared(previousState: WebPageState | null, newState: WebPageState): boolean;
|
|
95
99
|
/**
|
|
96
100
|
* Generate a basic hash for state comparison
|
|
97
101
|
*/
|
|
@@ -41,7 +41,10 @@ export class StateManager {
|
|
|
41
41
|
*/
|
|
42
42
|
emitStateChange(event) {
|
|
43
43
|
const state = event.toState;
|
|
44
|
-
|
|
44
|
+
const payload = { url: state.fullUrl || state.url, path: state.url, title: state.title, h1: state.h1 };
|
|
45
|
+
if (state.overlay?.present)
|
|
46
|
+
payload.region = state.overlay.name || state.overlay.type;
|
|
47
|
+
tag('data').log('state', payload);
|
|
45
48
|
this.stateChangeListeners.forEach((listener) => {
|
|
46
49
|
try {
|
|
47
50
|
listener(event);
|
|
@@ -63,8 +66,8 @@ export class StateManager {
|
|
|
63
66
|
if (newState.url)
|
|
64
67
|
this.allVisitedUrls.add(normalizeUrl(newState.url));
|
|
65
68
|
const hashChanged = actionResult.hash !== previousHash;
|
|
66
|
-
const
|
|
67
|
-
if (hashChanged ||
|
|
69
|
+
const regionAppeared = !hashChanged && this.hasRegionAppeared(previousState, newState);
|
|
70
|
+
if (hashChanged || regionAppeared) {
|
|
68
71
|
const transition = {
|
|
69
72
|
fromState: previousState,
|
|
70
73
|
toState: newState,
|
|
@@ -74,8 +77,8 @@ export class StateManager {
|
|
|
74
77
|
};
|
|
75
78
|
this.stateHistory.push(transition);
|
|
76
79
|
this.emitStateChange(transition);
|
|
77
|
-
if (
|
|
78
|
-
debugLog('State change detected:
|
|
80
|
+
if (regionAppeared) {
|
|
81
|
+
debugLog('State change detected: region of interest appeared');
|
|
79
82
|
}
|
|
80
83
|
}
|
|
81
84
|
debugLog(`State updated: ${this.currentState.url} (${this.currentState.hash})`);
|
|
@@ -113,10 +116,10 @@ export class StateManager {
|
|
|
113
116
|
debugLog(`State updated from navigation: ${this.currentState.url} (${this.currentState.hash})`);
|
|
114
117
|
return newState;
|
|
115
118
|
}
|
|
116
|
-
|
|
119
|
+
hasRegionAppeared(previousState, newState) {
|
|
117
120
|
const prevFocus = previousState?.overlay ?? Overlay.fromAria(previousState?.ariaSnapshot ?? null);
|
|
118
121
|
const newFocus = newState.overlay ?? Overlay.fromAria(newState.ariaSnapshot ?? null);
|
|
119
|
-
return !prevFocus.
|
|
122
|
+
return !prevFocus.present && newFocus.present;
|
|
120
123
|
}
|
|
121
124
|
/**
|
|
122
125
|
* Generate a basic hash for state comparison
|
package/dist/src/test-plan.d.ts
CHANGED
|
@@ -129,6 +129,7 @@ export declare class Plan {
|
|
|
129
129
|
title: string;
|
|
130
130
|
tests: Test[];
|
|
131
131
|
url?: string;
|
|
132
|
+
filePath?: string;
|
|
132
133
|
iteration: number;
|
|
133
134
|
parentPlan?: Plan;
|
|
134
135
|
changeListeners: PlanChangeListener[];
|
|
@@ -141,10 +142,12 @@ export declare class Plan {
|
|
|
141
142
|
getAllTests(): Test[];
|
|
142
143
|
listTests(): Test[];
|
|
143
144
|
getPendingTests(): Test[];
|
|
145
|
+
get startUrl(): string | undefined;
|
|
144
146
|
get isComplete(): boolean;
|
|
145
147
|
get allSuccessful(): boolean;
|
|
146
148
|
get allFailed(): boolean;
|
|
147
149
|
updateStatus(): void;
|
|
150
|
+
static loadFromFile(file: string, plansDir?: string): Plan | null;
|
|
148
151
|
static fromMarkdown(filePath: string): Plan;
|
|
149
152
|
saveToMarkdown(filePath: string): void;
|
|
150
153
|
static saveMultipleToMarkdown(plans: Plan[], filePath: string): void;
|
package/dist/src/test-plan.js
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
|
+
import { existsSync } from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { listSites } from "./global-config.js";
|
|
2
5
|
import { tag } from "./utils/logger.js";
|
|
3
6
|
import { parsePlanFromMarkdown, planToAiContext, savePlanToMarkdown, savePlansToMarkdown } from "./utils/test-plan-markdown.js";
|
|
4
7
|
import { uniqSessionName } from "./utils/unique-names.js";
|
|
@@ -320,10 +323,12 @@ export class Test extends Task {
|
|
|
320
323
|
.sort((a, b) => a.timestamp - b.timestamp);
|
|
321
324
|
}
|
|
322
325
|
}
|
|
326
|
+
const SITE_PLANS_DIR = ['output', 'plans'];
|
|
323
327
|
export class Plan {
|
|
324
328
|
title;
|
|
325
329
|
tests = [];
|
|
326
330
|
url;
|
|
331
|
+
filePath;
|
|
327
332
|
iteration = 0;
|
|
328
333
|
parentPlan;
|
|
329
334
|
changeListeners = [];
|
|
@@ -380,6 +385,9 @@ export class Plan {
|
|
|
380
385
|
getPendingTests() {
|
|
381
386
|
return this.tests.filter((test) => test.status === 'pending' && test.enabled);
|
|
382
387
|
}
|
|
388
|
+
get startUrl() {
|
|
389
|
+
return this.url || this.tests[0]?.startUrl;
|
|
390
|
+
}
|
|
383
391
|
get isComplete() {
|
|
384
392
|
return this.tests.length > 0 && this.tests.every((test) => test.hasFinished);
|
|
385
393
|
}
|
|
@@ -390,6 +398,25 @@ export class Plan {
|
|
|
390
398
|
return this.tests.length > 0 && this.tests.every((test) => test.hasFailed);
|
|
391
399
|
}
|
|
392
400
|
updateStatus() { }
|
|
401
|
+
static loadFromFile(file, plansDir) {
|
|
402
|
+
const names = [file];
|
|
403
|
+
if (!file.endsWith('.md'))
|
|
404
|
+
names.push(`${file}.md`);
|
|
405
|
+
const dirs = [process.cwd()];
|
|
406
|
+
if (plansDir)
|
|
407
|
+
dirs.push(plansDir);
|
|
408
|
+
if (!plansDir)
|
|
409
|
+
dirs.push(...listSites().map((site) => path.join(site.dir, ...SITE_PLANS_DIR)));
|
|
410
|
+
for (const dir of dirs) {
|
|
411
|
+
const filePath = names.map((name) => path.resolve(dir, name)).find(existsSync);
|
|
412
|
+
if (!filePath)
|
|
413
|
+
continue;
|
|
414
|
+
const loaded = parsePlanFromMarkdown(filePath);
|
|
415
|
+
loaded.filePath = filePath;
|
|
416
|
+
return loaded;
|
|
417
|
+
}
|
|
418
|
+
return null;
|
|
419
|
+
}
|
|
393
420
|
static fromMarkdown(filePath) {
|
|
394
421
|
return parsePlanFromMarkdown(filePath);
|
|
395
422
|
}
|