explorbot 0.1.9 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -1
- package/bin/explorbot-cli.ts +86 -15
- package/boat/api-tester/src/ai/curler-tools.ts +3 -3
- package/boat/api-tester/src/ai/curler.ts +1 -1
- package/boat/api-tester/src/apibot.ts +2 -2
- package/boat/api-tester/src/config.ts +1 -1
- package/dist/bin/explorbot-cli.js +85 -14
- package/dist/boat/api-tester/src/ai/curler-tools.js +2 -2
- package/dist/boat/api-tester/src/apibot.js +2 -2
- package/dist/package.json +2 -2
- package/dist/rules/navigator/output.md +9 -0
- package/dist/rules/navigator/verification-actions.md +2 -0
- package/dist/src/action-result.js +23 -1
- package/dist/src/action.js +46 -38
- package/dist/src/ai/bosun.js +16 -2
- package/dist/src/ai/conversation.js +39 -0
- package/dist/src/ai/experience-compactor.js +235 -50
- package/dist/src/ai/historian/codeceptjs.js +109 -0
- package/dist/src/ai/historian/experience.js +320 -0
- package/dist/src/ai/historian/mixin.js +2 -0
- package/dist/src/ai/historian/playwright.js +145 -0
- package/dist/src/ai/historian/utils.js +18 -0
- package/dist/src/ai/historian.js +19 -398
- package/dist/src/ai/navigator.js +133 -80
- package/dist/src/ai/pilot.js +254 -13
- package/dist/src/ai/planner/subpages.js +1 -30
- package/dist/src/ai/planner.js +33 -13
- package/dist/src/ai/provider.js +55 -18
- package/dist/src/ai/rerunner.js +3 -3
- package/dist/src/ai/researcher/deep-analysis.js +1 -1
- package/dist/src/ai/researcher/fingerprint-worker.js +1 -1
- package/dist/src/ai/researcher/locators.js +1 -1
- package/dist/src/ai/researcher/sections.js +8 -1
- package/dist/src/ai/researcher.js +43 -41
- package/dist/src/ai/rules.js +26 -14
- package/dist/src/ai/tester.js +90 -26
- package/dist/src/ai/tools.js +18 -10
- package/dist/src/api/request-store.js +20 -0
- package/dist/src/api/xhr-capture.js +19 -3
- package/dist/src/browser-server.js +16 -3
- package/dist/src/command-handler.js +1 -1
- package/dist/src/commands/add-rule-command.js +12 -9
- package/dist/src/commands/base-command.js +20 -0
- package/dist/src/commands/clean-command.js +3 -2
- package/dist/src/commands/compact-command.js +138 -0
- package/dist/src/commands/context-command.js +7 -1
- package/dist/src/commands/drill-command.js +4 -1
- package/dist/src/commands/experience-command.js +104 -0
- package/dist/src/commands/explore-command.js +54 -19
- package/dist/src/commands/freesail-command.js +2 -0
- package/dist/src/commands/index.js +7 -3
- package/dist/src/commands/init-command.js +11 -10
- package/dist/src/commands/learn-command.js +1 -1
- package/dist/src/commands/navigate-command.js +4 -1
- package/dist/src/commands/plan-clear-command.js +4 -1
- package/dist/src/commands/plan-command.js +43 -4
- package/dist/src/commands/plan-edit-command.js +1 -1
- package/dist/src/commands/plan-load-command.js +4 -1
- package/dist/src/commands/plan-reload-command.js +4 -1
- package/dist/src/commands/plan-save-command.js +20 -8
- package/dist/src/commands/rerun-command.js +4 -0
- package/dist/src/commands/research-command.js +5 -2
- package/dist/src/commands/start-command.js +5 -1
- package/dist/src/commands/test-command.js +7 -1
- package/dist/src/components/App.js +15 -5
- package/dist/src/execution-controller.js +13 -2
- package/dist/src/experience-tracker.js +174 -83
- package/dist/src/explorbot.js +31 -22
- package/dist/src/explorer.js +12 -5
- package/dist/src/observability.js +50 -99
- package/dist/src/playwright-recorder.js +309 -0
- package/dist/src/reporter.js +17 -2
- package/dist/src/stats.js +2 -0
- package/dist/src/suite.js +1 -1
- package/dist/src/test-plan.js +12 -0
- package/dist/src/utils/aria.js +37 -1
- package/dist/src/utils/error-page.js +30 -7
- package/dist/src/utils/logger.js +1 -1
- package/dist/src/utils/next-steps.js +37 -0
- package/dist/src/utils/rules-loader.js +1 -1
- package/dist/src/utils/test-files.js +1 -1
- package/dist/src/utils/url-matcher.js +50 -0
- package/package.json +2 -2
- package/rules/navigator/output.md +9 -0
- package/rules/navigator/verification-actions.md +2 -0
- package/src/action-result.ts +26 -1
- package/src/action.ts +44 -37
- package/src/ai/bosun.ts +16 -2
- package/src/ai/conversation.ts +37 -0
- package/src/ai/experience-compactor.ts +270 -63
- package/src/ai/historian/codeceptjs.ts +130 -0
- package/src/ai/historian/experience.ts +383 -0
- package/src/ai/historian/mixin.ts +4 -0
- package/src/ai/historian/playwright.ts +169 -0
- package/src/ai/historian/utils.ts +23 -0
- package/src/ai/historian.ts +35 -468
- package/src/ai/navigator.ts +140 -85
- package/src/ai/pilot.ts +259 -14
- package/src/ai/planner/subpages.ts +1 -24
- package/src/ai/planner.ts +34 -14
- package/src/ai/provider.ts +52 -18
- package/src/ai/rerunner.ts +3 -3
- package/src/ai/researcher/deep-analysis.ts +1 -1
- package/src/ai/researcher/fingerprint-worker.ts +1 -1
- package/src/ai/researcher/locators.ts +2 -2
- package/src/ai/researcher/sections.ts +7 -1
- package/src/ai/researcher.ts +47 -42
- package/src/ai/rules.ts +27 -14
- package/src/ai/task-agent.ts +1 -1
- package/src/ai/tester.ts +94 -26
- package/src/ai/tools.ts +53 -29
- package/src/api/request-store.ts +22 -0
- package/src/api/xhr-capture.ts +21 -3
- package/src/browser-server.ts +17 -3
- package/src/command-handler.ts +1 -1
- package/src/commands/add-rule-command.ts +13 -9
- package/src/commands/base-command.ts +26 -1
- package/src/commands/clean-command.ts +4 -3
- package/src/commands/compact-command.ts +156 -0
- package/src/commands/context-command.ts +8 -2
- package/src/commands/drill-command.ts +5 -2
- package/src/commands/experience-command.ts +125 -0
- package/src/commands/explore-command.ts +58 -21
- package/src/commands/freesail-command.ts +2 -0
- package/src/commands/index.ts +7 -3
- package/src/commands/init-command.ts +11 -10
- package/src/commands/learn-command.ts +2 -2
- package/src/commands/navigate-command.ts +5 -2
- package/src/commands/plan-clear-command.ts +5 -2
- package/src/commands/plan-command.ts +47 -5
- package/src/commands/plan-edit-command.ts +2 -2
- package/src/commands/plan-load-command.ts +5 -2
- package/src/commands/plan-reload-command.ts +5 -2
- package/src/commands/plan-save-command.ts +20 -9
- package/src/commands/rerun-command.ts +5 -0
- package/src/commands/research-command.ts +6 -3
- package/src/commands/start-command.ts +6 -2
- package/src/commands/test-command.ts +8 -2
- package/src/components/App.tsx +16 -5
- package/src/config.ts +6 -1
- package/src/execution-controller.ts +14 -3
- package/src/experience-tracker.ts +198 -100
- package/src/explorbot.ts +33 -23
- package/src/explorer.ts +14 -5
- package/src/observability.ts +50 -109
- package/src/playwright-recorder.ts +305 -0
- package/src/reporter.ts +17 -3
- package/src/stats.ts +4 -0
- package/src/suite.ts +1 -1
- package/src/test-plan.ts +12 -0
- package/src/utils/aria.ts +38 -1
- package/src/utils/error-page.ts +32 -7
- package/src/utils/logger.ts +1 -1
- package/src/utils/next-steps.ts +51 -0
- package/src/utils/rules-loader.ts +1 -1
- package/src/utils/test-files.ts +1 -1
- package/src/utils/url-matcher.ts +43 -0
|
@@ -10,6 +10,7 @@ export class ExecutionController extends EventEmitter {
|
|
|
10
10
|
private inputCallback: InputCallback | null = null;
|
|
11
11
|
private interruptResolvers: Array<() => void> = [];
|
|
12
12
|
private abortController: AbortController | null = null;
|
|
13
|
+
private awaitingInput = false;
|
|
13
14
|
|
|
14
15
|
private constructor() {
|
|
15
16
|
super();
|
|
@@ -48,6 +49,10 @@ export class ExecutionController extends EventEmitter {
|
|
|
48
49
|
this.emit('idle');
|
|
49
50
|
}
|
|
50
51
|
|
|
52
|
+
isAwaitingInput(): boolean {
|
|
53
|
+
return this.awaitingInput;
|
|
54
|
+
}
|
|
55
|
+
|
|
51
56
|
isInterrupted(): boolean {
|
|
52
57
|
return this.interrupted;
|
|
53
58
|
}
|
|
@@ -77,11 +82,16 @@ export class ExecutionController extends EventEmitter {
|
|
|
77
82
|
}
|
|
78
83
|
|
|
79
84
|
async requestInput(prompt: string): Promise<string | null> {
|
|
80
|
-
if (this.inputCallback) {
|
|
81
|
-
return await this.
|
|
85
|
+
if (!this.inputCallback) {
|
|
86
|
+
return await this.readlineInput(prompt);
|
|
82
87
|
}
|
|
83
88
|
|
|
84
|
-
|
|
89
|
+
this.awaitingInput = true;
|
|
90
|
+
try {
|
|
91
|
+
return await this.inputCallback(prompt);
|
|
92
|
+
} finally {
|
|
93
|
+
this.awaitingInput = false;
|
|
94
|
+
}
|
|
85
95
|
}
|
|
86
96
|
|
|
87
97
|
private async readlineInput(prompt: string): Promise<string | null> {
|
|
@@ -103,6 +113,7 @@ export class ExecutionController extends EventEmitter {
|
|
|
103
113
|
this.interrupted = false;
|
|
104
114
|
this.interruptResolvers = [];
|
|
105
115
|
this.abortController = null;
|
|
116
|
+
this.awaitingInput = false;
|
|
106
117
|
}
|
|
107
118
|
}
|
|
108
119
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs';
|
|
2
2
|
import { basename, dirname, join } from 'node:path';
|
|
3
3
|
import matter from 'gray-matter';
|
|
4
|
-
import {
|
|
4
|
+
import { type Tokens, marked } from 'marked';
|
|
5
5
|
import type { ActionResult } from './action-result.js';
|
|
6
6
|
import { ConfigParser } from './config.js';
|
|
7
7
|
import { KnowledgeTracker } from './knowledge-tracker.js';
|
|
@@ -13,6 +13,25 @@ import { extractStatePath } from './utils/url-matcher.js';
|
|
|
13
13
|
const debugLog = createDebug('explorbot:experience');
|
|
14
14
|
const DEFAULT_MAX_EXPERIENCE_LINES = 100;
|
|
15
15
|
|
|
16
|
+
export const RECENT_WINDOW_DAYS = 30;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Stores and reads per-page experience files (`./experience/<stateHash>.md`).
|
|
20
|
+
*
|
|
21
|
+
* Two writers, two contracts:
|
|
22
|
+
*
|
|
23
|
+
* writeFlow(state, body, relatedUrls?) — caller hands in a fully-formatted
|
|
24
|
+
* `## FLOW: <imperative title>` block (multi-step,
|
|
25
|
+
* `*` bullets + optional ```js``` + `>` discovery,
|
|
26
|
+
* ends with `---`). Tracker dedups + prepends.
|
|
27
|
+
* writeAction(state, ActionInput) — `## ACTION: <imperative title>`, single-step,
|
|
28
|
+
* optional `Solution:` line + one ```js``` code block.
|
|
29
|
+
* Title normalized via normalizeTitle().
|
|
30
|
+
*
|
|
31
|
+
* - Always h2. Never h3 for FLOW/ACTION.
|
|
32
|
+
* - On read (getSuccessfulExperience), headings are rendered as
|
|
33
|
+
* `## HOW to <title> (multi-step|single-step)` so prompts get natural phrasing.
|
|
34
|
+
*/
|
|
16
35
|
export class ExperienceTracker {
|
|
17
36
|
private experienceDir: string;
|
|
18
37
|
private disabled: boolean;
|
|
@@ -144,35 +163,57 @@ export class ExperienceTracker {
|
|
|
144
163
|
return this.knowledgeTracker.getRelevantKnowledge(state).some((k) => k.noExperienceWriting === true || k.noExperienceWriting === 'true');
|
|
145
164
|
}
|
|
146
165
|
|
|
147
|
-
|
|
166
|
+
writeAction(state: ActionResult, action: ActionInput): void {
|
|
148
167
|
if (this.disabled || this.isWritingDisabled(state)) return;
|
|
168
|
+
if (!action.code?.trim()) return;
|
|
149
169
|
|
|
150
170
|
this.ensureExperienceFile(state);
|
|
151
171
|
const stateHash = state.getStateHash();
|
|
152
172
|
const { content, data } = this.readExperienceFile(stateHash);
|
|
153
|
-
if (content.includes(code)) {
|
|
154
|
-
debugLog('Skipping duplicate
|
|
173
|
+
if (content.includes(action.code)) {
|
|
174
|
+
debugLog('Skipping duplicate action', action.code);
|
|
155
175
|
return;
|
|
156
176
|
}
|
|
157
177
|
|
|
158
|
-
const
|
|
159
|
-
|
|
178
|
+
const title = normalizeTitle(action.title.split('\n')[0]);
|
|
179
|
+
if (!title) return;
|
|
160
180
|
|
|
161
|
-
|
|
181
|
+
const filteredCode = action.code.replace(/I\.amOnPage\s*\([^)]*\)/gs, '');
|
|
182
|
+
const newEntry = generateActionContent(title, filteredCode, action.explanation);
|
|
183
|
+
const updatedContent = `${newEntry}\n\n${content}`;
|
|
184
|
+
this.writeExperienceFile(stateHash, updatedContent, data);
|
|
185
|
+
|
|
186
|
+
tag('substep').log(` Added ACTION to: ${stateHash}.md`);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
writeFlow(state: ActionResult, body: string, relatedUrls?: string[]): void {
|
|
190
|
+
if (this.disabled || this.isWritingDisabled(state)) return;
|
|
191
|
+
if (!body?.trim()) return;
|
|
162
192
|
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
193
|
+
this.ensureExperienceFile(state);
|
|
194
|
+
const stateHash = state.getStateHash();
|
|
195
|
+
const { content, data } = this.readExperienceFile(stateHash);
|
|
196
|
+
|
|
197
|
+
if (content.includes(body)) {
|
|
198
|
+
debugLog('Skipping duplicate flow body');
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
if (relatedUrls?.length) {
|
|
203
|
+
const currentPath = extractStatePath(state.url || '');
|
|
204
|
+
const existingRelated = Array.isArray(data.related) ? data.related : [];
|
|
205
|
+
const allRelated = [...new Set([...existingRelated, ...relatedUrls])];
|
|
206
|
+
data.related = allRelated.filter((url) => url !== currentPath);
|
|
207
|
+
}
|
|
167
208
|
|
|
168
|
-
const updatedContent = `${
|
|
209
|
+
const updatedContent = `${body}\n${content}`;
|
|
169
210
|
this.writeExperienceFile(stateHash, updatedContent, data);
|
|
170
211
|
|
|
171
|
-
tag('substep').log(`
|
|
212
|
+
tag('substep').log(`Added FLOW to: ${stateHash}.md`);
|
|
172
213
|
}
|
|
173
214
|
|
|
174
|
-
getAllExperience():
|
|
175
|
-
const allFiles:
|
|
215
|
+
getAllExperience(): ExperienceFile[] {
|
|
216
|
+
const allFiles: ExperienceFile[] = [];
|
|
176
217
|
|
|
177
218
|
for (const experienceDir of this.getExperienceDirectories()) {
|
|
178
219
|
if (!existsSync(experienceDir)) {
|
|
@@ -188,10 +229,12 @@ ${filteredCode}
|
|
|
188
229
|
try {
|
|
189
230
|
const content = readFileSync(file, 'utf8');
|
|
190
231
|
const parsed = matter(content);
|
|
232
|
+
const mtime = statSync(file).mtime;
|
|
191
233
|
allFiles.push({
|
|
192
234
|
filePath: file,
|
|
193
235
|
data: parsed.data,
|
|
194
236
|
content: parsed.content,
|
|
237
|
+
mtime,
|
|
195
238
|
});
|
|
196
239
|
} catch (error) {
|
|
197
240
|
debugLog(`Failed to read experience file ${file}:`, error);
|
|
@@ -205,7 +248,7 @@ ${filteredCode}
|
|
|
205
248
|
return allFiles;
|
|
206
249
|
}
|
|
207
250
|
|
|
208
|
-
getRelevantExperience(state: ActionResult, options?: { includeDescendantExperience?: boolean }):
|
|
251
|
+
getRelevantExperience(state: ActionResult, options?: { includeDescendantExperience?: boolean }): ExperienceFile[] {
|
|
209
252
|
const relevantKnowledge = this.knowledgeTracker.getRelevantKnowledge(state);
|
|
210
253
|
const readingDisabled = relevantKnowledge.some((knowledge) => knowledge.noExperienceReading === true || knowledge.noExperienceReading === 'true');
|
|
211
254
|
if (readingDisabled) {
|
|
@@ -236,79 +279,6 @@ ${filteredCode}
|
|
|
236
279
|
// The actual files will be cleaned up by test cleanup
|
|
237
280
|
}
|
|
238
281
|
|
|
239
|
-
saveSessionExperience(state: ActionResult, entry: SessionExperienceEntry): void {
|
|
240
|
-
if (this.disabled || this.isWritingDisabled(state)) return;
|
|
241
|
-
|
|
242
|
-
this.ensureExperienceFile(state);
|
|
243
|
-
const stateHash = state.getStateHash();
|
|
244
|
-
const { content, data } = this.readExperienceFile(stateHash);
|
|
245
|
-
|
|
246
|
-
if (entry.relatedUrls?.length) {
|
|
247
|
-
const currentPath = extractStatePath(state.url || '');
|
|
248
|
-
const existingRelated = Array.isArray(data.related) ? data.related : [];
|
|
249
|
-
const allRelated = [...new Set([...existingRelated, ...entry.relatedUrls])];
|
|
250
|
-
data.related = allRelated.filter((url) => url !== currentPath);
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
const sessionContent = this.trimSessionContent(this.generateSessionContent(entry));
|
|
254
|
-
if (!sessionContent) return;
|
|
255
|
-
const updatedContent = `${sessionContent}\n${content}`;
|
|
256
|
-
this.writeExperienceFile(stateHash, updatedContent, data);
|
|
257
|
-
|
|
258
|
-
tag('substep').log(`Added session experience to: ${stateHash}.md`);
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
private generateSessionContent(entry: SessionExperienceEntry): string {
|
|
262
|
-
let content = `## Successful Flow: ${entry.scenario}\n\n`;
|
|
263
|
-
|
|
264
|
-
for (const step of entry.steps) {
|
|
265
|
-
content += `* ${step.message}\n\n`;
|
|
266
|
-
if (step.code) {
|
|
267
|
-
content += '```js\n';
|
|
268
|
-
content += `${step.code}\n`;
|
|
269
|
-
content += '```\n\n';
|
|
270
|
-
}
|
|
271
|
-
if (step.discovery) {
|
|
272
|
-
const discoveries = step.discovery.split('\n').filter((d) => d.trim());
|
|
273
|
-
for (const discovery of discoveries) {
|
|
274
|
-
content += `> ${discovery.trim()}\n\n`;
|
|
275
|
-
}
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
content += '---\n';
|
|
280
|
-
return content;
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
private trimSessionContent(content: string): string | null {
|
|
284
|
-
const q = mdq(content);
|
|
285
|
-
if (q.query('heading').count() === 0) return null;
|
|
286
|
-
if (q.query('code').count() === 0) return null;
|
|
287
|
-
|
|
288
|
-
let result = content;
|
|
289
|
-
const codeBlocks = q.query('code').each();
|
|
290
|
-
if (codeBlocks.length > 2) {
|
|
291
|
-
for (const block of codeBlocks.slice(2)) {
|
|
292
|
-
result = result.replace(block.text(), '');
|
|
293
|
-
}
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
const blockquotes = mdq(result).query('blockquote').each();
|
|
297
|
-
if (blockquotes.length > 5) {
|
|
298
|
-
for (const bq of blockquotes.slice(5)) {
|
|
299
|
-
result = result.replace(bq.text(), '');
|
|
300
|
-
}
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
const lines = result.split('\n');
|
|
304
|
-
if (lines.length > 40) {
|
|
305
|
-
result = lines.slice(0, 40).join('\n');
|
|
306
|
-
}
|
|
307
|
-
|
|
308
|
-
if (!result.trim()) return null;
|
|
309
|
-
return result;
|
|
310
|
-
}
|
|
311
|
-
|
|
312
282
|
getSuccessfulExperience(state: ActionResult, options?: { includeDescendants?: boolean; stripCode?: boolean }): string[] {
|
|
313
283
|
const records = this.getRelevantExperience(state, {
|
|
314
284
|
includeDescendantExperience: options?.includeDescendants,
|
|
@@ -318,12 +288,14 @@ ${filteredCode}
|
|
|
318
288
|
for (const record of records) {
|
|
319
289
|
if (!record.content) continue;
|
|
320
290
|
|
|
321
|
-
const
|
|
322
|
-
const
|
|
323
|
-
let combined = [
|
|
291
|
+
const flows = mdq(record.content).query('section(~"FLOW:")').text();
|
|
292
|
+
const actions = mdq(record.content).query('section(~"ACTION:")').text();
|
|
293
|
+
let combined = [flows, actions].filter(Boolean).join('\n\n');
|
|
324
294
|
|
|
325
295
|
if (!combined.trim()) continue;
|
|
326
296
|
|
|
297
|
+
combined = renderAsHowTo(combined);
|
|
298
|
+
|
|
327
299
|
if (options?.stripCode) {
|
|
328
300
|
combined = mdq(combined).query('code').replace('');
|
|
329
301
|
}
|
|
@@ -383,6 +355,72 @@ ${filteredCode}
|
|
|
383
355
|
}
|
|
384
356
|
return null;
|
|
385
357
|
}
|
|
358
|
+
|
|
359
|
+
listAllExperienceToc(filter?: string, options?: { recency?: 'recent' | 'old' }): ExperienceTocEntry[] {
|
|
360
|
+
const records = this.getAllExperience();
|
|
361
|
+
if (records.length === 0) return [];
|
|
362
|
+
|
|
363
|
+
const trimmed = filter?.trim();
|
|
364
|
+
let matching = records;
|
|
365
|
+
|
|
366
|
+
if (trimmed) {
|
|
367
|
+
if (trimmed.endsWith('.md')) {
|
|
368
|
+
const bare = trimmed.slice(0, -3);
|
|
369
|
+
const byFilename = records.find((record) => basename(record.filePath, '.md') === bare);
|
|
370
|
+
matching = byFilename ? [byFilename] : [];
|
|
371
|
+
} else {
|
|
372
|
+
const lower = trimmed.toLowerCase();
|
|
373
|
+
matching = records.filter((record) => {
|
|
374
|
+
const url = ((record.data as WebPageState)?.url || '').toLowerCase();
|
|
375
|
+
if (!url) return false;
|
|
376
|
+
return url.includes(lower);
|
|
377
|
+
});
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
if (options?.recency) {
|
|
382
|
+
const cutoff = Date.now() - RECENT_WINDOW_DAYS * 24 * 60 * 60 * 1000;
|
|
383
|
+
matching = matching.filter((record) => {
|
|
384
|
+
const isRecent = record.mtime.getTime() >= cutoff;
|
|
385
|
+
return options.recency === 'recent' ? isRecent : !isRecent;
|
|
386
|
+
});
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
const sorted = matching.sort((a, b) => {
|
|
390
|
+
const aUrl = (a.data as WebPageState)?.url || '';
|
|
391
|
+
const bUrl = (b.data as WebPageState)?.url || '';
|
|
392
|
+
return aUrl.localeCompare(bUrl);
|
|
393
|
+
});
|
|
394
|
+
|
|
395
|
+
const toc: ExperienceTocEntry[] = [];
|
|
396
|
+
for (let i = 0; i < sorted.length; i++) {
|
|
397
|
+
const record = sorted[i];
|
|
398
|
+
const sections = listTocHeadings(record.content);
|
|
399
|
+
if (sections.length === 0) continue;
|
|
400
|
+
toc.push({
|
|
401
|
+
fileTag: indexToLetters(toc.length),
|
|
402
|
+
fileHash: basename(record.filePath, '.md'),
|
|
403
|
+
url: (record.data as WebPageState)?.url || '',
|
|
404
|
+
sections,
|
|
405
|
+
});
|
|
406
|
+
}
|
|
407
|
+
return toc;
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
getExperienceSectionByTag(fileTag: string, sectionIndex: number, filter?: string): { title: string; url: string; content: string; fileHash: string } | null {
|
|
411
|
+
const toc = this.listAllExperienceToc(filter);
|
|
412
|
+
const entry = toc.find((e) => e.fileTag === fileTag);
|
|
413
|
+
if (!entry) return null;
|
|
414
|
+
|
|
415
|
+
const filePath = this.findExperienceFileByHash(entry.fileHash);
|
|
416
|
+
if (!filePath) return null;
|
|
417
|
+
|
|
418
|
+
const { content } = this.readExperienceFile(entry.fileHash);
|
|
419
|
+
const extracted = extractHeadingSection(content, sectionIndex);
|
|
420
|
+
if (!extracted) return null;
|
|
421
|
+
|
|
422
|
+
return { title: extracted.title, url: entry.url, content: extracted.body, fileHash: entry.fileHash };
|
|
423
|
+
}
|
|
386
424
|
}
|
|
387
425
|
|
|
388
426
|
function listTocHeadings(content: string): { index: number; level: 2 | 3; title: string }[] {
|
|
@@ -448,7 +486,11 @@ export function renderExperienceToc(toc: ExperienceTocEntry[]): string {
|
|
|
448
486
|
|
|
449
487
|
const lines: string[] = [];
|
|
450
488
|
lines.push('<experience>');
|
|
451
|
-
lines.push('Past experience for this page
|
|
489
|
+
lines.push('Past experience for this page — recipes recorded from prior successful runs.');
|
|
490
|
+
lines.push('Locators and step ordering worked then; the page may have changed since.');
|
|
491
|
+
lines.push('Treat as a starting hypothesis, not ground truth. If a step fails, fall back to ARIA/UI-map.');
|
|
492
|
+
lines.push('FLOW: = multi-step recipe (bullets + code + discovery). ACTION: = single-step snippet (one code block).');
|
|
493
|
+
lines.push('Call learn_experience({ fileTag, sectionIndex }) to read a section when it looks relevant to the current step.');
|
|
452
494
|
lines.push('');
|
|
453
495
|
for (const entry of toc) {
|
|
454
496
|
lines.push(`File ${entry.fileTag} ${entry.url}:`);
|
|
@@ -462,6 +504,69 @@ export function renderExperienceToc(toc: ExperienceTocEntry[]): string {
|
|
|
462
504
|
return lines.join('\n');
|
|
463
505
|
}
|
|
464
506
|
|
|
507
|
+
function normalizeTitle(raw: string): string {
|
|
508
|
+
let t = (raw || '').trim();
|
|
509
|
+
for (const p of ['FLOW:', 'ACTION:']) {
|
|
510
|
+
if (t.toLowerCase().startsWith(p.toLowerCase())) {
|
|
511
|
+
t = t.slice(p.length).trim();
|
|
512
|
+
break;
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
while (t.length > 0 && '.!?,;:'.includes(t[t.length - 1])) {
|
|
516
|
+
t = t.slice(0, -1);
|
|
517
|
+
}
|
|
518
|
+
if (t.length > 0) t = t[0].toLowerCase() + t.slice(1);
|
|
519
|
+
return t;
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
function generateActionContent(title: string, code: string, explanation?: string): string {
|
|
523
|
+
const lines: string[] = [];
|
|
524
|
+
lines.push(`## ACTION: ${title}`);
|
|
525
|
+
lines.push('');
|
|
526
|
+
if (explanation) {
|
|
527
|
+
lines.push(`Solution: ${explanation}`);
|
|
528
|
+
lines.push('');
|
|
529
|
+
}
|
|
530
|
+
lines.push('```javascript');
|
|
531
|
+
lines.push(code);
|
|
532
|
+
lines.push('```');
|
|
533
|
+
lines.push('');
|
|
534
|
+
return lines.join('\n');
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
function renderAsHowTo(content: string): string {
|
|
538
|
+
const tokens = marked.lexer(content);
|
|
539
|
+
let result = '';
|
|
540
|
+
for (const token of tokens) {
|
|
541
|
+
if (token.type === 'heading' && (token as Tokens.Heading).depth === 2) {
|
|
542
|
+
const text = (token as Tokens.Heading).text.trim();
|
|
543
|
+
if (text.startsWith('FLOW:')) {
|
|
544
|
+
result += `## HOW to ${text.slice(5).trim()} (multi-step)\n\n`;
|
|
545
|
+
continue;
|
|
546
|
+
}
|
|
547
|
+
if (text.startsWith('ACTION:')) {
|
|
548
|
+
result += `## HOW to ${text.slice(7).trim()} (single-step)\n\n`;
|
|
549
|
+
continue;
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
result += (token as any).raw || '';
|
|
553
|
+
}
|
|
554
|
+
return result;
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
export interface ExperienceFile {
|
|
558
|
+
filePath: string;
|
|
559
|
+
data: { url?: string; title?: string; [key: string]: any };
|
|
560
|
+
content: string;
|
|
561
|
+
mtime: Date;
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
export interface ActionInput {
|
|
565
|
+
title: string;
|
|
566
|
+
code: string;
|
|
567
|
+
explanation?: string;
|
|
568
|
+
}
|
|
569
|
+
|
|
465
570
|
export interface SessionStep {
|
|
466
571
|
message: string;
|
|
467
572
|
status: 'passed' | 'failed' | 'neutral';
|
|
@@ -470,13 +575,6 @@ export interface SessionStep {
|
|
|
470
575
|
discovery?: string;
|
|
471
576
|
}
|
|
472
577
|
|
|
473
|
-
export interface SessionExperienceEntry {
|
|
474
|
-
scenario: string;
|
|
475
|
-
result: 'success' | 'partial' | 'failed';
|
|
476
|
-
steps: SessionStep[];
|
|
477
|
-
relatedUrls?: string[];
|
|
478
|
-
}
|
|
479
|
-
|
|
480
578
|
export interface ExperienceTocEntry {
|
|
481
579
|
fileTag: string;
|
|
482
580
|
fileHash: string;
|
package/src/explorbot.ts
CHANGED
|
@@ -1,9 +1,6 @@
|
|
|
1
1
|
import { existsSync, mkdirSync } from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { ActionResult } from './action-result.ts';
|
|
4
|
-
import { ApiClient } from './api/api-client.ts';
|
|
5
|
-
import { RequestStore } from './api/request-store.ts';
|
|
6
|
-
import { loadSpec } from './api/spec-reader.ts';
|
|
7
4
|
import { Bosun } from './ai/bosun.ts';
|
|
8
5
|
import { Captain } from './ai/captain.ts';
|
|
9
6
|
import { ExperienceCompactor } from './ai/experience-compactor.ts';
|
|
@@ -14,16 +11,20 @@ import { Pilot } from './ai/pilot.ts';
|
|
|
14
11
|
import { Planner } from './ai/planner.ts';
|
|
15
12
|
import { AIProvider } from './ai/provider.ts';
|
|
16
13
|
import { Quartermaster } from './ai/quartermaster.ts';
|
|
17
|
-
import { Researcher } from './ai/researcher.ts';
|
|
18
14
|
import { Rerunner } from './ai/rerunner.ts';
|
|
15
|
+
import { Researcher } from './ai/researcher.ts';
|
|
19
16
|
import { Tester } from './ai/tester.ts';
|
|
20
17
|
import { createAgentTools } from './ai/tools.ts';
|
|
18
|
+
import { ApiClient } from './api/api-client.ts';
|
|
19
|
+
import { RequestStore } from './api/request-store.ts';
|
|
20
|
+
import { loadSpec } from './api/spec-reader.ts';
|
|
21
21
|
import type { ExplorbotConfig } from './config.js';
|
|
22
22
|
import { ConfigParser } from './config.ts';
|
|
23
|
+
import { ExperienceTracker } from './experience-tracker.ts';
|
|
23
24
|
import Explorer from './explorer.ts';
|
|
24
|
-
import type { Suite } from './suite.ts';
|
|
25
25
|
import { KnowledgeTracker } from './knowledge-tracker.ts';
|
|
26
26
|
import { WebPageState } from './state-manager.ts';
|
|
27
|
+
import type { Suite } from './suite.ts';
|
|
27
28
|
import { Plan } from './test-plan.ts';
|
|
28
29
|
import { setVerboseMode, tag } from './utils/logger.ts';
|
|
29
30
|
import { sanitizeFilename } from './utils/strings.ts';
|
|
@@ -51,6 +52,8 @@ export class ExplorBot {
|
|
|
51
52
|
public needsInput = false;
|
|
52
53
|
private currentPlan?: Plan;
|
|
53
54
|
private planFeature?: string;
|
|
55
|
+
lastPlanError: Error | null = null;
|
|
56
|
+
lastSavedPlanPath: string | null = null;
|
|
54
57
|
private agents: Record<string, any> = {};
|
|
55
58
|
|
|
56
59
|
constructor(options: ExplorBotOptions = {}) {
|
|
@@ -76,13 +79,11 @@ export class ExplorBot {
|
|
|
76
79
|
}
|
|
77
80
|
|
|
78
81
|
try {
|
|
79
|
-
|
|
80
|
-
this.provider = new AIProvider(this.config.ai);
|
|
81
|
-
await this.provider.validateConnection();
|
|
82
|
+
await this.startProviderOnly();
|
|
82
83
|
this.explorer = new Explorer(this.config, this.provider, this.options);
|
|
83
84
|
await this.explorer.start();
|
|
84
85
|
if (!this.options.incognito) {
|
|
85
|
-
await this.agentExperienceCompactor().
|
|
86
|
+
await this.agentExperienceCompactor().autocompact();
|
|
86
87
|
}
|
|
87
88
|
if (this.userResolveFn) this.explorer.setUserResolve(this.userResolveFn);
|
|
88
89
|
} catch (error) {
|
|
@@ -92,9 +93,16 @@ export class ExplorBot {
|
|
|
92
93
|
}
|
|
93
94
|
}
|
|
94
95
|
|
|
96
|
+
async startProviderOnly(): Promise<void> {
|
|
97
|
+
if (this.provider) return;
|
|
98
|
+
this.config = await this.configParser.loadConfig(this.options);
|
|
99
|
+
this.provider = new AIProvider(this.config.ai);
|
|
100
|
+
await this.provider.validateConnection();
|
|
101
|
+
}
|
|
102
|
+
|
|
95
103
|
async stop(): Promise<void> {
|
|
96
104
|
this.agents.quartermaster?.stop();
|
|
97
|
-
await this.explorer
|
|
105
|
+
await this.explorer?.stop();
|
|
98
106
|
}
|
|
99
107
|
|
|
100
108
|
async visitInitialState(): Promise<void> {
|
|
@@ -125,6 +133,13 @@ export class ExplorBot {
|
|
|
125
133
|
return new KnowledgeTracker();
|
|
126
134
|
}
|
|
127
135
|
|
|
136
|
+
getExperienceTracker(): ExperienceTracker {
|
|
137
|
+
if (this.explorer) {
|
|
138
|
+
return this.explorer.getStateManager().getExperienceTracker();
|
|
139
|
+
}
|
|
140
|
+
return new ExperienceTracker();
|
|
141
|
+
}
|
|
142
|
+
|
|
128
143
|
getConfig(): ExplorbotConfig {
|
|
129
144
|
return this.config;
|
|
130
145
|
}
|
|
@@ -227,10 +242,7 @@ export class ExplorBot {
|
|
|
227
242
|
}
|
|
228
243
|
|
|
229
244
|
agentExperienceCompactor(): ExperienceCompactor {
|
|
230
|
-
return (this.agents.experienceCompactor ||= this.
|
|
231
|
-
const experienceTracker = explorer.getStateManager().getExperienceTracker();
|
|
232
|
-
return new ExperienceCompactor(ai, experienceTracker);
|
|
233
|
-
}));
|
|
245
|
+
return (this.agents.experienceCompactor ||= new ExperienceCompactor(this.provider, this.getExperienceTracker()));
|
|
234
246
|
}
|
|
235
247
|
|
|
236
248
|
agentQuartermaster(): Quartermaster | null {
|
|
@@ -247,10 +259,10 @@ export class ExplorBot {
|
|
|
247
259
|
}
|
|
248
260
|
|
|
249
261
|
agentHistorian(): Historian {
|
|
250
|
-
return (this.agents.historian ||= this.createAgent(({ ai, explorer }) => {
|
|
262
|
+
return (this.agents.historian ||= this.createAgent(({ ai, explorer, config }) => {
|
|
251
263
|
const experienceTracker = explorer.getStateManager().getExperienceTracker();
|
|
252
264
|
const reporter = explorer.getReporter();
|
|
253
|
-
return new Historian(ai, experienceTracker, reporter, explorer.getStateManager());
|
|
265
|
+
return new Historian(ai, experienceTracker, reporter, explorer.getStateManager(), config, explorer.getPlaywrightRecorder());
|
|
254
266
|
}));
|
|
255
267
|
}
|
|
256
268
|
|
|
@@ -348,20 +360,17 @@ export class ExplorBot {
|
|
|
348
360
|
if (this.currentPlan) {
|
|
349
361
|
planner.setPlan(this.currentPlan);
|
|
350
362
|
}
|
|
363
|
+
this.lastPlanError = null;
|
|
351
364
|
try {
|
|
352
365
|
this.currentPlan = await planner.plan(feature, opts.style, opts.extend, opts.completedPlans);
|
|
353
366
|
} catch (err) {
|
|
354
|
-
|
|
367
|
+
this.lastPlanError = err instanceof Error ? err : new Error(String(err));
|
|
368
|
+
tag('warning').log(`Planning failed: ${this.lastPlanError.message}`);
|
|
355
369
|
if (!this.currentPlan) return undefined;
|
|
356
370
|
return this.currentPlan;
|
|
357
371
|
}
|
|
358
372
|
|
|
359
|
-
|
|
360
|
-
if (savedPath) {
|
|
361
|
-
const relativePath = path.relative(process.cwd(), savedPath);
|
|
362
|
-
tag('info').log(`Plan saved to: ${relativePath}`);
|
|
363
|
-
tag('info').log(`Edit the plan file and run /plan:load ${relativePath} to reload it`);
|
|
364
|
-
}
|
|
373
|
+
this.savePlan();
|
|
365
374
|
|
|
366
375
|
return this.currentPlan;
|
|
367
376
|
}
|
|
@@ -387,6 +396,7 @@ export class ExplorBot {
|
|
|
387
396
|
const planFilename = filename || this.generatePlanFilename();
|
|
388
397
|
const planPath = path.join(plansDir, planFilename);
|
|
389
398
|
Plan.saveMultipleToMarkdown(plans, planPath);
|
|
399
|
+
this.lastSavedPlanPath = planPath;
|
|
390
400
|
return planPath;
|
|
391
401
|
}
|
|
392
402
|
|
package/src/explorer.ts
CHANGED
|
@@ -9,15 +9,16 @@ import { ActionResult } from './action-result.ts';
|
|
|
9
9
|
import Action from './action.js';
|
|
10
10
|
import { AIProvider } from './ai/provider.js';
|
|
11
11
|
import { visuallyAnnotateContainers } from './ai/researcher/coordinates.ts';
|
|
12
|
+
import { RequestStore } from './api/request-store.ts';
|
|
13
|
+
import { XhrCapture } from './api/xhr-capture.ts';
|
|
12
14
|
import type { ExplorbotConfig } from './config.js';
|
|
13
15
|
import { ConfigParser, outputPath } from './config.js';
|
|
14
16
|
import type { UserResolveFunction } from './explorbot.js';
|
|
15
17
|
import { KnowledgeTracker } from './knowledge-tracker.js';
|
|
18
|
+
import { PlaywrightRecorder } from './playwright-recorder.ts';
|
|
16
19
|
import { Reporter } from './reporter.ts';
|
|
17
20
|
import { StateManager } from './state-manager.js';
|
|
18
21
|
import { Test } from './test-plan.ts';
|
|
19
|
-
import { RequestStore } from './api/request-store.ts';
|
|
20
|
-
import { XhrCapture } from './api/xhr-capture.ts';
|
|
21
22
|
import { createDebug, log, tag } from './utils/logger.js';
|
|
22
23
|
import { WebElement, extractElementData } from './utils/web-element.ts';
|
|
23
24
|
|
|
@@ -60,6 +61,7 @@ class Explorer {
|
|
|
60
61
|
private _activeTest: Test | null = null;
|
|
61
62
|
private xhrCapture: XhrCapture | null = null;
|
|
62
63
|
private requestStore: RequestStore | null = null;
|
|
64
|
+
private playwrightRecorder: PlaywrightRecorder = new PlaywrightRecorder();
|
|
63
65
|
|
|
64
66
|
constructor(config: ExplorbotConfig, aiProvider: AIProvider, options?: { show?: boolean; headless?: boolean; incognito?: boolean; session?: string }) {
|
|
65
67
|
this.config = config;
|
|
@@ -68,7 +70,7 @@ class Explorer {
|
|
|
68
70
|
this.initializeContainer();
|
|
69
71
|
this.stateManager = new StateManager({ incognito: this.options?.incognito });
|
|
70
72
|
this.knowledgeTracker = new KnowledgeTracker();
|
|
71
|
-
this.reporter = new Reporter(config.reporter);
|
|
73
|
+
this.reporter = new Reporter(config.reporter, this.stateManager);
|
|
72
74
|
}
|
|
73
75
|
|
|
74
76
|
private initializeContainer() {
|
|
@@ -123,7 +125,7 @@ class Explorer {
|
|
|
123
125
|
tag('substep').log(debugInfo);
|
|
124
126
|
}
|
|
125
127
|
const PlaywrightConfig = {
|
|
126
|
-
timeout:
|
|
128
|
+
timeout: 3000,
|
|
127
129
|
highlightElement: true,
|
|
128
130
|
waitForAction: 500,
|
|
129
131
|
...playwrightConfig,
|
|
@@ -237,6 +239,7 @@ class Explorer {
|
|
|
237
239
|
const hasSession = this.options?.session && existsSync(this.options.session);
|
|
238
240
|
const contextOptions = hasSession ? { storageState: this.options!.session } : undefined;
|
|
239
241
|
await this.playwrightHelper._createContextPage(contextOptions);
|
|
242
|
+
await this.playwrightRecorder.start(this.playwrightHelper.browserContext);
|
|
240
243
|
this.setupXhrCapture();
|
|
241
244
|
if (hasSession) {
|
|
242
245
|
tag('info').log(`Session restored from ${path.relative(process.cwd(), this.options!.session!)}`);
|
|
@@ -273,7 +276,11 @@ class Explorer {
|
|
|
273
276
|
}
|
|
274
277
|
|
|
275
278
|
createAction() {
|
|
276
|
-
return new Action(this.actor, this.stateManager);
|
|
279
|
+
return new Action(this.actor, this.stateManager, this.playwrightRecorder);
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
getPlaywrightRecorder(): PlaywrightRecorder {
|
|
283
|
+
return this.playwrightRecorder;
|
|
277
284
|
}
|
|
278
285
|
|
|
279
286
|
async visit(url: string) {
|
|
@@ -489,6 +496,8 @@ class Explorer {
|
|
|
489
496
|
this.xhrCapture.detach(this.playwrightHelper.page);
|
|
490
497
|
}
|
|
491
498
|
|
|
499
|
+
await this.playwrightRecorder.stop();
|
|
500
|
+
|
|
492
501
|
if (this.options?.session && this.playwrightHelper?.browserContext) {
|
|
493
502
|
const dir = path.dirname(this.options.session);
|
|
494
503
|
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|