explorbot 0.1.8 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/explorbot-cli.ts +70 -8
- package/boat/api-tester/src/ai/curler-tools.ts +3 -3
- package/boat/api-tester/src/ai/curler.ts +1 -1
- package/boat/api-tester/src/apibot.ts +2 -2
- package/boat/api-tester/src/config.ts +1 -1
- package/dist/bin/explorbot-cli.js +70 -7
- package/dist/boat/api-tester/src/ai/curler-tools.js +2 -2
- package/dist/boat/api-tester/src/apibot.js +2 -2
- package/dist/package.json +1 -1
- package/dist/src/ai/bosun.js +5 -1
- package/dist/src/ai/experience-compactor.js +235 -50
- package/dist/src/ai/historian.js +13 -6
- package/dist/src/ai/navigator.js +62 -62
- package/dist/src/ai/pilot.js +22 -0
- package/dist/src/ai/planner/subpages.js +1 -30
- package/dist/src/ai/planner.js +4 -4
- package/dist/src/ai/provider.js +1 -1
- package/dist/src/ai/rerunner.js +3 -3
- package/dist/src/ai/researcher/deep-analysis.js +1 -1
- package/dist/src/ai/researcher/fingerprint-worker.js +1 -1
- package/dist/src/ai/researcher/locators.js +1 -1
- package/dist/src/ai/researcher/sections.js +8 -1
- package/dist/src/ai/researcher.js +4 -11
- package/dist/src/ai/tools.js +5 -3
- package/dist/src/api/request-store.js +20 -0
- package/dist/src/api/xhr-capture.js +19 -3
- package/dist/src/command-handler.js +1 -1
- package/dist/src/commands/add-rule-command.js +1 -1
- package/dist/src/commands/base-command.js +20 -0
- package/dist/src/commands/clean-command.js +1 -1
- package/dist/src/commands/compact-command.js +138 -0
- package/dist/src/commands/context-command.js +7 -1
- package/dist/src/commands/drill-command.js +4 -1
- package/dist/src/commands/experience-command.js +104 -0
- package/dist/src/commands/explore-command.js +33 -7
- package/dist/src/commands/freesail-command.js +2 -0
- package/dist/src/commands/index.js +7 -3
- package/dist/src/commands/init-command.js +2 -2
- package/dist/src/commands/learn-command.js +1 -1
- package/dist/src/commands/navigate-command.js +4 -1
- package/dist/src/commands/plan-clear-command.js +4 -1
- package/dist/src/commands/plan-command.js +11 -4
- package/dist/src/commands/plan-edit-command.js +1 -1
- package/dist/src/commands/plan-load-command.js +4 -1
- package/dist/src/commands/plan-reload-command.js +4 -1
- package/dist/src/commands/plan-save-command.js +1 -1
- package/dist/src/commands/research-command.js +5 -2
- package/dist/src/commands/start-command.js +5 -1
- package/dist/src/commands/test-command.js +7 -1
- package/dist/src/experience-tracker.js +191 -56
- package/dist/src/explorbot.js +26 -14
- package/dist/src/explorer.js +3 -3
- package/dist/src/reporter.js +17 -2
- package/dist/src/stats.js +2 -0
- package/dist/src/suite.js +1 -1
- package/dist/src/utils/error-page.js +10 -0
- package/dist/src/utils/logger.js +1 -1
- package/dist/src/utils/rules-loader.js +1 -1
- package/dist/src/utils/test-files.js +1 -1
- package/dist/src/utils/url-matcher.js +50 -0
- package/package.json +1 -1
- package/src/ai/bosun.ts +5 -1
- package/src/ai/experience-compactor.ts +270 -63
- package/src/ai/historian.ts +12 -7
- package/src/ai/navigator.ts +68 -66
- package/src/ai/pilot.ts +22 -0
- package/src/ai/planner/subpages.ts +1 -24
- package/src/ai/planner.ts +5 -5
- package/src/ai/provider.ts +1 -1
- package/src/ai/rerunner.ts +3 -3
- package/src/ai/researcher/deep-analysis.ts +1 -1
- package/src/ai/researcher/fingerprint-worker.ts +1 -1
- package/src/ai/researcher/locators.ts +2 -2
- package/src/ai/researcher/sections.ts +7 -1
- package/src/ai/researcher.ts +4 -11
- package/src/ai/task-agent.ts +1 -1
- package/src/ai/tools.ts +6 -4
- package/src/api/request-store.ts +22 -0
- package/src/api/xhr-capture.ts +21 -3
- package/src/command-handler.ts +1 -1
- package/src/commands/add-rule-command.ts +2 -2
- package/src/commands/base-command.ts +26 -1
- package/src/commands/clean-command.ts +2 -2
- package/src/commands/compact-command.ts +156 -0
- package/src/commands/context-command.ts +8 -2
- package/src/commands/drill-command.ts +5 -2
- package/src/commands/experience-command.ts +125 -0
- package/src/commands/explore-command.ts +35 -9
- package/src/commands/freesail-command.ts +2 -0
- package/src/commands/index.ts +7 -3
- package/src/commands/init-command.ts +2 -2
- package/src/commands/learn-command.ts +2 -2
- package/src/commands/navigate-command.ts +5 -2
- package/src/commands/plan-clear-command.ts +5 -2
- package/src/commands/plan-command.ts +12 -5
- package/src/commands/plan-edit-command.ts +2 -2
- package/src/commands/plan-load-command.ts +5 -2
- package/src/commands/plan-reload-command.ts +5 -2
- package/src/commands/plan-save-command.ts +2 -2
- package/src/commands/research-command.ts +6 -3
- package/src/commands/start-command.ts +6 -2
- package/src/commands/test-command.ts +8 -2
- package/src/experience-tracker.ts +220 -71
- package/src/explorbot.ts +28 -15
- package/src/explorer.ts +3 -3
- package/src/reporter.ts +17 -3
- package/src/stats.ts +4 -0
- package/src/suite.ts +1 -1
- package/src/utils/error-page.ts +10 -0
- package/src/utils/logger.ts +1 -1
- package/src/utils/rules-loader.ts +1 -1
- package/src/utils/test-files.ts +1 -1
- package/src/utils/url-matcher.ts +43 -0
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { BaseCommand } from './base-command.js';
|
|
1
|
+
import { BaseCommand, type Suggestion } from './base-command.js';
|
|
2
2
|
|
|
3
3
|
export class PlanEditCommand extends BaseCommand {
|
|
4
4
|
name = 'plan:edit';
|
|
5
5
|
description = 'Open test plan editor';
|
|
6
|
-
suggestions = ['
|
|
6
|
+
suggestions: Suggestion[] = [{ command: 'plan:edit', hint: 'toggle tests on/off' }];
|
|
7
7
|
|
|
8
8
|
async execute(_args: string): Promise<void> {}
|
|
9
9
|
}
|
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import { tag } from '../utils/logger.js';
|
|
2
|
-
import { BaseCommand } from './base-command.js';
|
|
2
|
+
import { BaseCommand, type Suggestion } from './base-command.js';
|
|
3
3
|
|
|
4
4
|
export class PlanLoadCommand extends BaseCommand {
|
|
5
5
|
name = 'plan:load';
|
|
6
6
|
description = 'Load plan from file';
|
|
7
|
-
suggestions = [
|
|
7
|
+
suggestions: Suggestion[] = [
|
|
8
|
+
{ command: 'test', hint: 'launch first test' },
|
|
9
|
+
{ command: 'test *', hint: 'launch all tests' },
|
|
10
|
+
];
|
|
8
11
|
|
|
9
12
|
async execute(args: string): Promise<void> {
|
|
10
13
|
const filename = args.trim();
|
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import { tag } from '../utils/logger.js';
|
|
2
|
-
import { BaseCommand } from './base-command.js';
|
|
2
|
+
import { BaseCommand, type Suggestion } from './base-command.js';
|
|
3
3
|
|
|
4
4
|
export class PlanReloadCommand extends BaseCommand {
|
|
5
5
|
name = 'plan:reload';
|
|
6
6
|
description = 'Clear current plan and regenerate';
|
|
7
|
-
suggestions = [
|
|
7
|
+
suggestions: Suggestion[] = [
|
|
8
|
+
{ command: 'test', hint: 'launch first test' },
|
|
9
|
+
{ command: 'test *', hint: 'launch all tests' },
|
|
10
|
+
];
|
|
8
11
|
|
|
9
12
|
async execute(args: string): Promise<void> {
|
|
10
13
|
const currentPlan = this.explorBot.getCurrentPlan();
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import path from 'node:path';
|
|
2
2
|
import { tag } from '../utils/logger.js';
|
|
3
|
-
import { BaseCommand } from './base-command.js';
|
|
3
|
+
import { BaseCommand, type Suggestion } from './base-command.js';
|
|
4
4
|
|
|
5
5
|
export class PlanSaveCommand extends BaseCommand {
|
|
6
6
|
name = 'plan:save';
|
|
7
7
|
description = 'Save current plan to file';
|
|
8
|
-
suggestions = ['
|
|
8
|
+
suggestions: Suggestion[] = [{ command: 'test', hint: 'launch first test' }];
|
|
9
9
|
|
|
10
10
|
async execute(args: string): Promise<void> {
|
|
11
11
|
const plan = this.explorBot.getCurrentPlan();
|
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
import { join } from 'node:path';
|
|
2
2
|
import { ConfigParser } from '../config.ts';
|
|
3
3
|
import { tag } from '../utils/logger.ts';
|
|
4
|
-
import { BaseCommand } from './base-command.js';
|
|
4
|
+
import { BaseCommand, type Suggestion } from './base-command.js';
|
|
5
5
|
|
|
6
6
|
export class ResearchCommand extends BaseCommand {
|
|
7
7
|
name = 'research';
|
|
8
8
|
description = 'Research current page or navigate to URI and research. Use --deep to explore interactive elements by clicking them. Use --data to include page data.';
|
|
9
|
-
suggestions = [
|
|
9
|
+
suggestions: Suggestion[] = [
|
|
10
|
+
{ command: 'navigate <page>', hint: 'go to another page' },
|
|
11
|
+
{ command: 'plan <feature>', hint: 'plan testing' },
|
|
12
|
+
];
|
|
10
13
|
options = [
|
|
11
14
|
{ flags: '--data', description: 'Include page data' },
|
|
12
15
|
{ flags: '--deep', description: 'Explore interactive elements by clicking them' },
|
|
@@ -45,7 +48,7 @@ export class ResearchCommand extends BaseCommand {
|
|
|
45
48
|
}
|
|
46
49
|
|
|
47
50
|
if (!enableDeep) {
|
|
48
|
-
this.suggestions = ['
|
|
51
|
+
this.suggestions = [{ command: 'research <page> --deep', hint: 'analyze page for all expandable elements and interactions' }];
|
|
49
52
|
}
|
|
50
53
|
}
|
|
51
54
|
}
|
|
@@ -1,10 +1,14 @@
|
|
|
1
|
-
import { BaseCommand } from './base-command.js';
|
|
1
|
+
import { BaseCommand, type Suggestion } from './base-command.js';
|
|
2
2
|
import { ExploreCommand } from './explore-command.js';
|
|
3
3
|
|
|
4
4
|
export class StartCommand extends BaseCommand {
|
|
5
5
|
name = 'start';
|
|
6
6
|
description = 'Start web exploration';
|
|
7
|
-
suggestions = [
|
|
7
|
+
suggestions: Suggestion[] = [
|
|
8
|
+
{ command: 'navigate <page>', hint: 'go to another page' },
|
|
9
|
+
{ command: 'research', hint: 'analyze current page' },
|
|
10
|
+
{ command: 'plan <feature>', hint: 'plan testing' },
|
|
11
|
+
];
|
|
8
12
|
|
|
9
13
|
async execute(args: string): Promise<void> {
|
|
10
14
|
await new ExploreCommand(this.explorBot).execute(args);
|
|
@@ -1,14 +1,20 @@
|
|
|
1
|
+
import { Stats } from '../stats.js';
|
|
1
2
|
import { Test } from '../test-plan.js';
|
|
2
3
|
import { tag } from '../utils/logger.js';
|
|
3
|
-
import { BaseCommand } from './base-command.js';
|
|
4
|
+
import { BaseCommand, type Suggestion } from './base-command.js';
|
|
4
5
|
|
|
5
6
|
export class TestCommand extends BaseCommand {
|
|
6
7
|
name = 'test';
|
|
7
8
|
description = 'Launch tester agent to execute test scenarios';
|
|
8
|
-
suggestions = [
|
|
9
|
+
suggestions: Suggestion[] = [
|
|
10
|
+
{ command: 'test', hint: 'run next test' },
|
|
11
|
+
{ command: 'plan', hint: 'create new plan' },
|
|
12
|
+
];
|
|
9
13
|
|
|
10
14
|
async execute(args: string): Promise<void> {
|
|
11
15
|
const plan = this.explorBot.getCurrentPlan();
|
|
16
|
+
Stats.mode = 'test';
|
|
17
|
+
Stats.focus = plan?.title;
|
|
12
18
|
const toExecute: Test[] = [];
|
|
13
19
|
|
|
14
20
|
const requirePlan = () => {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs';
|
|
2
2
|
import { basename, dirname, join } from 'node:path';
|
|
3
3
|
import matter from 'gray-matter';
|
|
4
|
-
import {
|
|
4
|
+
import { type Tokens, marked } from 'marked';
|
|
5
5
|
import type { ActionResult } from './action-result.js';
|
|
6
6
|
import { ConfigParser } from './config.js';
|
|
7
7
|
import { KnowledgeTracker } from './knowledge-tracker.js';
|
|
@@ -13,6 +13,23 @@ import { extractStatePath } from './utils/url-matcher.js';
|
|
|
13
13
|
const debugLog = createDebug('explorbot:experience');
|
|
14
14
|
const DEFAULT_MAX_EXPERIENCE_LINES = 100;
|
|
15
15
|
|
|
16
|
+
export const RECENT_WINDOW_DAYS = 30;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Stores and reads per-page experience files (`./experience/<stateHash>.md`).
|
|
20
|
+
*
|
|
21
|
+
* Format rules (enforced by writeFlow/writeAction — the only supported writers):
|
|
22
|
+
*
|
|
23
|
+
* ## FLOW: <imperative title> multi-step, `*` bullets + optional ```js``` + `>` discovery, ends with `---`
|
|
24
|
+
* ## ACTION: <imperative title> single-step, optional `Solution:` line + one ```js``` code block
|
|
25
|
+
*
|
|
26
|
+
* - Always h2. Never h3 for FLOW/ACTION.
|
|
27
|
+
* - Title is an imperative verb phrase, lowercase-first, no trailing punctuation.
|
|
28
|
+
* Writers normalize automatically (strip own `FLOW:` / `ACTION:` prefix if the caller included it,
|
|
29
|
+
* lowercase first char, trim trailing `.!?,;:`).
|
|
30
|
+
* - On read (getSuccessfulExperience), headings are rendered as
|
|
31
|
+
* `## HOW to <title> (multi-step|single-step)` so prompts get natural phrasing.
|
|
32
|
+
*/
|
|
16
33
|
export class ExperienceTracker {
|
|
17
34
|
private experienceDir: string;
|
|
18
35
|
private disabled: boolean;
|
|
@@ -144,35 +161,57 @@ export class ExperienceTracker {
|
|
|
144
161
|
return this.knowledgeTracker.getRelevantKnowledge(state).some((k) => k.noExperienceWriting === true || k.noExperienceWriting === 'true');
|
|
145
162
|
}
|
|
146
163
|
|
|
147
|
-
|
|
164
|
+
writeAction(state: ActionResult, action: ActionInput): void {
|
|
148
165
|
if (this.disabled || this.isWritingDisabled(state)) return;
|
|
166
|
+
if (!action.code?.trim()) return;
|
|
149
167
|
|
|
150
168
|
this.ensureExperienceFile(state);
|
|
151
169
|
const stateHash = state.getStateHash();
|
|
152
170
|
const { content, data } = this.readExperienceFile(stateHash);
|
|
153
|
-
if (content.includes(code)) {
|
|
154
|
-
debugLog('Skipping duplicate
|
|
171
|
+
if (content.includes(action.code)) {
|
|
172
|
+
debugLog('Skipping duplicate action', action.code);
|
|
155
173
|
return;
|
|
156
174
|
}
|
|
157
175
|
|
|
158
|
-
const
|
|
159
|
-
|
|
176
|
+
const title = normalizeTitle(action.title.split('\n')[0]);
|
|
177
|
+
if (!title) return;
|
|
160
178
|
|
|
161
|
-
|
|
179
|
+
const filteredCode = action.code.replace(/I\.amOnPage\s*\([^)]*\)/gs, '');
|
|
180
|
+
const newEntry = generateActionContent(title, filteredCode, action.explanation);
|
|
181
|
+
const updatedContent = `${newEntry}\n\n${content}`;
|
|
182
|
+
this.writeExperienceFile(stateHash, updatedContent, data);
|
|
183
|
+
|
|
184
|
+
tag('substep').log(` Added ACTION to: ${stateHash}.md`);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
writeFlow(state: ActionResult, flow: FlowInput): void {
|
|
188
|
+
if (this.disabled || this.isWritingDisabled(state)) return;
|
|
189
|
+
if (!flow.steps?.length) return;
|
|
190
|
+
|
|
191
|
+
this.ensureExperienceFile(state);
|
|
192
|
+
const stateHash = state.getStateHash();
|
|
193
|
+
const { content, data } = this.readExperienceFile(stateHash);
|
|
194
|
+
|
|
195
|
+
if (flow.relatedUrls?.length) {
|
|
196
|
+
const currentPath = extractStatePath(state.url || '');
|
|
197
|
+
const existingRelated = Array.isArray(data.related) ? data.related : [];
|
|
198
|
+
const allRelated = [...new Set([...existingRelated, ...flow.relatedUrls])];
|
|
199
|
+
data.related = allRelated.filter((url) => url !== currentPath);
|
|
200
|
+
}
|
|
162
201
|
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
\`\`\`
|
|
166
|
-
`;
|
|
202
|
+
const title = normalizeTitle(flow.scenario);
|
|
203
|
+
if (!title) return;
|
|
167
204
|
|
|
168
|
-
const
|
|
205
|
+
const sessionContent = this.trimSessionContent(generateFlowContent(title, flow.steps));
|
|
206
|
+
if (!sessionContent) return;
|
|
207
|
+
const updatedContent = `${sessionContent}\n${content}`;
|
|
169
208
|
this.writeExperienceFile(stateHash, updatedContent, data);
|
|
170
209
|
|
|
171
|
-
tag('substep').log(`
|
|
210
|
+
tag('substep').log(`Added FLOW to: ${stateHash}.md`);
|
|
172
211
|
}
|
|
173
212
|
|
|
174
|
-
getAllExperience():
|
|
175
|
-
const allFiles:
|
|
213
|
+
getAllExperience(): ExperienceFile[] {
|
|
214
|
+
const allFiles: ExperienceFile[] = [];
|
|
176
215
|
|
|
177
216
|
for (const experienceDir of this.getExperienceDirectories()) {
|
|
178
217
|
if (!existsSync(experienceDir)) {
|
|
@@ -188,10 +227,12 @@ ${filteredCode}
|
|
|
188
227
|
try {
|
|
189
228
|
const content = readFileSync(file, 'utf8');
|
|
190
229
|
const parsed = matter(content);
|
|
230
|
+
const mtime = statSync(file).mtime;
|
|
191
231
|
allFiles.push({
|
|
192
232
|
filePath: file,
|
|
193
233
|
data: parsed.data,
|
|
194
234
|
content: parsed.content,
|
|
235
|
+
mtime,
|
|
195
236
|
});
|
|
196
237
|
} catch (error) {
|
|
197
238
|
debugLog(`Failed to read experience file ${file}:`, error);
|
|
@@ -205,7 +246,7 @@ ${filteredCode}
|
|
|
205
246
|
return allFiles;
|
|
206
247
|
}
|
|
207
248
|
|
|
208
|
-
getRelevantExperience(state: ActionResult, options?: { includeDescendantExperience?: boolean }):
|
|
249
|
+
getRelevantExperience(state: ActionResult, options?: { includeDescendantExperience?: boolean }): ExperienceFile[] {
|
|
209
250
|
const relevantKnowledge = this.knowledgeTracker.getRelevantKnowledge(state);
|
|
210
251
|
const readingDisabled = relevantKnowledge.some((knowledge) => knowledge.noExperienceReading === true || knowledge.noExperienceReading === 'true');
|
|
211
252
|
if (readingDisabled) {
|
|
@@ -236,50 +277,6 @@ ${filteredCode}
|
|
|
236
277
|
// The actual files will be cleaned up by test cleanup
|
|
237
278
|
}
|
|
238
279
|
|
|
239
|
-
saveSessionExperience(state: ActionResult, entry: SessionExperienceEntry): void {
|
|
240
|
-
if (this.disabled || this.isWritingDisabled(state)) return;
|
|
241
|
-
|
|
242
|
-
this.ensureExperienceFile(state);
|
|
243
|
-
const stateHash = state.getStateHash();
|
|
244
|
-
const { content, data } = this.readExperienceFile(stateHash);
|
|
245
|
-
|
|
246
|
-
if (entry.relatedUrls?.length) {
|
|
247
|
-
const currentPath = extractStatePath(state.url || '');
|
|
248
|
-
const existingRelated = Array.isArray(data.related) ? data.related : [];
|
|
249
|
-
const allRelated = [...new Set([...existingRelated, ...entry.relatedUrls])];
|
|
250
|
-
data.related = allRelated.filter((url) => url !== currentPath);
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
const sessionContent = this.trimSessionContent(this.generateSessionContent(entry));
|
|
254
|
-
if (!sessionContent) return;
|
|
255
|
-
const updatedContent = `${sessionContent}\n${content}`;
|
|
256
|
-
this.writeExperienceFile(stateHash, updatedContent, data);
|
|
257
|
-
|
|
258
|
-
tag('substep').log(`Added session experience to: ${stateHash}.md`);
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
private generateSessionContent(entry: SessionExperienceEntry): string {
|
|
262
|
-
let content = `## Successful Flow: ${entry.scenario}\n\n`;
|
|
263
|
-
|
|
264
|
-
for (const step of entry.steps) {
|
|
265
|
-
content += `* ${step.message}\n\n`;
|
|
266
|
-
if (step.code) {
|
|
267
|
-
content += '```js\n';
|
|
268
|
-
content += `${step.code}\n`;
|
|
269
|
-
content += '```\n\n';
|
|
270
|
-
}
|
|
271
|
-
if (step.discovery) {
|
|
272
|
-
const discoveries = step.discovery.split('\n').filter((d) => d.trim());
|
|
273
|
-
for (const discovery of discoveries) {
|
|
274
|
-
content += `> ${discovery.trim()}\n\n`;
|
|
275
|
-
}
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
content += '---\n';
|
|
280
|
-
return content;
|
|
281
|
-
}
|
|
282
|
-
|
|
283
280
|
private trimSessionContent(content: string): string | null {
|
|
284
281
|
const q = mdq(content);
|
|
285
282
|
if (q.query('heading').count() === 0) return null;
|
|
@@ -318,12 +315,14 @@ ${filteredCode}
|
|
|
318
315
|
for (const record of records) {
|
|
319
316
|
if (!record.content) continue;
|
|
320
317
|
|
|
321
|
-
const
|
|
322
|
-
const
|
|
323
|
-
let combined = [
|
|
318
|
+
const flows = mdq(record.content).query('section(~"FLOW:")').text();
|
|
319
|
+
const actions = mdq(record.content).query('section(~"ACTION:")').text();
|
|
320
|
+
let combined = [flows, actions].filter(Boolean).join('\n\n');
|
|
324
321
|
|
|
325
322
|
if (!combined.trim()) continue;
|
|
326
323
|
|
|
324
|
+
combined = renderAsHowTo(combined);
|
|
325
|
+
|
|
327
326
|
if (options?.stripCode) {
|
|
328
327
|
combined = mdq(combined).query('code').replace('');
|
|
329
328
|
}
|
|
@@ -383,6 +382,72 @@ ${filteredCode}
|
|
|
383
382
|
}
|
|
384
383
|
return null;
|
|
385
384
|
}
|
|
385
|
+
|
|
386
|
+
listAllExperienceToc(filter?: string, options?: { recency?: 'recent' | 'old' }): ExperienceTocEntry[] {
|
|
387
|
+
const records = this.getAllExperience();
|
|
388
|
+
if (records.length === 0) return [];
|
|
389
|
+
|
|
390
|
+
const trimmed = filter?.trim();
|
|
391
|
+
let matching = records;
|
|
392
|
+
|
|
393
|
+
if (trimmed) {
|
|
394
|
+
if (trimmed.endsWith('.md')) {
|
|
395
|
+
const bare = trimmed.slice(0, -3);
|
|
396
|
+
const byFilename = records.find((record) => basename(record.filePath, '.md') === bare);
|
|
397
|
+
matching = byFilename ? [byFilename] : [];
|
|
398
|
+
} else {
|
|
399
|
+
const lower = trimmed.toLowerCase();
|
|
400
|
+
matching = records.filter((record) => {
|
|
401
|
+
const url = ((record.data as WebPageState)?.url || '').toLowerCase();
|
|
402
|
+
if (!url) return false;
|
|
403
|
+
return url.includes(lower);
|
|
404
|
+
});
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
if (options?.recency) {
|
|
409
|
+
const cutoff = Date.now() - RECENT_WINDOW_DAYS * 24 * 60 * 60 * 1000;
|
|
410
|
+
matching = matching.filter((record) => {
|
|
411
|
+
const isRecent = record.mtime.getTime() >= cutoff;
|
|
412
|
+
return options.recency === 'recent' ? isRecent : !isRecent;
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
const sorted = matching.sort((a, b) => {
|
|
417
|
+
const aUrl = (a.data as WebPageState)?.url || '';
|
|
418
|
+
const bUrl = (b.data as WebPageState)?.url || '';
|
|
419
|
+
return aUrl.localeCompare(bUrl);
|
|
420
|
+
});
|
|
421
|
+
|
|
422
|
+
const toc: ExperienceTocEntry[] = [];
|
|
423
|
+
for (let i = 0; i < sorted.length; i++) {
|
|
424
|
+
const record = sorted[i];
|
|
425
|
+
const sections = listTocHeadings(record.content);
|
|
426
|
+
if (sections.length === 0) continue;
|
|
427
|
+
toc.push({
|
|
428
|
+
fileTag: indexToLetters(toc.length),
|
|
429
|
+
fileHash: basename(record.filePath, '.md'),
|
|
430
|
+
url: (record.data as WebPageState)?.url || '',
|
|
431
|
+
sections,
|
|
432
|
+
});
|
|
433
|
+
}
|
|
434
|
+
return toc;
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
getExperienceSectionByTag(fileTag: string, sectionIndex: number, filter?: string): { title: string; url: string; content: string; fileHash: string } | null {
|
|
438
|
+
const toc = this.listAllExperienceToc(filter);
|
|
439
|
+
const entry = toc.find((e) => e.fileTag === fileTag);
|
|
440
|
+
if (!entry) return null;
|
|
441
|
+
|
|
442
|
+
const filePath = this.findExperienceFileByHash(entry.fileHash);
|
|
443
|
+
if (!filePath) return null;
|
|
444
|
+
|
|
445
|
+
const { content } = this.readExperienceFile(entry.fileHash);
|
|
446
|
+
const extracted = extractHeadingSection(content, sectionIndex);
|
|
447
|
+
if (!extracted) return null;
|
|
448
|
+
|
|
449
|
+
return { title: extracted.title, url: entry.url, content: extracted.body, fileHash: entry.fileHash };
|
|
450
|
+
}
|
|
386
451
|
}
|
|
387
452
|
|
|
388
453
|
function listTocHeadings(content: string): { index: number; level: 2 | 3; title: string }[] {
|
|
@@ -448,7 +513,9 @@ export function renderExperienceToc(toc: ExperienceTocEntry[]): string {
|
|
|
448
513
|
|
|
449
514
|
const lines: string[] = [];
|
|
450
515
|
lines.push('<experience>');
|
|
451
|
-
lines.push('Past experience for this page
|
|
516
|
+
lines.push('Past experience for this page — reusable recipes recorded from prior successful runs.');
|
|
517
|
+
lines.push('FLOW: = multi-step recipe (bullets + code + discovery). ACTION: = single-step snippet (one code block).');
|
|
518
|
+
lines.push('Call learn_experience({ fileTag, sectionIndex }) to read a section when it looks relevant to the current step.');
|
|
452
519
|
lines.push('');
|
|
453
520
|
for (const entry of toc) {
|
|
454
521
|
lines.push(`File ${entry.fileTag} ${entry.url}:`);
|
|
@@ -462,6 +529,95 @@ export function renderExperienceToc(toc: ExperienceTocEntry[]): string {
|
|
|
462
529
|
return lines.join('\n');
|
|
463
530
|
}
|
|
464
531
|
|
|
532
|
+
function normalizeTitle(raw: string): string {
|
|
533
|
+
let t = (raw || '').trim();
|
|
534
|
+
for (const p of ['FLOW:', 'ACTION:']) {
|
|
535
|
+
if (t.toLowerCase().startsWith(p.toLowerCase())) {
|
|
536
|
+
t = t.slice(p.length).trim();
|
|
537
|
+
break;
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
while (t.length > 0 && '.!?,;:'.includes(t[t.length - 1])) {
|
|
541
|
+
t = t.slice(0, -1);
|
|
542
|
+
}
|
|
543
|
+
if (t.length > 0) t = t[0].toLowerCase() + t.slice(1);
|
|
544
|
+
return t;
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
function generateActionContent(title: string, code: string, explanation?: string): string {
|
|
548
|
+
const lines: string[] = [];
|
|
549
|
+
lines.push(`## ACTION: ${title}`);
|
|
550
|
+
lines.push('');
|
|
551
|
+
if (explanation) {
|
|
552
|
+
lines.push(`Solution: ${explanation}`);
|
|
553
|
+
lines.push('');
|
|
554
|
+
}
|
|
555
|
+
lines.push('```javascript');
|
|
556
|
+
lines.push(code);
|
|
557
|
+
lines.push('```');
|
|
558
|
+
lines.push('');
|
|
559
|
+
return lines.join('\n');
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
function generateFlowContent(title: string, steps: SessionStep[]): string {
|
|
563
|
+
let content = `## FLOW: ${title}\n\n`;
|
|
564
|
+
for (const step of steps) {
|
|
565
|
+
content += `* ${step.message}\n\n`;
|
|
566
|
+
if (step.code) {
|
|
567
|
+
content += '```js\n';
|
|
568
|
+
content += `${step.code}\n`;
|
|
569
|
+
content += '```\n\n';
|
|
570
|
+
}
|
|
571
|
+
if (step.discovery) {
|
|
572
|
+
const discoveries = step.discovery.split('\n').filter((d) => d.trim());
|
|
573
|
+
for (const discovery of discoveries) {
|
|
574
|
+
content += `> ${discovery.trim()}\n\n`;
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
content += '---\n';
|
|
579
|
+
return content;
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
function renderAsHowTo(content: string): string {
|
|
583
|
+
const tokens = marked.lexer(content);
|
|
584
|
+
let result = '';
|
|
585
|
+
for (const token of tokens) {
|
|
586
|
+
if (token.type === 'heading' && (token as Tokens.Heading).depth === 2) {
|
|
587
|
+
const text = (token as Tokens.Heading).text.trim();
|
|
588
|
+
if (text.startsWith('FLOW:')) {
|
|
589
|
+
result += `## HOW to ${text.slice(5).trim()} (multi-step)\n\n`;
|
|
590
|
+
continue;
|
|
591
|
+
}
|
|
592
|
+
if (text.startsWith('ACTION:')) {
|
|
593
|
+
result += `## HOW to ${text.slice(7).trim()} (single-step)\n\n`;
|
|
594
|
+
continue;
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
result += (token as any).raw || '';
|
|
598
|
+
}
|
|
599
|
+
return result;
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
export interface ExperienceFile {
|
|
603
|
+
filePath: string;
|
|
604
|
+
data: { url?: string; title?: string; [key: string]: any };
|
|
605
|
+
content: string;
|
|
606
|
+
mtime: Date;
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
export interface FlowInput {
|
|
610
|
+
scenario: string;
|
|
611
|
+
steps: SessionStep[];
|
|
612
|
+
relatedUrls?: string[];
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
export interface ActionInput {
|
|
616
|
+
title: string;
|
|
617
|
+
code: string;
|
|
618
|
+
explanation?: string;
|
|
619
|
+
}
|
|
620
|
+
|
|
465
621
|
export interface SessionStep {
|
|
466
622
|
message: string;
|
|
467
623
|
status: 'passed' | 'failed' | 'neutral';
|
|
@@ -470,13 +626,6 @@ export interface SessionStep {
|
|
|
470
626
|
discovery?: string;
|
|
471
627
|
}
|
|
472
628
|
|
|
473
|
-
export interface SessionExperienceEntry {
|
|
474
|
-
scenario: string;
|
|
475
|
-
result: 'success' | 'partial' | 'failed';
|
|
476
|
-
steps: SessionStep[];
|
|
477
|
-
relatedUrls?: string[];
|
|
478
|
-
}
|
|
479
|
-
|
|
480
629
|
export interface ExperienceTocEntry {
|
|
481
630
|
fileTag: string;
|
|
482
631
|
fileHash: string;
|
package/src/explorbot.ts
CHANGED
|
@@ -1,9 +1,6 @@
|
|
|
1
1
|
import { existsSync, mkdirSync } from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { ActionResult } from './action-result.ts';
|
|
4
|
-
import { ApiClient } from './api/api-client.ts';
|
|
5
|
-
import { RequestStore } from './api/request-store.ts';
|
|
6
|
-
import { loadSpec } from './api/spec-reader.ts';
|
|
7
4
|
import { Bosun } from './ai/bosun.ts';
|
|
8
5
|
import { Captain } from './ai/captain.ts';
|
|
9
6
|
import { ExperienceCompactor } from './ai/experience-compactor.ts';
|
|
@@ -14,16 +11,20 @@ import { Pilot } from './ai/pilot.ts';
|
|
|
14
11
|
import { Planner } from './ai/planner.ts';
|
|
15
12
|
import { AIProvider } from './ai/provider.ts';
|
|
16
13
|
import { Quartermaster } from './ai/quartermaster.ts';
|
|
17
|
-
import { Researcher } from './ai/researcher.ts';
|
|
18
14
|
import { Rerunner } from './ai/rerunner.ts';
|
|
15
|
+
import { Researcher } from './ai/researcher.ts';
|
|
19
16
|
import { Tester } from './ai/tester.ts';
|
|
20
17
|
import { createAgentTools } from './ai/tools.ts';
|
|
18
|
+
import { ApiClient } from './api/api-client.ts';
|
|
19
|
+
import { RequestStore } from './api/request-store.ts';
|
|
20
|
+
import { loadSpec } from './api/spec-reader.ts';
|
|
21
21
|
import type { ExplorbotConfig } from './config.js';
|
|
22
22
|
import { ConfigParser } from './config.ts';
|
|
23
|
+
import { ExperienceTracker } from './experience-tracker.ts';
|
|
23
24
|
import Explorer from './explorer.ts';
|
|
24
|
-
import type { Suite } from './suite.ts';
|
|
25
25
|
import { KnowledgeTracker } from './knowledge-tracker.ts';
|
|
26
26
|
import { WebPageState } from './state-manager.ts';
|
|
27
|
+
import type { Suite } from './suite.ts';
|
|
27
28
|
import { Plan } from './test-plan.ts';
|
|
28
29
|
import { setVerboseMode, tag } from './utils/logger.ts';
|
|
29
30
|
import { sanitizeFilename } from './utils/strings.ts';
|
|
@@ -51,6 +52,7 @@ export class ExplorBot {
|
|
|
51
52
|
public needsInput = false;
|
|
52
53
|
private currentPlan?: Plan;
|
|
53
54
|
private planFeature?: string;
|
|
55
|
+
lastPlanError: Error | null = null;
|
|
54
56
|
private agents: Record<string, any> = {};
|
|
55
57
|
|
|
56
58
|
constructor(options: ExplorBotOptions = {}) {
|
|
@@ -76,13 +78,11 @@ export class ExplorBot {
|
|
|
76
78
|
}
|
|
77
79
|
|
|
78
80
|
try {
|
|
79
|
-
|
|
80
|
-
this.provider = new AIProvider(this.config.ai);
|
|
81
|
-
await this.provider.validateConnection();
|
|
81
|
+
await this.startProviderOnly();
|
|
82
82
|
this.explorer = new Explorer(this.config, this.provider, this.options);
|
|
83
83
|
await this.explorer.start();
|
|
84
84
|
if (!this.options.incognito) {
|
|
85
|
-
await this.agentExperienceCompactor().
|
|
85
|
+
await this.agentExperienceCompactor().autocompact();
|
|
86
86
|
}
|
|
87
87
|
if (this.userResolveFn) this.explorer.setUserResolve(this.userResolveFn);
|
|
88
88
|
} catch (error) {
|
|
@@ -92,9 +92,16 @@ export class ExplorBot {
|
|
|
92
92
|
}
|
|
93
93
|
}
|
|
94
94
|
|
|
95
|
+
async startProviderOnly(): Promise<void> {
|
|
96
|
+
if (this.provider) return;
|
|
97
|
+
this.config = await this.configParser.loadConfig(this.options);
|
|
98
|
+
this.provider = new AIProvider(this.config.ai);
|
|
99
|
+
await this.provider.validateConnection();
|
|
100
|
+
}
|
|
101
|
+
|
|
95
102
|
async stop(): Promise<void> {
|
|
96
103
|
this.agents.quartermaster?.stop();
|
|
97
|
-
await this.explorer
|
|
104
|
+
await this.explorer?.stop();
|
|
98
105
|
}
|
|
99
106
|
|
|
100
107
|
async visitInitialState(): Promise<void> {
|
|
@@ -125,6 +132,13 @@ export class ExplorBot {
|
|
|
125
132
|
return new KnowledgeTracker();
|
|
126
133
|
}
|
|
127
134
|
|
|
135
|
+
getExperienceTracker(): ExperienceTracker {
|
|
136
|
+
if (this.explorer) {
|
|
137
|
+
return this.explorer.getStateManager().getExperienceTracker();
|
|
138
|
+
}
|
|
139
|
+
return new ExperienceTracker();
|
|
140
|
+
}
|
|
141
|
+
|
|
128
142
|
getConfig(): ExplorbotConfig {
|
|
129
143
|
return this.config;
|
|
130
144
|
}
|
|
@@ -227,10 +241,7 @@ export class ExplorBot {
|
|
|
227
241
|
}
|
|
228
242
|
|
|
229
243
|
agentExperienceCompactor(): ExperienceCompactor {
|
|
230
|
-
return (this.agents.experienceCompactor ||= this.
|
|
231
|
-
const experienceTracker = explorer.getStateManager().getExperienceTracker();
|
|
232
|
-
return new ExperienceCompactor(ai, experienceTracker);
|
|
233
|
-
}));
|
|
244
|
+
return (this.agents.experienceCompactor ||= new ExperienceCompactor(this.provider, this.getExperienceTracker()));
|
|
234
245
|
}
|
|
235
246
|
|
|
236
247
|
agentQuartermaster(): Quartermaster | null {
|
|
@@ -348,10 +359,12 @@ export class ExplorBot {
|
|
|
348
359
|
if (this.currentPlan) {
|
|
349
360
|
planner.setPlan(this.currentPlan);
|
|
350
361
|
}
|
|
362
|
+
this.lastPlanError = null;
|
|
351
363
|
try {
|
|
352
364
|
this.currentPlan = await planner.plan(feature, opts.style, opts.extend, opts.completedPlans);
|
|
353
365
|
} catch (err) {
|
|
354
|
-
|
|
366
|
+
this.lastPlanError = err instanceof Error ? err : new Error(String(err));
|
|
367
|
+
tag('warning').log(`Planning failed: ${this.lastPlanError.message}`);
|
|
355
368
|
if (!this.currentPlan) return undefined;
|
|
356
369
|
return this.currentPlan;
|
|
357
370
|
}
|
package/src/explorer.ts
CHANGED
|
@@ -9,6 +9,8 @@ import { ActionResult } from './action-result.ts';
|
|
|
9
9
|
import Action from './action.js';
|
|
10
10
|
import { AIProvider } from './ai/provider.js';
|
|
11
11
|
import { visuallyAnnotateContainers } from './ai/researcher/coordinates.ts';
|
|
12
|
+
import { RequestStore } from './api/request-store.ts';
|
|
13
|
+
import { XhrCapture } from './api/xhr-capture.ts';
|
|
12
14
|
import type { ExplorbotConfig } from './config.js';
|
|
13
15
|
import { ConfigParser, outputPath } from './config.js';
|
|
14
16
|
import type { UserResolveFunction } from './explorbot.js';
|
|
@@ -16,8 +18,6 @@ import { KnowledgeTracker } from './knowledge-tracker.js';
|
|
|
16
18
|
import { Reporter } from './reporter.ts';
|
|
17
19
|
import { StateManager } from './state-manager.js';
|
|
18
20
|
import { Test } from './test-plan.ts';
|
|
19
|
-
import { RequestStore } from './api/request-store.ts';
|
|
20
|
-
import { XhrCapture } from './api/xhr-capture.ts';
|
|
21
21
|
import { createDebug, log, tag } from './utils/logger.js';
|
|
22
22
|
import { WebElement, extractElementData } from './utils/web-element.ts';
|
|
23
23
|
|
|
@@ -68,7 +68,7 @@ class Explorer {
|
|
|
68
68
|
this.initializeContainer();
|
|
69
69
|
this.stateManager = new StateManager({ incognito: this.options?.incognito });
|
|
70
70
|
this.knowledgeTracker = new KnowledgeTracker();
|
|
71
|
-
this.reporter = new Reporter(config.reporter);
|
|
71
|
+
this.reporter = new Reporter(config.reporter, this.stateManager);
|
|
72
72
|
}
|
|
73
73
|
|
|
74
74
|
private initializeContainer() {
|