explorbot 0.1.9 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/bin/explorbot-cli.ts +70 -8
  2. package/boat/api-tester/src/ai/curler-tools.ts +3 -3
  3. package/boat/api-tester/src/ai/curler.ts +1 -1
  4. package/boat/api-tester/src/apibot.ts +2 -2
  5. package/boat/api-tester/src/config.ts +1 -1
  6. package/dist/bin/explorbot-cli.js +70 -7
  7. package/dist/boat/api-tester/src/ai/curler-tools.js +2 -2
  8. package/dist/boat/api-tester/src/apibot.js +2 -2
  9. package/dist/package.json +1 -1
  10. package/dist/src/ai/bosun.js +5 -1
  11. package/dist/src/ai/experience-compactor.js +235 -50
  12. package/dist/src/ai/historian.js +13 -6
  13. package/dist/src/ai/navigator.js +62 -62
  14. package/dist/src/ai/pilot.js +22 -0
  15. package/dist/src/ai/planner/subpages.js +1 -30
  16. package/dist/src/ai/planner.js +4 -4
  17. package/dist/src/ai/provider.js +1 -1
  18. package/dist/src/ai/rerunner.js +3 -3
  19. package/dist/src/ai/researcher/deep-analysis.js +1 -1
  20. package/dist/src/ai/researcher/fingerprint-worker.js +1 -1
  21. package/dist/src/ai/researcher/locators.js +1 -1
  22. package/dist/src/ai/researcher/sections.js +8 -1
  23. package/dist/src/ai/researcher.js +4 -11
  24. package/dist/src/ai/tools.js +5 -3
  25. package/dist/src/api/request-store.js +20 -0
  26. package/dist/src/api/xhr-capture.js +19 -3
  27. package/dist/src/command-handler.js +1 -1
  28. package/dist/src/commands/add-rule-command.js +1 -1
  29. package/dist/src/commands/base-command.js +20 -0
  30. package/dist/src/commands/clean-command.js +1 -1
  31. package/dist/src/commands/compact-command.js +138 -0
  32. package/dist/src/commands/context-command.js +7 -1
  33. package/dist/src/commands/drill-command.js +4 -1
  34. package/dist/src/commands/experience-command.js +104 -0
  35. package/dist/src/commands/explore-command.js +28 -5
  36. package/dist/src/commands/freesail-command.js +2 -0
  37. package/dist/src/commands/index.js +7 -3
  38. package/dist/src/commands/init-command.js +2 -2
  39. package/dist/src/commands/learn-command.js +1 -1
  40. package/dist/src/commands/navigate-command.js +4 -1
  41. package/dist/src/commands/plan-clear-command.js +4 -1
  42. package/dist/src/commands/plan-command.js +11 -4
  43. package/dist/src/commands/plan-edit-command.js +1 -1
  44. package/dist/src/commands/plan-load-command.js +4 -1
  45. package/dist/src/commands/plan-reload-command.js +4 -1
  46. package/dist/src/commands/plan-save-command.js +1 -1
  47. package/dist/src/commands/research-command.js +5 -2
  48. package/dist/src/commands/start-command.js +5 -1
  49. package/dist/src/commands/test-command.js +7 -1
  50. package/dist/src/experience-tracker.js +191 -56
  51. package/dist/src/explorbot.js +26 -14
  52. package/dist/src/explorer.js +3 -3
  53. package/dist/src/reporter.js +17 -2
  54. package/dist/src/stats.js +2 -0
  55. package/dist/src/suite.js +1 -1
  56. package/dist/src/utils/error-page.js +10 -0
  57. package/dist/src/utils/logger.js +1 -1
  58. package/dist/src/utils/rules-loader.js +1 -1
  59. package/dist/src/utils/test-files.js +1 -1
  60. package/dist/src/utils/url-matcher.js +50 -0
  61. package/package.json +1 -1
  62. package/src/ai/bosun.ts +5 -1
  63. package/src/ai/experience-compactor.ts +270 -63
  64. package/src/ai/historian.ts +12 -7
  65. package/src/ai/navigator.ts +68 -66
  66. package/src/ai/pilot.ts +22 -0
  67. package/src/ai/planner/subpages.ts +1 -24
  68. package/src/ai/planner.ts +5 -5
  69. package/src/ai/provider.ts +1 -1
  70. package/src/ai/rerunner.ts +3 -3
  71. package/src/ai/researcher/deep-analysis.ts +1 -1
  72. package/src/ai/researcher/fingerprint-worker.ts +1 -1
  73. package/src/ai/researcher/locators.ts +2 -2
  74. package/src/ai/researcher/sections.ts +7 -1
  75. package/src/ai/researcher.ts +4 -11
  76. package/src/ai/task-agent.ts +1 -1
  77. package/src/ai/tools.ts +6 -4
  78. package/src/api/request-store.ts +22 -0
  79. package/src/api/xhr-capture.ts +21 -3
  80. package/src/command-handler.ts +1 -1
  81. package/src/commands/add-rule-command.ts +2 -2
  82. package/src/commands/base-command.ts +26 -1
  83. package/src/commands/clean-command.ts +2 -2
  84. package/src/commands/compact-command.ts +156 -0
  85. package/src/commands/context-command.ts +8 -2
  86. package/src/commands/drill-command.ts +5 -2
  87. package/src/commands/experience-command.ts +125 -0
  88. package/src/commands/explore-command.ts +30 -7
  89. package/src/commands/freesail-command.ts +2 -0
  90. package/src/commands/index.ts +7 -3
  91. package/src/commands/init-command.ts +2 -2
  92. package/src/commands/learn-command.ts +2 -2
  93. package/src/commands/navigate-command.ts +5 -2
  94. package/src/commands/plan-clear-command.ts +5 -2
  95. package/src/commands/plan-command.ts +12 -5
  96. package/src/commands/plan-edit-command.ts +2 -2
  97. package/src/commands/plan-load-command.ts +5 -2
  98. package/src/commands/plan-reload-command.ts +5 -2
  99. package/src/commands/plan-save-command.ts +2 -2
  100. package/src/commands/research-command.ts +6 -3
  101. package/src/commands/start-command.ts +6 -2
  102. package/src/commands/test-command.ts +8 -2
  103. package/src/experience-tracker.ts +220 -71
  104. package/src/explorbot.ts +28 -15
  105. package/src/explorer.ts +3 -3
  106. package/src/reporter.ts +17 -3
  107. package/src/stats.ts +4 -0
  108. package/src/suite.ts +1 -1
  109. package/src/utils/error-page.ts +10 -0
  110. package/src/utils/logger.ts +1 -1
  111. package/src/utils/rules-loader.ts +1 -1
  112. package/src/utils/test-files.ts +1 -1
  113. package/src/utils/url-matcher.ts +43 -0
@@ -1,9 +1,9 @@
1
- import { BaseCommand } from './base-command.js';
1
+ import { BaseCommand, type Suggestion } from './base-command.js';
2
2
 
3
3
  export class PlanEditCommand extends BaseCommand {
4
4
  name = 'plan:edit';
5
5
  description = 'Open test plan editor';
6
- suggestions = ['/plan:edit - toggle tests on/off'];
6
+ suggestions: Suggestion[] = [{ command: 'plan:edit', hint: 'toggle tests on/off' }];
7
7
 
8
8
  async execute(_args: string): Promise<void> {}
9
9
  }
@@ -1,10 +1,13 @@
1
1
  import { tag } from '../utils/logger.js';
2
- import { BaseCommand } from './base-command.js';
2
+ import { BaseCommand, type Suggestion } from './base-command.js';
3
3
 
4
4
  export class PlanLoadCommand extends BaseCommand {
5
5
  name = 'plan:load';
6
6
  description = 'Load plan from file';
7
- suggestions = ['/test - to launch first test', '/test * - to launch all tests'];
7
+ suggestions: Suggestion[] = [
8
+ { command: 'test', hint: 'launch first test' },
9
+ { command: 'test *', hint: 'launch all tests' },
10
+ ];
8
11
 
9
12
  async execute(args: string): Promise<void> {
10
13
  const filename = args.trim();
@@ -1,10 +1,13 @@
1
1
  import { tag } from '../utils/logger.js';
2
- import { BaseCommand } from './base-command.js';
2
+ import { BaseCommand, type Suggestion } from './base-command.js';
3
3
 
4
4
  export class PlanReloadCommand extends BaseCommand {
5
5
  name = 'plan:reload';
6
6
  description = 'Clear current plan and regenerate';
7
- suggestions = ['/test - to launch first test', '/test * - to launch all tests'];
7
+ suggestions: Suggestion[] = [
8
+ { command: 'test', hint: 'launch first test' },
9
+ { command: 'test *', hint: 'launch all tests' },
10
+ ];
8
11
 
9
12
  async execute(args: string): Promise<void> {
10
13
  const currentPlan = this.explorBot.getCurrentPlan();
@@ -1,11 +1,11 @@
1
1
  import path from 'node:path';
2
2
  import { tag } from '../utils/logger.js';
3
- import { BaseCommand } from './base-command.js';
3
+ import { BaseCommand, type Suggestion } from './base-command.js';
4
4
 
5
5
  export class PlanSaveCommand extends BaseCommand {
6
6
  name = 'plan:save';
7
7
  description = 'Save current plan to file';
8
- suggestions = ['/test - to launch first test'];
8
+ suggestions: Suggestion[] = [{ command: 'test', hint: 'launch first test' }];
9
9
 
10
10
  async execute(args: string): Promise<void> {
11
11
  const plan = this.explorBot.getCurrentPlan();
@@ -1,12 +1,15 @@
1
1
  import { join } from 'node:path';
2
2
  import { ConfigParser } from '../config.ts';
3
3
  import { tag } from '../utils/logger.ts';
4
- import { BaseCommand } from './base-command.js';
4
+ import { BaseCommand, type Suggestion } from './base-command.js';
5
5
 
6
6
  export class ResearchCommand extends BaseCommand {
7
7
  name = 'research';
8
8
  description = 'Research current page or navigate to URI and research. Use --deep to explore interactive elements by clicking them. Use --data to include page data.';
9
- suggestions = ['/navigate <page> - to go to another page', '/plan <feature> - to plan testing'];
9
+ suggestions: Suggestion[] = [
10
+ { command: 'navigate <page>', hint: 'go to another page' },
11
+ { command: 'plan <feature>', hint: 'plan testing' },
12
+ ];
10
13
  options = [
11
14
  { flags: '--data', description: 'Include page data' },
12
15
  { flags: '--deep', description: 'Explore interactive elements by clicking them' },
@@ -45,7 +48,7 @@ export class ResearchCommand extends BaseCommand {
45
48
  }
46
49
 
47
50
  if (!enableDeep) {
48
- this.suggestions = ['/research <page> --deep - analyze page for all expandable elements and interactions'];
51
+ this.suggestions = [{ command: 'research <page> --deep', hint: 'analyze page for all expandable elements and interactions' }];
49
52
  }
50
53
  }
51
54
  }
@@ -1,10 +1,14 @@
1
- import { BaseCommand } from './base-command.js';
1
+ import { BaseCommand, type Suggestion } from './base-command.js';
2
2
  import { ExploreCommand } from './explore-command.js';
3
3
 
4
4
  export class StartCommand extends BaseCommand {
5
5
  name = 'start';
6
6
  description = 'Start web exploration';
7
- suggestions = ['/navigate <page> - to go to another page', '/research - to analyze', '/plan <feature> - to plan testing'];
7
+ suggestions: Suggestion[] = [
8
+ { command: 'navigate <page>', hint: 'go to another page' },
9
+ { command: 'research', hint: 'analyze current page' },
10
+ { command: 'plan <feature>', hint: 'plan testing' },
11
+ ];
8
12
 
9
13
  async execute(args: string): Promise<void> {
10
14
  await new ExploreCommand(this.explorBot).execute(args);
@@ -1,14 +1,20 @@
1
+ import { Stats } from '../stats.js';
1
2
  import { Test } from '../test-plan.js';
2
3
  import { tag } from '../utils/logger.js';
3
- import { BaseCommand } from './base-command.js';
4
+ import { BaseCommand, type Suggestion } from './base-command.js';
4
5
 
5
6
  export class TestCommand extends BaseCommand {
6
7
  name = 'test';
7
8
  description = 'Launch tester agent to execute test scenarios';
8
- suggestions = ['/test - to run next test', '/plan - to create new plan'];
9
+ suggestions: Suggestion[] = [
10
+ { command: 'test', hint: 'run next test' },
11
+ { command: 'plan', hint: 'create new plan' },
12
+ ];
9
13
 
10
14
  async execute(args: string): Promise<void> {
11
15
  const plan = this.explorBot.getCurrentPlan();
16
+ Stats.mode = 'test';
17
+ Stats.focus = plan?.title;
12
18
  const toExecute: Test[] = [];
13
19
 
14
20
  const requirePlan = () => {
@@ -1,7 +1,7 @@
1
1
  import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs';
2
2
  import { basename, dirname, join } from 'node:path';
3
3
  import matter from 'gray-matter';
4
- import { marked, type Tokens } from 'marked';
4
+ import { type Tokens, marked } from 'marked';
5
5
  import type { ActionResult } from './action-result.js';
6
6
  import { ConfigParser } from './config.js';
7
7
  import { KnowledgeTracker } from './knowledge-tracker.js';
@@ -13,6 +13,23 @@ import { extractStatePath } from './utils/url-matcher.js';
13
13
  const debugLog = createDebug('explorbot:experience');
14
14
  const DEFAULT_MAX_EXPERIENCE_LINES = 100;
15
15
 
16
+ export const RECENT_WINDOW_DAYS = 30;
17
+
18
+ /**
19
+ * Stores and reads per-page experience files (`./experience/<stateHash>.md`).
20
+ *
21
+ * Format rules (enforced by writeFlow/writeAction — the only supported writers):
22
+ *
23
+ * ## FLOW: <imperative title> multi-step, `*` bullets + optional ```js``` + `>` discovery, ends with `---`
24
+ * ## ACTION: <imperative title> single-step, optional `Solution:` line + one ```js``` code block
25
+ *
26
+ * - Always h2. Never h3 for FLOW/ACTION.
27
+ * - Title is an imperative verb phrase, lowercase-first, no trailing punctuation.
28
+ * Writers normalize automatically (strip own `FLOW:` / `ACTION:` prefix if the caller included it,
29
+ * lowercase first char, trim trailing `.!?,;:`).
30
+ * - On read (getSuccessfulExperience), headings are rendered as
31
+ * `## HOW to <title> (multi-step|single-step)` so prompts get natural phrasing.
32
+ */
16
33
  export class ExperienceTracker {
17
34
  private experienceDir: string;
18
35
  private disabled: boolean;
@@ -144,35 +161,57 @@ export class ExperienceTracker {
144
161
  return this.knowledgeTracker.getRelevantKnowledge(state).some((k) => k.noExperienceWriting === true || k.noExperienceWriting === 'true');
145
162
  }
146
163
 
147
- async saveSuccessfulResolution(state: ActionResult, originalMessage: string, code: string, explanation?: string): Promise<void> {
164
+ writeAction(state: ActionResult, action: ActionInput): void {
148
165
  if (this.disabled || this.isWritingDisabled(state)) return;
166
+ if (!action.code?.trim()) return;
149
167
 
150
168
  this.ensureExperienceFile(state);
151
169
  const stateHash = state.getStateHash();
152
170
  const { content, data } = this.readExperienceFile(stateHash);
153
- if (content.includes(code)) {
154
- debugLog('Skipping duplicate successful resolution', code);
171
+ if (content.includes(action.code)) {
172
+ debugLog('Skipping duplicate action', action.code);
155
173
  return;
156
174
  }
157
175
 
158
- const filteredCode = code.replace(/I\.amOnPage\s*\([^)]*\)/gs, '');
159
- const newEntryContent = `### SUCCEEDED: ${originalMessage.split('\n')[0]}
176
+ const title = normalizeTitle(action.title.split('\n')[0]);
177
+ if (!title) return;
160
178
 
161
- ${explanation ? `Solution: ${explanation}` : ''}
179
+ const filteredCode = action.code.replace(/I\.amOnPage\s*\([^)]*\)/gs, '');
180
+ const newEntry = generateActionContent(title, filteredCode, action.explanation);
181
+ const updatedContent = `${newEntry}\n\n${content}`;
182
+ this.writeExperienceFile(stateHash, updatedContent, data);
183
+
184
+ tag('substep').log(` Added ACTION to: ${stateHash}.md`);
185
+ }
186
+
187
+ writeFlow(state: ActionResult, flow: FlowInput): void {
188
+ if (this.disabled || this.isWritingDisabled(state)) return;
189
+ if (!flow.steps?.length) return;
190
+
191
+ this.ensureExperienceFile(state);
192
+ const stateHash = state.getStateHash();
193
+ const { content, data } = this.readExperienceFile(stateHash);
194
+
195
+ if (flow.relatedUrls?.length) {
196
+ const currentPath = extractStatePath(state.url || '');
197
+ const existingRelated = Array.isArray(data.related) ? data.related : [];
198
+ const allRelated = [...new Set([...existingRelated, ...flow.relatedUrls])];
199
+ data.related = allRelated.filter((url) => url !== currentPath);
200
+ }
162
201
 
163
- \`\`\`javascript
164
- ${filteredCode}
165
- \`\`\`
166
- `;
202
+ const title = normalizeTitle(flow.scenario);
203
+ if (!title) return;
167
204
 
168
- const updatedContent = `${newEntryContent}\n\n${content}`;
205
+ const sessionContent = this.trimSessionContent(generateFlowContent(title, flow.steps));
206
+ if (!sessionContent) return;
207
+ const updatedContent = `${sessionContent}\n${content}`;
169
208
  this.writeExperienceFile(stateHash, updatedContent, data);
170
209
 
171
- tag('substep').log(` Added successful resolution to: ${stateHash}.md`);
210
+ tag('substep').log(`Added FLOW to: ${stateHash}.md`);
172
211
  }
173
212
 
174
- getAllExperience(): { filePath: string; data: any; content: string }[] {
175
- const allFiles: { filePath: string; data: any; content: string }[] = [];
213
+ getAllExperience(): ExperienceFile[] {
214
+ const allFiles: ExperienceFile[] = [];
176
215
 
177
216
  for (const experienceDir of this.getExperienceDirectories()) {
178
217
  if (!existsSync(experienceDir)) {
@@ -188,10 +227,12 @@ ${filteredCode}
188
227
  try {
189
228
  const content = readFileSync(file, 'utf8');
190
229
  const parsed = matter(content);
230
+ const mtime = statSync(file).mtime;
191
231
  allFiles.push({
192
232
  filePath: file,
193
233
  data: parsed.data,
194
234
  content: parsed.content,
235
+ mtime,
195
236
  });
196
237
  } catch (error) {
197
238
  debugLog(`Failed to read experience file ${file}:`, error);
@@ -205,7 +246,7 @@ ${filteredCode}
205
246
  return allFiles;
206
247
  }
207
248
 
208
- getRelevantExperience(state: ActionResult, options?: { includeDescendantExperience?: boolean }): { filePath: string; data: any; content: string }[] {
249
+ getRelevantExperience(state: ActionResult, options?: { includeDescendantExperience?: boolean }): ExperienceFile[] {
209
250
  const relevantKnowledge = this.knowledgeTracker.getRelevantKnowledge(state);
210
251
  const readingDisabled = relevantKnowledge.some((knowledge) => knowledge.noExperienceReading === true || knowledge.noExperienceReading === 'true');
211
252
  if (readingDisabled) {
@@ -236,50 +277,6 @@ ${filteredCode}
236
277
  // The actual files will be cleaned up by test cleanup
237
278
  }
238
279
 
239
- saveSessionExperience(state: ActionResult, entry: SessionExperienceEntry): void {
240
- if (this.disabled || this.isWritingDisabled(state)) return;
241
-
242
- this.ensureExperienceFile(state);
243
- const stateHash = state.getStateHash();
244
- const { content, data } = this.readExperienceFile(stateHash);
245
-
246
- if (entry.relatedUrls?.length) {
247
- const currentPath = extractStatePath(state.url || '');
248
- const existingRelated = Array.isArray(data.related) ? data.related : [];
249
- const allRelated = [...new Set([...existingRelated, ...entry.relatedUrls])];
250
- data.related = allRelated.filter((url) => url !== currentPath);
251
- }
252
-
253
- const sessionContent = this.trimSessionContent(this.generateSessionContent(entry));
254
- if (!sessionContent) return;
255
- const updatedContent = `${sessionContent}\n${content}`;
256
- this.writeExperienceFile(stateHash, updatedContent, data);
257
-
258
- tag('substep').log(`Added session experience to: ${stateHash}.md`);
259
- }
260
-
261
- private generateSessionContent(entry: SessionExperienceEntry): string {
262
- let content = `## Successful Flow: ${entry.scenario}\n\n`;
263
-
264
- for (const step of entry.steps) {
265
- content += `* ${step.message}\n\n`;
266
- if (step.code) {
267
- content += '```js\n';
268
- content += `${step.code}\n`;
269
- content += '```\n\n';
270
- }
271
- if (step.discovery) {
272
- const discoveries = step.discovery.split('\n').filter((d) => d.trim());
273
- for (const discovery of discoveries) {
274
- content += `> ${discovery.trim()}\n\n`;
275
- }
276
- }
277
- }
278
-
279
- content += '---\n';
280
- return content;
281
- }
282
-
283
280
  private trimSessionContent(content: string): string | null {
284
281
  const q = mdq(content);
285
282
  if (q.query('heading').count() === 0) return null;
@@ -318,12 +315,14 @@ ${filteredCode}
318
315
  for (const record of records) {
319
316
  if (!record.content) continue;
320
317
 
321
- const successFlows = mdq(record.content).query('section(~"Successful Flow")').text();
322
- const succeeded = mdq(record.content).query('section(~"SUCCEEDED")').text();
323
- let combined = [successFlows, succeeded].filter(Boolean).join('\n\n');
318
+ const flows = mdq(record.content).query('section(~"FLOW:")').text();
319
+ const actions = mdq(record.content).query('section(~"ACTION:")').text();
320
+ let combined = [flows, actions].filter(Boolean).join('\n\n');
324
321
 
325
322
  if (!combined.trim()) continue;
326
323
 
324
+ combined = renderAsHowTo(combined);
325
+
327
326
  if (options?.stripCode) {
328
327
  combined = mdq(combined).query('code').replace('');
329
328
  }
@@ -383,6 +382,72 @@ ${filteredCode}
383
382
  }
384
383
  return null;
385
384
  }
385
+
386
+ listAllExperienceToc(filter?: string, options?: { recency?: 'recent' | 'old' }): ExperienceTocEntry[] {
387
+ const records = this.getAllExperience();
388
+ if (records.length === 0) return [];
389
+
390
+ const trimmed = filter?.trim();
391
+ let matching = records;
392
+
393
+ if (trimmed) {
394
+ if (trimmed.endsWith('.md')) {
395
+ const bare = trimmed.slice(0, -3);
396
+ const byFilename = records.find((record) => basename(record.filePath, '.md') === bare);
397
+ matching = byFilename ? [byFilename] : [];
398
+ } else {
399
+ const lower = trimmed.toLowerCase();
400
+ matching = records.filter((record) => {
401
+ const url = ((record.data as WebPageState)?.url || '').toLowerCase();
402
+ if (!url) return false;
403
+ return url.includes(lower);
404
+ });
405
+ }
406
+ }
407
+
408
+ if (options?.recency) {
409
+ const cutoff = Date.now() - RECENT_WINDOW_DAYS * 24 * 60 * 60 * 1000;
410
+ matching = matching.filter((record) => {
411
+ const isRecent = record.mtime.getTime() >= cutoff;
412
+ return options.recency === 'recent' ? isRecent : !isRecent;
413
+ });
414
+ }
415
+
416
+ const sorted = matching.sort((a, b) => {
417
+ const aUrl = (a.data as WebPageState)?.url || '';
418
+ const bUrl = (b.data as WebPageState)?.url || '';
419
+ return aUrl.localeCompare(bUrl);
420
+ });
421
+
422
+ const toc: ExperienceTocEntry[] = [];
423
+ for (let i = 0; i < sorted.length; i++) {
424
+ const record = sorted[i];
425
+ const sections = listTocHeadings(record.content);
426
+ if (sections.length === 0) continue;
427
+ toc.push({
428
+ fileTag: indexToLetters(toc.length),
429
+ fileHash: basename(record.filePath, '.md'),
430
+ url: (record.data as WebPageState)?.url || '',
431
+ sections,
432
+ });
433
+ }
434
+ return toc;
435
+ }
436
+
437
+ getExperienceSectionByTag(fileTag: string, sectionIndex: number, filter?: string): { title: string; url: string; content: string; fileHash: string } | null {
438
+ const toc = this.listAllExperienceToc(filter);
439
+ const entry = toc.find((e) => e.fileTag === fileTag);
440
+ if (!entry) return null;
441
+
442
+ const filePath = this.findExperienceFileByHash(entry.fileHash);
443
+ if (!filePath) return null;
444
+
445
+ const { content } = this.readExperienceFile(entry.fileHash);
446
+ const extracted = extractHeadingSection(content, sectionIndex);
447
+ if (!extracted) return null;
448
+
449
+ return { title: extracted.title, url: entry.url, content: extracted.body, fileHash: entry.fileHash };
450
+ }
386
451
  }
387
452
 
388
453
  function listTocHeadings(content: string): { index: number; level: 2 | 3; title: string }[] {
@@ -448,7 +513,9 @@ export function renderExperienceToc(toc: ExperienceTocEntry[]): string {
448
513
 
449
514
  const lines: string[] = [];
450
515
  lines.push('<experience>');
451
- lines.push('Past experience for this page. Call learn_experience({ fileTag, sectionIndex }) to read a section.');
516
+ lines.push('Past experience for this page reusable recipes recorded from prior successful runs.');
517
+ lines.push('FLOW: = multi-step recipe (bullets + code + discovery). ACTION: = single-step snippet (one code block).');
518
+ lines.push('Call learn_experience({ fileTag, sectionIndex }) to read a section when it looks relevant to the current step.');
452
519
  lines.push('');
453
520
  for (const entry of toc) {
454
521
  lines.push(`File ${entry.fileTag} ${entry.url}:`);
@@ -462,6 +529,95 @@ export function renderExperienceToc(toc: ExperienceTocEntry[]): string {
462
529
  return lines.join('\n');
463
530
  }
464
531
 
532
+ function normalizeTitle(raw: string): string {
533
+ let t = (raw || '').trim();
534
+ for (const p of ['FLOW:', 'ACTION:']) {
535
+ if (t.toLowerCase().startsWith(p.toLowerCase())) {
536
+ t = t.slice(p.length).trim();
537
+ break;
538
+ }
539
+ }
540
+ while (t.length > 0 && '.!?,;:'.includes(t[t.length - 1])) {
541
+ t = t.slice(0, -1);
542
+ }
543
+ if (t.length > 0) t = t[0].toLowerCase() + t.slice(1);
544
+ return t;
545
+ }
546
+
547
+ function generateActionContent(title: string, code: string, explanation?: string): string {
548
+ const lines: string[] = [];
549
+ lines.push(`## ACTION: ${title}`);
550
+ lines.push('');
551
+ if (explanation) {
552
+ lines.push(`Solution: ${explanation}`);
553
+ lines.push('');
554
+ }
555
+ lines.push('```javascript');
556
+ lines.push(code);
557
+ lines.push('```');
558
+ lines.push('');
559
+ return lines.join('\n');
560
+ }
561
+
562
+ function generateFlowContent(title: string, steps: SessionStep[]): string {
563
+ let content = `## FLOW: ${title}\n\n`;
564
+ for (const step of steps) {
565
+ content += `* ${step.message}\n\n`;
566
+ if (step.code) {
567
+ content += '```js\n';
568
+ content += `${step.code}\n`;
569
+ content += '```\n\n';
570
+ }
571
+ if (step.discovery) {
572
+ const discoveries = step.discovery.split('\n').filter((d) => d.trim());
573
+ for (const discovery of discoveries) {
574
+ content += `> ${discovery.trim()}\n\n`;
575
+ }
576
+ }
577
+ }
578
+ content += '---\n';
579
+ return content;
580
+ }
581
+
582
+ function renderAsHowTo(content: string): string {
583
+ const tokens = marked.lexer(content);
584
+ let result = '';
585
+ for (const token of tokens) {
586
+ if (token.type === 'heading' && (token as Tokens.Heading).depth === 2) {
587
+ const text = (token as Tokens.Heading).text.trim();
588
+ if (text.startsWith('FLOW:')) {
589
+ result += `## HOW to ${text.slice(5).trim()} (multi-step)\n\n`;
590
+ continue;
591
+ }
592
+ if (text.startsWith('ACTION:')) {
593
+ result += `## HOW to ${text.slice(7).trim()} (single-step)\n\n`;
594
+ continue;
595
+ }
596
+ }
597
+ result += (token as any).raw || '';
598
+ }
599
+ return result;
600
+ }
601
+
602
+ export interface ExperienceFile {
603
+ filePath: string;
604
+ data: { url?: string; title?: string; [key: string]: any };
605
+ content: string;
606
+ mtime: Date;
607
+ }
608
+
609
+ export interface FlowInput {
610
+ scenario: string;
611
+ steps: SessionStep[];
612
+ relatedUrls?: string[];
613
+ }
614
+
615
+ export interface ActionInput {
616
+ title: string;
617
+ code: string;
618
+ explanation?: string;
619
+ }
620
+
465
621
  export interface SessionStep {
466
622
  message: string;
467
623
  status: 'passed' | 'failed' | 'neutral';
@@ -470,13 +626,6 @@ export interface SessionStep {
470
626
  discovery?: string;
471
627
  }
472
628
 
473
- export interface SessionExperienceEntry {
474
- scenario: string;
475
- result: 'success' | 'partial' | 'failed';
476
- steps: SessionStep[];
477
- relatedUrls?: string[];
478
- }
479
-
480
629
  export interface ExperienceTocEntry {
481
630
  fileTag: string;
482
631
  fileHash: string;
package/src/explorbot.ts CHANGED
@@ -1,9 +1,6 @@
1
1
  import { existsSync, mkdirSync } from 'node:fs';
2
2
  import path from 'node:path';
3
3
  import { ActionResult } from './action-result.ts';
4
- import { ApiClient } from './api/api-client.ts';
5
- import { RequestStore } from './api/request-store.ts';
6
- import { loadSpec } from './api/spec-reader.ts';
7
4
  import { Bosun } from './ai/bosun.ts';
8
5
  import { Captain } from './ai/captain.ts';
9
6
  import { ExperienceCompactor } from './ai/experience-compactor.ts';
@@ -14,16 +11,20 @@ import { Pilot } from './ai/pilot.ts';
14
11
  import { Planner } from './ai/planner.ts';
15
12
  import { AIProvider } from './ai/provider.ts';
16
13
  import { Quartermaster } from './ai/quartermaster.ts';
17
- import { Researcher } from './ai/researcher.ts';
18
14
  import { Rerunner } from './ai/rerunner.ts';
15
+ import { Researcher } from './ai/researcher.ts';
19
16
  import { Tester } from './ai/tester.ts';
20
17
  import { createAgentTools } from './ai/tools.ts';
18
+ import { ApiClient } from './api/api-client.ts';
19
+ import { RequestStore } from './api/request-store.ts';
20
+ import { loadSpec } from './api/spec-reader.ts';
21
21
  import type { ExplorbotConfig } from './config.js';
22
22
  import { ConfigParser } from './config.ts';
23
+ import { ExperienceTracker } from './experience-tracker.ts';
23
24
  import Explorer from './explorer.ts';
24
- import type { Suite } from './suite.ts';
25
25
  import { KnowledgeTracker } from './knowledge-tracker.ts';
26
26
  import { WebPageState } from './state-manager.ts';
27
+ import type { Suite } from './suite.ts';
27
28
  import { Plan } from './test-plan.ts';
28
29
  import { setVerboseMode, tag } from './utils/logger.ts';
29
30
  import { sanitizeFilename } from './utils/strings.ts';
@@ -51,6 +52,7 @@ export class ExplorBot {
51
52
  public needsInput = false;
52
53
  private currentPlan?: Plan;
53
54
  private planFeature?: string;
55
+ lastPlanError: Error | null = null;
54
56
  private agents: Record<string, any> = {};
55
57
 
56
58
  constructor(options: ExplorBotOptions = {}) {
@@ -76,13 +78,11 @@ export class ExplorBot {
76
78
  }
77
79
 
78
80
  try {
79
- this.config = await this.configParser.loadConfig(this.options);
80
- this.provider = new AIProvider(this.config.ai);
81
- await this.provider.validateConnection();
81
+ await this.startProviderOnly();
82
82
  this.explorer = new Explorer(this.config, this.provider, this.options);
83
83
  await this.explorer.start();
84
84
  if (!this.options.incognito) {
85
- await this.agentExperienceCompactor().compactAllExperiences();
85
+ await this.agentExperienceCompactor().autocompact();
86
86
  }
87
87
  if (this.userResolveFn) this.explorer.setUserResolve(this.userResolveFn);
88
88
  } catch (error) {
@@ -92,9 +92,16 @@ export class ExplorBot {
92
92
  }
93
93
  }
94
94
 
95
+ async startProviderOnly(): Promise<void> {
96
+ if (this.provider) return;
97
+ this.config = await this.configParser.loadConfig(this.options);
98
+ this.provider = new AIProvider(this.config.ai);
99
+ await this.provider.validateConnection();
100
+ }
101
+
95
102
  async stop(): Promise<void> {
96
103
  this.agents.quartermaster?.stop();
97
- await this.explorer.stop();
104
+ await this.explorer?.stop();
98
105
  }
99
106
 
100
107
  async visitInitialState(): Promise<void> {
@@ -125,6 +132,13 @@ export class ExplorBot {
125
132
  return new KnowledgeTracker();
126
133
  }
127
134
 
135
+ getExperienceTracker(): ExperienceTracker {
136
+ if (this.explorer) {
137
+ return this.explorer.getStateManager().getExperienceTracker();
138
+ }
139
+ return new ExperienceTracker();
140
+ }
141
+
128
142
  getConfig(): ExplorbotConfig {
129
143
  return this.config;
130
144
  }
@@ -227,10 +241,7 @@ export class ExplorBot {
227
241
  }
228
242
 
229
243
  agentExperienceCompactor(): ExperienceCompactor {
230
- return (this.agents.experienceCompactor ||= this.createAgent(({ ai, explorer }) => {
231
- const experienceTracker = explorer.getStateManager().getExperienceTracker();
232
- return new ExperienceCompactor(ai, experienceTracker);
233
- }));
244
+ return (this.agents.experienceCompactor ||= new ExperienceCompactor(this.provider, this.getExperienceTracker()));
234
245
  }
235
246
 
236
247
  agentQuartermaster(): Quartermaster | null {
@@ -348,10 +359,12 @@ export class ExplorBot {
348
359
  if (this.currentPlan) {
349
360
  planner.setPlan(this.currentPlan);
350
361
  }
362
+ this.lastPlanError = null;
351
363
  try {
352
364
  this.currentPlan = await planner.plan(feature, opts.style, opts.extend, opts.completedPlans);
353
365
  } catch (err) {
354
- tag('warning').log(`Planning failed: ${err instanceof Error ? err.message : err}`);
366
+ this.lastPlanError = err instanceof Error ? err : new Error(String(err));
367
+ tag('warning').log(`Planning failed: ${this.lastPlanError.message}`);
355
368
  if (!this.currentPlan) return undefined;
356
369
  return this.currentPlan;
357
370
  }
package/src/explorer.ts CHANGED
@@ -9,6 +9,8 @@ import { ActionResult } from './action-result.ts';
9
9
  import Action from './action.js';
10
10
  import { AIProvider } from './ai/provider.js';
11
11
  import { visuallyAnnotateContainers } from './ai/researcher/coordinates.ts';
12
+ import { RequestStore } from './api/request-store.ts';
13
+ import { XhrCapture } from './api/xhr-capture.ts';
12
14
  import type { ExplorbotConfig } from './config.js';
13
15
  import { ConfigParser, outputPath } from './config.js';
14
16
  import type { UserResolveFunction } from './explorbot.js';
@@ -16,8 +18,6 @@ import { KnowledgeTracker } from './knowledge-tracker.js';
16
18
  import { Reporter } from './reporter.ts';
17
19
  import { StateManager } from './state-manager.js';
18
20
  import { Test } from './test-plan.ts';
19
- import { RequestStore } from './api/request-store.ts';
20
- import { XhrCapture } from './api/xhr-capture.ts';
21
21
  import { createDebug, log, tag } from './utils/logger.js';
22
22
  import { WebElement, extractElementData } from './utils/web-element.ts';
23
23
 
@@ -68,7 +68,7 @@ class Explorer {
68
68
  this.initializeContainer();
69
69
  this.stateManager = new StateManager({ incognito: this.options?.incognito });
70
70
  this.knowledgeTracker = new KnowledgeTracker();
71
- this.reporter = new Reporter(config.reporter);
71
+ this.reporter = new Reporter(config.reporter, this.stateManager);
72
72
  }
73
73
 
74
74
  private initializeContainer() {