explorbot 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/README.md +27 -1
  2. package/bin/explorbot-cli.ts +86 -15
  3. package/boat/api-tester/src/ai/curler-tools.ts +3 -3
  4. package/boat/api-tester/src/ai/curler.ts +1 -1
  5. package/boat/api-tester/src/apibot.ts +2 -2
  6. package/boat/api-tester/src/config.ts +1 -1
  7. package/dist/bin/explorbot-cli.js +85 -14
  8. package/dist/boat/api-tester/src/ai/curler-tools.js +2 -2
  9. package/dist/boat/api-tester/src/apibot.js +2 -2
  10. package/dist/package.json +2 -2
  11. package/dist/rules/navigator/output.md +9 -0
  12. package/dist/rules/navigator/verification-actions.md +2 -0
  13. package/dist/src/action-result.js +23 -1
  14. package/dist/src/action.js +46 -38
  15. package/dist/src/ai/bosun.js +16 -2
  16. package/dist/src/ai/conversation.js +39 -0
  17. package/dist/src/ai/experience-compactor.js +235 -50
  18. package/dist/src/ai/historian/codeceptjs.js +109 -0
  19. package/dist/src/ai/historian/experience.js +320 -0
  20. package/dist/src/ai/historian/mixin.js +2 -0
  21. package/dist/src/ai/historian/playwright.js +145 -0
  22. package/dist/src/ai/historian/utils.js +18 -0
  23. package/dist/src/ai/historian.js +19 -398
  24. package/dist/src/ai/navigator.js +133 -80
  25. package/dist/src/ai/pilot.js +254 -13
  26. package/dist/src/ai/planner/subpages.js +1 -30
  27. package/dist/src/ai/planner.js +33 -13
  28. package/dist/src/ai/provider.js +55 -18
  29. package/dist/src/ai/rerunner.js +3 -3
  30. package/dist/src/ai/researcher/deep-analysis.js +1 -1
  31. package/dist/src/ai/researcher/fingerprint-worker.js +1 -1
  32. package/dist/src/ai/researcher/locators.js +1 -1
  33. package/dist/src/ai/researcher/sections.js +8 -1
  34. package/dist/src/ai/researcher.js +43 -41
  35. package/dist/src/ai/rules.js +26 -14
  36. package/dist/src/ai/tester.js +90 -26
  37. package/dist/src/ai/tools.js +18 -10
  38. package/dist/src/api/request-store.js +20 -0
  39. package/dist/src/api/xhr-capture.js +19 -3
  40. package/dist/src/browser-server.js +16 -3
  41. package/dist/src/command-handler.js +1 -1
  42. package/dist/src/commands/add-rule-command.js +12 -9
  43. package/dist/src/commands/base-command.js +20 -0
  44. package/dist/src/commands/clean-command.js +3 -2
  45. package/dist/src/commands/compact-command.js +138 -0
  46. package/dist/src/commands/context-command.js +7 -1
  47. package/dist/src/commands/drill-command.js +4 -1
  48. package/dist/src/commands/experience-command.js +104 -0
  49. package/dist/src/commands/explore-command.js +54 -19
  50. package/dist/src/commands/freesail-command.js +2 -0
  51. package/dist/src/commands/index.js +7 -3
  52. package/dist/src/commands/init-command.js +11 -10
  53. package/dist/src/commands/learn-command.js +1 -1
  54. package/dist/src/commands/navigate-command.js +4 -1
  55. package/dist/src/commands/plan-clear-command.js +4 -1
  56. package/dist/src/commands/plan-command.js +43 -4
  57. package/dist/src/commands/plan-edit-command.js +1 -1
  58. package/dist/src/commands/plan-load-command.js +4 -1
  59. package/dist/src/commands/plan-reload-command.js +4 -1
  60. package/dist/src/commands/plan-save-command.js +20 -8
  61. package/dist/src/commands/rerun-command.js +4 -0
  62. package/dist/src/commands/research-command.js +5 -2
  63. package/dist/src/commands/start-command.js +5 -1
  64. package/dist/src/commands/test-command.js +7 -1
  65. package/dist/src/components/App.js +15 -5
  66. package/dist/src/execution-controller.js +13 -2
  67. package/dist/src/experience-tracker.js +174 -83
  68. package/dist/src/explorbot.js +31 -22
  69. package/dist/src/explorer.js +12 -5
  70. package/dist/src/observability.js +50 -99
  71. package/dist/src/playwright-recorder.js +309 -0
  72. package/dist/src/reporter.js +17 -2
  73. package/dist/src/stats.js +2 -0
  74. package/dist/src/suite.js +1 -1
  75. package/dist/src/test-plan.js +12 -0
  76. package/dist/src/utils/aria.js +37 -1
  77. package/dist/src/utils/error-page.js +30 -7
  78. package/dist/src/utils/logger.js +1 -1
  79. package/dist/src/utils/next-steps.js +37 -0
  80. package/dist/src/utils/rules-loader.js +1 -1
  81. package/dist/src/utils/test-files.js +1 -1
  82. package/dist/src/utils/url-matcher.js +50 -0
  83. package/package.json +2 -2
  84. package/rules/navigator/output.md +9 -0
  85. package/rules/navigator/verification-actions.md +2 -0
  86. package/src/action-result.ts +26 -1
  87. package/src/action.ts +44 -37
  88. package/src/ai/bosun.ts +16 -2
  89. package/src/ai/conversation.ts +37 -0
  90. package/src/ai/experience-compactor.ts +270 -63
  91. package/src/ai/historian/codeceptjs.ts +130 -0
  92. package/src/ai/historian/experience.ts +383 -0
  93. package/src/ai/historian/mixin.ts +4 -0
  94. package/src/ai/historian/playwright.ts +169 -0
  95. package/src/ai/historian/utils.ts +23 -0
  96. package/src/ai/historian.ts +35 -468
  97. package/src/ai/navigator.ts +140 -85
  98. package/src/ai/pilot.ts +259 -14
  99. package/src/ai/planner/subpages.ts +1 -24
  100. package/src/ai/planner.ts +34 -14
  101. package/src/ai/provider.ts +52 -18
  102. package/src/ai/rerunner.ts +3 -3
  103. package/src/ai/researcher/deep-analysis.ts +1 -1
  104. package/src/ai/researcher/fingerprint-worker.ts +1 -1
  105. package/src/ai/researcher/locators.ts +2 -2
  106. package/src/ai/researcher/sections.ts +7 -1
  107. package/src/ai/researcher.ts +47 -42
  108. package/src/ai/rules.ts +27 -14
  109. package/src/ai/task-agent.ts +1 -1
  110. package/src/ai/tester.ts +94 -26
  111. package/src/ai/tools.ts +53 -29
  112. package/src/api/request-store.ts +22 -0
  113. package/src/api/xhr-capture.ts +21 -3
  114. package/src/browser-server.ts +17 -3
  115. package/src/command-handler.ts +1 -1
  116. package/src/commands/add-rule-command.ts +13 -9
  117. package/src/commands/base-command.ts +26 -1
  118. package/src/commands/clean-command.ts +4 -3
  119. package/src/commands/compact-command.ts +156 -0
  120. package/src/commands/context-command.ts +8 -2
  121. package/src/commands/drill-command.ts +5 -2
  122. package/src/commands/experience-command.ts +125 -0
  123. package/src/commands/explore-command.ts +58 -21
  124. package/src/commands/freesail-command.ts +2 -0
  125. package/src/commands/index.ts +7 -3
  126. package/src/commands/init-command.ts +11 -10
  127. package/src/commands/learn-command.ts +2 -2
  128. package/src/commands/navigate-command.ts +5 -2
  129. package/src/commands/plan-clear-command.ts +5 -2
  130. package/src/commands/plan-command.ts +47 -5
  131. package/src/commands/plan-edit-command.ts +2 -2
  132. package/src/commands/plan-load-command.ts +5 -2
  133. package/src/commands/plan-reload-command.ts +5 -2
  134. package/src/commands/plan-save-command.ts +20 -9
  135. package/src/commands/rerun-command.ts +5 -0
  136. package/src/commands/research-command.ts +6 -3
  137. package/src/commands/start-command.ts +6 -2
  138. package/src/commands/test-command.ts +8 -2
  139. package/src/components/App.tsx +16 -5
  140. package/src/config.ts +6 -1
  141. package/src/execution-controller.ts +14 -3
  142. package/src/experience-tracker.ts +198 -100
  143. package/src/explorbot.ts +33 -23
  144. package/src/explorer.ts +14 -5
  145. package/src/observability.ts +50 -109
  146. package/src/playwright-recorder.ts +305 -0
  147. package/src/reporter.ts +17 -3
  148. package/src/stats.ts +4 -0
  149. package/src/suite.ts +1 -1
  150. package/src/test-plan.ts +12 -0
  151. package/src/utils/aria.ts +38 -1
  152. package/src/utils/error-page.ts +32 -7
  153. package/src/utils/logger.ts +1 -1
  154. package/src/utils/next-steps.ts +51 -0
  155. package/src/utils/rules-loader.ts +1 -1
  156. package/src/utils/test-files.ts +1 -1
  157. package/src/utils/url-matcher.ts +43 -0
@@ -10,6 +10,7 @@ export class ExecutionController extends EventEmitter {
10
10
  private inputCallback: InputCallback | null = null;
11
11
  private interruptResolvers: Array<() => void> = [];
12
12
  private abortController: AbortController | null = null;
13
+ private awaitingInput = false;
13
14
 
14
15
  private constructor() {
15
16
  super();
@@ -48,6 +49,10 @@ export class ExecutionController extends EventEmitter {
48
49
  this.emit('idle');
49
50
  }
50
51
 
52
+ isAwaitingInput(): boolean {
53
+ return this.awaitingInput;
54
+ }
55
+
51
56
  isInterrupted(): boolean {
52
57
  return this.interrupted;
53
58
  }
@@ -77,11 +82,16 @@ export class ExecutionController extends EventEmitter {
77
82
  }
78
83
 
79
84
  async requestInput(prompt: string): Promise<string | null> {
80
- if (this.inputCallback) {
81
- return await this.inputCallback(prompt);
85
+ if (!this.inputCallback) {
86
+ return await this.readlineInput(prompt);
82
87
  }
83
88
 
84
- return await this.readlineInput(prompt);
89
+ this.awaitingInput = true;
90
+ try {
91
+ return await this.inputCallback(prompt);
92
+ } finally {
93
+ this.awaitingInput = false;
94
+ }
85
95
  }
86
96
 
87
97
  private async readlineInput(prompt: string): Promise<string | null> {
@@ -103,6 +113,7 @@ export class ExecutionController extends EventEmitter {
103
113
  this.interrupted = false;
104
114
  this.interruptResolvers = [];
105
115
  this.abortController = null;
116
+ this.awaitingInput = false;
106
117
  }
107
118
  }
108
119
 
@@ -1,7 +1,7 @@
1
1
  import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs';
2
2
  import { basename, dirname, join } from 'node:path';
3
3
  import matter from 'gray-matter';
4
- import { marked, type Tokens } from 'marked';
4
+ import { type Tokens, marked } from 'marked';
5
5
  import type { ActionResult } from './action-result.js';
6
6
  import { ConfigParser } from './config.js';
7
7
  import { KnowledgeTracker } from './knowledge-tracker.js';
@@ -13,6 +13,25 @@ import { extractStatePath } from './utils/url-matcher.js';
13
13
  const debugLog = createDebug('explorbot:experience');
14
14
  const DEFAULT_MAX_EXPERIENCE_LINES = 100;
15
15
 
16
+ export const RECENT_WINDOW_DAYS = 30;
17
+
18
+ /**
19
+ * Stores and reads per-page experience files (`./experience/<stateHash>.md`).
20
+ *
21
+ * Two writers, two contracts:
22
+ *
23
+ * writeFlow(state, body, relatedUrls?) — caller hands in a fully-formatted
24
+ * `## FLOW: <imperative title>` block (multi-step,
25
+ * `*` bullets + optional ```js``` + `>` discovery,
26
+ * ends with `---`). Tracker dedups + prepends.
27
+ * writeAction(state, ActionInput) — `## ACTION: <imperative title>`, single-step,
28
+ * optional `Solution:` line + one ```js``` code block.
29
+ * Title normalized via normalizeTitle().
30
+ *
31
+ * - Always h2. Never h3 for FLOW/ACTION.
32
+ * - On read (getSuccessfulExperience), headings are rendered as
33
+ * `## HOW to <title> (multi-step|single-step)` so prompts get natural phrasing.
34
+ */
16
35
  export class ExperienceTracker {
17
36
  private experienceDir: string;
18
37
  private disabled: boolean;
@@ -144,35 +163,57 @@ export class ExperienceTracker {
144
163
  return this.knowledgeTracker.getRelevantKnowledge(state).some((k) => k.noExperienceWriting === true || k.noExperienceWriting === 'true');
145
164
  }
146
165
 
147
- async saveSuccessfulResolution(state: ActionResult, originalMessage: string, code: string, explanation?: string): Promise<void> {
166
+ writeAction(state: ActionResult, action: ActionInput): void {
148
167
  if (this.disabled || this.isWritingDisabled(state)) return;
168
+ if (!action.code?.trim()) return;
149
169
 
150
170
  this.ensureExperienceFile(state);
151
171
  const stateHash = state.getStateHash();
152
172
  const { content, data } = this.readExperienceFile(stateHash);
153
- if (content.includes(code)) {
154
- debugLog('Skipping duplicate successful resolution', code);
173
+ if (content.includes(action.code)) {
174
+ debugLog('Skipping duplicate action', action.code);
155
175
  return;
156
176
  }
157
177
 
158
- const filteredCode = code.replace(/I\.amOnPage\s*\([^)]*\)/gs, '');
159
- const newEntryContent = `### SUCCEEDED: ${originalMessage.split('\n')[0]}
178
+ const title = normalizeTitle(action.title.split('\n')[0]);
179
+ if (!title) return;
160
180
 
161
- ${explanation ? `Solution: ${explanation}` : ''}
181
+ const filteredCode = action.code.replace(/I\.amOnPage\s*\([^)]*\)/gs, '');
182
+ const newEntry = generateActionContent(title, filteredCode, action.explanation);
183
+ const updatedContent = `${newEntry}\n\n${content}`;
184
+ this.writeExperienceFile(stateHash, updatedContent, data);
185
+
186
+ tag('substep').log(` Added ACTION to: ${stateHash}.md`);
187
+ }
188
+
189
+ writeFlow(state: ActionResult, body: string, relatedUrls?: string[]): void {
190
+ if (this.disabled || this.isWritingDisabled(state)) return;
191
+ if (!body?.trim()) return;
162
192
 
163
- \`\`\`javascript
164
- ${filteredCode}
165
- \`\`\`
166
- `;
193
+ this.ensureExperienceFile(state);
194
+ const stateHash = state.getStateHash();
195
+ const { content, data } = this.readExperienceFile(stateHash);
196
+
197
+ if (content.includes(body)) {
198
+ debugLog('Skipping duplicate flow body');
199
+ return;
200
+ }
201
+
202
+ if (relatedUrls?.length) {
203
+ const currentPath = extractStatePath(state.url || '');
204
+ const existingRelated = Array.isArray(data.related) ? data.related : [];
205
+ const allRelated = [...new Set([...existingRelated, ...relatedUrls])];
206
+ data.related = allRelated.filter((url) => url !== currentPath);
207
+ }
167
208
 
168
- const updatedContent = `${newEntryContent}\n\n${content}`;
209
+ const updatedContent = `${body}\n${content}`;
169
210
  this.writeExperienceFile(stateHash, updatedContent, data);
170
211
 
171
- tag('substep').log(` Added successful resolution to: ${stateHash}.md`);
212
+ tag('substep').log(`Added FLOW to: ${stateHash}.md`);
172
213
  }
173
214
 
174
- getAllExperience(): { filePath: string; data: any; content: string }[] {
175
- const allFiles: { filePath: string; data: any; content: string }[] = [];
215
+ getAllExperience(): ExperienceFile[] {
216
+ const allFiles: ExperienceFile[] = [];
176
217
 
177
218
  for (const experienceDir of this.getExperienceDirectories()) {
178
219
  if (!existsSync(experienceDir)) {
@@ -188,10 +229,12 @@ ${filteredCode}
188
229
  try {
189
230
  const content = readFileSync(file, 'utf8');
190
231
  const parsed = matter(content);
232
+ const mtime = statSync(file).mtime;
191
233
  allFiles.push({
192
234
  filePath: file,
193
235
  data: parsed.data,
194
236
  content: parsed.content,
237
+ mtime,
195
238
  });
196
239
  } catch (error) {
197
240
  debugLog(`Failed to read experience file ${file}:`, error);
@@ -205,7 +248,7 @@ ${filteredCode}
205
248
  return allFiles;
206
249
  }
207
250
 
208
- getRelevantExperience(state: ActionResult, options?: { includeDescendantExperience?: boolean }): { filePath: string; data: any; content: string }[] {
251
+ getRelevantExperience(state: ActionResult, options?: { includeDescendantExperience?: boolean }): ExperienceFile[] {
209
252
  const relevantKnowledge = this.knowledgeTracker.getRelevantKnowledge(state);
210
253
  const readingDisabled = relevantKnowledge.some((knowledge) => knowledge.noExperienceReading === true || knowledge.noExperienceReading === 'true');
211
254
  if (readingDisabled) {
@@ -236,79 +279,6 @@ ${filteredCode}
236
279
  // The actual files will be cleaned up by test cleanup
237
280
  }
238
281
 
239
- saveSessionExperience(state: ActionResult, entry: SessionExperienceEntry): void {
240
- if (this.disabled || this.isWritingDisabled(state)) return;
241
-
242
- this.ensureExperienceFile(state);
243
- const stateHash = state.getStateHash();
244
- const { content, data } = this.readExperienceFile(stateHash);
245
-
246
- if (entry.relatedUrls?.length) {
247
- const currentPath = extractStatePath(state.url || '');
248
- const existingRelated = Array.isArray(data.related) ? data.related : [];
249
- const allRelated = [...new Set([...existingRelated, ...entry.relatedUrls])];
250
- data.related = allRelated.filter((url) => url !== currentPath);
251
- }
252
-
253
- const sessionContent = this.trimSessionContent(this.generateSessionContent(entry));
254
- if (!sessionContent) return;
255
- const updatedContent = `${sessionContent}\n${content}`;
256
- this.writeExperienceFile(stateHash, updatedContent, data);
257
-
258
- tag('substep').log(`Added session experience to: ${stateHash}.md`);
259
- }
260
-
261
- private generateSessionContent(entry: SessionExperienceEntry): string {
262
- let content = `## Successful Flow: ${entry.scenario}\n\n`;
263
-
264
- for (const step of entry.steps) {
265
- content += `* ${step.message}\n\n`;
266
- if (step.code) {
267
- content += '```js\n';
268
- content += `${step.code}\n`;
269
- content += '```\n\n';
270
- }
271
- if (step.discovery) {
272
- const discoveries = step.discovery.split('\n').filter((d) => d.trim());
273
- for (const discovery of discoveries) {
274
- content += `> ${discovery.trim()}\n\n`;
275
- }
276
- }
277
- }
278
-
279
- content += '---\n';
280
- return content;
281
- }
282
-
283
- private trimSessionContent(content: string): string | null {
284
- const q = mdq(content);
285
- if (q.query('heading').count() === 0) return null;
286
- if (q.query('code').count() === 0) return null;
287
-
288
- let result = content;
289
- const codeBlocks = q.query('code').each();
290
- if (codeBlocks.length > 2) {
291
- for (const block of codeBlocks.slice(2)) {
292
- result = result.replace(block.text(), '');
293
- }
294
- }
295
-
296
- const blockquotes = mdq(result).query('blockquote').each();
297
- if (blockquotes.length > 5) {
298
- for (const bq of blockquotes.slice(5)) {
299
- result = result.replace(bq.text(), '');
300
- }
301
- }
302
-
303
- const lines = result.split('\n');
304
- if (lines.length > 40) {
305
- result = lines.slice(0, 40).join('\n');
306
- }
307
-
308
- if (!result.trim()) return null;
309
- return result;
310
- }
311
-
312
282
  getSuccessfulExperience(state: ActionResult, options?: { includeDescendants?: boolean; stripCode?: boolean }): string[] {
313
283
  const records = this.getRelevantExperience(state, {
314
284
  includeDescendantExperience: options?.includeDescendants,
@@ -318,12 +288,14 @@ ${filteredCode}
318
288
  for (const record of records) {
319
289
  if (!record.content) continue;
320
290
 
321
- const successFlows = mdq(record.content).query('section(~"Successful Flow")').text();
322
- const succeeded = mdq(record.content).query('section(~"SUCCEEDED")').text();
323
- let combined = [successFlows, succeeded].filter(Boolean).join('\n\n');
291
+ const flows = mdq(record.content).query('section(~"FLOW:")').text();
292
+ const actions = mdq(record.content).query('section(~"ACTION:")').text();
293
+ let combined = [flows, actions].filter(Boolean).join('\n\n');
324
294
 
325
295
  if (!combined.trim()) continue;
326
296
 
297
+ combined = renderAsHowTo(combined);
298
+
327
299
  if (options?.stripCode) {
328
300
  combined = mdq(combined).query('code').replace('');
329
301
  }
@@ -383,6 +355,72 @@ ${filteredCode}
383
355
  }
384
356
  return null;
385
357
  }
358
+
359
+ listAllExperienceToc(filter?: string, options?: { recency?: 'recent' | 'old' }): ExperienceTocEntry[] {
360
+ const records = this.getAllExperience();
361
+ if (records.length === 0) return [];
362
+
363
+ const trimmed = filter?.trim();
364
+ let matching = records;
365
+
366
+ if (trimmed) {
367
+ if (trimmed.endsWith('.md')) {
368
+ const bare = trimmed.slice(0, -3);
369
+ const byFilename = records.find((record) => basename(record.filePath, '.md') === bare);
370
+ matching = byFilename ? [byFilename] : [];
371
+ } else {
372
+ const lower = trimmed.toLowerCase();
373
+ matching = records.filter((record) => {
374
+ const url = ((record.data as WebPageState)?.url || '').toLowerCase();
375
+ if (!url) return false;
376
+ return url.includes(lower);
377
+ });
378
+ }
379
+ }
380
+
381
+ if (options?.recency) {
382
+ const cutoff = Date.now() - RECENT_WINDOW_DAYS * 24 * 60 * 60 * 1000;
383
+ matching = matching.filter((record) => {
384
+ const isRecent = record.mtime.getTime() >= cutoff;
385
+ return options.recency === 'recent' ? isRecent : !isRecent;
386
+ });
387
+ }
388
+
389
+ const sorted = matching.sort((a, b) => {
390
+ const aUrl = (a.data as WebPageState)?.url || '';
391
+ const bUrl = (b.data as WebPageState)?.url || '';
392
+ return aUrl.localeCompare(bUrl);
393
+ });
394
+
395
+ const toc: ExperienceTocEntry[] = [];
396
+ for (let i = 0; i < sorted.length; i++) {
397
+ const record = sorted[i];
398
+ const sections = listTocHeadings(record.content);
399
+ if (sections.length === 0) continue;
400
+ toc.push({
401
+ fileTag: indexToLetters(toc.length),
402
+ fileHash: basename(record.filePath, '.md'),
403
+ url: (record.data as WebPageState)?.url || '',
404
+ sections,
405
+ });
406
+ }
407
+ return toc;
408
+ }
409
+
410
+ getExperienceSectionByTag(fileTag: string, sectionIndex: number, filter?: string): { title: string; url: string; content: string; fileHash: string } | null {
411
+ const toc = this.listAllExperienceToc(filter);
412
+ const entry = toc.find((e) => e.fileTag === fileTag);
413
+ if (!entry) return null;
414
+
415
+ const filePath = this.findExperienceFileByHash(entry.fileHash);
416
+ if (!filePath) return null;
417
+
418
+ const { content } = this.readExperienceFile(entry.fileHash);
419
+ const extracted = extractHeadingSection(content, sectionIndex);
420
+ if (!extracted) return null;
421
+
422
+ return { title: extracted.title, url: entry.url, content: extracted.body, fileHash: entry.fileHash };
423
+ }
386
424
  }
387
425
 
388
426
  function listTocHeadings(content: string): { index: number; level: 2 | 3; title: string }[] {
@@ -448,7 +486,11 @@ export function renderExperienceToc(toc: ExperienceTocEntry[]): string {
448
486
 
449
487
  const lines: string[] = [];
450
488
  lines.push('<experience>');
451
- lines.push('Past experience for this page. Call learn_experience({ fileTag, sectionIndex }) to read a section.');
489
+ lines.push('Past experience for this page recipes recorded from prior successful runs.');
490
+ lines.push('Locators and step ordering worked then; the page may have changed since.');
491
+ lines.push('Treat as a starting hypothesis, not ground truth. If a step fails, fall back to ARIA/UI-map.');
492
+ lines.push('FLOW: = multi-step recipe (bullets + code + discovery). ACTION: = single-step snippet (one code block).');
493
+ lines.push('Call learn_experience({ fileTag, sectionIndex }) to read a section when it looks relevant to the current step.');
452
494
  lines.push('');
453
495
  for (const entry of toc) {
454
496
  lines.push(`File ${entry.fileTag} ${entry.url}:`);
@@ -462,6 +504,69 @@ export function renderExperienceToc(toc: ExperienceTocEntry[]): string {
462
504
  return lines.join('\n');
463
505
  }
464
506
 
507
+ function normalizeTitle(raw: string): string {
508
+ let t = (raw || '').trim();
509
+ for (const p of ['FLOW:', 'ACTION:']) {
510
+ if (t.toLowerCase().startsWith(p.toLowerCase())) {
511
+ t = t.slice(p.length).trim();
512
+ break;
513
+ }
514
+ }
515
+ while (t.length > 0 && '.!?,;:'.includes(t[t.length - 1])) {
516
+ t = t.slice(0, -1);
517
+ }
518
+ if (t.length > 0) t = t[0].toLowerCase() + t.slice(1);
519
+ return t;
520
+ }
521
+
522
+ function generateActionContent(title: string, code: string, explanation?: string): string {
523
+ const lines: string[] = [];
524
+ lines.push(`## ACTION: ${title}`);
525
+ lines.push('');
526
+ if (explanation) {
527
+ lines.push(`Solution: ${explanation}`);
528
+ lines.push('');
529
+ }
530
+ lines.push('```javascript');
531
+ lines.push(code);
532
+ lines.push('```');
533
+ lines.push('');
534
+ return lines.join('\n');
535
+ }
536
+
537
+ function renderAsHowTo(content: string): string {
538
+ const tokens = marked.lexer(content);
539
+ let result = '';
540
+ for (const token of tokens) {
541
+ if (token.type === 'heading' && (token as Tokens.Heading).depth === 2) {
542
+ const text = (token as Tokens.Heading).text.trim();
543
+ if (text.startsWith('FLOW:')) {
544
+ result += `## HOW to ${text.slice(5).trim()} (multi-step)\n\n`;
545
+ continue;
546
+ }
547
+ if (text.startsWith('ACTION:')) {
548
+ result += `## HOW to ${text.slice(7).trim()} (single-step)\n\n`;
549
+ continue;
550
+ }
551
+ }
552
+ result += (token as any).raw || '';
553
+ }
554
+ return result;
555
+ }
556
+
557
+ export interface ExperienceFile {
558
+ filePath: string;
559
+ data: { url?: string; title?: string; [key: string]: any };
560
+ content: string;
561
+ mtime: Date;
562
+ }
563
+
564
+ export interface ActionInput {
565
+ title: string;
566
+ code: string;
567
+ explanation?: string;
568
+ }
569
+
465
570
  export interface SessionStep {
466
571
  message: string;
467
572
  status: 'passed' | 'failed' | 'neutral';
@@ -470,13 +575,6 @@ export interface SessionStep {
470
575
  discovery?: string;
471
576
  }
472
577
 
473
- export interface SessionExperienceEntry {
474
- scenario: string;
475
- result: 'success' | 'partial' | 'failed';
476
- steps: SessionStep[];
477
- relatedUrls?: string[];
478
- }
479
-
480
578
  export interface ExperienceTocEntry {
481
579
  fileTag: string;
482
580
  fileHash: string;
package/src/explorbot.ts CHANGED
@@ -1,9 +1,6 @@
1
1
  import { existsSync, mkdirSync } from 'node:fs';
2
2
  import path from 'node:path';
3
3
  import { ActionResult } from './action-result.ts';
4
- import { ApiClient } from './api/api-client.ts';
5
- import { RequestStore } from './api/request-store.ts';
6
- import { loadSpec } from './api/spec-reader.ts';
7
4
  import { Bosun } from './ai/bosun.ts';
8
5
  import { Captain } from './ai/captain.ts';
9
6
  import { ExperienceCompactor } from './ai/experience-compactor.ts';
@@ -14,16 +11,20 @@ import { Pilot } from './ai/pilot.ts';
14
11
  import { Planner } from './ai/planner.ts';
15
12
  import { AIProvider } from './ai/provider.ts';
16
13
  import { Quartermaster } from './ai/quartermaster.ts';
17
- import { Researcher } from './ai/researcher.ts';
18
14
  import { Rerunner } from './ai/rerunner.ts';
15
+ import { Researcher } from './ai/researcher.ts';
19
16
  import { Tester } from './ai/tester.ts';
20
17
  import { createAgentTools } from './ai/tools.ts';
18
+ import { ApiClient } from './api/api-client.ts';
19
+ import { RequestStore } from './api/request-store.ts';
20
+ import { loadSpec } from './api/spec-reader.ts';
21
21
  import type { ExplorbotConfig } from './config.js';
22
22
  import { ConfigParser } from './config.ts';
23
+ import { ExperienceTracker } from './experience-tracker.ts';
23
24
  import Explorer from './explorer.ts';
24
- import type { Suite } from './suite.ts';
25
25
  import { KnowledgeTracker } from './knowledge-tracker.ts';
26
26
  import { WebPageState } from './state-manager.ts';
27
+ import type { Suite } from './suite.ts';
27
28
  import { Plan } from './test-plan.ts';
28
29
  import { setVerboseMode, tag } from './utils/logger.ts';
29
30
  import { sanitizeFilename } from './utils/strings.ts';
@@ -51,6 +52,8 @@ export class ExplorBot {
51
52
  public needsInput = false;
52
53
  private currentPlan?: Plan;
53
54
  private planFeature?: string;
55
+ lastPlanError: Error | null = null;
56
+ lastSavedPlanPath: string | null = null;
54
57
  private agents: Record<string, any> = {};
55
58
 
56
59
  constructor(options: ExplorBotOptions = {}) {
@@ -76,13 +79,11 @@ export class ExplorBot {
76
79
  }
77
80
 
78
81
  try {
79
- this.config = await this.configParser.loadConfig(this.options);
80
- this.provider = new AIProvider(this.config.ai);
81
- await this.provider.validateConnection();
82
+ await this.startProviderOnly();
82
83
  this.explorer = new Explorer(this.config, this.provider, this.options);
83
84
  await this.explorer.start();
84
85
  if (!this.options.incognito) {
85
- await this.agentExperienceCompactor().compactAllExperiences();
86
+ await this.agentExperienceCompactor().autocompact();
86
87
  }
87
88
  if (this.userResolveFn) this.explorer.setUserResolve(this.userResolveFn);
88
89
  } catch (error) {
@@ -92,9 +93,16 @@ export class ExplorBot {
92
93
  }
93
94
  }
94
95
 
96
+ async startProviderOnly(): Promise<void> {
97
+ if (this.provider) return;
98
+ this.config = await this.configParser.loadConfig(this.options);
99
+ this.provider = new AIProvider(this.config.ai);
100
+ await this.provider.validateConnection();
101
+ }
102
+
95
103
  async stop(): Promise<void> {
96
104
  this.agents.quartermaster?.stop();
97
- await this.explorer.stop();
105
+ await this.explorer?.stop();
98
106
  }
99
107
 
100
108
  async visitInitialState(): Promise<void> {
@@ -125,6 +133,13 @@ export class ExplorBot {
125
133
  return new KnowledgeTracker();
126
134
  }
127
135
 
136
+ getExperienceTracker(): ExperienceTracker {
137
+ if (this.explorer) {
138
+ return this.explorer.getStateManager().getExperienceTracker();
139
+ }
140
+ return new ExperienceTracker();
141
+ }
142
+
128
143
  getConfig(): ExplorbotConfig {
129
144
  return this.config;
130
145
  }
@@ -227,10 +242,7 @@ export class ExplorBot {
227
242
  }
228
243
 
229
244
  agentExperienceCompactor(): ExperienceCompactor {
230
- return (this.agents.experienceCompactor ||= this.createAgent(({ ai, explorer }) => {
231
- const experienceTracker = explorer.getStateManager().getExperienceTracker();
232
- return new ExperienceCompactor(ai, experienceTracker);
233
- }));
245
+ return (this.agents.experienceCompactor ||= new ExperienceCompactor(this.provider, this.getExperienceTracker()));
234
246
  }
235
247
 
236
248
  agentQuartermaster(): Quartermaster | null {
@@ -247,10 +259,10 @@ export class ExplorBot {
247
259
  }
248
260
 
249
261
  agentHistorian(): Historian {
250
- return (this.agents.historian ||= this.createAgent(({ ai, explorer }) => {
262
+ return (this.agents.historian ||= this.createAgent(({ ai, explorer, config }) => {
251
263
  const experienceTracker = explorer.getStateManager().getExperienceTracker();
252
264
  const reporter = explorer.getReporter();
253
- return new Historian(ai, experienceTracker, reporter, explorer.getStateManager());
265
+ return new Historian(ai, experienceTracker, reporter, explorer.getStateManager(), config, explorer.getPlaywrightRecorder());
254
266
  }));
255
267
  }
256
268
 
@@ -348,20 +360,17 @@ export class ExplorBot {
348
360
  if (this.currentPlan) {
349
361
  planner.setPlan(this.currentPlan);
350
362
  }
363
+ this.lastPlanError = null;
351
364
  try {
352
365
  this.currentPlan = await planner.plan(feature, opts.style, opts.extend, opts.completedPlans);
353
366
  } catch (err) {
354
- tag('warning').log(`Planning failed: ${err instanceof Error ? err.message : err}`);
367
+ this.lastPlanError = err instanceof Error ? err : new Error(String(err));
368
+ tag('warning').log(`Planning failed: ${this.lastPlanError.message}`);
355
369
  if (!this.currentPlan) return undefined;
356
370
  return this.currentPlan;
357
371
  }
358
372
 
359
- const savedPath = this.savePlan();
360
- if (savedPath) {
361
- const relativePath = path.relative(process.cwd(), savedPath);
362
- tag('info').log(`Plan saved to: ${relativePath}`);
363
- tag('info').log(`Edit the plan file and run /plan:load ${relativePath} to reload it`);
364
- }
373
+ this.savePlan();
365
374
 
366
375
  return this.currentPlan;
367
376
  }
@@ -387,6 +396,7 @@ export class ExplorBot {
387
396
  const planFilename = filename || this.generatePlanFilename();
388
397
  const planPath = path.join(plansDir, planFilename);
389
398
  Plan.saveMultipleToMarkdown(plans, planPath);
399
+ this.lastSavedPlanPath = planPath;
390
400
  return planPath;
391
401
  }
392
402
 
package/src/explorer.ts CHANGED
@@ -9,15 +9,16 @@ import { ActionResult } from './action-result.ts';
9
9
  import Action from './action.js';
10
10
  import { AIProvider } from './ai/provider.js';
11
11
  import { visuallyAnnotateContainers } from './ai/researcher/coordinates.ts';
12
+ import { RequestStore } from './api/request-store.ts';
13
+ import { XhrCapture } from './api/xhr-capture.ts';
12
14
  import type { ExplorbotConfig } from './config.js';
13
15
  import { ConfigParser, outputPath } from './config.js';
14
16
  import type { UserResolveFunction } from './explorbot.js';
15
17
  import { KnowledgeTracker } from './knowledge-tracker.js';
18
+ import { PlaywrightRecorder } from './playwright-recorder.ts';
16
19
  import { Reporter } from './reporter.ts';
17
20
  import { StateManager } from './state-manager.js';
18
21
  import { Test } from './test-plan.ts';
19
- import { RequestStore } from './api/request-store.ts';
20
- import { XhrCapture } from './api/xhr-capture.ts';
21
22
  import { createDebug, log, tag } from './utils/logger.js';
22
23
  import { WebElement, extractElementData } from './utils/web-element.ts';
23
24
 
@@ -60,6 +61,7 @@ class Explorer {
60
61
  private _activeTest: Test | null = null;
61
62
  private xhrCapture: XhrCapture | null = null;
62
63
  private requestStore: RequestStore | null = null;
64
+ private playwrightRecorder: PlaywrightRecorder = new PlaywrightRecorder();
63
65
 
64
66
  constructor(config: ExplorbotConfig, aiProvider: AIProvider, options?: { show?: boolean; headless?: boolean; incognito?: boolean; session?: string }) {
65
67
  this.config = config;
@@ -68,7 +70,7 @@ class Explorer {
68
70
  this.initializeContainer();
69
71
  this.stateManager = new StateManager({ incognito: this.options?.incognito });
70
72
  this.knowledgeTracker = new KnowledgeTracker();
71
- this.reporter = new Reporter(config.reporter);
73
+ this.reporter = new Reporter(config.reporter, this.stateManager);
72
74
  }
73
75
 
74
76
  private initializeContainer() {
@@ -123,7 +125,7 @@ class Explorer {
123
125
  tag('substep').log(debugInfo);
124
126
  }
125
127
  const PlaywrightConfig = {
126
- timeout: 1000,
128
+ timeout: 3000,
127
129
  highlightElement: true,
128
130
  waitForAction: 500,
129
131
  ...playwrightConfig,
@@ -237,6 +239,7 @@ class Explorer {
237
239
  const hasSession = this.options?.session && existsSync(this.options.session);
238
240
  const contextOptions = hasSession ? { storageState: this.options!.session } : undefined;
239
241
  await this.playwrightHelper._createContextPage(contextOptions);
242
+ await this.playwrightRecorder.start(this.playwrightHelper.browserContext);
240
243
  this.setupXhrCapture();
241
244
  if (hasSession) {
242
245
  tag('info').log(`Session restored from ${path.relative(process.cwd(), this.options!.session!)}`);
@@ -273,7 +276,11 @@ class Explorer {
273
276
  }
274
277
 
275
278
  createAction() {
276
- return new Action(this.actor, this.stateManager);
279
+ return new Action(this.actor, this.stateManager, this.playwrightRecorder);
280
+ }
281
+
282
+ getPlaywrightRecorder(): PlaywrightRecorder {
283
+ return this.playwrightRecorder;
277
284
  }
278
285
 
279
286
  async visit(url: string) {
@@ -489,6 +496,8 @@ class Explorer {
489
496
  this.xhrCapture.detach(this.playwrightHelper.page);
490
497
  }
491
498
 
499
+ await this.playwrightRecorder.stop();
500
+
492
501
  if (this.options?.session && this.playwrightHelper?.browserContext) {
493
502
  const dir = path.dirname(this.options.session);
494
503
  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });