explorbot 0.1.32-beta.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/README.md +14 -14
  2. package/bin/explorbot-cli.ts +4 -3
  3. package/boat/doc-collector/src/ai/documentarian.ts +22 -27
  4. package/boat/doc-collector/src/ai/tools.ts +67 -10
  5. package/boat/doc-collector/src/docbot.ts +28 -7
  6. package/boat/doc-collector/src/docs-renderer.ts +19 -24
  7. package/boat/doc-collector/src/screenshots.ts +33 -1
  8. package/boat/doc-collector/src/state-diagram.ts +281 -0
  9. package/dist/bin/explorbot-cli.js +3 -3
  10. package/dist/boat/doc-collector/src/ai/documentarian.js +16 -6
  11. package/dist/boat/doc-collector/src/ai/tools.js +47 -10
  12. package/dist/boat/doc-collector/src/docbot.js +25 -6
  13. package/dist/boat/doc-collector/src/docs-renderer.js +18 -3
  14. package/dist/boat/doc-collector/src/screenshots.js +29 -0
  15. package/dist/boat/doc-collector/src/state-diagram.js +200 -0
  16. package/dist/package.json +20 -3
  17. package/dist/src/action-result.d.ts +8 -10
  18. package/dist/src/action-result.js +72 -164
  19. package/dist/src/action.d.ts +3 -10
  20. package/dist/src/action.js +18 -111
  21. package/dist/src/ai/captain/web-mode.js +0 -7
  22. package/dist/src/ai/captain.d.ts +1 -3
  23. package/dist/src/ai/captain.js +4 -19
  24. package/dist/src/ai/conversation.d.ts +2 -5
  25. package/dist/src/ai/conversation.js +9 -28
  26. package/dist/src/ai/driller.d.ts +0 -4
  27. package/dist/src/ai/driller.js +3 -12
  28. package/dist/src/ai/experience-compactor.d.ts +2 -2
  29. package/dist/src/ai/experience-compactor.js +16 -33
  30. package/dist/src/ai/fisherman-tools.js +1 -1
  31. package/dist/src/ai/fisherman.d.ts +0 -1
  32. package/dist/src/ai/fisherman.js +0 -3
  33. package/dist/src/ai/historian/codeceptjs.d.ts +2 -0
  34. package/dist/src/ai/historian/codeceptjs.js +3 -2
  35. package/dist/src/ai/historian/playwright.d.ts +2 -0
  36. package/dist/src/ai/historian/playwright.js +3 -2
  37. package/dist/src/ai/historian/utils.d.ts +0 -1
  38. package/dist/src/ai/historian/utils.js +0 -1
  39. package/dist/src/ai/historian.d.ts +5 -4
  40. package/dist/src/ai/historian.js +1 -5
  41. package/dist/src/ai/navigator.d.ts +3 -5
  42. package/dist/src/ai/navigator.js +11 -46
  43. package/dist/src/ai/pilot.d.ts +1 -3
  44. package/dist/src/ai/pilot.js +6 -17
  45. package/dist/src/ai/planner.d.ts +3 -3
  46. package/dist/src/ai/planner.js +49 -37
  47. package/dist/src/ai/provider.d.ts +4 -1
  48. package/dist/src/ai/provider.js +77 -136
  49. package/dist/src/ai/quartermaster.d.ts +21 -22
  50. package/dist/src/ai/quartermaster.js +4 -10
  51. package/dist/src/ai/rerunner.d.ts +1 -2
  52. package/dist/src/ai/rerunner.js +14 -22
  53. package/dist/src/ai/researcher/cache.js +11 -18
  54. package/dist/src/ai/researcher/deep-analysis.js +3 -6
  55. package/dist/src/ai/researcher/parser.js +1 -1
  56. package/dist/src/ai/researcher/research-result.d.ts +0 -1
  57. package/dist/src/ai/researcher/research-result.js +0 -19
  58. package/dist/src/ai/researcher/sections.js +5 -1
  59. package/dist/src/ai/researcher.js +12 -53
  60. package/dist/src/ai/rules.d.ts +1 -0
  61. package/dist/src/ai/rules.js +22 -15
  62. package/dist/src/ai/session-analyst.js +8 -5
  63. package/dist/src/ai/task-agent.d.ts +1 -1
  64. package/dist/src/ai/task-agent.js +2 -27
  65. package/dist/src/ai/tester.d.ts +0 -2
  66. package/dist/src/ai/tester.js +10 -61
  67. package/dist/src/ai/tools.d.ts +18 -7
  68. package/dist/src/ai/tools.js +119 -101
  69. package/dist/src/commands/compact-command.d.ts +1 -1
  70. package/dist/src/commands/compact-command.js +6 -6
  71. package/dist/src/commands/context-command.js +1 -1
  72. package/dist/src/commands/context-knowledge-command.js +1 -1
  73. package/dist/src/commands/experience-command.js +1 -1
  74. package/dist/src/commands/knows-command.js +1 -1
  75. package/dist/src/commands/learn-command.js +3 -1
  76. package/dist/src/components/AddKnowledge.d.ts +2 -0
  77. package/dist/src/components/AddKnowledge.js +3 -7
  78. package/dist/src/components/App.js +2 -2
  79. package/dist/src/config.d.ts +2 -0
  80. package/dist/src/config.js +12 -0
  81. package/dist/src/experience-tracker.d.ts +11 -6
  82. package/dist/src/experience-tracker.js +81 -154
  83. package/dist/src/explorbot.d.ts +4 -5
  84. package/dist/src/explorbot.js +17 -51
  85. package/dist/src/explorer.d.ts +4 -15
  86. package/dist/src/explorer.js +18 -68
  87. package/dist/src/knowledge-tracker.d.ts +2 -2
  88. package/dist/src/knowledge-tracker.js +38 -45
  89. package/dist/src/state-manager.d.ts +6 -31
  90. package/dist/src/state-manager.js +10 -129
  91. package/dist/src/utils/aria.d.ts +6 -1
  92. package/dist/src/utils/aria.js +4 -1
  93. package/dist/src/utils/cache.d.ts +15 -0
  94. package/dist/src/utils/cache.js +34 -0
  95. package/dist/src/utils/context-formatter.d.ts +6 -0
  96. package/dist/src/utils/context-formatter.js +15 -27
  97. package/dist/src/utils/html.d.ts +5 -0
  98. package/dist/src/utils/html.js +71 -0
  99. package/dist/src/utils/markdown-files.d.ts +10 -0
  100. package/dist/src/utils/markdown-files.js +21 -0
  101. package/dist/src/utils/markdown-query.d.ts +6 -0
  102. package/dist/src/utils/markdown-query.js +14 -1
  103. package/dist/src/utils/page-readiness.d.ts +1 -0
  104. package/dist/src/utils/page-readiness.js +1 -1
  105. package/dist/src/utils/secrets.js +2 -2
  106. package/dist/src/utils/strings.d.ts +2 -0
  107. package/dist/src/utils/strings.js +10 -0
  108. package/dist/src/utils/web-element.d.ts +2 -1
  109. package/dist/src/utils/web-element.js +3 -0
  110. package/dist/src/utils/web-sandbox.d.ts +4 -0
  111. package/dist/src/utils/web-sandbox.js +37 -0
  112. package/package.json +20 -3
  113. package/src/action-result.ts +70 -173
  114. package/src/action.ts +16 -130
  115. package/src/ai/captain/web-mode.ts +0 -7
  116. package/src/ai/captain.ts +5 -17
  117. package/src/ai/conversation.ts +9 -32
  118. package/src/ai/driller.ts +3 -16
  119. package/src/ai/experience-compactor.ts +16 -35
  120. package/src/ai/fisherman-tools.ts +1 -1
  121. package/src/ai/fisherman.ts +0 -4
  122. package/src/ai/historian/codeceptjs.ts +4 -2
  123. package/src/ai/historian/playwright.ts +4 -2
  124. package/src/ai/historian/utils.ts +0 -1
  125. package/src/ai/historian.ts +4 -7
  126. package/src/ai/navigator.ts +13 -49
  127. package/src/ai/pilot.ts +6 -16
  128. package/src/ai/planner.ts +56 -44
  129. package/src/ai/provider.ts +85 -145
  130. package/src/ai/quartermaster.ts +34 -41
  131. package/src/ai/rerunner.ts +17 -21
  132. package/src/ai/researcher/cache.ts +10 -16
  133. package/src/ai/researcher/deep-analysis.ts +3 -6
  134. package/src/ai/researcher/parser.ts +1 -1
  135. package/src/ai/researcher/research-result.ts +0 -18
  136. package/src/ai/researcher/sections.ts +5 -1
  137. package/src/ai/researcher.ts +11 -59
  138. package/src/ai/rules.ts +24 -13
  139. package/src/ai/session-analyst.ts +8 -5
  140. package/src/ai/task-agent.ts +3 -30
  141. package/src/ai/tester.ts +12 -73
  142. package/src/ai/tools.ts +119 -106
  143. package/src/commands/compact-command.ts +7 -8
  144. package/src/commands/context-command.ts +1 -1
  145. package/src/commands/context-knowledge-command.ts +1 -1
  146. package/src/commands/experience-command.ts +1 -1
  147. package/src/commands/knows-command.ts +1 -1
  148. package/src/commands/learn-command.ts +3 -1
  149. package/src/components/AddKnowledge.tsx +5 -7
  150. package/src/components/App.tsx +2 -2
  151. package/src/config.ts +12 -0
  152. package/src/experience-tracker.ts +88 -151
  153. package/src/explorbot.ts +17 -51
  154. package/src/explorer.ts +20 -72
  155. package/src/knowledge-tracker.ts +42 -49
  156. package/src/state-manager.ts +15 -156
  157. package/src/utils/aria.ts +11 -2
  158. package/src/utils/cache.ts +40 -0
  159. package/src/utils/context-formatter.ts +12 -15
  160. package/src/utils/html.ts +79 -0
  161. package/src/utils/markdown-files.ts +30 -0
  162. package/src/utils/markdown-query.ts +15 -1
  163. package/src/utils/page-readiness.ts +1 -1
  164. package/src/utils/secrets.ts +2 -3
  165. package/src/utils/strings.ts +12 -0
  166. package/src/utils/web-element.ts +4 -0
  167. package/src/utils/web-sandbox.ts +43 -0
  168. package/dist/boat/api-tester/example/apibot.config.js +0 -30
package/src/explorer.ts CHANGED
@@ -14,17 +14,17 @@ import { visuallyAnnotateContainers } from './ai/researcher/coordinates.ts';
14
14
  import { RequestStore } from './api/request-store.ts';
15
15
  import { XhrCapture } from './api/xhr-capture.ts';
16
16
  import type { ExplorbotConfig } from './config.js';
17
- import { ConfigParser, outputPath } from './config.js';
18
- import type { UserResolveFunction } from './explorbot.js';
17
+ import { ConfigParser } from './config.js';
18
+ import type { ExperienceTracker } from './experience-tracker.js';
19
19
  import { KnowledgeTracker } from './knowledge-tracker.js';
20
20
  import { PlaywrightRecorder } from './playwright-recorder.ts';
21
21
  import { Reporter } from './reporter.ts';
22
22
  import { StateManager } from './state-manager.js';
23
23
  import { Test, TestResult } from './test-plan.ts';
24
- import { BrowserRecoveryError, isFatalBrowserError, isNavigationTransitionError } from './utils/browser-errors.ts';
24
+ import { BrowserRecoveryError, browserErrorMessage, isFatalBrowserError, isNavigationTransitionError } from './utils/browser-errors.ts';
25
25
  import { ELEMENT_EXTRACTION_CONFIG, getElementDataExtractorSource } from './utils/html.ts';
26
26
  import { createDebug, log, tag } from './utils/logger.js';
27
- import { waitForPageReadiness } from './utils/page-readiness.ts';
27
+ import { sleep, waitForPageReadiness } from './utils/page-readiness.ts';
28
28
  import { WebElement } from './utils/web-element.ts';
29
29
 
30
30
  declare global {
@@ -64,7 +64,6 @@ class Explorer {
64
64
  private stateManager!: StateManager;
65
65
  private knowledgeTracker!: KnowledgeTracker;
66
66
  config: ExplorbotConfig;
67
- private userResolveFn: UserResolveFunction | null = null;
68
67
  private options?: { show?: boolean; headless?: boolean; incognito?: boolean; session?: string };
69
68
  private reporter!: Reporter;
70
69
  private otherTabs: TabInfo[] = [];
@@ -77,13 +76,13 @@ class Explorer {
77
76
  private testConsoleHandler: ((message: any) => void) | null = null;
78
77
  private testDialogHandler: ((dialog: any) => void) | null = null;
79
78
 
80
- constructor(config: ExplorbotConfig, aiProvider: AIProvider, options?: { show?: boolean; headless?: boolean; incognito?: boolean; session?: string }) {
79
+ constructor(config: ExplorbotConfig, aiProvider: AIProvider, options: { show?: boolean; headless?: boolean; incognito?: boolean; session?: string } | undefined, experienceTracker: ExperienceTracker, knowledgeTracker: KnowledgeTracker) {
81
80
  this.config = config;
82
81
  this.aiProvider = aiProvider;
83
82
  this.options = options;
84
83
  this.initializeContainer();
85
- this.stateManager = new StateManager({ incognito: this.options?.incognito });
86
- this.knowledgeTracker = new KnowledgeTracker();
84
+ this.knowledgeTracker = knowledgeTracker;
85
+ this.stateManager = new StateManager(experienceTracker, knowledgeTracker);
87
86
  this.reporter = new Reporter(config.reporter, this.stateManager);
88
87
  }
89
88
 
@@ -91,8 +90,7 @@ class Explorer {
91
90
  try {
92
91
  // Use project root for output directory, not current working directory
93
92
  const configParser = ConfigParser.getInstance();
94
- const configPath = configParser.getConfigPath();
95
- const projectRoot = configPath ? path.dirname(configPath) : process.cwd();
93
+ const projectRoot = configParser.getProjectRoot();
96
94
  (global as any).output_dir = path.join(projectRoot, 'output', 'states');
97
95
  (global as any).codecept_dir = projectRoot;
98
96
 
@@ -155,8 +153,7 @@ class Explorer {
155
153
  };
156
154
 
157
155
  if (this.config.stepsFile) {
158
- const configPath = ConfigParser.getInstance().getConfigPath();
159
- const projectRoot = configPath ? path.dirname(configPath) : process.cwd();
156
+ const projectRoot = ConfigParser.getInstance().getProjectRoot();
160
157
  codeceptConfig.include = { I: path.resolve(projectRoot, this.config.stepsFile) };
161
158
  }
162
159
 
@@ -164,17 +161,9 @@ class Explorer {
164
161
  }
165
162
 
166
163
  public getConfig(): ExplorbotConfig {
167
- if (!this.config) {
168
- throw new Error('Configuration not loaded. Call run() first.');
169
- }
170
164
  return this.config;
171
165
  }
172
166
 
173
- public getConfigPath(): string | null {
174
- const configParser = ConfigParser.getInstance();
175
- return configParser.getConfigPath();
176
- }
177
-
178
167
  public getAIProvider(): AIProvider {
179
168
  return this.aiProvider;
180
169
  }
@@ -183,10 +172,6 @@ class Explorer {
183
172
  return this.stateManager;
184
173
  }
185
174
 
186
- public getCurrentUrl(): string {
187
- return this.stateManager.getCurrentState()!.url || '?';
188
- }
189
-
190
175
  public getKnowledgeTracker(): KnowledgeTracker {
191
176
  return this.knowledgeTracker;
192
177
  }
@@ -227,10 +212,6 @@ class Explorer {
227
212
  return;
228
213
  }
229
214
 
230
- if (!this.config) {
231
- await this.initializeContainer();
232
- }
233
-
234
215
  await codeceptjs.recorder.start();
235
216
  await codeceptjs.container.started(null);
236
217
  storeListener();
@@ -359,8 +340,8 @@ class Explorer {
359
340
  return this.runWithBrowserRecovery('executeAction', () => this.createAction().execute(code));
360
341
  }
361
342
 
362
- async attemptAction(code: string, originalMessage?: string, experience = true): Promise<boolean> {
363
- return this.runWithBrowserRecovery('attemptAction', () => this.createAction().attempt(code, originalMessage, experience));
343
+ async attemptAction(code: string, originalMessage?: string): Promise<boolean> {
344
+ return this.runWithBrowserRecovery('attemptAction', () => this.createAction().attempt(code, originalMessage));
364
345
  }
365
346
 
366
347
  getPlaywrightRecorder(): PlaywrightRecorder {
@@ -388,7 +369,7 @@ class Explorer {
388
369
  try {
389
370
  await action.execute(`I.amOnPage(${serializedUrl})`);
390
371
  } catch (err) {
391
- const msg = err instanceof Error ? err.message : String(err);
372
+ const msg = browserErrorMessage(err);
392
373
  if (!RECOVERABLE_NAVIGATION_ERRORS.test(msg)) throw err;
393
374
  tag('warning').log(`Navigation warning (continuing after load): ${msg.split('\n')[0]}`);
394
375
  await this.waitForPageReadiness();
@@ -437,7 +418,7 @@ class Explorer {
437
418
  return result;
438
419
  } catch (error) {
439
420
  if (this.isFatalBrowserError(error)) {
440
- tag('warning').log(`getEidxInContainer: ${error instanceof Error ? error.message : error}`);
421
+ tag('warning').log(`getEidxInContainer: ${browserErrorMessage(error)}`);
441
422
  await this.recoverFromBrowserError();
442
423
  }
443
424
  return [];
@@ -452,18 +433,13 @@ class Explorer {
452
433
  return await el.first().getAttribute(ELEMENT_EXTRACTION_CONFIG.attrs.eidx);
453
434
  } catch (error) {
454
435
  if (this.isFatalBrowserError(error)) {
455
- tag('warning').log(`getEidxByLocator: ${error instanceof Error ? error.message : error}`);
436
+ tag('warning').log(`getEidxByLocator: ${browserErrorMessage(error)}`);
456
437
  await this.recoverFromBrowserError();
457
438
  }
458
439
  return null;
459
440
  }
460
441
  }
461
442
 
462
- async reload() {
463
- await this.closeOtherTabs();
464
- await this.playwrightHelper.page.reload();
465
- }
466
-
467
443
  private resolveBrowserUrl(url?: string): string | null {
468
444
  if (!url) return null;
469
445
  try {
@@ -508,7 +484,7 @@ class Explorer {
508
484
  await this.playwrightHelper.page.reload({ waitUntil: 'domcontentloaded', timeout: 10000 });
509
485
  return this.waitForPageReadiness();
510
486
  } catch (err) {
511
- tag('error').log(`Browser recovery failed: ${err instanceof Error ? err.message : err}`);
487
+ tag('error').log(`Browser recovery failed: ${browserErrorMessage(err)}`);
512
488
  return false;
513
489
  }
514
490
  }
@@ -552,7 +528,7 @@ class Explorer {
552
528
  tag('success').log('Browser restarted');
553
529
  return true;
554
530
  } catch (err) {
555
- tag('error').log(`Browser restart failed: ${err instanceof Error ? err.message : err}`);
531
+ tag('error').log(`Browser restart failed: ${browserErrorMessage(err)}`);
556
532
  return false;
557
533
  }
558
534
  }
@@ -584,7 +560,7 @@ class Explorer {
584
560
  return await page.evaluate(() => window.top !== window.self);
585
561
  } catch (error) {
586
562
  if (this.isFatalBrowserError(error)) {
587
- tag('warning').log(`isInsideIframe: ${error instanceof Error ? error.message : error}`);
563
+ tag('warning').log(`isInsideIframe: ${browserErrorMessage(error)}`);
588
564
  await this.recoverFromBrowserError();
589
565
  }
590
566
  return false;
@@ -608,10 +584,6 @@ class Explorer {
608
584
  this.otherTabs = [];
609
585
  }
610
586
 
611
- setUserResolve(userResolveFn: UserResolveFunction): void {
612
- this.userResolveFn = userResolveFn;
613
- }
614
-
615
587
  private listenToStateChanged(): void {
616
588
  if (!this.playwrightHelper) {
617
589
  debugLog('Playwright helper not available for state monitoring');
@@ -760,12 +732,8 @@ class Explorer {
760
732
  return true;
761
733
  }
762
734
 
763
- async ensureActiveTestPageAvailable(): Promise<boolean> {
764
- return this.ensurePageAvailable();
765
- }
766
-
767
735
  async handleExecutionError(error: unknown): Promise<BrowserExecutionErrorResult> {
768
- const message = error instanceof Error ? error.message : String(error);
736
+ const message = browserErrorMessage(error);
769
737
  tag('error').log(`Browser execution error: ${message}`);
770
738
 
771
739
  if (error instanceof Error && error.name === 'AbortError') {
@@ -873,33 +841,13 @@ class Explorer {
873
841
  this.observedTestPages.clear();
874
842
  }
875
843
 
876
- async hasPlaywrightLocator(locatorFn: (page: any) => any, opts: { multiple?: boolean; contents?: boolean; success?: (locator: any) => Promise<void> | void } = {}): Promise<boolean> {
877
- try {
878
- const pwLocator = locatorFn(this.playwrightHelper.page);
879
- const count = await pwLocator.count();
880
- if (opts.multiple ? count === 0 : count !== 1) return false;
881
- if (opts.contents) {
882
- const html = await pwLocator.first().innerHTML();
883
- if (!html?.trim()) return false;
884
- }
885
- if (opts.success) await opts.success(pwLocator);
886
- return true;
887
- } catch (error) {
888
- if (this.isFatalBrowserError(error)) {
889
- tag('warning').log(`hasPlaywrightLocator: ${error instanceof Error ? error.message : error}`);
890
- await this.recoverFromBrowserError();
891
- }
892
- return false;
893
- }
894
- }
895
-
896
844
  async playwrightLocatorCount(locatorFn: (page: any) => any): Promise<number> {
897
845
  try {
898
846
  const pwLocator = locatorFn(this.playwrightHelper.page);
899
847
  return await pwLocator.count();
900
848
  } catch (error) {
901
849
  if (this.isFatalBrowserError(error)) {
902
- tag('warning').log(`playwrightLocatorCount: ${error instanceof Error ? error.message : error}`);
850
+ tag('warning').log(`playwrightLocatorCount: ${browserErrorMessage(error)}`);
903
851
  await this.recoverFromBrowserError();
904
852
  }
905
853
  throw error;
@@ -921,7 +869,7 @@ class Explorer {
921
869
 
922
870
  this.stateManager.updateStateFromBasic(newUrl, newTitle, 'navigation');
923
871
 
924
- await new Promise((resolve) => setTimeout(resolve, 500));
872
+ await sleep(500);
925
873
  });
926
874
  }
927
875
 
@@ -1,15 +1,19 @@
1
- import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
2
- import { dirname, join, resolve } from 'node:path';
1
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
2
+ import { join } from 'node:path';
3
+ import dedent from 'dedent';
3
4
  import matter from 'gray-matter';
4
5
  import { ActionResult } from './action-result.js';
5
6
  import { ConfigParser } from './config.js';
6
7
  import { getCliName } from './utils/cli-name.ts';
7
- import { createDebug } from './utils/logger.js';
8
+ import { createDebug, pluralize, tag } from './utils/logger.js';
9
+ import { loadMarkdownFiles } from './utils/markdown-files.js';
10
+ import { mdq } from './utils/markdown-query.js';
8
11
  import { isSecretName, registerSecret } from './utils/secrets.js';
12
+ import { slugify } from './utils/strings.js';
9
13
 
10
14
  const debugLog = createDebug('explorbot:knowledge-tracker');
11
15
 
12
- interface Knowledge {
16
+ export interface Knowledge {
13
17
  filePath: string;
14
18
  url: string;
15
19
  content: string;
@@ -24,14 +28,7 @@ export class KnowledgeTracker {
24
28
  constructor() {
25
29
  const configParser = ConfigParser.getInstance();
26
30
  const config = configParser.getConfig();
27
- const configPath = configParser.getConfigPath();
28
-
29
- if (configPath) {
30
- const projectRoot = dirname(configPath);
31
- this.knowledgeDir = join(projectRoot, config.dirs?.knowledge || 'knowledge');
32
- } else {
33
- this.knowledgeDir = config.dirs?.knowledge || 'knowledge';
34
- }
31
+ this.knowledgeDir = configParser.resolveProjectDir(config.dirs?.knowledge || 'knowledge');
35
32
 
36
33
  if (!existsSync(this.knowledgeDir)) {
37
34
  mkdirSync(this.knowledgeDir, { recursive: true });
@@ -43,29 +40,13 @@ export class KnowledgeTracker {
43
40
 
44
41
  this.knowledgeFiles = [];
45
42
 
46
- if (!existsSync(this.knowledgeDir)) {
47
- return;
48
- }
49
-
50
- const files = readdirSync(this.knowledgeDir)
51
- .filter((file) => file.endsWith('.md'))
52
- .map((file) => join(this.knowledgeDir, file));
53
-
54
- for (const filePath of files) {
55
- try {
56
- const fileContent = readFileSync(filePath, 'utf8');
57
- const parsed = matter(fileContent);
58
- const urlPattern = parsed.data.url || parsed.data.path || '*';
59
-
60
- this.knowledgeFiles.push({
61
- filePath,
62
- url: urlPattern,
63
- content: this.interpolateVars(parsed.content),
64
- ...parsed.data,
65
- });
66
- } catch (error) {
67
- // Skip invalid files
68
- }
43
+ for (const entry of loadMarkdownFiles(this.knowledgeDir, { recursive: true })) {
44
+ this.knowledgeFiles.push({
45
+ filePath: entry.filePath,
46
+ url: entry.data.url || entry.data.path || '*',
47
+ content: this.interpolateVars(entry.content),
48
+ ...entry.data,
49
+ });
69
50
  }
70
51
 
71
52
  this.isLoaded = true;
@@ -79,25 +60,40 @@ export class KnowledgeTracker {
79
60
  });
80
61
  }
81
62
 
63
+ renderRelevantKnowledge(state: ActionResult): string {
64
+ const knowledgeFiles = this.getRelevantKnowledge(state);
65
+ if (knowledgeFiles.length === 0) return '';
66
+
67
+ const knowledgeContent = knowledgeFiles
68
+ .map((k) => k.content)
69
+ .filter((k) => !!k)
70
+ .join('\n\n');
71
+
72
+ tag('operation').log(`Found ${knowledgeFiles.length} relevant knowledge ${pluralize(knowledgeFiles.length, 'file')}`);
73
+ return dedent`
74
+ <knowledge>
75
+ Here is relevant knowledge for this page:
76
+
77
+ ${knowledgeContent}
78
+ </knowledge>
79
+ `;
80
+ }
81
+
82
82
  addKnowledge(urlPattern: string, description: string): { filename: string; filePath: string; isNewFile: boolean } {
83
83
  const configParser = ConfigParser.getInstance();
84
- const config = configParser.getConfig();
85
84
  const configPath = configParser.getConfigPath();
86
85
 
87
86
  if (!configPath) {
88
87
  throw new Error(`No explorbot configuration found. Please run "${getCliName()} init" first.`);
89
88
  }
90
89
 
91
- const projectRoot = dirname(configPath);
92
- const knowledgeDir = join(projectRoot, config.dirs?.knowledge || 'knowledge');
93
-
94
- if (!existsSync(knowledgeDir)) {
95
- mkdirSync(knowledgeDir, { recursive: true });
90
+ if (!existsSync(this.knowledgeDir)) {
91
+ mkdirSync(this.knowledgeDir, { recursive: true });
96
92
  }
97
93
 
98
94
  const normalizedUrl = this.normalizeUrl(urlPattern);
99
95
  const filename = this.generateFilename(normalizedUrl);
100
- const filePath = join(knowledgeDir, filename);
96
+ const filePath = join(this.knowledgeDir, filename);
101
97
 
102
98
  const isNewFile = !existsSync(filePath);
103
99
 
@@ -128,6 +124,8 @@ export class KnowledgeTracker {
128
124
  writeFileSync(filePath, fileContent, 'utf8');
129
125
  }
130
126
 
127
+ this.isLoaded = false;
128
+
131
129
  return { filename, filePath, isNewFile };
132
130
  }
133
131
 
@@ -158,12 +156,7 @@ export class KnowledgeTracker {
158
156
  }
159
157
 
160
158
  private generateFilename(url: string): string {
161
- let filename = url
162
- .replace(/https?:\/\//g, '')
163
- .replace(/[^a-zA-Z0-9_]/g, '_')
164
- .replace(/_+/g, '_')
165
- .replace(/^_|_$/g, '')
166
- .toLowerCase();
159
+ let filename = slugify(url.replace(/https?:\/\//g, ''));
167
160
 
168
161
  if (!filename || filename === '*') {
169
162
  filename = 'general';
@@ -194,7 +187,7 @@ export class KnowledgeTracker {
194
187
 
195
188
  return this.knowledgeFiles.map((knowledge) => {
196
189
  const content = knowledge.content.trim();
197
- const firstLine = content.split('\n')[0]?.trim() || '';
190
+ const firstLine = mdq(content).meta()[0]?.text.split('\n')[0]?.trim() || '';
198
191
  return {
199
192
  url: knowledge.url,
200
193
  firstLine,
@@ -1,11 +1,9 @@
1
- import { existsSync, readFileSync, readdirSync } from 'node:fs';
2
- import { dirname, join } from 'node:path';
3
- import matter from 'gray-matter';
4
1
  import { ActionResult } from './action-result.js';
5
- import { ConfigParser, outputPath } from './config.js';
6
- import { ExperienceTracker } from './experience-tracker.js';
2
+ import type { ExperienceTracker } from './experience-tracker.js';
3
+ import type { KnowledgeTracker, Knowledge } from './knowledge-tracker.js';
7
4
  import { detectFocusArea } from './utils/aria.js';
8
- import { createDebug, tag } from './utils/logger.js';
5
+ import { createDebug } from './utils/logger.js';
6
+ import { slugify } from './utils/strings.js';
9
7
  import { extractStatePath } from './utils/url-matcher.js';
10
8
 
11
9
  const debugLog = createDebug('explorbot:state');
@@ -69,43 +67,30 @@ export interface StateTransition {
69
67
 
70
68
  export type StateChangeListener = (event: StateTransition) => void;
71
69
 
72
- export interface Knowledge extends WebPageState {
73
- /** File path */
74
- filePath: string;
75
- /** Markdown content */
76
- content: string;
77
- }
70
+ export type { Knowledge };
78
71
 
79
72
  export class StateManager {
80
73
  private currentState: WebPageState | null = null;
81
74
  private stateHistory: StateTransition[] = [];
82
75
  private allVisitedUrls: Set<string> = new Set();
83
- private knowledgeCache: Knowledge[] = [];
84
- private lastKnowledgeScan: Date | null = null;
85
76
  private stateChangeListeners: StateChangeListener[] = [];
86
77
  private experienceTracker!: ExperienceTracker;
87
- private knowledgeDir: string;
78
+ private knowledgeTracker: KnowledgeTracker;
88
79
  private nextStateId = 1;
89
80
 
90
- constructor() {
91
- this.experienceTracker = new ExperienceTracker();
92
- const configParser = ConfigParser.getInstance();
93
- const config = configParser.getConfig();
94
- const configPath = configParser.getConfigPath();
95
-
96
- // Resolve knowledge directory relative to the config file location (project root)
97
- if (configPath) {
98
- const projectRoot = dirname(configPath);
99
- this.knowledgeDir = join(projectRoot, config.dirs?.knowledge || 'knowledge');
100
- } else {
101
- this.knowledgeDir = config.dirs?.knowledge || 'knowledge';
102
- }
81
+ constructor(experienceTracker: ExperienceTracker, knowledgeTracker: KnowledgeTracker) {
82
+ this.experienceTracker = experienceTracker;
83
+ this.knowledgeTracker = knowledgeTracker;
103
84
  }
104
85
 
105
86
  getExperienceTracker(): ExperienceTracker {
106
87
  return this.experienceTracker;
107
88
  }
108
89
 
90
+ getKnowledgeTracker(): KnowledgeTracker {
91
+ return this.knowledgeTracker;
92
+ }
93
+
109
94
  /**
110
95
  * Subscribe to state change events
111
96
  */
@@ -134,11 +119,6 @@ export class StateManager {
134
119
  });
135
120
  }
136
121
 
137
- /**
138
- * Extract state path from full URL
139
- * Removes domain, port, protocol
140
- * Keeps path, query, and hash: /path/to/page?tab=users#section
141
- */
142
122
  /**
143
123
  * Update current state from ActionResult and record transition if state changed
144
124
  */
@@ -230,13 +210,7 @@ export class StateManager {
230
210
  parts.push(`title_${title}`);
231
211
  }
232
212
 
233
- return parts
234
- .join('_')
235
- .replace(/[^a-zA-Z0-9_]/g, '_')
236
- .replace(/_+/g, '_')
237
- .replace(/^_|_$/g, '')
238
- .toLowerCase()
239
- .substring(0, 200);
213
+ return slugify(parts.join('_')).substring(0, 200);
240
214
  }
241
215
 
242
216
  /**
@@ -327,65 +301,14 @@ export class StateManager {
327
301
  return this.stateHistory.slice(-count);
328
302
  }
329
303
 
330
- /**
331
- * Scan knowledge directory for .md files and cache them
332
- */
333
- private scanKnowledgeFiles(): void {
334
- const now = new Date();
335
-
336
- // Only rescan every 30 seconds to avoid excessive file I/O
337
- if (this.lastKnowledgeScan && now.getTime() - this.lastKnowledgeScan.getTime() < 30000) {
338
- return;
339
- }
340
-
341
- this.knowledgeCache = [];
342
-
343
- if (!existsSync(this.knowledgeDir)) {
344
- debugLog(`Knowledge directory not found: ${this.knowledgeDir}`);
345
- return;
346
- }
347
-
348
- try {
349
- const files = readdirSync(this.knowledgeDir, { recursive: true })
350
- .filter((file) => typeof file === 'string' && file.endsWith('.md'))
351
- .map((file) => join(this.knowledgeDir, file as string));
352
-
353
- for (const filePath of files) {
354
- try {
355
- const fileContent = readFileSync(filePath, 'utf8');
356
- const parsed = matter(fileContent);
357
-
358
- const urlPattern = parsed.data.url || parsed.data.path || '*';
359
-
360
- this.knowledgeCache.push({
361
- filePath,
362
- url: urlPattern,
363
- ...parsed.data,
364
- content: parsed.content,
365
- });
366
-
367
- debugLog(`Loaded knowledge file: ${filePath} (pattern: ${urlPattern})`);
368
- } catch (error) {
369
- debugLog(`Failed to load knowledge file ${filePath}:`, error);
370
- }
371
- }
372
-
373
- this.lastKnowledgeScan = now;
374
- debugLog(`Scanned ${this.knowledgeCache.length} knowledge files`);
375
- } catch (error) {
376
- debugLog('Failed to scan knowledge directory:', error);
377
- }
378
- }
379
304
  /**
380
305
  * Get relevant knowledge files for current state
381
306
  */
382
307
  getRelevantKnowledge(): Knowledge[] {
383
308
  if (!this.currentState) return [];
384
309
 
385
- this.scanKnowledgeFiles();
386
-
387
310
  const actionResult = ActionResult.fromState(this.currentState);
388
- return this.knowledgeCache.filter((knowledge) => actionResult.isMatchedBy(knowledge));
311
+ return this.knowledgeTracker.getRelevantKnowledge(actionResult);
389
312
  }
390
313
 
391
314
  /**
@@ -402,24 +325,6 @@ export class StateManager {
402
325
  /**
403
326
  * Get all context for current state (knowledge + experience)
404
327
  */
405
- getCurrentContext(): {
406
- state: WebPageState;
407
- knowledge: Knowledge[];
408
- experience: string[];
409
- recentTransitions: StateTransition[];
410
- } {
411
- if (!this.currentState) {
412
- throw new Error('No current state available');
413
- }
414
-
415
- return {
416
- state: this.currentState,
417
- knowledge: this.getRelevantKnowledge(),
418
- experience: this.getRelevantExperience(),
419
- recentTransitions: this.getRecentTransitions(),
420
- };
421
- }
422
-
423
328
  /**
424
329
  * Check if we've been in this state before
425
330
  */
@@ -455,45 +360,6 @@ export class StateManager {
455
360
  * If the last transition changed URL, returns the fromState (for URL change detection).
456
361
  * Otherwise returns the most recent toState with content (for diffing).
457
362
  */
458
- getPreviousState(): WebPageState | null {
459
- if (this.stateHistory.length === 0) return null;
460
-
461
- const lastTransition = this.stateHistory[this.stateHistory.length - 1];
462
-
463
- if (lastTransition.fromState?.url !== lastTransition.toState?.url) {
464
- return lastTransition.fromState;
465
- }
466
-
467
- for (let i = this.stateHistory.length - 1; i >= 0; i--) {
468
- const toState = this.stateHistory[i].toState;
469
-
470
- if (!toState) continue;
471
- if (toState.id === this.currentState?.id) continue;
472
-
473
- if (toState.html || toState.ariaSnapshot) {
474
- return toState;
475
- }
476
- }
477
-
478
- return null;
479
- }
480
-
481
- /**
482
- * Load HTML content from file
483
- */
484
- loadHtmlFromFile(htmlFile: string): string | null {
485
- try {
486
- const filePath = outputPath('states', htmlFile);
487
- if (existsSync(filePath)) {
488
- return readFileSync(filePath, 'utf8');
489
- }
490
- return null;
491
- } catch (error) {
492
- debugLog('Failed to load HTML from file:', error);
493
- return null;
494
- }
495
- }
496
-
497
363
  /**
498
364
  * Clear state history (useful for testing or reset)
499
365
  * Note: This preserves the current state, only clears navigation history
@@ -527,15 +393,8 @@ export class StateManager {
527
393
  this.stateHistory = [];
528
394
  this.allVisitedUrls.clear();
529
395
  this.stateChangeListeners = [];
530
- this.knowledgeCache = [];
531
- this.lastKnowledgeScan = null;
532
396
  this.nextStateId = 1;
533
397
 
534
- // Clean up experience tracker if it has cleanup method
535
- if (this.experienceTracker && typeof this.experienceTracker.cleanup === 'function') {
536
- this.experienceTracker.cleanup();
537
- }
538
-
539
398
  debugLog('StateManager cleanup completed');
540
399
  }
541
400
  }
package/src/utils/aria.ts CHANGED
@@ -483,7 +483,7 @@ export const compactAriaSnapshot = (snapshot: string | null, keepNamed = false):
483
483
  return renderTree(tree);
484
484
  };
485
485
 
486
- export const diffAriaSnapshots = (previous: string | null, current: string | null): string | null => {
486
+ export const diffAriaSnapshots = (previous: string | null, current: string | null): AriaDiff => {
487
487
  const flat = (snap: string | null): FlatEntry[] => {
488
488
  let tree = parseSnapshot(snap);
489
489
  tree = unwrapIgnored(tree);
@@ -500,7 +500,9 @@ export const diffAriaSnapshots = (previous: string | null, current: string | nul
500
500
  const currTotals = countBy(curr.map((e) => e.summary));
501
501
  const byCount = diffByCount(prevTotals, currTotals);
502
502
  const renames = detectRenames(prev, curr, prevTotals, currTotals);
503
- return formatDiff([...byCount.added, ...renames.added], [...byCount.removed, ...renames.removed], toggled);
503
+ const added = [...byCount.added, ...renames.added];
504
+ const removed = [...byCount.removed, ...renames.removed];
505
+ return { text: formatDiff(added, removed, toggled), count: added.length + removed.length + toggled.length };
504
506
  };
505
507
 
506
508
  export const detectFocusArea = (snapshot: string | null): FocusAreaResult => {
@@ -570,10 +572,17 @@ export function parseAriaLocator(ariaStr: string): { role: string; text: string
570
572
  return { role: match[1], text: match[2] };
571
573
  }
572
574
 
575
+ export const LARGE_ARIA_CHANGE_THRESHOLD = 50;
576
+
573
577
  // ─────────────────────────────────────────────────────────────────
574
578
  // Types
575
579
  // ─────────────────────────────────────────────────────────────────
576
580
 
581
+ export interface AriaDiff {
582
+ text: string | null;
583
+ count: number;
584
+ }
585
+
577
586
  type AriaNode = {
578
587
  role: string;
579
588
  name?: string;