explorbot 0.1.31 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (288) hide show
  1. package/README.md +14 -14
  2. package/bin/explorbot-cli.ts +4 -3
  3. package/boat/doc-collector/src/ai/documentarian.ts +22 -27
  4. package/boat/doc-collector/src/ai/tools.ts +67 -10
  5. package/boat/doc-collector/src/docbot.ts +28 -7
  6. package/boat/doc-collector/src/docs-renderer.ts +19 -24
  7. package/boat/doc-collector/src/screenshots.ts +33 -1
  8. package/boat/doc-collector/src/state-diagram.ts +281 -0
  9. package/dist/bin/explorbot-cli.js +3 -3
  10. package/dist/boat/doc-collector/src/ai/documentarian.js +16 -6
  11. package/dist/boat/doc-collector/src/ai/tools.js +47 -10
  12. package/dist/boat/doc-collector/src/docbot.js +25 -6
  13. package/dist/boat/doc-collector/src/docs-renderer.js +18 -3
  14. package/dist/boat/doc-collector/src/screenshots.js +29 -0
  15. package/dist/boat/doc-collector/src/state-diagram.js +200 -0
  16. package/dist/package.json +8 -6
  17. package/dist/src/action-result.d.ts +131 -0
  18. package/dist/src/action-result.js +82 -159
  19. package/dist/src/action.d.ts +43 -0
  20. package/dist/src/action.js +18 -106
  21. package/dist/src/activity.d.ts +26 -0
  22. package/dist/src/ai/agent.d.ts +3 -0
  23. package/dist/src/ai/captain/file-tools.d.ts +28 -0
  24. package/dist/src/ai/captain/idle-mode.d.ts +11 -0
  25. package/dist/src/ai/captain/mixin.d.ts +10 -0
  26. package/dist/src/ai/captain/test-mode.d.ts +13 -0
  27. package/dist/src/ai/captain/web-mode.d.ts +13 -0
  28. package/dist/src/ai/captain/web-mode.js +0 -7
  29. package/dist/src/ai/captain.d.ts +141 -0
  30. package/dist/src/ai/captain.js +4 -19
  31. package/dist/src/ai/conversation.d.ts +28 -0
  32. package/dist/src/ai/conversation.js +9 -28
  33. package/dist/src/ai/driller.d.ts +157 -0
  34. package/dist/src/ai/driller.js +3 -12
  35. package/dist/src/ai/experience-compactor.d.ts +53 -0
  36. package/dist/src/ai/experience-compactor.js +16 -33
  37. package/dist/src/ai/fisherman-tools.d.ts +90 -0
  38. package/dist/src/ai/fisherman-tools.js +1 -1
  39. package/dist/src/ai/fisherman.d.ts +29 -0
  40. package/dist/src/ai/fisherman.js +0 -3
  41. package/dist/src/ai/historian/codeceptjs.d.ts +17 -0
  42. package/dist/src/ai/historian/codeceptjs.js +3 -2
  43. package/dist/src/ai/historian/experience.d.ts +40 -0
  44. package/dist/src/ai/historian/mixin.d.ts +2 -0
  45. package/dist/src/ai/historian/playwright.d.ts +22 -0
  46. package/dist/src/ai/historian/playwright.js +3 -2
  47. package/dist/src/ai/historian/screencast.d.ts +32 -0
  48. package/dist/src/ai/historian/utils.d.ts +3 -0
  49. package/dist/src/ai/historian/utils.js +0 -1
  50. package/dist/src/ai/historian.d.ts +109 -0
  51. package/dist/src/ai/historian.js +1 -5
  52. package/dist/src/ai/navigator.d.ts +54 -0
  53. package/dist/src/ai/navigator.js +11 -46
  54. package/dist/src/ai/pilot.d.ts +81 -0
  55. package/dist/src/ai/pilot.js +6 -17
  56. package/dist/src/ai/planner/session-dedup.d.ts +12 -0
  57. package/dist/src/ai/planner/styles.d.ts +6 -0
  58. package/dist/src/ai/planner/subpages.d.ts +35 -0
  59. package/dist/src/ai/planner.d.ts +79 -0
  60. package/dist/src/ai/planner.js +49 -37
  61. package/dist/src/ai/provider.d.ts +53 -0
  62. package/dist/src/ai/provider.js +78 -130
  63. package/dist/src/ai/quartermaster.d.ts +56 -0
  64. package/dist/src/ai/quartermaster.js +4 -10
  65. package/dist/src/ai/rerunner.d.ts +48 -0
  66. package/dist/src/ai/rerunner.js +14 -22
  67. package/dist/src/ai/researcher/cache.d.ts +6 -0
  68. package/dist/src/ai/researcher/cache.js +11 -18
  69. package/dist/src/ai/researcher/coordinates.d.ts +56 -0
  70. package/dist/src/ai/researcher/deep-analysis.d.ts +74 -0
  71. package/dist/src/ai/researcher/deep-analysis.js +3 -6
  72. package/dist/src/ai/researcher/fingerprint-worker.d.ts +1 -0
  73. package/dist/src/ai/researcher/focus.d.ts +8 -0
  74. package/dist/src/ai/researcher/locators.d.ts +36 -0
  75. package/dist/src/ai/researcher/mixin.d.ts +2 -0
  76. package/dist/src/ai/researcher/parser.d.ts +35 -0
  77. package/dist/src/ai/researcher/parser.js +1 -1
  78. package/dist/src/ai/researcher/research-result.d.ts +15 -0
  79. package/dist/src/ai/researcher/research-result.js +0 -19
  80. package/dist/src/ai/researcher/sections.d.ts +19 -0
  81. package/dist/src/ai/researcher/sections.js +5 -1
  82. package/dist/src/ai/researcher.js +12 -53
  83. package/dist/src/ai/rules.d.ts +16 -0
  84. package/dist/src/ai/rules.js +22 -15
  85. package/dist/src/ai/session-analyst.d.ts +11 -0
  86. package/dist/src/ai/session-analyst.js +8 -5
  87. package/dist/src/ai/task-agent.d.ts +28 -0
  88. package/dist/src/ai/task-agent.js +2 -27
  89. package/dist/src/ai/tester.d.ts +101 -0
  90. package/dist/src/ai/tester.js +10 -61
  91. package/dist/src/ai/tools.d.ts +52 -0
  92. package/dist/src/ai/tools.js +119 -101
  93. package/dist/src/api/api-client.d.ts +17 -0
  94. package/dist/src/api/request-result.d.ts +44 -0
  95. package/dist/src/api/request-store.d.ts +29 -0
  96. package/dist/src/api/spec-reader.d.ts +5 -0
  97. package/dist/src/api/xhr-capture.d.ts +10 -0
  98. package/dist/src/browser-server.d.ts +10 -0
  99. package/dist/src/command-handler.d.ts +68 -0
  100. package/dist/src/commands/add-rule-command.d.ts +11 -0
  101. package/dist/src/commands/base-command.d.ts +26 -0
  102. package/dist/src/commands/clean-command.d.ts +12 -0
  103. package/dist/src/commands/compact-command.d.ts +20 -0
  104. package/dist/src/commands/compact-command.js +6 -6
  105. package/dist/src/commands/context-aria-command.d.ts +6 -0
  106. package/dist/src/commands/context-command.d.ts +11 -0
  107. package/dist/src/commands/context-command.js +1 -1
  108. package/dist/src/commands/context-data-command.d.ts +6 -0
  109. package/dist/src/commands/context-experience-command.d.ts +6 -0
  110. package/dist/src/commands/context-html-command.d.ts +6 -0
  111. package/dist/src/commands/context-knowledge-command.d.ts +6 -0
  112. package/dist/src/commands/context-knowledge-command.js +1 -1
  113. package/dist/src/commands/debug-command.d.ts +6 -0
  114. package/dist/src/commands/drill-command.d.ts +10 -0
  115. package/dist/src/commands/exit-command.d.ts +7 -0
  116. package/dist/src/commands/experience-command.d.ts +15 -0
  117. package/dist/src/commands/experience-command.js +1 -1
  118. package/dist/src/commands/explore-command.d.ts +51 -0
  119. package/dist/src/commands/freesail-command.d.ts +12 -0
  120. package/dist/src/commands/help-command.d.ts +6 -0
  121. package/dist/src/commands/index.d.ts +4 -0
  122. package/dist/src/commands/init-command.d.ts +7 -0
  123. package/dist/src/commands/knows-command.d.ts +10 -0
  124. package/dist/src/commands/knows-command.js +1 -1
  125. package/dist/src/commands/learn-command.d.ts +7 -0
  126. package/dist/src/commands/learn-command.js +3 -1
  127. package/dist/src/commands/navigate-command.d.ts +7 -0
  128. package/dist/src/commands/path-command.d.ts +10 -0
  129. package/dist/src/commands/plan-clear-command.d.ts +7 -0
  130. package/dist/src/commands/plan-command.d.ts +14 -0
  131. package/dist/src/commands/plan-edit-command.d.ts +7 -0
  132. package/dist/src/commands/plan-load-command.d.ts +7 -0
  133. package/dist/src/commands/plan-reload-command.d.ts +7 -0
  134. package/dist/src/commands/plan-save-command.d.ts +7 -0
  135. package/dist/src/commands/plans-command.d.ts +21 -0
  136. package/dist/src/commands/rerun-command.d.ts +7 -0
  137. package/dist/src/commands/research-command.d.ts +11 -0
  138. package/dist/src/commands/runs-command.d.ts +7 -0
  139. package/dist/src/commands/start-command.d.ts +7 -0
  140. package/dist/src/commands/status-command.d.ts +6 -0
  141. package/dist/src/commands/test-command.d.ts +11 -0
  142. package/dist/src/components/ActivityPane.d.ts +6 -0
  143. package/dist/src/components/AddKnowledge.d.ts +10 -0
  144. package/dist/src/components/AddKnowledge.js +3 -7
  145. package/dist/src/components/AddRule.d.ts +9 -0
  146. package/dist/src/components/App.d.ts +9 -0
  147. package/dist/src/components/App.js +2 -2
  148. package/dist/src/components/Autocomplete.d.ts +3 -0
  149. package/dist/src/components/InputPane.d.ts +13 -0
  150. package/dist/src/components/InputReadline.d.ts +17 -0
  151. package/dist/src/components/LogPane.d.ts +6 -0
  152. package/dist/src/components/PlanEditor.d.ts +9 -0
  153. package/dist/src/components/PlanPane.d.ts +13 -0
  154. package/dist/src/components/SessionTimer.d.ts +6 -0
  155. package/dist/src/components/StateTransitionPane.d.ts +8 -0
  156. package/dist/src/components/StatusPane.d.ts +4 -0
  157. package/dist/src/components/TaskPane.d.ts +9 -0
  158. package/dist/src/components/Welcome.d.ts +3 -0
  159. package/dist/src/components/WelcomeChecklist.d.ts +9 -0
  160. package/dist/src/components/WelcomeCommands.d.ts +6 -0
  161. package/dist/src/components/autocomplete-store.d.ts +10 -0
  162. package/dist/src/components/parse-keypress.d.ts +12 -0
  163. package/dist/src/config.d.ts +253 -0
  164. package/dist/src/config.js +12 -0
  165. package/dist/src/execution-controller.d.ts +25 -0
  166. package/dist/src/experience-tracker.d.ts +112 -0
  167. package/dist/src/experience-tracker.js +83 -155
  168. package/dist/src/explorbot.d.ts +106 -0
  169. package/dist/src/explorbot.js +17 -51
  170. package/dist/src/explorer.d.ts +129 -0
  171. package/dist/src/explorer.js +31 -77
  172. package/dist/src/index.d.ts +6 -0
  173. package/dist/src/index.js +2 -55
  174. package/dist/src/knowledge-tracker.d.ts +33 -0
  175. package/dist/src/knowledge-tracker.js +47 -47
  176. package/dist/src/observability.d.ts +13 -0
  177. package/dist/src/observability.js +2 -1
  178. package/dist/src/playwright-recorder.d.ts +31 -0
  179. package/dist/src/reporter.d.ts +34 -0
  180. package/dist/src/state-manager.d.ts +160 -0
  181. package/dist/src/state-manager.js +10 -136
  182. package/dist/src/stats.d.ts +24 -0
  183. package/dist/src/suite.d.ts +24 -0
  184. package/dist/src/test-plan.d.ts +166 -0
  185. package/dist/src/utils/aria.d.ts +25 -0
  186. package/dist/src/utils/aria.js +4 -1
  187. package/dist/src/utils/browser-errors.d.ts +8 -0
  188. package/dist/src/utils/cache.d.ts +15 -0
  189. package/dist/src/utils/cache.js +34 -0
  190. package/dist/src/utils/cli-name.d.ts +1 -0
  191. package/dist/src/utils/code-extractor.d.ts +1 -0
  192. package/dist/src/utils/context-formatter.d.ts +31 -0
  193. package/dist/src/utils/context-formatter.js +15 -27
  194. package/dist/src/utils/error-page.d.ts +12 -0
  195. package/dist/src/utils/expandable.d.ts +4 -0
  196. package/dist/src/utils/hooks-runner.d.ts +15 -0
  197. package/dist/src/utils/html-diff.d.ts +19 -0
  198. package/dist/src/utils/html.d.ts +129 -0
  199. package/dist/src/utils/html.js +71 -0
  200. package/dist/src/utils/log-filters.d.ts +6 -0
  201. package/dist/src/utils/logger.d.ts +35 -0
  202. package/dist/src/utils/loop.d.ts +35 -0
  203. package/dist/src/utils/markdown-files.d.ts +10 -0
  204. package/dist/src/utils/markdown-files.js +21 -0
  205. package/dist/src/utils/markdown-parser.d.ts +17 -0
  206. package/dist/src/utils/markdown-query.d.ts +46 -0
  207. package/dist/src/utils/markdown-query.js +14 -1
  208. package/dist/src/utils/markdown-terminal.d.ts +1 -0
  209. package/dist/src/utils/next-steps.d.ts +11 -0
  210. package/dist/src/utils/page-readiness.d.ts +7 -0
  211. package/dist/src/utils/page-readiness.js +1 -1
  212. package/dist/src/utils/research-parser.d.ts +1 -0
  213. package/dist/src/utils/retry.d.ts +8 -0
  214. package/dist/src/utils/rules-loader.d.ts +12 -0
  215. package/dist/src/utils/secrets.d.ts +4 -0
  216. package/dist/src/utils/secrets.js +28 -0
  217. package/dist/src/utils/step-analyzer.d.ts +11 -0
  218. package/dist/src/utils/strings.d.ts +5 -0
  219. package/dist/src/utils/strings.js +10 -0
  220. package/dist/src/utils/test-files.d.ts +3 -0
  221. package/dist/src/utils/test-plan-markdown.d.ts +9 -0
  222. package/dist/src/utils/throttle.d.ts +2 -0
  223. package/dist/src/utils/unique-names.d.ts +2 -0
  224. package/dist/src/utils/url-matcher.d.ts +6 -0
  225. package/dist/src/utils/web-element.d.ts +45 -0
  226. package/dist/src/utils/web-element.js +3 -0
  227. package/dist/src/utils/web-sandbox.d.ts +4 -0
  228. package/dist/src/utils/web-sandbox.js +37 -0
  229. package/dist/src/utils/xpath.d.ts +17 -0
  230. package/package.json +8 -6
  231. package/src/action-result.ts +80 -168
  232. package/src/action.ts +16 -123
  233. package/src/ai/captain/web-mode.ts +0 -7
  234. package/src/ai/captain.ts +5 -17
  235. package/src/ai/conversation.ts +9 -32
  236. package/src/ai/driller.ts +3 -16
  237. package/src/ai/experience-compactor.ts +16 -35
  238. package/src/ai/fisherman-tools.ts +1 -1
  239. package/src/ai/fisherman.ts +0 -4
  240. package/src/ai/historian/codeceptjs.ts +4 -2
  241. package/src/ai/historian/playwright.ts +4 -2
  242. package/src/ai/historian/utils.ts +0 -1
  243. package/src/ai/historian.ts +4 -7
  244. package/src/ai/navigator.ts +13 -49
  245. package/src/ai/pilot.ts +6 -16
  246. package/src/ai/planner.ts +56 -44
  247. package/src/ai/provider.ts +86 -139
  248. package/src/ai/quartermaster.ts +34 -41
  249. package/src/ai/rerunner.ts +17 -21
  250. package/src/ai/researcher/cache.ts +10 -16
  251. package/src/ai/researcher/deep-analysis.ts +3 -6
  252. package/src/ai/researcher/parser.ts +1 -1
  253. package/src/ai/researcher/research-result.ts +0 -18
  254. package/src/ai/researcher/sections.ts +5 -1
  255. package/src/ai/researcher.ts +11 -59
  256. package/src/ai/rules.ts +24 -13
  257. package/src/ai/session-analyst.ts +8 -5
  258. package/src/ai/task-agent.ts +3 -30
  259. package/src/ai/tester.ts +12 -73
  260. package/src/ai/tools.ts +119 -106
  261. package/src/commands/compact-command.ts +7 -8
  262. package/src/commands/context-command.ts +1 -1
  263. package/src/commands/context-knowledge-command.ts +1 -1
  264. package/src/commands/experience-command.ts +1 -1
  265. package/src/commands/knows-command.ts +1 -1
  266. package/src/commands/learn-command.ts +3 -1
  267. package/src/components/AddKnowledge.tsx +5 -7
  268. package/src/components/App.tsx +2 -2
  269. package/src/config.ts +12 -0
  270. package/src/experience-tracker.ts +90 -152
  271. package/src/explorbot.ts +17 -51
  272. package/src/explorer.ts +34 -81
  273. package/src/index.ts +9 -0
  274. package/src/knowledge-tracker.ts +49 -50
  275. package/src/observability.ts +2 -1
  276. package/src/state-manager.ts +15 -164
  277. package/src/utils/aria.ts +11 -2
  278. package/src/utils/cache.ts +40 -0
  279. package/src/utils/context-formatter.ts +12 -15
  280. package/src/utils/html.ts +79 -0
  281. package/src/utils/markdown-files.ts +30 -0
  282. package/src/utils/markdown-query.ts +15 -1
  283. package/src/utils/page-readiness.ts +1 -1
  284. package/src/utils/secrets.ts +28 -0
  285. package/src/utils/strings.ts +12 -0
  286. package/src/utils/web-element.ts +4 -0
  287. package/src/utils/web-sandbox.ts +43 -0
  288. package/src/index.tsx +0 -62
@@ -1,12 +1,9 @@
1
- import { existsSync, readFileSync, readdirSync } from 'node:fs';
2
- import { dirname, join } from 'node:path';
3
- import matter from 'gray-matter';
4
1
  import { ActionResult } from './action-result.js';
5
- import { ConfigParser, outputPath } from './config.js';
6
- import { ExperienceTracker } from './experience-tracker.js';
2
+ import type { ExperienceTracker } from './experience-tracker.js';
3
+ import type { KnowledgeTracker, Knowledge } from './knowledge-tracker.js';
7
4
  import { detectFocusArea } from './utils/aria.js';
8
- import { htmlTextSnapshot } from './utils/html.js';
9
- import { createDebug, tag } from './utils/logger.js';
5
+ import { createDebug } from './utils/logger.js';
6
+ import { slugify } from './utils/strings.js';
10
7
  import { extractStatePath } from './utils/url-matcher.js';
11
8
 
12
9
  const debugLog = createDebug('explorbot:state');
@@ -70,43 +67,30 @@ export interface StateTransition {
70
67
 
71
68
  export type StateChangeListener = (event: StateTransition) => void;
72
69
 
73
- export interface Knowledge extends WebPageState {
74
- /** File path */
75
- filePath: string;
76
- /** Markdown content */
77
- content: string;
78
- }
70
+ export type { Knowledge };
79
71
 
80
72
  export class StateManager {
81
73
  private currentState: WebPageState | null = null;
82
74
  private stateHistory: StateTransition[] = [];
83
75
  private allVisitedUrls: Set<string> = new Set();
84
- private knowledgeCache: Knowledge[] = [];
85
- private lastKnowledgeScan: Date | null = null;
86
76
  private stateChangeListeners: StateChangeListener[] = [];
87
77
  private experienceTracker!: ExperienceTracker;
88
- private knowledgeDir: string;
78
+ private knowledgeTracker: KnowledgeTracker;
89
79
  private nextStateId = 1;
90
80
 
91
- constructor() {
92
- this.experienceTracker = new ExperienceTracker();
93
- const configParser = ConfigParser.getInstance();
94
- const config = configParser.getConfig();
95
- const configPath = configParser.getConfigPath();
96
-
97
- // Resolve knowledge directory relative to the config file location (project root)
98
- if (configPath) {
99
- const projectRoot = dirname(configPath);
100
- this.knowledgeDir = join(projectRoot, config.dirs?.knowledge || 'knowledge');
101
- } else {
102
- this.knowledgeDir = config.dirs?.knowledge || 'knowledge';
103
- }
81
+ constructor(experienceTracker: ExperienceTracker, knowledgeTracker: KnowledgeTracker) {
82
+ this.experienceTracker = experienceTracker;
83
+ this.knowledgeTracker = knowledgeTracker;
104
84
  }
105
85
 
106
86
  getExperienceTracker(): ExperienceTracker {
107
87
  return this.experienceTracker;
108
88
  }
109
89
 
90
+ getKnowledgeTracker(): KnowledgeTracker {
91
+ return this.knowledgeTracker;
92
+ }
93
+
110
94
  /**
111
95
  * Subscribe to state change events
112
96
  */
@@ -126,13 +110,6 @@ export class StateManager {
126
110
  * Emit state change event to all listeners
127
111
  */
128
112
  private emitStateChange(event: StateTransition): void {
129
- // Log HTML content when state changes
130
- if (event.toState.html && event.toState.html !== event.fromState?.html) {
131
- let htmlContent = event?.toState?.html ?? '';
132
- htmlContent = htmlTextSnapshot(htmlContent);
133
- // tag('html').log(`Page HTML for ${event.toState.url}:\n${htmlContent}`);
134
- }
135
-
136
113
  this.stateChangeListeners.forEach((listener) => {
137
114
  try {
138
115
  listener(event);
@@ -142,11 +119,6 @@ export class StateManager {
142
119
  });
143
120
  }
144
121
 
145
- /**
146
- * Extract state path from full URL
147
- * Removes domain, port, protocol
148
- * Keeps path, query, and hash: /path/to/page?tab=users#section
149
- */
150
122
  /**
151
123
  * Update current state from ActionResult and record transition if state changed
152
124
  */
@@ -238,13 +210,7 @@ export class StateManager {
238
210
  parts.push(`title_${title}`);
239
211
  }
240
212
 
241
- return parts
242
- .join('_')
243
- .replace(/[^a-zA-Z0-9_]/g, '_')
244
- .replace(/_+/g, '_')
245
- .replace(/^_|_$/g, '')
246
- .toLowerCase()
247
- .substring(0, 200);
213
+ return slugify(parts.join('_')).substring(0, 200);
248
214
  }
249
215
 
250
216
  /**
@@ -335,65 +301,14 @@ export class StateManager {
335
301
  return this.stateHistory.slice(-count);
336
302
  }
337
303
 
338
- /**
339
- * Scan knowledge directory for .md files and cache them
340
- */
341
- private scanKnowledgeFiles(): void {
342
- const now = new Date();
343
-
344
- // Only rescan every 30 seconds to avoid excessive file I/O
345
- if (this.lastKnowledgeScan && now.getTime() - this.lastKnowledgeScan.getTime() < 30000) {
346
- return;
347
- }
348
-
349
- this.knowledgeCache = [];
350
-
351
- if (!existsSync(this.knowledgeDir)) {
352
- debugLog(`Knowledge directory not found: ${this.knowledgeDir}`);
353
- return;
354
- }
355
-
356
- try {
357
- const files = readdirSync(this.knowledgeDir, { recursive: true })
358
- .filter((file) => typeof file === 'string' && file.endsWith('.md'))
359
- .map((file) => join(this.knowledgeDir, file as string));
360
-
361
- for (const filePath of files) {
362
- try {
363
- const fileContent = readFileSync(filePath, 'utf8');
364
- const parsed = matter(fileContent);
365
-
366
- const urlPattern = parsed.data.url || parsed.data.path || '*';
367
-
368
- this.knowledgeCache.push({
369
- filePath,
370
- url: urlPattern,
371
- ...parsed.data,
372
- content: parsed.content,
373
- });
374
-
375
- debugLog(`Loaded knowledge file: ${filePath} (pattern: ${urlPattern})`);
376
- } catch (error) {
377
- debugLog(`Failed to load knowledge file ${filePath}:`, error);
378
- }
379
- }
380
-
381
- this.lastKnowledgeScan = now;
382
- debugLog(`Scanned ${this.knowledgeCache.length} knowledge files`);
383
- } catch (error) {
384
- debugLog('Failed to scan knowledge directory:', error);
385
- }
386
- }
387
304
  /**
388
305
  * Get relevant knowledge files for current state
389
306
  */
390
307
  getRelevantKnowledge(): Knowledge[] {
391
308
  if (!this.currentState) return [];
392
309
 
393
- this.scanKnowledgeFiles();
394
-
395
310
  const actionResult = ActionResult.fromState(this.currentState);
396
- return this.knowledgeCache.filter((knowledge) => actionResult.isMatchedBy(knowledge));
311
+ return this.knowledgeTracker.getRelevantKnowledge(actionResult);
397
312
  }
398
313
 
399
314
  /**
@@ -410,24 +325,6 @@ export class StateManager {
410
325
  /**
411
326
  * Get all context for current state (knowledge + experience)
412
327
  */
413
- getCurrentContext(): {
414
- state: WebPageState;
415
- knowledge: Knowledge[];
416
- experience: string[];
417
- recentTransitions: StateTransition[];
418
- } {
419
- if (!this.currentState) {
420
- throw new Error('No current state available');
421
- }
422
-
423
- return {
424
- state: this.currentState,
425
- knowledge: this.getRelevantKnowledge(),
426
- experience: this.getRelevantExperience(),
427
- recentTransitions: this.getRecentTransitions(),
428
- };
429
- }
430
-
431
328
  /**
432
329
  * Check if we've been in this state before
433
330
  */
@@ -463,45 +360,6 @@ export class StateManager {
463
360
  * If the last transition changed URL, returns the fromState (for URL change detection).
464
361
  * Otherwise returns the most recent toState with content (for diffing).
465
362
  */
466
- getPreviousState(): WebPageState | null {
467
- if (this.stateHistory.length === 0) return null;
468
-
469
- const lastTransition = this.stateHistory[this.stateHistory.length - 1];
470
-
471
- if (lastTransition.fromState?.url !== lastTransition.toState?.url) {
472
- return lastTransition.fromState;
473
- }
474
-
475
- for (let i = this.stateHistory.length - 1; i >= 0; i--) {
476
- const toState = this.stateHistory[i].toState;
477
-
478
- if (!toState) continue;
479
- if (toState.id === this.currentState?.id) continue;
480
-
481
- if (toState.html || toState.ariaSnapshot) {
482
- return toState;
483
- }
484
- }
485
-
486
- return null;
487
- }
488
-
489
- /**
490
- * Load HTML content from file
491
- */
492
- loadHtmlFromFile(htmlFile: string): string | null {
493
- try {
494
- const filePath = outputPath('states', htmlFile);
495
- if (existsSync(filePath)) {
496
- return readFileSync(filePath, 'utf8');
497
- }
498
- return null;
499
- } catch (error) {
500
- debugLog('Failed to load HTML from file:', error);
501
- return null;
502
- }
503
- }
504
-
505
363
  /**
506
364
  * Clear state history (useful for testing or reset)
507
365
  * Note: This preserves the current state, only clears navigation history
@@ -535,15 +393,8 @@ export class StateManager {
535
393
  this.stateHistory = [];
536
394
  this.allVisitedUrls.clear();
537
395
  this.stateChangeListeners = [];
538
- this.knowledgeCache = [];
539
- this.lastKnowledgeScan = null;
540
396
  this.nextStateId = 1;
541
397
 
542
- // Clean up experience tracker if it has cleanup method
543
- if (this.experienceTracker && typeof this.experienceTracker.cleanup === 'function') {
544
- this.experienceTracker.cleanup();
545
- }
546
-
547
398
  debugLog('StateManager cleanup completed');
548
399
  }
549
400
  }
package/src/utils/aria.ts CHANGED
@@ -483,7 +483,7 @@ export const compactAriaSnapshot = (snapshot: string | null, keepNamed = false):
483
483
  return renderTree(tree);
484
484
  };
485
485
 
486
- export const diffAriaSnapshots = (previous: string | null, current: string | null): string | null => {
486
+ export const diffAriaSnapshots = (previous: string | null, current: string | null): AriaDiff => {
487
487
  const flat = (snap: string | null): FlatEntry[] => {
488
488
  let tree = parseSnapshot(snap);
489
489
  tree = unwrapIgnored(tree);
@@ -500,7 +500,9 @@ export const diffAriaSnapshots = (previous: string | null, current: string | nul
500
500
  const currTotals = countBy(curr.map((e) => e.summary));
501
501
  const byCount = diffByCount(prevTotals, currTotals);
502
502
  const renames = detectRenames(prev, curr, prevTotals, currTotals);
503
- return formatDiff([...byCount.added, ...renames.added], [...byCount.removed, ...renames.removed], toggled);
503
+ const added = [...byCount.added, ...renames.added];
504
+ const removed = [...byCount.removed, ...renames.removed];
505
+ return { text: formatDiff(added, removed, toggled), count: added.length + removed.length + toggled.length };
504
506
  };
505
507
 
506
508
  export const detectFocusArea = (snapshot: string | null): FocusAreaResult => {
@@ -570,10 +572,17 @@ export function parseAriaLocator(ariaStr: string): { role: string; text: string
570
572
  return { role: match[1], text: match[2] };
571
573
  }
572
574
 
575
+ export const LARGE_ARIA_CHANGE_THRESHOLD = 50;
576
+
573
577
  // ─────────────────────────────────────────────────────────────────
574
578
  // Types
575
579
  // ─────────────────────────────────────────────────────────────────
576
580
 
581
+ export interface AriaDiff {
582
+ text: string | null;
583
+ count: number;
584
+ }
585
+
577
586
  type AriaNode = {
578
587
  role: string;
579
588
  name?: string;
@@ -0,0 +1,40 @@
1
+ export class TTLCache<T> {
2
+ private store = new Map<string, CacheEntry<T>>();
3
+ private ttlMs: number;
4
+
5
+ constructor(ttlMs = Number.POSITIVE_INFINITY) {
6
+ this.ttlMs = ttlMs;
7
+ }
8
+
9
+ get(key: string): T | undefined {
10
+ const entry = this.store.get(key);
11
+ if (!entry) return undefined;
12
+ if (Date.now() - entry.at > this.ttlMs) {
13
+ this.store.delete(key);
14
+ return undefined;
15
+ }
16
+ return entry.value;
17
+ }
18
+
19
+ set(key: string, value: T): void {
20
+ this.store.set(key, { value, at: Date.now() });
21
+ }
22
+
23
+ async fetch(key: string, compute: () => Promise<T>): Promise<T> {
24
+ const cached = this.get(key);
25
+ if (cached !== undefined) return cached;
26
+ const value = await compute();
27
+ this.set(key, value);
28
+ return value;
29
+ }
30
+
31
+ delete(key: string): void {
32
+ this.store.delete(key);
33
+ }
34
+
35
+ clear(): void {
36
+ this.store.clear();
37
+ }
38
+ }
39
+
40
+ type CacheEntry<T> = { value: T; at: number };
@@ -29,6 +29,15 @@ function indent(text: string): string {
29
29
  .join('\n');
30
30
  }
31
31
 
32
+ export function formatHeadings(headings: { h1?: string; h2?: string; h3?: string; h4?: string }): string[] {
33
+ const lines: string[] = [];
34
+ if (headings.h1) lines.push(`H1: ${headings.h1}`);
35
+ if (headings.h2) lines.push(`H2: ${headings.h2}`);
36
+ if (headings.h3) lines.push(`H3: ${headings.h3}`);
37
+ if (headings.h4) lines.push(`H4: ${headings.h4}`);
38
+ return lines;
39
+ }
40
+
32
41
  export type ContextMode = 'attached' | 'compact' | 'full';
33
42
 
34
43
  export function formatContextSummary(data: ContextData, mode: ContextMode = 'compact'): string {
@@ -51,11 +60,7 @@ function formatContextAttached(data: ContextData): string {
51
60
  }
52
61
  lines.push('');
53
62
 
54
- const headingLines: string[] = [];
55
- if (data.headings.h1) headingLines.push(`H1: ${data.headings.h1}`);
56
- if (data.headings.h2) headingLines.push(`H2: ${data.headings.h2}`);
57
- if (data.headings.h3) headingLines.push(`H3: ${data.headings.h3}`);
58
- if (data.headings.h4) headingLines.push(`H4: ${data.headings.h4}`);
63
+ const headingLines = formatHeadings(data.headings);
59
64
 
60
65
  if (headingLines.length > 0) {
61
66
  lines.push(section('HEADINGS') + chalk.gray(' (Auto-attached)'));
@@ -116,11 +121,7 @@ function formatContextCompact(data: ContextData): string {
116
121
  }
117
122
  lines.push('');
118
123
 
119
- const headingLines: string[] = [];
120
- if (data.headings.h1) headingLines.push(`H1: ${data.headings.h1}`);
121
- if (data.headings.h2) headingLines.push(`H2: ${data.headings.h2}`);
122
- if (data.headings.h3) headingLines.push(`H3: ${data.headings.h3}`);
123
- if (data.headings.h4) headingLines.push(`H4: ${data.headings.h4}`);
124
+ const headingLines = formatHeadings(data.headings);
124
125
 
125
126
  if (headingLines.length > 0) {
126
127
  lines.push(section('HEADINGS') + chalk.gray(' (Auto-attached)'));
@@ -177,11 +178,7 @@ function formatContextFull(data: ContextData): string {
177
178
  }
178
179
  lines.push('');
179
180
 
180
- const headingLines: string[] = [];
181
- if (data.headings.h1) headingLines.push(`H1: ${data.headings.h1}`);
182
- if (data.headings.h2) headingLines.push(`H2: ${data.headings.h2}`);
183
- if (data.headings.h3) headingLines.push(`H3: ${data.headings.h3}`);
184
- if (data.headings.h4) headingLines.push(`H4: ${data.headings.h4}`);
181
+ const headingLines = formatHeadings(data.headings);
185
182
 
186
183
  if (headingLines.length > 0) {
187
184
  lines.push(section('HEADINGS') + chalk.gray(' (Auto-attached)'));
package/src/utils/html.ts CHANGED
@@ -85,8 +85,10 @@ const HIDDEN_CLASSES = new Set(['hidden', 'invisible', 'd-none', 'hide', 'dn', '
85
85
 
86
86
  export const EXPLORBOT_ATTRS = {
87
87
  area: 'data-explorbot-area',
88
+ coveredBy: 'data-explorbot-covered-by',
88
89
  context: 'data-explorbot-context',
89
90
  eidx: 'data-explorbot-eidx',
91
+ hit: 'data-explorbot-hit',
90
92
  variant: 'data-explorbot-variant',
91
93
  } as const;
92
94
 
@@ -169,14 +171,49 @@ export type ComponentScopeExtractionConfig = {
169
171
  limits: typeof HTML_EXTRACTION_LIMITS;
170
172
  };
171
173
 
174
+ export function captureHtmlForSnapshot(): string {
175
+ const clone = document.documentElement.cloneNode(true) as HTMLElement;
176
+ const liveControls = Array.from(document.querySelectorAll('input, textarea, select')) as Array<HTMLInputElement | HTMLTextAreaElement | HTMLSelectElement>;
177
+ const clonedControls = Array.from(clone.querySelectorAll('input, textarea, select')) as HTMLElement[];
178
+
179
+ for (let i = 0; i < liveControls.length; i++) {
180
+ const source = liveControls[i];
181
+ const target = clonedControls[i];
182
+ if (!target) continue;
183
+
184
+ if (source instanceof HTMLInputElement || source instanceof HTMLTextAreaElement) {
185
+ target.setAttribute('data-explorbot-value', source.value);
186
+ }
187
+
188
+ if (source instanceof HTMLInputElement && (source.type === 'checkbox' || source.type === 'radio')) {
189
+ let checked = 'false';
190
+ if (source.checked) checked = 'true';
191
+ target.setAttribute('data-explorbot-checked', checked);
192
+ }
193
+
194
+ if (source instanceof HTMLSelectElement) {
195
+ target.setAttribute(
196
+ 'data-explorbot-value',
197
+ Array.from(source.selectedOptions)
198
+ .map((option) => option.value)
199
+ .join(',')
200
+ );
201
+ }
202
+ }
203
+
204
+ return clone.outerHTML;
205
+ }
206
+
172
207
  export function extractElementData(el: Element, config?: ElementExtractionConfig) {
173
208
  const cfg =
174
209
  config ||
175
210
  ({
176
211
  attrs: {
177
212
  area: 'data-explorbot-area',
213
+ coveredBy: 'data-explorbot-covered-by',
178
214
  context: 'data-explorbot-context',
179
215
  eidx: 'data-explorbot-eidx',
216
+ hit: 'data-explorbot-hit',
180
217
  variant: 'data-explorbot-variant',
181
218
  },
182
219
  codeEditorMarkers: ['monaco', 'codemirror', 'ace', 'ace_editor', 'code'],
@@ -246,6 +283,45 @@ export function extractElementData(el: Element, config?: ElementExtractionConfig
246
283
  return Array.from(tokens).slice(0, 8);
247
284
  }
248
285
 
286
+ function describeOverlappingElement(target: Element): string {
287
+ const parts = [target.tagName.toLowerCase()];
288
+ const role = target.getAttribute('role');
289
+ const ariaLabel = target.getAttribute('aria-label');
290
+ const id = target.getAttribute('id');
291
+ const className = (target.getAttribute('class') || '').split(/\s+/).filter(Boolean).slice(0, 3).join('.');
292
+
293
+ if (id) parts.push(`#${id.slice(0, 40)}`);
294
+ if (className) parts.push(`.${className.slice(0, 80)}`);
295
+ if (role) parts.push(`[role="${role.slice(0, 40)}"]`);
296
+ if (ariaLabel) parts.push(`[aria-label="${ariaLabel.slice(0, 80)}"]`);
297
+
298
+ return parts.join('');
299
+ }
300
+
301
+ function inspectHitTarget(target: Element, rect: DOMRect): { hit: string; coveredBy?: string } {
302
+ const viewportPoints = [
303
+ { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 },
304
+ { x: rect.left + Math.min(rect.width - 1, Math.max(1, rect.width * 0.25)), y: rect.top + Math.min(rect.height - 1, Math.max(1, rect.height * 0.25)) },
305
+ { x: rect.left + Math.min(rect.width - 1, Math.max(1, rect.width * 0.75)), y: rect.top + Math.min(rect.height - 1, Math.max(1, rect.height * 0.75)) },
306
+ ].filter((point) => point.x >= 0 && point.y >= 0 && point.x <= window.innerWidth && point.y <= window.innerHeight);
307
+
308
+ if (viewportPoints.length === 0) {
309
+ return { hit: 'offscreen' };
310
+ }
311
+
312
+ for (const point of viewportPoints) {
313
+ const top = document.elementFromPoint(point.x, point.y);
314
+ if (!top) continue;
315
+ if (top === target || target.contains(top)) {
316
+ return { hit: 'target' };
317
+ }
318
+ }
319
+
320
+ const centerPoint = viewportPoints[0];
321
+ const top = document.elementFromPoint(centerPoint.x, centerPoint.y);
322
+ return { hit: 'covered', coveredBy: top ? describeOverlappingElement(top) : undefined };
323
+ }
324
+
249
325
  function isEmbeddedCodeEditorFrame(target: Element): boolean {
250
326
  const src = (target.getAttribute('src') || '').toLowerCase();
251
327
  const markerSelector = cfg.codeEditorMarkers.map((marker) => `[class*="${marker}"]`).join(', ');
@@ -354,6 +430,9 @@ export function extractElementData(el: Element, config?: ElementExtractionConfig
354
430
  allAttrs[cfg.attrs.area] = areaHints.join('|');
355
431
  allAttrs[cfg.attrs.context] = findContextLabel(el);
356
432
  allAttrs[cfg.attrs.variant] = collectVariantHints(el).join('|');
433
+ const hitTarget = inspectHitTarget(el, rect);
434
+ allAttrs[cfg.attrs.hit] = hitTarget.hit;
435
+ if (hitTarget.coveredBy) allAttrs[cfg.attrs.coveredBy] = hitTarget.coveredBy;
357
436
 
358
437
  return {
359
438
  tag: el.tagName.toLowerCase(),
@@ -0,0 +1,30 @@
1
+ import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
2
+ import { join } from 'node:path';
3
+ import matter from 'gray-matter';
4
+
5
+ export function loadMarkdownFiles(dir: string, options: { recursive?: boolean; onError?: (filePath: string, error: unknown) => void } = {}): MarkdownFile[] {
6
+ if (!existsSync(dir)) return [];
7
+
8
+ const results: MarkdownFile[] = [];
9
+ const files = readdirSync(dir, { recursive: options.recursive === true })
10
+ .filter((file): file is string => typeof file === 'string' && file.endsWith('.md'))
11
+ .map((file) => join(dir, file));
12
+
13
+ for (const filePath of files) {
14
+ try {
15
+ const parsed = matter(readFileSync(filePath, 'utf8'));
16
+ results.push({ filePath, content: parsed.content, data: parsed.data, mtime: statSync(filePath).mtime });
17
+ } catch (error) {
18
+ options.onError?.(filePath, error);
19
+ }
20
+ }
21
+
22
+ return results;
23
+ }
24
+
25
+ export interface MarkdownFile {
26
+ filePath: string;
27
+ content: string;
28
+ data: Record<string, any>;
29
+ mtime: Date;
30
+ }
@@ -408,6 +408,10 @@ export class MarkdownQuery {
408
408
  }
409
409
 
410
410
  replace(content: string): string {
411
+ return this.replaceEach(() => content);
412
+ }
413
+
414
+ replaceEach(replacer: (match: MarkdownQuery, index: number) => string): string {
411
415
  const sorted = [...this.matches].sort((a, b) => a.start - b.start);
412
416
 
413
417
  const kept: MatchedRange[] = [];
@@ -418,10 +422,11 @@ export class MarkdownQuery {
418
422
  lastEnd = range.start + range.length;
419
423
  }
420
424
 
425
+ const replacements = kept.map((range, index) => replacer(new MarkdownQuery(this.source, [range]), index));
421
426
  let result = this.source;
422
427
  for (let i = kept.length - 1; i >= 0; i--) {
423
428
  const range = kept[i];
424
- result = result.slice(0, range.start) + content + result.slice(range.start + range.length);
429
+ result = result.slice(0, range.start) + replacements[i] + result.slice(range.start + range.length);
425
430
  }
426
431
 
427
432
  return result;
@@ -459,6 +464,15 @@ export class MarkdownQuery {
459
464
  each(): MarkdownQuery[] {
460
465
  return this.matches.map((m) => new MarkdownQuery(this.source, [m]));
461
466
  }
467
+
468
+ meta(): Array<{ type: string; depth: number | null; text: string }> {
469
+ return this.matches.map((range) => {
470
+ const token = range.token as any;
471
+ let depth: number | null = null;
472
+ if (token.type === 'heading') depth = token.depth;
473
+ return { type: token.type, depth, text: getTokenText(range.token) };
474
+ });
475
+ }
462
476
  }
463
477
 
464
478
  export function mdq(source: string): MarkdownQuery {
@@ -49,7 +49,7 @@ function waitForPageBodyContent(page: any, timeout: number): Promise<void> {
49
49
  .catch(() => {});
50
50
  }
51
51
 
52
- function sleep(ms: number): Promise<void> {
52
+ export function sleep(ms: number): Promise<void> {
53
53
  return new Promise((resolve) => setTimeout(resolve, ms));
54
54
  }
55
55
 
@@ -0,0 +1,28 @@
1
+ const MIN_SECRET_LENGTH = 4;
2
+ const SECRET_NAME_TOKENS = ['password', 'passwd', 'secret', 'token', 'apikey', 'api_key', 'credential', 'private_key', 'privatekey', 'access_key', 'jwt', 'auth', 'bearer', 'authorization', 'cert', 'certificate'];
3
+ const secretValues = new Set<string>();
4
+
5
+ export function registerSecret(value: string): void {
6
+ if (!value) return;
7
+ if (value.length < MIN_SECRET_LENGTH) return;
8
+ secretValues.add(value);
9
+ }
10
+
11
+ export function redactSecrets(text: string): string {
12
+ if (!text) return text;
13
+ let result = text;
14
+ for (const value of [...secretValues].sort((a, b) => b.length - a.length)) {
15
+ if (!result.includes(value)) continue;
16
+ result = result.split(value).join('***REDACTED***');
17
+ }
18
+ return result;
19
+ }
20
+
21
+ export function clearRegisteredSecrets(): void {
22
+ secretValues.clear();
23
+ }
24
+
25
+ export function isSecretName(name: string): boolean {
26
+ const lower = name.toLowerCase();
27
+ return SECRET_NAME_TOKENS.some((token) => lower.includes(token));
28
+ }
@@ -6,6 +6,18 @@ export function truncateJson(input: any): string {
6
6
  return str.length <= 80 ? str : `${str.slice(0, 77)}...`;
7
7
  }
8
8
 
9
+ export function slugify(text: string): string {
10
+ return text
11
+ .replace(/[^a-zA-Z0-9_]/g, '_')
12
+ .replace(/_+/g, '_')
13
+ .replace(/^_|_$/g, '')
14
+ .toLowerCase();
15
+ }
16
+
17
+ export function normalizeInlineText(text: string): string {
18
+ return text.normalize('NFKC').replace(/\s+/g, ' ').trim();
19
+ }
20
+
9
21
  export function sanitizeFilename(name: string): string {
10
22
  return name
11
23
  .toLowerCase()
@@ -47,6 +47,10 @@ export class WebElement {
47
47
  return this.attrs[EXPLORBOT_ATTRS.eidx] || this.attrs.eidx || null;
48
48
  }
49
49
 
50
+ ourAttr(name: keyof typeof EXPLORBOT_ATTRS): string | undefined {
51
+ return this.attrs[EXPLORBOT_ATTRS[name]];
52
+ }
53
+
50
54
  get isNavigationLink(): boolean {
51
55
  if (this.tag !== 'a') return false;
52
56
  const href = this.attrs.href || '';