explorbot 0.1.32-beta.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -14
- package/bin/explorbot-cli.ts +4 -3
- package/boat/doc-collector/src/ai/documentarian.ts +22 -27
- package/boat/doc-collector/src/ai/tools.ts +67 -10
- package/boat/doc-collector/src/docbot.ts +28 -7
- package/boat/doc-collector/src/docs-renderer.ts +19 -24
- package/boat/doc-collector/src/screenshots.ts +33 -1
- package/boat/doc-collector/src/state-diagram.ts +281 -0
- package/dist/bin/explorbot-cli.js +3 -3
- package/dist/boat/doc-collector/src/ai/documentarian.js +16 -6
- package/dist/boat/doc-collector/src/ai/tools.js +47 -10
- package/dist/boat/doc-collector/src/docbot.js +25 -6
- package/dist/boat/doc-collector/src/docs-renderer.js +18 -3
- package/dist/boat/doc-collector/src/screenshots.js +29 -0
- package/dist/boat/doc-collector/src/state-diagram.js +200 -0
- package/dist/package.json +20 -3
- package/dist/src/action-result.d.ts +8 -10
- package/dist/src/action-result.js +72 -164
- package/dist/src/action.d.ts +3 -10
- package/dist/src/action.js +18 -111
- package/dist/src/ai/captain/web-mode.js +0 -7
- package/dist/src/ai/captain.d.ts +1 -3
- package/dist/src/ai/captain.js +4 -19
- package/dist/src/ai/conversation.d.ts +2 -5
- package/dist/src/ai/conversation.js +9 -28
- package/dist/src/ai/driller.d.ts +0 -4
- package/dist/src/ai/driller.js +3 -12
- package/dist/src/ai/experience-compactor.d.ts +2 -2
- package/dist/src/ai/experience-compactor.js +16 -33
- package/dist/src/ai/fisherman-tools.js +1 -1
- package/dist/src/ai/fisherman.d.ts +0 -1
- package/dist/src/ai/fisherman.js +0 -3
- package/dist/src/ai/historian/codeceptjs.d.ts +2 -0
- package/dist/src/ai/historian/codeceptjs.js +3 -2
- package/dist/src/ai/historian/playwright.d.ts +2 -0
- package/dist/src/ai/historian/playwright.js +3 -2
- package/dist/src/ai/historian/utils.d.ts +0 -1
- package/dist/src/ai/historian/utils.js +0 -1
- package/dist/src/ai/historian.d.ts +5 -4
- package/dist/src/ai/historian.js +1 -5
- package/dist/src/ai/navigator.d.ts +3 -5
- package/dist/src/ai/navigator.js +11 -46
- package/dist/src/ai/pilot.d.ts +1 -3
- package/dist/src/ai/pilot.js +6 -17
- package/dist/src/ai/planner.d.ts +3 -3
- package/dist/src/ai/planner.js +49 -37
- package/dist/src/ai/provider.d.ts +4 -1
- package/dist/src/ai/provider.js +77 -136
- package/dist/src/ai/quartermaster.d.ts +21 -22
- package/dist/src/ai/quartermaster.js +4 -10
- package/dist/src/ai/rerunner.d.ts +1 -2
- package/dist/src/ai/rerunner.js +14 -22
- package/dist/src/ai/researcher/cache.js +11 -18
- package/dist/src/ai/researcher/deep-analysis.js +3 -6
- package/dist/src/ai/researcher/parser.js +1 -1
- package/dist/src/ai/researcher/research-result.d.ts +0 -1
- package/dist/src/ai/researcher/research-result.js +0 -19
- package/dist/src/ai/researcher/sections.js +5 -1
- package/dist/src/ai/researcher.js +12 -53
- package/dist/src/ai/rules.d.ts +1 -0
- package/dist/src/ai/rules.js +22 -15
- package/dist/src/ai/session-analyst.js +8 -5
- package/dist/src/ai/task-agent.d.ts +1 -1
- package/dist/src/ai/task-agent.js +2 -27
- package/dist/src/ai/tester.d.ts +0 -2
- package/dist/src/ai/tester.js +10 -61
- package/dist/src/ai/tools.d.ts +18 -7
- package/dist/src/ai/tools.js +119 -101
- package/dist/src/commands/compact-command.d.ts +1 -1
- package/dist/src/commands/compact-command.js +6 -6
- package/dist/src/commands/context-command.js +1 -1
- package/dist/src/commands/context-knowledge-command.js +1 -1
- package/dist/src/commands/experience-command.js +1 -1
- package/dist/src/commands/knows-command.js +1 -1
- package/dist/src/commands/learn-command.js +3 -1
- package/dist/src/components/AddKnowledge.d.ts +2 -0
- package/dist/src/components/AddKnowledge.js +3 -7
- package/dist/src/components/App.js +2 -2
- package/dist/src/config.d.ts +2 -0
- package/dist/src/config.js +12 -0
- package/dist/src/experience-tracker.d.ts +11 -6
- package/dist/src/experience-tracker.js +81 -154
- package/dist/src/explorbot.d.ts +4 -5
- package/dist/src/explorbot.js +17 -51
- package/dist/src/explorer.d.ts +4 -15
- package/dist/src/explorer.js +18 -68
- package/dist/src/knowledge-tracker.d.ts +2 -2
- package/dist/src/knowledge-tracker.js +38 -45
- package/dist/src/state-manager.d.ts +6 -31
- package/dist/src/state-manager.js +10 -129
- package/dist/src/utils/aria.d.ts +6 -1
- package/dist/src/utils/aria.js +4 -1
- package/dist/src/utils/cache.d.ts +15 -0
- package/dist/src/utils/cache.js +34 -0
- package/dist/src/utils/context-formatter.d.ts +6 -0
- package/dist/src/utils/context-formatter.js +15 -27
- package/dist/src/utils/html.d.ts +5 -0
- package/dist/src/utils/html.js +71 -0
- package/dist/src/utils/markdown-files.d.ts +10 -0
- package/dist/src/utils/markdown-files.js +21 -0
- package/dist/src/utils/markdown-query.d.ts +6 -0
- package/dist/src/utils/markdown-query.js +14 -1
- package/dist/src/utils/page-readiness.d.ts +1 -0
- package/dist/src/utils/page-readiness.js +1 -1
- package/dist/src/utils/secrets.js +2 -2
- package/dist/src/utils/strings.d.ts +2 -0
- package/dist/src/utils/strings.js +10 -0
- package/dist/src/utils/web-element.d.ts +2 -1
- package/dist/src/utils/web-element.js +3 -0
- package/dist/src/utils/web-sandbox.d.ts +4 -0
- package/dist/src/utils/web-sandbox.js +37 -0
- package/package.json +20 -3
- package/src/action-result.ts +70 -173
- package/src/action.ts +16 -130
- package/src/ai/captain/web-mode.ts +0 -7
- package/src/ai/captain.ts +5 -17
- package/src/ai/conversation.ts +9 -32
- package/src/ai/driller.ts +3 -16
- package/src/ai/experience-compactor.ts +16 -35
- package/src/ai/fisherman-tools.ts +1 -1
- package/src/ai/fisherman.ts +0 -4
- package/src/ai/historian/codeceptjs.ts +4 -2
- package/src/ai/historian/playwright.ts +4 -2
- package/src/ai/historian/utils.ts +0 -1
- package/src/ai/historian.ts +4 -7
- package/src/ai/navigator.ts +13 -49
- package/src/ai/pilot.ts +6 -16
- package/src/ai/planner.ts +56 -44
- package/src/ai/provider.ts +85 -145
- package/src/ai/quartermaster.ts +34 -41
- package/src/ai/rerunner.ts +17 -21
- package/src/ai/researcher/cache.ts +10 -16
- package/src/ai/researcher/deep-analysis.ts +3 -6
- package/src/ai/researcher/parser.ts +1 -1
- package/src/ai/researcher/research-result.ts +0 -18
- package/src/ai/researcher/sections.ts +5 -1
- package/src/ai/researcher.ts +11 -59
- package/src/ai/rules.ts +24 -13
- package/src/ai/session-analyst.ts +8 -5
- package/src/ai/task-agent.ts +3 -30
- package/src/ai/tester.ts +12 -73
- package/src/ai/tools.ts +119 -106
- package/src/commands/compact-command.ts +7 -8
- package/src/commands/context-command.ts +1 -1
- package/src/commands/context-knowledge-command.ts +1 -1
- package/src/commands/experience-command.ts +1 -1
- package/src/commands/knows-command.ts +1 -1
- package/src/commands/learn-command.ts +3 -1
- package/src/components/AddKnowledge.tsx +5 -7
- package/src/components/App.tsx +2 -2
- package/src/config.ts +12 -0
- package/src/experience-tracker.ts +88 -151
- package/src/explorbot.ts +17 -51
- package/src/explorer.ts +20 -72
- package/src/knowledge-tracker.ts +42 -49
- package/src/state-manager.ts +15 -156
- package/src/utils/aria.ts +11 -2
- package/src/utils/cache.ts +40 -0
- package/src/utils/context-formatter.ts +12 -15
- package/src/utils/html.ts +79 -0
- package/src/utils/markdown-files.ts +30 -0
- package/src/utils/markdown-query.ts +15 -1
- package/src/utils/page-readiness.ts +1 -1
- package/src/utils/secrets.ts +2 -3
- package/src/utils/strings.ts +12 -0
- package/src/utils/web-element.ts +4 -0
- package/src/utils/web-sandbox.ts +43 -0
- package/dist/boat/api-tester/example/apibot.config.js +0 -30
package/src/explorer.ts
CHANGED
|
@@ -14,17 +14,17 @@ import { visuallyAnnotateContainers } from './ai/researcher/coordinates.ts';
|
|
|
14
14
|
import { RequestStore } from './api/request-store.ts';
|
|
15
15
|
import { XhrCapture } from './api/xhr-capture.ts';
|
|
16
16
|
import type { ExplorbotConfig } from './config.js';
|
|
17
|
-
import { ConfigParser
|
|
18
|
-
import type {
|
|
17
|
+
import { ConfigParser } from './config.js';
|
|
18
|
+
import type { ExperienceTracker } from './experience-tracker.js';
|
|
19
19
|
import { KnowledgeTracker } from './knowledge-tracker.js';
|
|
20
20
|
import { PlaywrightRecorder } from './playwright-recorder.ts';
|
|
21
21
|
import { Reporter } from './reporter.ts';
|
|
22
22
|
import { StateManager } from './state-manager.js';
|
|
23
23
|
import { Test, TestResult } from './test-plan.ts';
|
|
24
|
-
import { BrowserRecoveryError, isFatalBrowserError, isNavigationTransitionError } from './utils/browser-errors.ts';
|
|
24
|
+
import { BrowserRecoveryError, browserErrorMessage, isFatalBrowserError, isNavigationTransitionError } from './utils/browser-errors.ts';
|
|
25
25
|
import { ELEMENT_EXTRACTION_CONFIG, getElementDataExtractorSource } from './utils/html.ts';
|
|
26
26
|
import { createDebug, log, tag } from './utils/logger.js';
|
|
27
|
-
import { waitForPageReadiness } from './utils/page-readiness.ts';
|
|
27
|
+
import { sleep, waitForPageReadiness } from './utils/page-readiness.ts';
|
|
28
28
|
import { WebElement } from './utils/web-element.ts';
|
|
29
29
|
|
|
30
30
|
declare global {
|
|
@@ -64,7 +64,6 @@ class Explorer {
|
|
|
64
64
|
private stateManager!: StateManager;
|
|
65
65
|
private knowledgeTracker!: KnowledgeTracker;
|
|
66
66
|
config: ExplorbotConfig;
|
|
67
|
-
private userResolveFn: UserResolveFunction | null = null;
|
|
68
67
|
private options?: { show?: boolean; headless?: boolean; incognito?: boolean; session?: string };
|
|
69
68
|
private reporter!: Reporter;
|
|
70
69
|
private otherTabs: TabInfo[] = [];
|
|
@@ -77,13 +76,13 @@ class Explorer {
|
|
|
77
76
|
private testConsoleHandler: ((message: any) => void) | null = null;
|
|
78
77
|
private testDialogHandler: ((dialog: any) => void) | null = null;
|
|
79
78
|
|
|
80
|
-
constructor(config: ExplorbotConfig, aiProvider: AIProvider, options
|
|
79
|
+
constructor(config: ExplorbotConfig, aiProvider: AIProvider, options: { show?: boolean; headless?: boolean; incognito?: boolean; session?: string } | undefined, experienceTracker: ExperienceTracker, knowledgeTracker: KnowledgeTracker) {
|
|
81
80
|
this.config = config;
|
|
82
81
|
this.aiProvider = aiProvider;
|
|
83
82
|
this.options = options;
|
|
84
83
|
this.initializeContainer();
|
|
85
|
-
this.
|
|
86
|
-
this.
|
|
84
|
+
this.knowledgeTracker = knowledgeTracker;
|
|
85
|
+
this.stateManager = new StateManager(experienceTracker, knowledgeTracker);
|
|
87
86
|
this.reporter = new Reporter(config.reporter, this.stateManager);
|
|
88
87
|
}
|
|
89
88
|
|
|
@@ -91,8 +90,7 @@ class Explorer {
|
|
|
91
90
|
try {
|
|
92
91
|
// Use project root for output directory, not current working directory
|
|
93
92
|
const configParser = ConfigParser.getInstance();
|
|
94
|
-
const
|
|
95
|
-
const projectRoot = configPath ? path.dirname(configPath) : process.cwd();
|
|
93
|
+
const projectRoot = configParser.getProjectRoot();
|
|
96
94
|
(global as any).output_dir = path.join(projectRoot, 'output', 'states');
|
|
97
95
|
(global as any).codecept_dir = projectRoot;
|
|
98
96
|
|
|
@@ -155,8 +153,7 @@ class Explorer {
|
|
|
155
153
|
};
|
|
156
154
|
|
|
157
155
|
if (this.config.stepsFile) {
|
|
158
|
-
const
|
|
159
|
-
const projectRoot = configPath ? path.dirname(configPath) : process.cwd();
|
|
156
|
+
const projectRoot = ConfigParser.getInstance().getProjectRoot();
|
|
160
157
|
codeceptConfig.include = { I: path.resolve(projectRoot, this.config.stepsFile) };
|
|
161
158
|
}
|
|
162
159
|
|
|
@@ -164,17 +161,9 @@ class Explorer {
|
|
|
164
161
|
}
|
|
165
162
|
|
|
166
163
|
public getConfig(): ExplorbotConfig {
|
|
167
|
-
if (!this.config) {
|
|
168
|
-
throw new Error('Configuration not loaded. Call run() first.');
|
|
169
|
-
}
|
|
170
164
|
return this.config;
|
|
171
165
|
}
|
|
172
166
|
|
|
173
|
-
public getConfigPath(): string | null {
|
|
174
|
-
const configParser = ConfigParser.getInstance();
|
|
175
|
-
return configParser.getConfigPath();
|
|
176
|
-
}
|
|
177
|
-
|
|
178
167
|
public getAIProvider(): AIProvider {
|
|
179
168
|
return this.aiProvider;
|
|
180
169
|
}
|
|
@@ -183,10 +172,6 @@ class Explorer {
|
|
|
183
172
|
return this.stateManager;
|
|
184
173
|
}
|
|
185
174
|
|
|
186
|
-
public getCurrentUrl(): string {
|
|
187
|
-
return this.stateManager.getCurrentState()!.url || '?';
|
|
188
|
-
}
|
|
189
|
-
|
|
190
175
|
public getKnowledgeTracker(): KnowledgeTracker {
|
|
191
176
|
return this.knowledgeTracker;
|
|
192
177
|
}
|
|
@@ -227,10 +212,6 @@ class Explorer {
|
|
|
227
212
|
return;
|
|
228
213
|
}
|
|
229
214
|
|
|
230
|
-
if (!this.config) {
|
|
231
|
-
await this.initializeContainer();
|
|
232
|
-
}
|
|
233
|
-
|
|
234
215
|
await codeceptjs.recorder.start();
|
|
235
216
|
await codeceptjs.container.started(null);
|
|
236
217
|
storeListener();
|
|
@@ -359,8 +340,8 @@ class Explorer {
|
|
|
359
340
|
return this.runWithBrowserRecovery('executeAction', () => this.createAction().execute(code));
|
|
360
341
|
}
|
|
361
342
|
|
|
362
|
-
async attemptAction(code: string, originalMessage?: string
|
|
363
|
-
return this.runWithBrowserRecovery('attemptAction', () => this.createAction().attempt(code, originalMessage
|
|
343
|
+
async attemptAction(code: string, originalMessage?: string): Promise<boolean> {
|
|
344
|
+
return this.runWithBrowserRecovery('attemptAction', () => this.createAction().attempt(code, originalMessage));
|
|
364
345
|
}
|
|
365
346
|
|
|
366
347
|
getPlaywrightRecorder(): PlaywrightRecorder {
|
|
@@ -388,7 +369,7 @@ class Explorer {
|
|
|
388
369
|
try {
|
|
389
370
|
await action.execute(`I.amOnPage(${serializedUrl})`);
|
|
390
371
|
} catch (err) {
|
|
391
|
-
const msg =
|
|
372
|
+
const msg = browserErrorMessage(err);
|
|
392
373
|
if (!RECOVERABLE_NAVIGATION_ERRORS.test(msg)) throw err;
|
|
393
374
|
tag('warning').log(`Navigation warning (continuing after load): ${msg.split('\n')[0]}`);
|
|
394
375
|
await this.waitForPageReadiness();
|
|
@@ -437,7 +418,7 @@ class Explorer {
|
|
|
437
418
|
return result;
|
|
438
419
|
} catch (error) {
|
|
439
420
|
if (this.isFatalBrowserError(error)) {
|
|
440
|
-
tag('warning').log(`getEidxInContainer: ${error
|
|
421
|
+
tag('warning').log(`getEidxInContainer: ${browserErrorMessage(error)}`);
|
|
441
422
|
await this.recoverFromBrowserError();
|
|
442
423
|
}
|
|
443
424
|
return [];
|
|
@@ -452,18 +433,13 @@ class Explorer {
|
|
|
452
433
|
return await el.first().getAttribute(ELEMENT_EXTRACTION_CONFIG.attrs.eidx);
|
|
453
434
|
} catch (error) {
|
|
454
435
|
if (this.isFatalBrowserError(error)) {
|
|
455
|
-
tag('warning').log(`getEidxByLocator: ${error
|
|
436
|
+
tag('warning').log(`getEidxByLocator: ${browserErrorMessage(error)}`);
|
|
456
437
|
await this.recoverFromBrowserError();
|
|
457
438
|
}
|
|
458
439
|
return null;
|
|
459
440
|
}
|
|
460
441
|
}
|
|
461
442
|
|
|
462
|
-
async reload() {
|
|
463
|
-
await this.closeOtherTabs();
|
|
464
|
-
await this.playwrightHelper.page.reload();
|
|
465
|
-
}
|
|
466
|
-
|
|
467
443
|
private resolveBrowserUrl(url?: string): string | null {
|
|
468
444
|
if (!url) return null;
|
|
469
445
|
try {
|
|
@@ -508,7 +484,7 @@ class Explorer {
|
|
|
508
484
|
await this.playwrightHelper.page.reload({ waitUntil: 'domcontentloaded', timeout: 10000 });
|
|
509
485
|
return this.waitForPageReadiness();
|
|
510
486
|
} catch (err) {
|
|
511
|
-
tag('error').log(`Browser recovery failed: ${err
|
|
487
|
+
tag('error').log(`Browser recovery failed: ${browserErrorMessage(err)}`);
|
|
512
488
|
return false;
|
|
513
489
|
}
|
|
514
490
|
}
|
|
@@ -552,7 +528,7 @@ class Explorer {
|
|
|
552
528
|
tag('success').log('Browser restarted');
|
|
553
529
|
return true;
|
|
554
530
|
} catch (err) {
|
|
555
|
-
tag('error').log(`Browser restart failed: ${err
|
|
531
|
+
tag('error').log(`Browser restart failed: ${browserErrorMessage(err)}`);
|
|
556
532
|
return false;
|
|
557
533
|
}
|
|
558
534
|
}
|
|
@@ -584,7 +560,7 @@ class Explorer {
|
|
|
584
560
|
return await page.evaluate(() => window.top !== window.self);
|
|
585
561
|
} catch (error) {
|
|
586
562
|
if (this.isFatalBrowserError(error)) {
|
|
587
|
-
tag('warning').log(`isInsideIframe: ${error
|
|
563
|
+
tag('warning').log(`isInsideIframe: ${browserErrorMessage(error)}`);
|
|
588
564
|
await this.recoverFromBrowserError();
|
|
589
565
|
}
|
|
590
566
|
return false;
|
|
@@ -608,10 +584,6 @@ class Explorer {
|
|
|
608
584
|
this.otherTabs = [];
|
|
609
585
|
}
|
|
610
586
|
|
|
611
|
-
setUserResolve(userResolveFn: UserResolveFunction): void {
|
|
612
|
-
this.userResolveFn = userResolveFn;
|
|
613
|
-
}
|
|
614
|
-
|
|
615
587
|
private listenToStateChanged(): void {
|
|
616
588
|
if (!this.playwrightHelper) {
|
|
617
589
|
debugLog('Playwright helper not available for state monitoring');
|
|
@@ -760,12 +732,8 @@ class Explorer {
|
|
|
760
732
|
return true;
|
|
761
733
|
}
|
|
762
734
|
|
|
763
|
-
async ensureActiveTestPageAvailable(): Promise<boolean> {
|
|
764
|
-
return this.ensurePageAvailable();
|
|
765
|
-
}
|
|
766
|
-
|
|
767
735
|
async handleExecutionError(error: unknown): Promise<BrowserExecutionErrorResult> {
|
|
768
|
-
const message =
|
|
736
|
+
const message = browserErrorMessage(error);
|
|
769
737
|
tag('error').log(`Browser execution error: ${message}`);
|
|
770
738
|
|
|
771
739
|
if (error instanceof Error && error.name === 'AbortError') {
|
|
@@ -873,33 +841,13 @@ class Explorer {
|
|
|
873
841
|
this.observedTestPages.clear();
|
|
874
842
|
}
|
|
875
843
|
|
|
876
|
-
async hasPlaywrightLocator(locatorFn: (page: any) => any, opts: { multiple?: boolean; contents?: boolean; success?: (locator: any) => Promise<void> | void } = {}): Promise<boolean> {
|
|
877
|
-
try {
|
|
878
|
-
const pwLocator = locatorFn(this.playwrightHelper.page);
|
|
879
|
-
const count = await pwLocator.count();
|
|
880
|
-
if (opts.multiple ? count === 0 : count !== 1) return false;
|
|
881
|
-
if (opts.contents) {
|
|
882
|
-
const html = await pwLocator.first().innerHTML();
|
|
883
|
-
if (!html?.trim()) return false;
|
|
884
|
-
}
|
|
885
|
-
if (opts.success) await opts.success(pwLocator);
|
|
886
|
-
return true;
|
|
887
|
-
} catch (error) {
|
|
888
|
-
if (this.isFatalBrowserError(error)) {
|
|
889
|
-
tag('warning').log(`hasPlaywrightLocator: ${error instanceof Error ? error.message : error}`);
|
|
890
|
-
await this.recoverFromBrowserError();
|
|
891
|
-
}
|
|
892
|
-
return false;
|
|
893
|
-
}
|
|
894
|
-
}
|
|
895
|
-
|
|
896
844
|
async playwrightLocatorCount(locatorFn: (page: any) => any): Promise<number> {
|
|
897
845
|
try {
|
|
898
846
|
const pwLocator = locatorFn(this.playwrightHelper.page);
|
|
899
847
|
return await pwLocator.count();
|
|
900
848
|
} catch (error) {
|
|
901
849
|
if (this.isFatalBrowserError(error)) {
|
|
902
|
-
tag('warning').log(`playwrightLocatorCount: ${error
|
|
850
|
+
tag('warning').log(`playwrightLocatorCount: ${browserErrorMessage(error)}`);
|
|
903
851
|
await this.recoverFromBrowserError();
|
|
904
852
|
}
|
|
905
853
|
throw error;
|
|
@@ -921,7 +869,7 @@ class Explorer {
|
|
|
921
869
|
|
|
922
870
|
this.stateManager.updateStateFromBasic(newUrl, newTitle, 'navigation');
|
|
923
871
|
|
|
924
|
-
await
|
|
872
|
+
await sleep(500);
|
|
925
873
|
});
|
|
926
874
|
}
|
|
927
875
|
|
package/src/knowledge-tracker.ts
CHANGED
|
@@ -1,15 +1,19 @@
|
|
|
1
|
-
import { existsSync, mkdirSync, readFileSync,
|
|
2
|
-
import {
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import dedent from 'dedent';
|
|
3
4
|
import matter from 'gray-matter';
|
|
4
5
|
import { ActionResult } from './action-result.js';
|
|
5
6
|
import { ConfigParser } from './config.js';
|
|
6
7
|
import { getCliName } from './utils/cli-name.ts';
|
|
7
|
-
import { createDebug } from './utils/logger.js';
|
|
8
|
+
import { createDebug, pluralize, tag } from './utils/logger.js';
|
|
9
|
+
import { loadMarkdownFiles } from './utils/markdown-files.js';
|
|
10
|
+
import { mdq } from './utils/markdown-query.js';
|
|
8
11
|
import { isSecretName, registerSecret } from './utils/secrets.js';
|
|
12
|
+
import { slugify } from './utils/strings.js';
|
|
9
13
|
|
|
10
14
|
const debugLog = createDebug('explorbot:knowledge-tracker');
|
|
11
15
|
|
|
12
|
-
interface Knowledge {
|
|
16
|
+
export interface Knowledge {
|
|
13
17
|
filePath: string;
|
|
14
18
|
url: string;
|
|
15
19
|
content: string;
|
|
@@ -24,14 +28,7 @@ export class KnowledgeTracker {
|
|
|
24
28
|
constructor() {
|
|
25
29
|
const configParser = ConfigParser.getInstance();
|
|
26
30
|
const config = configParser.getConfig();
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
if (configPath) {
|
|
30
|
-
const projectRoot = dirname(configPath);
|
|
31
|
-
this.knowledgeDir = join(projectRoot, config.dirs?.knowledge || 'knowledge');
|
|
32
|
-
} else {
|
|
33
|
-
this.knowledgeDir = config.dirs?.knowledge || 'knowledge';
|
|
34
|
-
}
|
|
31
|
+
this.knowledgeDir = configParser.resolveProjectDir(config.dirs?.knowledge || 'knowledge');
|
|
35
32
|
|
|
36
33
|
if (!existsSync(this.knowledgeDir)) {
|
|
37
34
|
mkdirSync(this.knowledgeDir, { recursive: true });
|
|
@@ -43,29 +40,13 @@ export class KnowledgeTracker {
|
|
|
43
40
|
|
|
44
41
|
this.knowledgeFiles = [];
|
|
45
42
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
for (const filePath of files) {
|
|
55
|
-
try {
|
|
56
|
-
const fileContent = readFileSync(filePath, 'utf8');
|
|
57
|
-
const parsed = matter(fileContent);
|
|
58
|
-
const urlPattern = parsed.data.url || parsed.data.path || '*';
|
|
59
|
-
|
|
60
|
-
this.knowledgeFiles.push({
|
|
61
|
-
filePath,
|
|
62
|
-
url: urlPattern,
|
|
63
|
-
content: this.interpolateVars(parsed.content),
|
|
64
|
-
...parsed.data,
|
|
65
|
-
});
|
|
66
|
-
} catch (error) {
|
|
67
|
-
// Skip invalid files
|
|
68
|
-
}
|
|
43
|
+
for (const entry of loadMarkdownFiles(this.knowledgeDir, { recursive: true })) {
|
|
44
|
+
this.knowledgeFiles.push({
|
|
45
|
+
filePath: entry.filePath,
|
|
46
|
+
url: entry.data.url || entry.data.path || '*',
|
|
47
|
+
content: this.interpolateVars(entry.content),
|
|
48
|
+
...entry.data,
|
|
49
|
+
});
|
|
69
50
|
}
|
|
70
51
|
|
|
71
52
|
this.isLoaded = true;
|
|
@@ -79,25 +60,40 @@ export class KnowledgeTracker {
|
|
|
79
60
|
});
|
|
80
61
|
}
|
|
81
62
|
|
|
63
|
+
renderRelevantKnowledge(state: ActionResult): string {
|
|
64
|
+
const knowledgeFiles = this.getRelevantKnowledge(state);
|
|
65
|
+
if (knowledgeFiles.length === 0) return '';
|
|
66
|
+
|
|
67
|
+
const knowledgeContent = knowledgeFiles
|
|
68
|
+
.map((k) => k.content)
|
|
69
|
+
.filter((k) => !!k)
|
|
70
|
+
.join('\n\n');
|
|
71
|
+
|
|
72
|
+
tag('operation').log(`Found ${knowledgeFiles.length} relevant knowledge ${pluralize(knowledgeFiles.length, 'file')}`);
|
|
73
|
+
return dedent`
|
|
74
|
+
<knowledge>
|
|
75
|
+
Here is relevant knowledge for this page:
|
|
76
|
+
|
|
77
|
+
${knowledgeContent}
|
|
78
|
+
</knowledge>
|
|
79
|
+
`;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
82
|
addKnowledge(urlPattern: string, description: string): { filename: string; filePath: string; isNewFile: boolean } {
|
|
83
83
|
const configParser = ConfigParser.getInstance();
|
|
84
|
-
const config = configParser.getConfig();
|
|
85
84
|
const configPath = configParser.getConfigPath();
|
|
86
85
|
|
|
87
86
|
if (!configPath) {
|
|
88
87
|
throw new Error(`No explorbot configuration found. Please run "${getCliName()} init" first.`);
|
|
89
88
|
}
|
|
90
89
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
if (!existsSync(knowledgeDir)) {
|
|
95
|
-
mkdirSync(knowledgeDir, { recursive: true });
|
|
90
|
+
if (!existsSync(this.knowledgeDir)) {
|
|
91
|
+
mkdirSync(this.knowledgeDir, { recursive: true });
|
|
96
92
|
}
|
|
97
93
|
|
|
98
94
|
const normalizedUrl = this.normalizeUrl(urlPattern);
|
|
99
95
|
const filename = this.generateFilename(normalizedUrl);
|
|
100
|
-
const filePath = join(knowledgeDir, filename);
|
|
96
|
+
const filePath = join(this.knowledgeDir, filename);
|
|
101
97
|
|
|
102
98
|
const isNewFile = !existsSync(filePath);
|
|
103
99
|
|
|
@@ -128,6 +124,8 @@ export class KnowledgeTracker {
|
|
|
128
124
|
writeFileSync(filePath, fileContent, 'utf8');
|
|
129
125
|
}
|
|
130
126
|
|
|
127
|
+
this.isLoaded = false;
|
|
128
|
+
|
|
131
129
|
return { filename, filePath, isNewFile };
|
|
132
130
|
}
|
|
133
131
|
|
|
@@ -158,12 +156,7 @@ export class KnowledgeTracker {
|
|
|
158
156
|
}
|
|
159
157
|
|
|
160
158
|
private generateFilename(url: string): string {
|
|
161
|
-
let filename = url
|
|
162
|
-
.replace(/https?:\/\//g, '')
|
|
163
|
-
.replace(/[^a-zA-Z0-9_]/g, '_')
|
|
164
|
-
.replace(/_+/g, '_')
|
|
165
|
-
.replace(/^_|_$/g, '')
|
|
166
|
-
.toLowerCase();
|
|
159
|
+
let filename = slugify(url.replace(/https?:\/\//g, ''));
|
|
167
160
|
|
|
168
161
|
if (!filename || filename === '*') {
|
|
169
162
|
filename = 'general';
|
|
@@ -194,7 +187,7 @@ export class KnowledgeTracker {
|
|
|
194
187
|
|
|
195
188
|
return this.knowledgeFiles.map((knowledge) => {
|
|
196
189
|
const content = knowledge.content.trim();
|
|
197
|
-
const firstLine = content.split('\n')[0]?.trim() || '';
|
|
190
|
+
const firstLine = mdq(content).meta()[0]?.text.split('\n')[0]?.trim() || '';
|
|
198
191
|
return {
|
|
199
192
|
url: knowledge.url,
|
|
200
193
|
firstLine,
|
package/src/state-manager.ts
CHANGED
|
@@ -1,11 +1,9 @@
|
|
|
1
|
-
import { existsSync, readFileSync, readdirSync } from 'node:fs';
|
|
2
|
-
import { dirname, join } from 'node:path';
|
|
3
|
-
import matter from 'gray-matter';
|
|
4
1
|
import { ActionResult } from './action-result.js';
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
2
|
+
import type { ExperienceTracker } from './experience-tracker.js';
|
|
3
|
+
import type { KnowledgeTracker, Knowledge } from './knowledge-tracker.js';
|
|
7
4
|
import { detectFocusArea } from './utils/aria.js';
|
|
8
|
-
import { createDebug
|
|
5
|
+
import { createDebug } from './utils/logger.js';
|
|
6
|
+
import { slugify } from './utils/strings.js';
|
|
9
7
|
import { extractStatePath } from './utils/url-matcher.js';
|
|
10
8
|
|
|
11
9
|
const debugLog = createDebug('explorbot:state');
|
|
@@ -69,43 +67,30 @@ export interface StateTransition {
|
|
|
69
67
|
|
|
70
68
|
export type StateChangeListener = (event: StateTransition) => void;
|
|
71
69
|
|
|
72
|
-
export
|
|
73
|
-
/** File path */
|
|
74
|
-
filePath: string;
|
|
75
|
-
/** Markdown content */
|
|
76
|
-
content: string;
|
|
77
|
-
}
|
|
70
|
+
export type { Knowledge };
|
|
78
71
|
|
|
79
72
|
export class StateManager {
|
|
80
73
|
private currentState: WebPageState | null = null;
|
|
81
74
|
private stateHistory: StateTransition[] = [];
|
|
82
75
|
private allVisitedUrls: Set<string> = new Set();
|
|
83
|
-
private knowledgeCache: Knowledge[] = [];
|
|
84
|
-
private lastKnowledgeScan: Date | null = null;
|
|
85
76
|
private stateChangeListeners: StateChangeListener[] = [];
|
|
86
77
|
private experienceTracker!: ExperienceTracker;
|
|
87
|
-
private
|
|
78
|
+
private knowledgeTracker: KnowledgeTracker;
|
|
88
79
|
private nextStateId = 1;
|
|
89
80
|
|
|
90
|
-
constructor() {
|
|
91
|
-
this.experienceTracker =
|
|
92
|
-
|
|
93
|
-
const config = configParser.getConfig();
|
|
94
|
-
const configPath = configParser.getConfigPath();
|
|
95
|
-
|
|
96
|
-
// Resolve knowledge directory relative to the config file location (project root)
|
|
97
|
-
if (configPath) {
|
|
98
|
-
const projectRoot = dirname(configPath);
|
|
99
|
-
this.knowledgeDir = join(projectRoot, config.dirs?.knowledge || 'knowledge');
|
|
100
|
-
} else {
|
|
101
|
-
this.knowledgeDir = config.dirs?.knowledge || 'knowledge';
|
|
102
|
-
}
|
|
81
|
+
constructor(experienceTracker: ExperienceTracker, knowledgeTracker: KnowledgeTracker) {
|
|
82
|
+
this.experienceTracker = experienceTracker;
|
|
83
|
+
this.knowledgeTracker = knowledgeTracker;
|
|
103
84
|
}
|
|
104
85
|
|
|
105
86
|
getExperienceTracker(): ExperienceTracker {
|
|
106
87
|
return this.experienceTracker;
|
|
107
88
|
}
|
|
108
89
|
|
|
90
|
+
getKnowledgeTracker(): KnowledgeTracker {
|
|
91
|
+
return this.knowledgeTracker;
|
|
92
|
+
}
|
|
93
|
+
|
|
109
94
|
/**
|
|
110
95
|
* Subscribe to state change events
|
|
111
96
|
*/
|
|
@@ -134,11 +119,6 @@ export class StateManager {
|
|
|
134
119
|
});
|
|
135
120
|
}
|
|
136
121
|
|
|
137
|
-
/**
|
|
138
|
-
* Extract state path from full URL
|
|
139
|
-
* Removes domain, port, protocol
|
|
140
|
-
* Keeps path, query, and hash: /path/to/page?tab=users#section
|
|
141
|
-
*/
|
|
142
122
|
/**
|
|
143
123
|
* Update current state from ActionResult and record transition if state changed
|
|
144
124
|
*/
|
|
@@ -230,13 +210,7 @@ export class StateManager {
|
|
|
230
210
|
parts.push(`title_${title}`);
|
|
231
211
|
}
|
|
232
212
|
|
|
233
|
-
return parts
|
|
234
|
-
.join('_')
|
|
235
|
-
.replace(/[^a-zA-Z0-9_]/g, '_')
|
|
236
|
-
.replace(/_+/g, '_')
|
|
237
|
-
.replace(/^_|_$/g, '')
|
|
238
|
-
.toLowerCase()
|
|
239
|
-
.substring(0, 200);
|
|
213
|
+
return slugify(parts.join('_')).substring(0, 200);
|
|
240
214
|
}
|
|
241
215
|
|
|
242
216
|
/**
|
|
@@ -327,65 +301,14 @@ export class StateManager {
|
|
|
327
301
|
return this.stateHistory.slice(-count);
|
|
328
302
|
}
|
|
329
303
|
|
|
330
|
-
/**
|
|
331
|
-
* Scan knowledge directory for .md files and cache them
|
|
332
|
-
*/
|
|
333
|
-
private scanKnowledgeFiles(): void {
|
|
334
|
-
const now = new Date();
|
|
335
|
-
|
|
336
|
-
// Only rescan every 30 seconds to avoid excessive file I/O
|
|
337
|
-
if (this.lastKnowledgeScan && now.getTime() - this.lastKnowledgeScan.getTime() < 30000) {
|
|
338
|
-
return;
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
this.knowledgeCache = [];
|
|
342
|
-
|
|
343
|
-
if (!existsSync(this.knowledgeDir)) {
|
|
344
|
-
debugLog(`Knowledge directory not found: ${this.knowledgeDir}`);
|
|
345
|
-
return;
|
|
346
|
-
}
|
|
347
|
-
|
|
348
|
-
try {
|
|
349
|
-
const files = readdirSync(this.knowledgeDir, { recursive: true })
|
|
350
|
-
.filter((file) => typeof file === 'string' && file.endsWith('.md'))
|
|
351
|
-
.map((file) => join(this.knowledgeDir, file as string));
|
|
352
|
-
|
|
353
|
-
for (const filePath of files) {
|
|
354
|
-
try {
|
|
355
|
-
const fileContent = readFileSync(filePath, 'utf8');
|
|
356
|
-
const parsed = matter(fileContent);
|
|
357
|
-
|
|
358
|
-
const urlPattern = parsed.data.url || parsed.data.path || '*';
|
|
359
|
-
|
|
360
|
-
this.knowledgeCache.push({
|
|
361
|
-
filePath,
|
|
362
|
-
url: urlPattern,
|
|
363
|
-
...parsed.data,
|
|
364
|
-
content: parsed.content,
|
|
365
|
-
});
|
|
366
|
-
|
|
367
|
-
debugLog(`Loaded knowledge file: ${filePath} (pattern: ${urlPattern})`);
|
|
368
|
-
} catch (error) {
|
|
369
|
-
debugLog(`Failed to load knowledge file ${filePath}:`, error);
|
|
370
|
-
}
|
|
371
|
-
}
|
|
372
|
-
|
|
373
|
-
this.lastKnowledgeScan = now;
|
|
374
|
-
debugLog(`Scanned ${this.knowledgeCache.length} knowledge files`);
|
|
375
|
-
} catch (error) {
|
|
376
|
-
debugLog('Failed to scan knowledge directory:', error);
|
|
377
|
-
}
|
|
378
|
-
}
|
|
379
304
|
/**
|
|
380
305
|
* Get relevant knowledge files for current state
|
|
381
306
|
*/
|
|
382
307
|
getRelevantKnowledge(): Knowledge[] {
|
|
383
308
|
if (!this.currentState) return [];
|
|
384
309
|
|
|
385
|
-
this.scanKnowledgeFiles();
|
|
386
|
-
|
|
387
310
|
const actionResult = ActionResult.fromState(this.currentState);
|
|
388
|
-
return this.
|
|
311
|
+
return this.knowledgeTracker.getRelevantKnowledge(actionResult);
|
|
389
312
|
}
|
|
390
313
|
|
|
391
314
|
/**
|
|
@@ -402,24 +325,6 @@ export class StateManager {
|
|
|
402
325
|
/**
|
|
403
326
|
* Get all context for current state (knowledge + experience)
|
|
404
327
|
*/
|
|
405
|
-
getCurrentContext(): {
|
|
406
|
-
state: WebPageState;
|
|
407
|
-
knowledge: Knowledge[];
|
|
408
|
-
experience: string[];
|
|
409
|
-
recentTransitions: StateTransition[];
|
|
410
|
-
} {
|
|
411
|
-
if (!this.currentState) {
|
|
412
|
-
throw new Error('No current state available');
|
|
413
|
-
}
|
|
414
|
-
|
|
415
|
-
return {
|
|
416
|
-
state: this.currentState,
|
|
417
|
-
knowledge: this.getRelevantKnowledge(),
|
|
418
|
-
experience: this.getRelevantExperience(),
|
|
419
|
-
recentTransitions: this.getRecentTransitions(),
|
|
420
|
-
};
|
|
421
|
-
}
|
|
422
|
-
|
|
423
328
|
/**
|
|
424
329
|
* Check if we've been in this state before
|
|
425
330
|
*/
|
|
@@ -455,45 +360,6 @@ export class StateManager {
|
|
|
455
360
|
* If the last transition changed URL, returns the fromState (for URL change detection).
|
|
456
361
|
* Otherwise returns the most recent toState with content (for diffing).
|
|
457
362
|
*/
|
|
458
|
-
getPreviousState(): WebPageState | null {
|
|
459
|
-
if (this.stateHistory.length === 0) return null;
|
|
460
|
-
|
|
461
|
-
const lastTransition = this.stateHistory[this.stateHistory.length - 1];
|
|
462
|
-
|
|
463
|
-
if (lastTransition.fromState?.url !== lastTransition.toState?.url) {
|
|
464
|
-
return lastTransition.fromState;
|
|
465
|
-
}
|
|
466
|
-
|
|
467
|
-
for (let i = this.stateHistory.length - 1; i >= 0; i--) {
|
|
468
|
-
const toState = this.stateHistory[i].toState;
|
|
469
|
-
|
|
470
|
-
if (!toState) continue;
|
|
471
|
-
if (toState.id === this.currentState?.id) continue;
|
|
472
|
-
|
|
473
|
-
if (toState.html || toState.ariaSnapshot) {
|
|
474
|
-
return toState;
|
|
475
|
-
}
|
|
476
|
-
}
|
|
477
|
-
|
|
478
|
-
return null;
|
|
479
|
-
}
|
|
480
|
-
|
|
481
|
-
/**
|
|
482
|
-
* Load HTML content from file
|
|
483
|
-
*/
|
|
484
|
-
loadHtmlFromFile(htmlFile: string): string | null {
|
|
485
|
-
try {
|
|
486
|
-
const filePath = outputPath('states', htmlFile);
|
|
487
|
-
if (existsSync(filePath)) {
|
|
488
|
-
return readFileSync(filePath, 'utf8');
|
|
489
|
-
}
|
|
490
|
-
return null;
|
|
491
|
-
} catch (error) {
|
|
492
|
-
debugLog('Failed to load HTML from file:', error);
|
|
493
|
-
return null;
|
|
494
|
-
}
|
|
495
|
-
}
|
|
496
|
-
|
|
497
363
|
/**
|
|
498
364
|
* Clear state history (useful for testing or reset)
|
|
499
365
|
* Note: This preserves the current state, only clears navigation history
|
|
@@ -527,15 +393,8 @@ export class StateManager {
|
|
|
527
393
|
this.stateHistory = [];
|
|
528
394
|
this.allVisitedUrls.clear();
|
|
529
395
|
this.stateChangeListeners = [];
|
|
530
|
-
this.knowledgeCache = [];
|
|
531
|
-
this.lastKnowledgeScan = null;
|
|
532
396
|
this.nextStateId = 1;
|
|
533
397
|
|
|
534
|
-
// Clean up experience tracker if it has cleanup method
|
|
535
|
-
if (this.experienceTracker && typeof this.experienceTracker.cleanup === 'function') {
|
|
536
|
-
this.experienceTracker.cleanup();
|
|
537
|
-
}
|
|
538
|
-
|
|
539
398
|
debugLog('StateManager cleanup completed');
|
|
540
399
|
}
|
|
541
400
|
}
|
package/src/utils/aria.ts
CHANGED
|
@@ -483,7 +483,7 @@ export const compactAriaSnapshot = (snapshot: string | null, keepNamed = false):
|
|
|
483
483
|
return renderTree(tree);
|
|
484
484
|
};
|
|
485
485
|
|
|
486
|
-
export const diffAriaSnapshots = (previous: string | null, current: string | null):
|
|
486
|
+
export const diffAriaSnapshots = (previous: string | null, current: string | null): AriaDiff => {
|
|
487
487
|
const flat = (snap: string | null): FlatEntry[] => {
|
|
488
488
|
let tree = parseSnapshot(snap);
|
|
489
489
|
tree = unwrapIgnored(tree);
|
|
@@ -500,7 +500,9 @@ export const diffAriaSnapshots = (previous: string | null, current: string | nul
|
|
|
500
500
|
const currTotals = countBy(curr.map((e) => e.summary));
|
|
501
501
|
const byCount = diffByCount(prevTotals, currTotals);
|
|
502
502
|
const renames = detectRenames(prev, curr, prevTotals, currTotals);
|
|
503
|
-
|
|
503
|
+
const added = [...byCount.added, ...renames.added];
|
|
504
|
+
const removed = [...byCount.removed, ...renames.removed];
|
|
505
|
+
return { text: formatDiff(added, removed, toggled), count: added.length + removed.length + toggled.length };
|
|
504
506
|
};
|
|
505
507
|
|
|
506
508
|
export const detectFocusArea = (snapshot: string | null): FocusAreaResult => {
|
|
@@ -570,10 +572,17 @@ export function parseAriaLocator(ariaStr: string): { role: string; text: string
|
|
|
570
572
|
return { role: match[1], text: match[2] };
|
|
571
573
|
}
|
|
572
574
|
|
|
575
|
+
export const LARGE_ARIA_CHANGE_THRESHOLD = 50;
|
|
576
|
+
|
|
573
577
|
// ─────────────────────────────────────────────────────────────────
|
|
574
578
|
// Types
|
|
575
579
|
// ─────────────────────────────────────────────────────────────────
|
|
576
580
|
|
|
581
|
+
export interface AriaDiff {
|
|
582
|
+
text: string | null;
|
|
583
|
+
count: number;
|
|
584
|
+
}
|
|
585
|
+
|
|
577
586
|
type AriaNode = {
|
|
578
587
|
role: string;
|
|
579
588
|
name?: string;
|