explorbot 0.2.1 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/README.md +1 -1
  2. package/bin/explorbot-cli.ts +52 -37
  3. package/boat/api-tester/src/apibot.ts +4 -2
  4. package/boat/api-tester/src/cli.ts +2 -2
  5. package/boat/api-tester/src/config.ts +39 -8
  6. package/boat/doc-collector/src/ai/documentarian.ts +3 -0
  7. package/boat/doc-collector/src/ai/tools.ts +17 -5
  8. package/boat/doc-collector/src/cli.ts +2 -0
  9. package/boat/doc-collector/src/config.ts +2 -0
  10. package/boat/doc-collector/src/docbot.ts +39 -11
  11. package/boat/doc-collector/src/docs-renderer.ts +18 -4
  12. package/boat/doc-collector/src/interaction-screenshots.ts +160 -0
  13. package/boat/doc-collector/src/screenshots.ts +22 -14
  14. package/boat/doc-collector/src/state-diagram.ts +61 -14
  15. package/boat/prima/bin/prima-cli.ts +5 -0
  16. package/boat/prima/package.json +16 -0
  17. package/boat/prima/src/cli.ts +222 -0
  18. package/boat/prima/src/envelope.ts +141 -0
  19. package/boat/prima/src/prima.ts +705 -0
  20. package/boat/prima/src/pw-parser.ts +17 -0
  21. package/boat/prima/src/pw-registry.ts +75 -0
  22. package/dist/bin/explorbot-cli.js +44 -31
  23. package/dist/boat/api-tester/src/apibot.js +3 -2
  24. package/dist/boat/api-tester/src/cli.js +2 -2
  25. package/dist/boat/api-tester/src/config.js +36 -8
  26. package/dist/boat/doc-collector/src/ai/documentarian.js +3 -0
  27. package/dist/boat/doc-collector/src/ai/tools.js +10 -4
  28. package/dist/boat/doc-collector/src/cli.js +2 -0
  29. package/dist/boat/doc-collector/src/config.js +1 -0
  30. package/dist/boat/doc-collector/src/docbot.js +36 -10
  31. package/dist/boat/doc-collector/src/docs-renderer.js +17 -3
  32. package/dist/boat/doc-collector/src/interaction-screenshots.js +156 -0
  33. package/dist/boat/doc-collector/src/screenshots.js +23 -14
  34. package/dist/boat/doc-collector/src/state-diagram.js +57 -13
  35. package/dist/boat/prima/bin/prima-cli.js +4 -0
  36. package/dist/boat/prima/src/cli.js +200 -0
  37. package/dist/boat/prima/src/envelope.js +116 -0
  38. package/dist/boat/prima/src/prima.js +635 -0
  39. package/dist/boat/prima/src/pw-parser.js +18 -0
  40. package/dist/boat/prima/src/pw-registry.js +66 -0
  41. package/dist/models.json +3 -0
  42. package/dist/package.json +9 -2
  43. package/dist/src/action.d.ts +5 -2
  44. package/dist/src/action.js +5 -5
  45. package/dist/src/ai/captain/mixin.js +3 -4
  46. package/dist/src/ai/captain/web-mode.js +1 -1
  47. package/dist/src/ai/navigator.d.ts +4 -0
  48. package/dist/src/ai/navigator.js +11 -6
  49. package/dist/src/ai/planner.d.ts +1 -0
  50. package/dist/src/ai/planner.js +6 -0
  51. package/dist/src/ai/researcher/locators.js +1 -1
  52. package/dist/src/ai/researcher.js +1 -1
  53. package/dist/src/ai/task-agent.js +1 -1
  54. package/dist/src/ai/tester.d.ts +1 -0
  55. package/dist/src/ai/tester.js +13 -0
  56. package/dist/src/application-spec-contract.d.ts +8 -0
  57. package/dist/src/application-spec-contract.js +8 -0
  58. package/dist/src/application-spec.d.ts +15 -0
  59. package/dist/src/application-spec.js +71 -0
  60. package/dist/src/browser-server.d.ts +12 -6
  61. package/dist/src/browser-server.js +74 -19
  62. package/dist/src/commands/clean-command.js +2 -7
  63. package/dist/src/commands/init-command.d.ts +5 -0
  64. package/dist/src/commands/init-command.js +119 -1
  65. package/dist/src/commands/navigate-command.js +1 -1
  66. package/dist/src/commands/research-command.js +1 -1
  67. package/dist/src/commands/sites-command.d.ts +6 -0
  68. package/dist/src/commands/sites-command.js +23 -0
  69. package/dist/src/components/InitWizard.d.ts +10 -0
  70. package/dist/src/components/InitWizard.js +133 -0
  71. package/dist/src/components/InputReadline.d.ts +1 -0
  72. package/dist/src/components/InputReadline.js +7 -4
  73. package/dist/src/config.d.ts +24 -5
  74. package/dist/src/config.js +146 -37
  75. package/dist/src/explorbot.d.ts +9 -0
  76. package/dist/src/explorbot.js +24 -5
  77. package/dist/src/explorer.d.ts +5 -1
  78. package/dist/src/explorer.js +54 -19
  79. package/dist/src/global-config.d.ts +22 -0
  80. package/dist/src/global-config.js +117 -0
  81. package/dist/src/knowledge-tracker.d.ts +5 -1
  82. package/dist/src/knowledge-tracker.js +14 -1
  83. package/dist/src/utils/cli-name.js +6 -2
  84. package/dist/src/utils/test-files.js +1 -2
  85. package/dist/src/utils/url-matcher.d.ts +1 -0
  86. package/dist/src/utils/url-matcher.js +9 -0
  87. package/models.json +3 -0
  88. package/package.json +9 -2
  89. package/src/action.ts +9 -5
  90. package/src/ai/captain/mixin.ts +3 -3
  91. package/src/ai/captain/web-mode.ts +1 -1
  92. package/src/ai/navigator.ts +12 -7
  93. package/src/ai/planner.ts +7 -0
  94. package/src/ai/researcher/locators.ts +1 -1
  95. package/src/ai/researcher.ts +1 -1
  96. package/src/ai/task-agent.ts +1 -1
  97. package/src/ai/tester.ts +15 -0
  98. package/src/application-spec-contract.ts +10 -0
  99. package/src/application-spec.ts +87 -0
  100. package/src/browser-server.ts +74 -19
  101. package/src/commands/clean-command.ts +1 -6
  102. package/src/commands/init-command.ts +146 -1
  103. package/src/commands/navigate-command.ts +1 -1
  104. package/src/commands/research-command.ts +1 -1
  105. package/src/commands/sites-command.ts +27 -0
  106. package/src/components/InitWizard.tsx +166 -0
  107. package/src/components/InputReadline.tsx +8 -4
  108. package/src/config.ts +162 -39
  109. package/src/explorbot.ts +30 -5
  110. package/src/explorer.ts +59 -20
  111. package/src/global-config.ts +148 -0
  112. package/src/knowledge-tracker.ts +17 -1
  113. package/src/utils/cli-name.ts +5 -2
  114. package/src/utils/test-files.ts +1 -2
  115. package/src/utils/url-matcher.ts +10 -0
@@ -0,0 +1,17 @@
1
+ const FUNCTION_SHAPE = /^(async\s+)?(function\b|\()/;
2
+
3
+ export function isFunctionExpression(expr: string): { valid: boolean; error?: string } {
4
+ const trimmed = expr.trim();
5
+ if (!trimmed) return { valid: false, error: 'empty expression; pass a function like ({ page }) => ...' };
6
+ if (!FUNCTION_SHAPE.test(trimmed)) return { valid: false, error: 'expression must be a function like ({ page }) => ... destructuring the playwright objects it needs' };
7
+ try {
8
+ new Function(`return (${trimmed})`);
9
+ } catch (e) {
10
+ return { valid: false, error: `not a valid function expression: ${(e as Error).message}` };
11
+ }
12
+ return { valid: true };
13
+ }
14
+
15
+ export function toCodeceptWrapper(expr: string): string {
16
+ return `I.usePlaywrightTo('pw', async (playwright) => (${expr.trim()})(playwright))`;
17
+ }
@@ -0,0 +1,75 @@
1
+ import { readFileSync, readdirSync } from 'node:fs';
2
+ import os from 'node:os';
3
+ import path from 'node:path';
4
+
5
+ const DEFAULT_TITLE = 'default';
6
+ const DEFAULT_BROWSER = 'chromium';
7
+
8
+ export function registryDir(): string {
9
+ if (process.platform === 'darwin') return path.join(os.homedir(), 'Library', 'Caches', 'ms-playwright', 'b');
10
+ if (process.platform === 'win32') return path.join(process.env.LOCALAPPDATA || path.join(os.homedir(), 'AppData', 'Local'), 'ms-playwright', 'b');
11
+ return path.join(process.env.XDG_CACHE_HOME || path.join(os.homedir(), '.cache'), 'ms-playwright', 'b');
12
+ }
13
+
14
+ export function readDescriptors(dir = registryDir()): PwServerDescriptor[] {
15
+ const descriptors: PwServerDescriptor[] = [];
16
+ for (const fileName of listFiles(dir)) {
17
+ const descriptor = parseDescriptor(path.join(dir, fileName));
18
+ if (descriptor) descriptors.push(descriptor);
19
+ }
20
+ return descriptors;
21
+ }
22
+
23
+ export function selectDescriptor(descriptors: PwServerDescriptor[], opts: { workspaceDir: string; title?: string }): { match?: PwServerDescriptor; candidates: PwServerDescriptor[] } {
24
+ const workspaceDir = path.resolve(opts.workspaceDir);
25
+ const candidates = descriptors.filter((descriptor) => path.resolve(descriptor.workspaceDir) === workspaceDir);
26
+ if (!candidates.length) return { candidates };
27
+
28
+ if (opts.title) {
29
+ const titled = candidates.find((descriptor) => descriptor.title === opts.title);
30
+ if (titled) return { match: titled, candidates };
31
+ return { candidates };
32
+ }
33
+
34
+ const preferred = candidates.find((descriptor) => descriptor.title === DEFAULT_TITLE);
35
+ if (preferred) return { match: preferred, candidates };
36
+ if (candidates.length === 1) return { match: candidates[0], candidates };
37
+ return { candidates };
38
+ }
39
+
40
+ function listFiles(dir: string): string[] {
41
+ try {
42
+ return readdirSync(dir);
43
+ } catch {
44
+ return [];
45
+ }
46
+ }
47
+
48
+ function parseDescriptor(file: string): PwServerDescriptor | null {
49
+ let data: any;
50
+ try {
51
+ data = JSON.parse(readFileSync(file, 'utf8'));
52
+ } catch {
53
+ return null;
54
+ }
55
+
56
+ if (!data?.endpoint || !data?.title || !data?.workspaceDir) return null;
57
+
58
+ return {
59
+ file,
60
+ title: data.title,
61
+ endpoint: data.endpoint,
62
+ workspaceDir: data.workspaceDir,
63
+ browserName: data.browser?.browserName || DEFAULT_BROWSER,
64
+ playwrightLib: data.playwrightLib || '',
65
+ };
66
+ }
67
+
68
+ export interface PwServerDescriptor {
69
+ file: string;
70
+ title: string;
71
+ endpoint: string;
72
+ workspaceDir: string;
73
+ browserName: string;
74
+ playwrightLib: string;
75
+ }
@@ -36,6 +36,7 @@ function buildExplorBotOptions(from, options) {
36
36
  headless: options.headless,
37
37
  incognito: options.incognito,
38
38
  session: options.session,
39
+ applicationSpec: options.spec,
39
40
  };
40
41
  }
41
42
  function addCommonOptions(cmd) {
@@ -47,6 +48,7 @@ function addCommonOptions(cmd) {
47
48
  .option('-s, --show', 'Show browser window')
48
49
  .option('--headless', 'Run browser in headless mode')
49
50
  .option('--incognito', 'Run without recording experiences')
51
+ .option('--spec <path>', 'Use a Docbot application spec directory or index.md')
50
52
  .option('--session [file]', 'Save/restore browser session from file');
51
53
  }
52
54
  async function startTUI(explorBot) {
@@ -341,6 +343,13 @@ program
341
343
  process.exit(1);
342
344
  }
343
345
  });
346
+ program
347
+ .command('sites')
348
+ .description('List sites registered in the global installation')
349
+ .action(async () => {
350
+ const { SitesCommand } = await import('../src/commands/sites-command.js');
351
+ await new SitesCommand(new ExplorBot()).execute('');
352
+ });
344
353
  addCommonOptions(program.command('rerun <filename> [index]').description('Re-run generated tests with AI auto-healing')).action(async (filename, index, options) => {
345
354
  try {
346
355
  const explorBot = new ExplorBot(buildExplorBotOptions(undefined, options));
@@ -373,17 +382,29 @@ addCommonOptions(program
373
382
  });
374
383
  program
375
384
  .command('init')
376
- .description('Initialize a new project with configuration')
377
- .option('-c, --config-path <path>', 'Path for the config file', './explorbot.config.js')
385
+ .description('Initialize configuration for a project or for this machine')
386
+ .option('-c, --config-path <path>', 'Path for the config file')
378
387
  .option('-f, --force', 'Overwrite existing config file')
379
388
  .option('-p, --path <path>', 'Working directory for initialization')
389
+ .option('-g, --global', 'Configure explorbot in ~/.explorbot to run from anywhere')
390
+ .option('--provider <name>', `AI provider for the global config: ${Object.keys(PROVIDERS).join(', ')}`)
391
+ .option('--api-key <key>', 'API key stored in ~/.explorbot/.env')
380
392
  .action(async (options) => {
381
- const { runInitCommand } = await import('../src/commands/init-command.js');
382
- runInitCommand({
383
- configPath: options.configPath,
384
- force: options.force,
385
- path: options.path,
386
- });
393
+ try {
394
+ const { runInit } = await import('../src/commands/init-command.js');
395
+ await runInit({
396
+ configPath: options.configPath,
397
+ force: options.force,
398
+ path: options.path,
399
+ global: options.global,
400
+ provider: options.provider,
401
+ apiKey: options.apiKey,
402
+ });
403
+ }
404
+ catch (error) {
405
+ console.error('Failed:', error instanceof Error ? error.message : 'Unknown error');
406
+ process.exit(1);
407
+ }
387
408
  });
388
409
  program
389
410
  .command('clean [target]')
@@ -626,7 +647,7 @@ program
626
647
  };
627
648
  const explorBot = new ExplorBot(mainOptions);
628
649
  await explorBot.start();
629
- await explorBot.agentNavigator().visit(url);
650
+ await explorBot.visit(url);
630
651
  const { ContextCommand } = await import('../src/commands/context-command.js');
631
652
  const argParts = [];
632
653
  if (options.full)
@@ -650,7 +671,7 @@ addCommonOptions(program.command('shell <url> <command>').description('Execute a
650
671
  try {
651
672
  const explorBot = new ExplorBot(buildExplorBotOptions(url, options));
652
673
  await explorBot.start();
653
- await explorBot.agentNavigator().visit(url);
674
+ await explorBot.visit(url);
654
675
  const action = explorBot.getExplorer().action();
655
676
  await action.execute(command);
656
677
  log('Command executed successfully');
@@ -671,10 +692,11 @@ browserCmd
671
692
  .description('Launch a persistent browser server')
672
693
  .option('-s, --show', 'Launch browser in headed mode (visible window)')
673
694
  .option('--headless', 'Launch browser in headless mode')
695
+ .option('--instance <name>', 'Named browser instance (lowercase letters, digits, dashes)')
674
696
  .option('-c, --config <path>', 'Path to configuration file')
675
697
  .option('-p, --path <path>', 'Working directory path')
676
698
  .action(async (options) => {
677
- const { launchServer, removeEndpointFile } = await import('../src/browser-server.js');
699
+ const { launchServer, removeEndpointFile, keepServerRunning } = await import('../src/browser-server.js');
678
700
  await ConfigParser.getInstance().loadConfig({
679
701
  config: options.config,
680
702
  path: options.path,
@@ -688,45 +710,34 @@ browserCmd
688
710
  const server = await launchServer({
689
711
  browser: config.playwright.browser,
690
712
  show,
713
+ }, options.instance);
714
+ await keepServerRunning(async () => {
715
+ await server.close();
716
+ removeEndpointFile(options.instance);
691
717
  });
692
- console.log('Browser server is running. Press Ctrl+C to stop.');
693
- const cleanup = () => {
694
- console.log('\nStopping browser server...');
695
- server.close();
696
- removeEndpointFile();
697
- process.exit(0);
698
- };
699
- process.on('SIGINT', cleanup);
700
- process.on('SIGTERM', cleanup);
701
718
  });
702
719
  browserCmd
703
720
  .command('stop')
704
721
  .description('Stop a running browser server')
722
+ .option('--instance <name>', 'Named browser instance (lowercase letters, digits, dashes)')
705
723
  .option('-c, --config <path>', 'Path to configuration file')
706
724
  .option('-p, --path <path>', 'Working directory path')
707
725
  .action(async (options) => {
708
- const { getAliveEndpoint, removeEndpointFile } = await import('../src/browser-server.js');
726
+ const { stopServer } = await import('../src/browser-server.js');
709
727
  await ConfigParser.getInstance().loadConfig({
710
728
  config: options.config,
711
729
  path: options.path,
712
730
  });
713
- const endpoint = await getAliveEndpoint();
714
- if (!endpoint) {
731
+ if (!(await stopServer(options.instance))) {
715
732
  console.log('No running browser server found.');
716
733
  process.exit(0);
717
734
  }
718
- try {
719
- const { chromium } = await import('playwright-core');
720
- const browser = await chromium.connect(endpoint, { timeout: 3000 });
721
- await browser.close();
722
- }
723
- catch { }
724
- removeEndpointFile();
725
735
  console.log('Browser server stopped.');
726
736
  });
727
737
  browserCmd
728
738
  .command('status')
729
739
  .description('Check if a browser server is running')
740
+ .option('--instance <name>', 'Named browser instance (lowercase letters, digits, dashes)')
730
741
  .option('-c, --config <path>', 'Path to configuration file')
731
742
  .option('-p, --path <path>', 'Working directory path')
732
743
  .action(async (options) => {
@@ -735,7 +746,7 @@ browserCmd
735
746
  config: options.config,
736
747
  path: options.path,
737
748
  });
738
- const endpoint = await getAliveEndpoint();
749
+ const endpoint = await getAliveEndpoint(options.instance);
739
750
  if (endpoint) {
740
751
  console.log(`Browser server is running at: ${endpoint}`);
741
752
  }
@@ -790,8 +801,10 @@ program
790
801
  });
791
802
  import { createApiCommands } from "../boat/api-tester/src/cli.js";
792
803
  import { createDocsCommands } from "../boat/doc-collector/src/cli.js";
804
+ import { createPrimaCommands } from "../boat/prima/src/cli.js";
793
805
  program.addCommand(createApiCommands('api'));
794
806
  program.addCommand(createDocsCommands('docs'));
807
+ program.addCommand(createPrimaCommands('prima'));
795
808
  const envHelp = () => {
796
809
  const width = Math.max(...EXPLORBOT_ENV_VARS.map((v) => v.name.length));
797
810
  const rows = EXPLORBOT_ENV_VARS.map((v) => ` ${v.name.padEnd(width)} ${v.description}`).join('\n');
@@ -30,7 +30,7 @@ export class ApiBot {
30
30
  }
31
31
  }
32
32
  async start() {
33
- this.config = await this.configParser.loadConfig({ config: this.options.config, path: this.options.path });
33
+ this.config = await this.configParser.loadConfig({ config: this.options.config, path: this.options.path, endpoint: this.options.endpoint });
34
34
  this.provider = new AIProvider(this.config.ai);
35
35
  await this.provider.validateConnection();
36
36
  this.apiClient = new ApiClient(this.config.api.baseEndpoint, this.config.api.headers || {}, {
@@ -86,11 +86,12 @@ export class ApiBot {
86
86
  agentCurler() {
87
87
  return (this.agents.curler ||= this.createAgent(({ ai, apiClient, requestState }) => new Curler(ai, apiClient, requestState, this.reporter)));
88
88
  }
89
- async plan(endpoint, opts = {}) {
89
+ async plan(target, opts = {}) {
90
90
  if (opts.fresh) {
91
91
  this.currentPlan = undefined;
92
92
  this.agents.chief = undefined;
93
93
  }
94
+ const endpoint = this.configParser.resolveEndpointPath(target);
94
95
  const chief = this.agentChief();
95
96
  const specDefinition = this.getEndpointDefinition(endpoint);
96
97
  this.currentPlan = await chief.plan(endpoint, { style: opts.style, specDefinition });
@@ -40,7 +40,7 @@ export function createApiCommands(name = 'api') {
40
40
  addCommonOptions(cmd.command('plan <endpoint>').description('Generate test plan for an API endpoint').option('--style <style>', 'Planning style: basename of a file in rules/chief/styles/').option('--fresh', 'Start planning from scratch')).action(async (endpoint, options) => {
41
41
  setPreserveConsoleLogs(true);
42
42
  try {
43
- const bot = new ApiBot(buildOptions(options));
43
+ const bot = new ApiBot({ ...buildOptions(options), endpoint });
44
44
  await bot.start();
45
45
  await bot.plan(endpoint, { style: options.style, fresh: options.fresh });
46
46
  const plan = bot.getCurrentPlan();
@@ -104,7 +104,7 @@ export function createApiCommands(name = 'api') {
104
104
  addCommonOptions(cmd.command('explore <endpoint>').description('Full cycle: plan all styles, execute tests, re-plan')).action(async (endpoint, options) => {
105
105
  setPreserveConsoleLogs(true);
106
106
  try {
107
- const bot = new ApiBot(buildOptions(options));
107
+ const bot = new ApiBot({ ...buildOptions(options), endpoint });
108
108
  await bot.start();
109
109
  const styles = Object.keys(getStyles());
110
110
  let totalPassed = 0;
@@ -9,11 +9,13 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
9
9
  import { existsSync, mkdirSync, readFileSync } from 'node:fs';
10
10
  import path, { resolve } from 'node:path';
11
11
  import { parseEnv } from 'node:util';
12
- import { EXPLORBOT_CONFIG_PATHS, createModel, materializeKnowledge, resolveModel, resolveOutputRoot } from "../../../src/config.js";
12
+ import { ConfigMissingError, EXPLORBOT_CONFIG_PATHS, createModel, envConfigRequested, materializeKnowledge, missingConfigMessage, resolveConfigModels, resolveModel, resolveOutputRoot } from "../../../src/config.js";
13
+ import { findGlobalConfig, globalEnvPath, isGlobalConfigPath, registerSite, resolveSiteTarget } from "../../../src/global-config.js";
13
14
  export class ApibotConfigParser {
14
15
  static instance;
15
16
  config = null;
16
17
  configPath = null;
18
+ site = null;
17
19
  constructor() { }
18
20
  static getInstance() {
19
21
  if (!ApibotConfigParser.instance) {
@@ -34,6 +36,7 @@ export class ApibotConfigParser {
34
36
  if (options?.path) {
35
37
  process.chdir(resolve(options.path));
36
38
  }
39
+ ApibotConfigParser.loadEnv(globalEnvPath());
37
40
  ApibotConfigParser.loadEnv('.env');
38
41
  const resolvedPath = options?.config || this.findConfigFile();
39
42
  if (!resolvedPath) {
@@ -59,7 +62,12 @@ export class ApibotConfigParser {
59
62
  };
60
63
  }
61
64
  this.config = this.mergeWithDefaults(loadedConfig);
65
+ await resolveConfigModels(this.config.ai);
62
66
  this.configPath = resolvedPath;
67
+ this.site = null;
68
+ if (isGlobalConfigPath(resolvedPath)) {
69
+ this.enterGlobalMode(this.config, options?.endpoint);
70
+ }
63
71
  this.validateConfig(this.config);
64
72
  return this.config;
65
73
  }
@@ -79,10 +87,23 @@ export class ApibotConfigParser {
79
87
  }
80
88
  getOutputDir() {
81
89
  const config = this.getConfig();
90
+ return path.join(this.getProjectRoot(), config.dirs?.output || 'output');
91
+ }
92
+ getProjectRoot() {
93
+ if (this.site)
94
+ return this.site.dir;
82
95
  const configPath = this.getConfigPath();
83
96
  if (!configPath)
84
97
  throw new Error('Config path not found');
85
- return path.join(path.dirname(configPath), config.dirs?.output || 'output');
98
+ return path.dirname(configPath);
99
+ }
100
+ resolveEndpointPath(endpoint) {
101
+ if (!this.site)
102
+ return endpoint;
103
+ const resolved = resolveSiteTarget(endpoint, this.site.url);
104
+ if (resolved.baseUrl !== this.site.url)
105
+ return endpoint;
106
+ return resolved.path;
86
107
  }
87
108
  getPlansDir() {
88
109
  return path.join(this.getOutputDir(), 'plans');
@@ -92,21 +113,26 @@ export class ApibotConfigParser {
92
113
  }
93
114
  getKnowledgeDir() {
94
115
  const config = this.getConfig();
95
- const configPath = this.getConfigPath();
96
- if (!configPath)
97
- throw new Error('Config path not found');
98
- return path.join(path.dirname(configPath), config.dirs?.knowledge || 'knowledge');
116
+ return path.join(this.getProjectRoot(), config.dirs?.knowledge || 'knowledge');
99
117
  }
100
118
  ensureDirectory(dirPath) {
101
119
  if (!existsSync(dirPath)) {
102
120
  mkdirSync(dirPath, { recursive: true });
103
121
  }
104
122
  }
123
+ enterGlobalMode(config, endpoint) {
124
+ const site = resolveSiteTarget(endpoint);
125
+ this.site = registerSite(site.baseUrl);
126
+ config.dirs = { output: 'output', knowledge: 'knowledge' };
127
+ config.api = { ...config.api, baseEndpoint: site.baseUrl };
128
+ if (process.env.EXPLORBOT_API_SPEC)
129
+ config.api.spec = [process.env.EXPLORBOT_API_SPEC];
130
+ }
105
131
  async loadEnvConfig() {
106
132
  const provider = process.env.EXPLORBOT_AI_PROVIDER;
107
133
  const modelSpec = process.env.EXPLORBOT_AI_MODEL;
108
134
  if (!provider && !modelSpec) {
109
- throw new Error('No configuration file found. Create apibot.config.js or set EXPLORBOT_URL and EXPLORBOT_AI_PROVIDER environment variables');
135
+ throw new ConfigMissingError(missingConfigMessage('apibot.config.js'));
110
136
  }
111
137
  if (modelSpec && !provider && !modelSpec.includes('/')) {
112
138
  throw new Error('EXPLORBOT_AI_MODEL needs a provider — set EXPLORBOT_AI_PROVIDER, or write it as "provider/model-id"');
@@ -149,7 +175,9 @@ export class ApibotConfigParser {
149
175
  if (existsSync(fullPath))
150
176
  return fullPath;
151
177
  }
152
- return null;
178
+ if (envConfigRequested())
179
+ return null;
180
+ return findGlobalConfig();
153
181
  }
154
182
  async loadConfigModule(configPath) {
155
183
  const ext = configPath.split('.').pop();
@@ -77,6 +77,9 @@ class Documentarian {
77
77
  if ((interaction.changes?.newElements || 0) > 0) {
78
78
  return true;
79
79
  }
80
+ if ((interaction.changes?.removedElements || 0) > 0) {
81
+ return true;
82
+ }
80
83
  return (interaction.discoveredUrls || []).length > 0;
81
84
  });
82
85
  }
@@ -58,6 +58,7 @@ async function executeInteraction(explorer, stateManager, candidate, restoreUrl,
58
58
  if (!beforeState) {
59
59
  return null;
60
60
  }
61
+ const beforeScreenshot = await captureState?.before();
61
62
  const executed = await attemptInteraction(explorer, candidate);
62
63
  if (!executed) {
63
64
  return null;
@@ -75,12 +76,12 @@ async function executeInteraction(explorer, stateManager, candidate, restoreUrl,
75
76
  removedElements: ariaChanges.removedCount,
76
77
  });
77
78
  if (captureState && isMeaningfulStateTransition(transition)) {
78
- const screenshot = await captureState(afterState, transition);
79
+ const screenshot = await captureState.after(beforeScreenshot ?? null, afterState, transition);
79
80
  if (screenshot) {
80
81
  transition.screenshot = screenshot;
81
82
  }
82
83
  }
83
- if (urlChanged || ariaChanges.newCount > 0) {
84
+ if (urlChanged || ariaChanges.newCount > 0 || ariaChanges.removedCount > 0) {
84
85
  await restoreInteractionState(explorer, restoreUrl);
85
86
  }
86
87
  return transition;
@@ -108,11 +109,16 @@ async function restoreInteractionState(explorer, restoreUrl, primaryCommand) {
108
109
  await action.attempt(`I.amOnPage(${JSON.stringify(restoreUrl)})`, `Restore page ${restoreUrl}`);
109
110
  }
110
111
  function buildTransition(candidate, beforeState, afterState, changes) {
112
+ const existingUrls = new Set(collectLinks(beforeState).map((link) => link.url));
111
113
  const transition = {
112
114
  action: describeAction(candidate),
113
115
  before: summarizeInteractiveState(beforeState),
114
116
  after: summarizeInteractiveState(afterState),
115
- discoveredUrls: collectLinks(afterState).map((link) => link.url),
117
+ discoveredUrls: [
118
+ ...new Set(collectLinks(afterState)
119
+ .map((link) => link.url)
120
+ .filter((url) => !existingUrls.has(url))),
121
+ ],
116
122
  newCapabilities: collectDiscoveryNotes(afterState, changes),
117
123
  element: buildInteractionElement(candidate),
118
124
  changes,
@@ -150,7 +156,7 @@ function isMeaningfulStateTransition(transition) {
150
156
  if (transition.targetUrl || transition.changes?.urlChanged) {
151
157
  return true;
152
158
  }
153
- return (transition.changes?.newElements || 0) > 0;
159
+ return (transition.changes?.newElements || 0) > 0 || (transition.changes?.removedElements || 0) > 0;
154
160
  }
155
161
  function buildInteractionElement(candidate) {
156
162
  const element = {
@@ -47,6 +47,7 @@ export function createDocsCommands(name = 'docs') {
47
47
  console.log(`Skipped ${result.skipped.length} page(s)`);
48
48
  console.log(`Spec index: ${result.indexPath}`);
49
49
  console.log(`Pages dir: ${path.join(result.outputDir, 'pages')}`);
50
+ console.log(`Use in Explorbot: npx explorbot start ${startPath} --spec "${result.outputDir}"`);
50
51
  await bot.stop();
51
52
  process.exit(0);
52
53
  }
@@ -80,6 +81,7 @@ export function createDocsCommands(name = 'docs') {
80
81
  output: 'docs',
81
82
  screenshot: true,
82
83
  interactive: false,
84
+ ignoreErrors: true,
83
85
  collapseDynamicPages: true,
84
86
  scope: 'site',
85
87
  includePaths: [],
@@ -105,6 +105,7 @@ class DocbotConfigParser {
105
105
  output: 'docs',
106
106
  screenshot: true,
107
107
  interactive: false,
108
+ ignoreErrors: true,
108
109
  collapseDynamicPages: true,
109
110
  scope: 'site',
110
111
  includePaths: [],
@@ -9,7 +9,7 @@ import { DocbotConfigParser } from "./config.js";
9
9
  import { renderPageDocumentation, renderSpecIndex } from "./docs-renderer.js";
10
10
  import { getDocPageKey, shouldCrawlDocPath } from "./path-filter.js";
11
11
  import { extractResearchNavigationTargets } from "./research-navigation.js";
12
- import { captureDocumentationScreenshots, captureInteractionScreenshot } from "./screenshots.js";
12
+ import { captureBeforeInteraction, captureDocumentationScreenshots, captureInteractionScreenshot } from "./screenshots.js";
13
13
  import { renderMermaidBody } from "./state-diagram.js";
14
14
  class DocBot {
15
15
  explorBot;
@@ -98,16 +98,17 @@ class DocBot {
98
98
  force: true,
99
99
  });
100
100
  const pagePath = this.getPageFilePath(state.url);
101
- const documentation = await this.documentarian.document(state, research, async (interactionState, transition) => {
102
- if (!this.shouldUseScreenshots()) {
103
- return null;
101
+ const captureState = this.shouldUseScreenshots()
102
+ ? {
103
+ before: () => captureBeforeInteraction(this.explorBot.getExplorer()),
104
+ after: (beforeScreenshot, interactionState, transition) => captureInteractionScreenshot(this.explorBot.getExplorer(), beforeScreenshot, interactionState, transition, {
105
+ pageFilePath: pagePath,
106
+ screenshotsDir: this.getScreenshotsDir(),
107
+ config: this.config,
108
+ }),
104
109
  }
105
- return captureInteractionScreenshot(this.explorBot.getExplorer(), interactionState, transition, {
106
- pageFilePath: pagePath,
107
- screenshotsDir: this.getScreenshotsDir(),
108
- config: this.config,
109
- });
110
- });
110
+ : undefined;
111
+ const documentation = await this.documentarian.document(state, research, captureState);
111
112
  const lowSignalReason = this.getLowSignalReason(documentation, research);
112
113
  if (lowSignalReason) {
113
114
  skipped.push({
@@ -147,6 +148,9 @@ class DocBot {
147
148
  }
148
149
  catch (error) {
149
150
  const reason = error instanceof Error ? error.message : String(error);
151
+ if (!this.shouldIgnoreError(error)) {
152
+ throw error;
153
+ }
150
154
  tag('warning').log(`Skipping ${target}: ${reason}`);
151
155
  skipped.push({
152
156
  url: target,
@@ -360,6 +364,28 @@ class DocBot {
360
364
  }
361
365
  return `low-signal page: only ${documentation.can.length} proven actions and ${interactiveCount} interactive elements`;
362
366
  }
367
+ shouldIgnoreError(error) {
368
+ const ignoreErrors = this.config.docs?.ignoreErrors;
369
+ if (ignoreErrors === undefined || ignoreErrors === true)
370
+ return true;
371
+ if (ignoreErrors === false)
372
+ return false;
373
+ const details = [error instanceof Error ? error.name : '', error instanceof Error ? error.message : String(error)];
374
+ if (typeof error === 'object' && error && 'code' in error) {
375
+ details.push(String(error.code));
376
+ }
377
+ const normalized = details
378
+ .join(' ')
379
+ .toLowerCase()
380
+ .replaceAll(/[\W_]+/g, ' ');
381
+ return ignoreErrors.some((pattern) => {
382
+ const normalizedPattern = pattern
383
+ .trim()
384
+ .toLowerCase()
385
+ .replaceAll(/[\W_]+/g, ' ');
386
+ return normalizedPattern.length > 0 && normalized.includes(normalizedPattern);
387
+ });
388
+ }
363
389
  countInteractiveElements(research) {
364
390
  const matches = [...research.matchAll(/\((\d+) elements?\)/g)];
365
391
  return matches.reduce((sum, match) => sum + Number.parseInt(match[1], 10), 0);
@@ -1,6 +1,8 @@
1
1
  import path from 'node:path';
2
- import { buildStateGraph, renderMermaidFromGraph, renderStateMapFromGraph } from "./state-diagram.js";
2
+ import matter from 'gray-matter';
3
+ import { APPLICATION_SPEC_FORMAT, APPLICATION_SPEC_VERSION } from "../../../src/application-spec-contract.js";
3
4
  import { normalizeInlineText } from "../../../src/utils/strings.js";
5
+ import { buildStateGraph, renderMermaidFromGraph, renderPageStateDiagram, renderStateMapFromGraph } from "./state-diagram.js";
4
6
  function renderPageDocumentation(state, documentation, screenshots = []) {
5
7
  const lines = [];
6
8
  lines.push(`# ${state.url}`);
@@ -25,6 +27,13 @@ function renderPageDocumentation(state, documentation, screenshots = []) {
25
27
  }
26
28
  }
27
29
  const interactions = documentation.interactions;
30
+ const pageStateDiagram = renderPageStateDiagram(state.title || state.url || 'page', state.url || '', interactions || []);
31
+ if (pageStateDiagram) {
32
+ lines.push('## State Map');
33
+ lines.push('');
34
+ lines.push(`\`\`\`mermaid\n${pageStateDiagram}\n\`\`\``);
35
+ lines.push('');
36
+ }
28
37
  if (interactions && interactions.length > 0) {
29
38
  lines.push('## State Transitions');
30
39
  lines.push('');
@@ -83,7 +92,11 @@ function renderPageDocumentation(state, documentation, screenshots = []) {
83
92
  }
84
93
  lines.push('');
85
94
  }
86
- return `${lines.join('\n').trimEnd()}\n`;
95
+ return matter.stringify(`${lines.join('\n').trimEnd()}\n`, {
96
+ url: state.url,
97
+ format: APPLICATION_SPEC_FORMAT,
98
+ version: APPLICATION_SPEC_VERSION,
99
+ });
87
100
  }
88
101
  function renderSpecIndex(outputDir, startPath, pages, skipped, maxPages) {
89
102
  const lines = [];
@@ -97,9 +110,10 @@ function renderSpecIndex(outputDir, startPath, pages, skipped, maxPages) {
97
110
  lines.push(`Max pages: ${maxPages}`);
98
111
  lines.push('');
99
112
  const graph = buildStateGraph(outputDir, pages);
113
+ const mermaid = renderMermaidFromGraph(graph, true);
100
114
  lines.push('## State Transitions');
101
115
  lines.push('');
102
- lines.push(`\`\`\`mermaid\n${renderMermaidFromGraph(graph)}\n\`\`\``);
116
+ lines.push(`\`\`\`mermaid\n${mermaid}\n\`\`\``);
103
117
  lines.push('');
104
118
  const stateMap = renderStateMapFromGraph(graph);
105
119
  if (stateMap) {