explorbot 0.4.6 → 0.4.7

package/boat/api-tester/src/ai/curler.ts

@@ -3,6 +3,7 @@ import { z } from 'zod';
 import type { AIProvider } from '../../../../src/ai/provider.ts';
 import type { RequestStore } from '../../../../src/api/request-store.ts';
 import type { KnowledgeTracker } from '../../../../src/knowledge-tracker.ts';
+import { Observability } from '../../../../src/observability.ts';
 import type { Reporter } from '../../../../src/reporter.ts';
 import { type Test, TestResult } from '../../../../src/test-plan.ts';
 import { createDebug, tag } from '../../../../src/utils/logger.ts';
@@ -46,75 +47,77 @@ export class Curler {
     const initialPrompt = this.buildTestPrompt(test, opts?.specDefinition, opts?.baseEndpoint);
     conversation.addUserText(initialPrompt);
 
-    await loop(
-      async ({ stop, iteration }) => {
-        debugLog(`Iteration ${iteration}`);
-
-        if (iteration > 1) {
-          const requestLog = this.requestState.toLog();
-          const nextStep = dedent`
-            <request_log>
-            ${requestLog || 'No requests made yet'}
-            </request_log>
-
-            <task>
-            Continue testing. Review the request log above and proceed with the next step.
-            </task>
-
-            <notes>
-            ${test.notesToString() || 'No notes yet'}
-            </notes>
-          `;
-          conversation.addUserText(nextStep);
-        }
-
-        const result = await this.provider.invokeConversation(conversation, tools, {
-          maxToolRoundtrips: 5,
-          toolChoice: 'required',
-          agentName: 'curler',
-        });
-
-        if (!result) throw new Error('Failed to get response from provider');
-
-        const toolNames = result.toolExecutions?.map((e: any) => e.toolName) || [];
-        debugLog('Tool calls:', toolNames.join(', '));
-
-        if (test.hasFinished) {
-          stop();
-          return;
-        }
-
-        if (iteration >= MAX_ITERATIONS) {
-          tag('warning').log('Max iterations reached, running final review...');
-          stop();
-        }
-      },
+    await Observability.run(
+      `curler: ${test.scenario}`,
       {
-        maxAttempts: MAX_ITERATIONS,
-        observability: {
-          name: `curler: ${test.scenario}`,
-          agent: 'curler',
-          sessionId: test.sessionName,
-          metadata: {
-            input: {
-              scenario: test.scenario,
-              startUrl: test.startUrl,
-              expected: test.expected,
-            },
-          },
-        },
-        catch: async ({ error, stop }) => {
-          tag('error').log(`Test execution error: ${error}`);
-          stop();
+        sessionId: test.sessionName,
+        tags: ['curler'],
+        input: {
+          scenario: test.scenario,
+          startUrl: test.startUrl,
+          expected: test.expected,
         },
+      },
+      async () => {
+        await loop(
+          async ({ stop, iteration }) => {
+            debugLog(`Iteration ${iteration}`);
+
+            if (iteration > 1) {
+              const requestLog = this.requestState.toLog();
+              const nextStep = dedent`
+                <request_log>
+                ${requestLog || 'No requests made yet'}
+                </request_log>
+
+                <task>
+                Continue testing. Review the request log above and proceed with the next step.
+                </task>
+
+                <notes>
+                ${test.notesToString() || 'No notes yet'}
+                </notes>
+              `;
+              conversation.addUserText(nextStep);
+            }
+
+            const result = await this.provider.invokeConversation(conversation, tools, {
+              maxToolRoundtrips: 5,
+              toolChoice: 'required',
+              agentName: 'curler',
+            });
+
+            if (!result) throw new Error('Failed to get response from provider');
+
+            const toolNames = result.toolExecutions?.map((e: any) => e.toolName) || [];
+            debugLog('Tool calls:', toolNames.join(', '));
+
+            if (test.hasFinished) {
+              stop();
+              return;
+            }
+
+            if (iteration >= MAX_ITERATIONS) {
+              tag('warning').log('Max iterations reached, running final review...');
+              stop();
+            }
+          },
+          {
+            maxAttempts: MAX_ITERATIONS,
+            catch: async ({ error, stop }) => {
+              tag('error').log(`Test execution error: ${error}`);
+              stop();
+            },
+          }
+        );
+
+        try {
+          await this.finalReview(test);
+        } catch (error) {
+          tag('error').log(`Final review failed: ${error}`);
+        }
       }
     );
-
-    try {
-      await this.finalReview(test);
-    } catch (error) {
-      tag('error').log(`Final review failed: ${error}`);
-    }
     this.finishTest(test);
     const meta: Record<string, string | undefined> = {
       endpoint: test.startUrl,
@@ -190,7 +193,8 @@ export class Curler {
         },
       ],
       schema,
-      model
+      model,
+      { agentName: 'curler', telemetryFunctionId: 'curler.finalReview' }
     );
 
     const result = response?.object;

package/boat/api-tester/src/apibot.ts

@@ -85,6 +85,7 @@ export class ApiBot {
   async stop(): Promise<void> {
     await this.reporter?.finishRun();
     await this.apiClient?.teardown();
+    await this.provider?.stop();
   }
 
   createAgent<T>(factory: (deps: { ai: AIProvider; config: ApibotConfig; apiClient: ApiClient; requestState: RequestStore; knowledge: KnowledgeTracker }) => T): T {

package/boat/api-tester/src/cli.ts

@@ -1,4 +1,5 @@
 import { Command } from 'commander';
+import { flushTelemetry } from '../../../src/ai/provider.ts';
 import { ConfigCommand } from '../../../src/commands/config-command.ts';
 import { RecommendedModelsCommand } from '../../../src/commands/recommended-models-command.ts';
 import { listSites } from '../../../src/global-config.ts';
@@ -111,6 +112,7 @@ async function run(name: string, options: any, endpoint: string | undefined, bod
     process.exit(code);
   } catch (error) {
     console.error('Failed:', error instanceof Error ? error.message : 'Unknown error');
+    await flushTelemetry();
     process.exit(1);
   }
 }

package/boat/api-tester/src/config.ts

@@ -2,7 +2,22 @@ import { existsSync, mkdirSync, readFileSync } from 'node:fs';
 import path, { resolve } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { parseEnv } from 'node:util';
-import { type AIConfig, type ApiHookFn, type ApiConfig as BaseApiConfig, ConfigMissingError, EXPLORBOT_CONFIG_PATHS, createModel, envConfigRequested, materializeKnowledge, missingConfigMessage, resolveConfigModels, resolveModel, resolveOutputRoot, setOutputDir } from '../../../src/config.ts';
+import {
+  type AIConfig,
+  type ApiHookFn,
+  type ApiConfig as BaseApiConfig,
+  ConfigMissingError,
+  EXPLORBOT_CONFIG_PATHS,
+  createModel,
+  envConfigRequested,
+  materializeKnowledge,
+  missingConfigMessage,
+  resolveConfigModels,
+  resolveLangfuse,
+  resolveModel,
+  resolveOutputRoot,
+  setOutputDir,
+} from '../../../src/config.ts';
 import { type SiteRecord, findGlobalConfig, globalEnvPath, isGlobalConfigPath, registerSite, resolveSiteTarget } from '../../../src/global-config.ts';
 
 export type { AIConfig };
@@ -101,6 +116,7 @@ export class ApibotConfigParser {
       this.applyEnvHeaders(this.config.api);
       if (options?.baseEndpoint) this.config.api.baseEndpoint = options.baseEndpoint.replace(/\/$/, '');
       await resolveConfigModels(this.config.ai);
+      resolveLangfuse(this.config.ai);
       this.configPath = resolvedPath;
       this.site = null;
 
@@ -236,6 +252,7 @@ export class ApibotConfigParser {
       api,
       dirs: { output: '.', knowledge: 'knowledge' },
     };
+    resolveLangfuse(this.config.ai);
     this.configPath = path.join(outputRoot, 'apibot.config.js');
     this.validateConfig(this.config);
     setOutputDir(this.getOutputDir());

package/dist/boat/api-tester/src/ai/curler.js

@@ -1,5 +1,6 @@
 import dedent from 'dedent';
 import { z } from 'zod';
+import { Observability } from "../../../../src/observability.js";
 import { TestResult } from "../../../../src/test-plan.js";
 import { createDebug, tag } from "../../../../src/utils/logger.js";
 import { loop } from "../../../../src/utils/loop.js";
@@ -32,67 +33,65 @@ export class Curler {
             conversation.addUserText(knowledge);
         const initialPrompt = this.buildTestPrompt(test, opts?.specDefinition, opts?.baseEndpoint);
         conversation.addUserText(initialPrompt);
-        await loop(async ({ stop, iteration }) => {
-            debugLog(`Iteration ${iteration}`);
-            if (iteration > 1) {
-                const requestLog = this.requestState.toLog();
-                const nextStep = dedent `
-            <request_log>
-            ${requestLog || 'No requests made yet'}
-            </request_log>
+        await Observability.run(`curler: ${test.scenario}`, {
+            sessionId: test.sessionName,
+            tags: ['curler'],
+            input: {
+                scenario: test.scenario,
+                startUrl: test.startUrl,
+                expected: test.expected,
+            },
+        }, async () => {
+            await loop(async ({ stop, iteration }) => {
+                debugLog(`Iteration ${iteration}`);
+                if (iteration > 1) {
+                    const requestLog = this.requestState.toLog();
+                    const nextStep = dedent `
+                <request_log>
+                ${requestLog || 'No requests made yet'}
+                </request_log>
 
-            <task>
-            Continue testing. Review the request log above and proceed with the next step.
-            </task>
+                <task>
+                Continue testing. Review the request log above and proceed with the next step.
+                </task>
 
-            <notes>
-            ${test.notesToString() || 'No notes yet'}
-            </notes>
-          `;
-                conversation.addUserText(nextStep);
-            }
-            const result = await this.provider.invokeConversation(conversation, tools, {
-                maxToolRoundtrips: 5,
-                toolChoice: 'required',
-                agentName: 'curler',
+                <notes>
+                ${test.notesToString() || 'No notes yet'}
+                </notes>
+              `;
+                    conversation.addUserText(nextStep);
+                }
+                const result = await this.provider.invokeConversation(conversation, tools, {
+                    maxToolRoundtrips: 5,
+                    toolChoice: 'required',
+                    agentName: 'curler',
+                });
+                if (!result)
+                    throw new Error('Failed to get response from provider');
+                const toolNames = result.toolExecutions?.map((e) => e.toolName) || [];
+                debugLog('Tool calls:', toolNames.join(', '));
+                if (test.hasFinished) {
+                    stop();
+                    return;
+                }
+                if (iteration >= MAX_ITERATIONS) {
+                    tag('warning').log('Max iterations reached, running final review...');
+                    stop();
+                }
+            }, {
+                maxAttempts: MAX_ITERATIONS,
+                catch: async ({ error, stop }) => {
+                    tag('error').log(`Test execution error: ${error}`);
+                    stop();
+                },
             });
-            if (!result)
-                throw new Error('Failed to get response from provider');
-            const toolNames = result.toolExecutions?.map((e) => e.toolName) || [];
-            debugLog('Tool calls:', toolNames.join(', '));
-            if (test.hasFinished) {
-                stop();
-                return;
+            try {
+                await this.finalReview(test);
             }
-            if (iteration >= MAX_ITERATIONS) {
-                tag('warning').log('Max iterations reached, running final review...');
-                stop();
+            catch (error) {
+                tag('error').log(`Final review failed: ${error}`);
             }
-        }, {
-            maxAttempts: MAX_ITERATIONS,
-            observability: {
-                name: `curler: ${test.scenario}`,
-                agent: 'curler',
-                sessionId: test.sessionName,
-                metadata: {
-                    input: {
-                        scenario: test.scenario,
-                        startUrl: test.startUrl,
-                        expected: test.expected,
-                    },
-                },
-            },
-            catch: async ({ error, stop }) => {
-                tag('error').log(`Test execution error: ${error}`);
-                stop();
-            },
         });
-        try {
-            await this.finalReview(test);
-        }
-        catch (error) {
-            tag('error').log(`Final review failed: ${error}`);
-        }
         this.finishTest(test);
         const meta = {
             endpoint: test.startUrl,
@@ -160,7 +159,7 @@ export class Curler {
             3. Should the test pass or fail overall?
           `,
             },
-        ], schema, model);
+        ], schema, model, { agentName: 'curler', telemetryFunctionId: 'curler.finalReview' });
         const result = response?.object;
         if (!result)
             return;

package/dist/boat/api-tester/src/apibot.js

@@ -74,6 +74,7 @@ export class ApiBot {
     async stop() {
         await this.reporter?.finishRun();
         await this.apiClient?.teardown();
+        await this.provider?.stop();
     }
     createAgent(factory) {
         return factory({

package/dist/boat/api-tester/src/cli.js

@@ -1,4 +1,5 @@
 import { Command } from 'commander';
+import { flushTelemetry } from "../../../src/ai/provider.js";
 import { ConfigCommand } from "../../../src/commands/config-command.js";
 import { RecommendedModelsCommand } from "../../../src/commands/recommended-models-command.js";
 import { listSites } from "../../../src/global-config.js";
@@ -108,6 +109,7 @@ async function run(name, options, endpoint, body) {
     }
     catch (error) {
         console.error('Failed:', error instanceof Error ? error.message : 'Unknown error');
+        await flushTelemetry();
         process.exit(1);
     }
 }

package/dist/boat/api-tester/src/config.js

@@ -10,7 +10,7 @@ import { existsSync, mkdirSync, readFileSync } from 'node:fs';
 import path, { resolve } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { parseEnv } from 'node:util';
-import { ConfigMissingError, EXPLORBOT_CONFIG_PATHS, createModel, envConfigRequested, materializeKnowledge, missingConfigMessage, resolveConfigModels, resolveModel, resolveOutputRoot, setOutputDir } from "../../../src/config.js";
+import { ConfigMissingError, EXPLORBOT_CONFIG_PATHS, createModel, envConfigRequested, materializeKnowledge, missingConfigMessage, resolveConfigModels, resolveLangfuse, resolveModel, resolveOutputRoot, setOutputDir, } from "../../../src/config.js";
 import { findGlobalConfig, globalEnvPath, isGlobalConfigPath, registerSite, resolveSiteTarget } from "../../../src/global-config.js";
 function isAbsoluteEndpoint(value) {
     return !!value && (value.startsWith('http://') || value.startsWith('https://'));
@@ -82,6 +82,7 @@ export class ApibotConfigParser {
             if (options?.baseEndpoint)
                 this.config.api.baseEndpoint = options.baseEndpoint.replace(/\/$/, '');
             await resolveConfigModels(this.config.ai);
+            resolveLangfuse(this.config.ai);
             this.configPath = resolvedPath;
             this.site = null;
             if (isGlobalConfigPath(resolvedPath)) {
@@ -208,6 +209,7 @@ export class ApibotConfigParser {
             api,
             dirs: { output: '.', knowledge: 'knowledge' },
         };
+        resolveLangfuse(this.config.ai);
         this.configPath = path.join(outputRoot, 'apibot.config.js');
         this.validateConfig(this.config);
         setOutputDir(this.getOutputDir());

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "explorbot",
-  "version": "0.4.6",
+  "version": "0.4.7",
   "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
   "license": "Elastic-2.0",
   "type": "module",
@@ -101,7 +101,7 @@
     "bash-tool": "^1.3.15",
     "chalk": "^5.6.2",
     "cli-highlight": "^2.1.11",
-    "codeceptjs": "4.2.0-beta.2",
+    "codeceptjs": "^4.2.0-beta.3",
     "commander": "^14.0.1",
     "debug": "^4.4.3",
     "dedent": "^1.6.0",

package/dist/rules/researcher/pagination.md

@@ -0,0 +1,6 @@
+<pagination>
+When a section is a list that continues beyond what is shown, add one line under its `> Container:` line:
+`> Pagination: controls` — it has page numbers (1, 2, 3), prev/next arrows, or a "load more" button.
+`> Pagination: infinite` — it has none of those and loads more as it is scrolled.
+Sorting, filtering and switching tabs are not pagination — omit the line then.
+</pagination>

package/dist/src/action-result.d.ts

@@ -37,6 +37,8 @@ export interface PageDiff {
     currentUrl: string;
     ariaChanges?: string | null;
     ariaChangeCount?: number;
+    ariaAdded?: number;
+    ariaRemoved?: number;
     messages?: string[];
     requests?: NetworkCall[];
     consoleErrors?: string[];
@@ -137,6 +139,8 @@ export declare class Diff {
     _messages: string[];
     _ariaDiffResult: string | null;
     _ariaChangeCount: number;
+    _ariaAdded: number;
+    _ariaRemoved: number;
     _isSameUrl: boolean;
     constructor(current: ActionResult, previous: ActionResult | null);
     static create(current: ActionResult, previous: ActionResult | null): Promise<Diff>;
@@ -147,6 +151,8 @@ export declare class Diff {
     cleanedHtmlParts(): Promise<HtmlDiffPart[]>;
     get ariaChanged(): string | null;
     get ariaChangeCount(): number;
+    get ariaAdded(): number;
+    get ariaRemoved(): number;
     get htmlDiff(): HtmlDiffResult | null;
     get messages(): string[];
     get similarity(): number;

package/dist/src/action-result.js

@@ -451,6 +451,8 @@ export class ActionResult {
         if (diff.ariaChanged) {
             pageDiff.ariaChanges = diff.ariaChanged;
             pageDiff.ariaChangeCount = diff.ariaChangeCount;
+            pageDiff.ariaAdded = diff.ariaAdded;
+            pageDiff.ariaRemoved = diff.ariaRemoved;
         }
         if (this.overlay.isOpen && (!previousState.overlay.isOpen || previousState.overlay.name !== this.overlay.name)) {
             pageDiff.areaOfInterest = this.overlay.describe();
@@ -542,6 +544,8 @@ export class Diff {
     _messages = [];
     _ariaDiffResult = null;
     _ariaChangeCount = 0;
+    _ariaAdded = 0;
+    _ariaRemoved = 0;
     _isSameUrl;
     constructor(current, previous) {
         this.current = current;
@@ -590,6 +594,12 @@ export class Diff {
     get ariaChangeCount() {
         return this._ariaChangeCount;
     }
+    get ariaAdded() {
+        return this._ariaAdded;
+    }
+    get ariaRemoved() {
+        return this._ariaRemoved;
+    }
     get htmlDiff() {
         return this._htmlDiffResult;
     }
@@ -614,5 +624,7 @@ export class Diff {
         const ariaDiff = diffAriaSnapshots(this.previous.ariaSnapshot, this.current.ariaSnapshot);
         this._ariaDiffResult = ariaDiff.text;
         this._ariaChangeCount = ariaDiff.count;
+        this._ariaAdded = ariaDiff.added;
+        this._ariaRemoved = ariaDiff.removed;
     }
 }

package/dist/src/ai/planner.js

@@ -404,12 +404,16 @@ export class Planner extends PlannerBase {
         `);
             }
         }
+        let activeRegion = '';
+        if (state.overlay.isOpen)
+            activeRegion = `Active region: ${state.overlay.describe()} — the user's current focus area. Plan tests for the controls inside it first.`;
         conversation.addUserText(dedent `
       ${this.buildApproach(style)}
 
       <context>
       URL: ${state.url || 'Unknown'}
       Title: ${state.title || 'Unknown'}
+      ${activeRegion}
       </context>
     `);
         if (this.fisherman) {

package/dist/src/ai/researcher/locators.js

@@ -281,7 +281,7 @@ export function WithLocators(Base) {
             if (sectionQuery.count() === 0)
                 sectionQuery = mdq(result.text).query(`section3(~"${escaped}")`);
             if (newCss) {
-                result.text = sectionQuery.query('blockquote[0]').replace(`Container: '${newCss}'`);
+                result.text = sectionQuery.query('blockquote[0]').setKeyValue('Container', `'${newCss}'`);
             }
             else {
                 result.text = sectionQuery.query('blockquote[0]').replace('');

package/dist/src/ai/researcher/pagination.d.ts

@@ -0,0 +1,16 @@
+import type Explorer from '../../explorer.js';
+import { type ListMeasure, type PaginationStrategy } from '../../utils/pagination.js';
+import { type Constructor } from './mixin.js';
+import type { ResearchResult } from './research-result.js';
+export declare function WithPagination<T extends Constructor>(Base: T): {
+    new (...args: any[]): {
+        explorer: Explorer;
+        detectPagination(result: ResearchResult): Promise<void>;
+        probeSection(css: string): Promise<PaginationStrategy | null>;
+        measure(css: string): Promise<ListMeasure | null>;
+        recordPagination(result: ResearchResult, name: string, strategy: PaginationStrategy): void;
+    };
+} & T;
+export interface PaginationMethods {
+    detectPagination(result: ResearchResult): Promise<void>;
+}

package/dist/src/ai/researcher/pagination.js

@@ -0,0 +1,62 @@
+import { mdq } from "../../utils/markdown-query.js";
+import { inspectList, restoreScroll } from "../../utils/pagination.js";
+import { debugLog } from "./mixin.js";
+import { extractPaginationFromBlockquote, parseDataSections, parseResearchSections } from "./parser.js";
+export function WithPagination(Base) {
+    return class extends Base {
+        async detectPagination(result) {
+            const sections = [...parseResearchSections(result.text), ...parseDataSections(result.text)];
+            for (const section of sections) {
+                const css = section.containerCss;
+                if (!css)
+                    continue;
+                if (extractPaginationFromBlockquote(section.rawMarkdown))
+                    continue;
+                const strategy = await this.probeSection(css);
+                if (!strategy)
+                    continue;
+                this.recordPagination(result, section.name, strategy);
+                debugLog(`Pagination in "${section.name}": ${strategy}`);
+            }
+        }
+        async probeSection(css) {
+            const before = await this.measure(css);
+            if (!before)
+                return null;
+            if (before.hasPagingControls)
+                return 'controls';
+            if (before.isFeed)
+                return 'infinite';
+            if (!before.scrolls)
+                return null;
+            const action = this.explorer.action();
+            const scrolled = await action.attempt(`I.scrollTo('${css} > *:last-child')`).catch(() => false);
+            if (!scrolled)
+                return null;
+            const after = await this.measure(css);
+            await this.explorer.withPage((page) => page.evaluate(restoreScroll, { css, scrollTop: before.scrollTop, pageScrollY: before.pageScrollY })).catch(() => { });
+            if (!after)
+                return null;
+            if (after.items > before.items)
+                return 'infinite';
+            return null;
+        }
+        measure(css) {
+            return this.explorer
+                .withPage((page) => page.evaluate(inspectList, css))
+                .catch((err) => {
+                debugLog(`List measurement failed for '${css}': ${err.message}`);
+                return null;
+            });
+        }
+        recordPagination(result, name, strategy) {
+            const escaped = name.replace(/"/g, '\\"');
+            let sectionQuery = mdq(result.text).query(`section2(~"${escaped}")`);
+            if (sectionQuery.count() === 0)
+                sectionQuery = mdq(result.text).query(`section3(~"${escaped}")`);
+            if (sectionQuery.count() === 0)
+                return;
+            result.text = sectionQuery.query('blockquote[0]').setKeyValue('Pagination', strategy);
+        }
+    };
+}

package/dist/src/ai/researcher/parser.d.ts

@@ -1,3 +1,4 @@
+import type { PaginationStrategy } from '../../utils/pagination.js';
 export interface ResearchElement {
     name: string;
     type: string | null;
@@ -23,6 +24,8 @@ export declare const RESEARCH_COLUMN_ORDER: string[];
 export declare function mapRowToElement(row: Record<string, string>): ResearchElement | null;
 export declare function extractContainerFromBlockquote(sectionMarkdown: string): string | null;
 export declare function parseResearchSections(markdown: string): ResearchSection[];
+export declare function parseDataSections(markdown: string): ResearchSection[];
+export declare function extractPaginationFromBlockquote(sectionMarkdown: string): PaginationStrategy | null;
 export declare function extractValidContainers(researchText: string, opts?: {
     exclude?: string[];
 }): Array<{

package/dist/src/ai/researcher/parser.js

@@ -57,13 +57,10 @@ export function mapRowToElement(row) {
     };
 }
 export function extractContainerFromBlockquote(sectionMarkdown) {
-    const bq = mdq(sectionMarkdown).query('blockquote[0]').text().trim();
-    if (!bq)
+    const entry = mdq(sectionMarkdown).query('blockquote[0]').keyValue().container;
+    if (!entry)
         return null;
-    const match = bq.match(/Container:\s*(.+)/i);
-    if (!match)
-        return null;
-    const css = normalizeLocatorValue(match[1]);
+    const css = normalizeLocatorValue(entry);
     if (!css || !/^[.#\[\w]/.test(css))
         return null;
     return css;
@@ -80,6 +77,25 @@ export function parseResearchSections(markdown) {
         return { name: section.name, containerCss, elements, rawMarkdown: section.rawMarkdown, isExtended };
     });
 }
+export function parseDataSections(markdown) {
+    return parseSections(markdown)
+        .filter((s) => s.name.toLowerCase().startsWith('data:'))
+        .map((section) => ({
+        name: section.name,
+        containerCss: extractContainerFromBlockquote(section.rawMarkdown),
+        elements: [],
+        rawMarkdown: section.rawMarkdown,
+        isExtended: false,
+    }));
+}
+export function extractPaginationFromBlockquote(sectionMarkdown) {
+    const value = mdq(sectionMarkdown).query('blockquote[0]').keyValue().pagination?.toLowerCase();
+    if (value === 'controls')
+        return 'controls';
+    if (value === 'infinite')
+        return 'infinite';
+    return null;
+}
 export function extractValidContainers(researchText, opts) {
     const exclude = opts?.exclude || [];
     return parseResearchSections(researchText)