explorbot 0.1.30 → 0.1.32-beta.1

package/dist/src/index.js

@@ -1,55 +1,2 @@
-#!/usr/bin/env node
-import { Command } from 'commander';
-import { addKnowledgeCommand } from './commands/add-knowledge.js';
-import { cleanCommand } from './commands/clean.js';
-import { exploreCommand } from './commands/explore.js';
-import { initCommand } from './commands/init.js';
-const program = new Command();
-program
-    .name('explorbot')
-    .description('AI-powered web exploration tool')
-    .option('-f, --from <url>', 'Start exploration from a specific URL')
-    .option('-v, --verbose', 'Enable verbose logging')
-    .option('--debug', 'Enable debug logging (same as --verbose)')
-    .option('-c, --config <path>', 'Path to configuration file')
-    .option('-p, --path <path>', 'Working directory path')
-    .helpOption('-h, --help', 'Show this help message');
-program
-    .command('clean')
-    .description('Clean up artifacts or experience folders')
-    .option('-t, --type <type>', 'Type of cleanup (artifacts|experience|all)', 'artifacts')
-    .option('-p, --path <path>', 'Custom path to clean')
-    .action(async (options, command) => {
-    const globalOptions = command.parent.opts();
-    const allOptions = { ...globalOptions, ...options };
-    await cleanCommand(allOptions);
-});
-program
-    .command('init')
-    .description('Initialize a new project with configuration')
-    .option('-c, --config-path <path>', 'Path for the config file', './explorbot.config.js')
-    .option('-f, --force', 'Overwrite existing config file', false)
-    .option('-p, --path <path>', 'Working directory for initialization')
-    .action(async (options, command) => {
-    const globalOptions = command.parent.opts();
-    const allOptions = { ...globalOptions, ...options };
-    await initCommand(allOptions);
-});
-program
-    .command('add-knowledge')
-    .alias('learn')
-    .description('Add knowledge for specific URLs')
-    .option('-p, --path <path>', 'Knowledge directory path')
-    .action(async (options) => {
-    await addKnowledgeCommand(options);
-});
-program.parse();
-const options = program.opts();
-const command = program.args[0];
-if (command === 'clean' || command === 'init' || command === 'add-knowledge' || command === 'learn') {
-    // These commands are handled by their respective actions
-}
-else {
-    // Default to explore command
-    exploreCommand(options);
-}
+export { ExplorBot } from "./explorbot.js";
+export { Plan, Test, TestResult, TestStatus } from "./test-plan.js";

package/dist/src/knowledge-tracker.d.ts

@@ -0,0 +1,33 @@
+import { ActionResult } from './action-result.js';
+interface Knowledge {
+    filePath: string;
+    url: string;
+    content: string;
+    [key: string]: any;
+}
+export declare class KnowledgeTracker {
+    knowledgeDir: string;
+    knowledgeFiles: Knowledge[];
+    isLoaded: boolean;
+    constructor();
+    loadKnowledgeFiles(): void;
+    getRelevantKnowledge(state: ActionResult): Knowledge[];
+    addKnowledge(urlPattern: string, description: string): {
+        filename: string;
+        filePath: string;
+        isNewFile: boolean;
+    };
+    interpolateVars(content: string): string;
+    generateFilename(url: string): string;
+    getExistingUrls(): string[];
+    getKnowledgeForUrl(urlPattern: string): string[];
+    listAllKnowledge(): Array<{
+        url: string;
+        firstLine: string;
+        filePath: string;
+    }>;
+    getMatchingKnowledge(url: string): Knowledge[];
+    normalizeUrl(url: string): string;
+    getStateParameters(state: ActionResult, keys: string[]): Record<string, any>;
+}
+export {};

package/dist/src/knowledge-tracker.js

@@ -5,6 +5,7 @@ import { ActionResult } from './action-result.js';
 import { ConfigParser } from './config.js';
 import { getCliName } from "./utils/cli-name.js";
 import { createDebug } from './utils/logger.js';
+import { isSecretName, registerSecret } from './utils/secrets.js';
 const debugLog = createDebug('explorbot:knowledge-tracker');
 export class KnowledgeTracker {
     knowledgeDir;
@@ -109,9 +110,15 @@ export class KnowledgeTracker {
                 return match;
             const namespace = expr.slice(0, dotIndex);
             const key = expr.slice(dotIndex + 1);
-            if (namespace === 'env')
-                return process.env[key] ?? '';
+            if (namespace === 'env') {
+                const value = process.env[key] ?? '';
+                if (isSecretName(key))
+                    registerSecret(value);
+                return value;
+            }
             if (namespace === 'config') {
+                if (isSecretName(key))
+                    return '';
                 const config = ConfigParser.getInstance().getConfig();
                 const value = key.split('.').reduce((obj, k) => obj?.[k], config);
                 if (value !== undefined && typeof value !== 'object')

package/dist/src/observability.d.ts

@@ -0,0 +1,13 @@
+import { type Span } from '@opentelemetry/api';
+type TelemetryMetadata = Record<string, unknown>;
+export declare const Observability: {
+    run<T>(name: string, metadata: TelemetryMetadata, fn: () => Promise<T>): Promise<T>;
+    getTelemetry(): {
+        isEnabled: boolean;
+        functionId: string;
+        metadata: Record<string, unknown>;
+    };
+    isTracing(): boolean;
+    getSpan(): Span;
+};
+export {};

package/dist/src/observability.js

@@ -1,4 +1,5 @@
 import { trace } from '@opentelemetry/api';
+import { redactSecrets } from './utils/secrets.js';
 let current = null;
 export const Observability = {
     async run(name, metadata, fn) {
@@ -70,7 +71,7 @@ function buildRootSpanAttributes(name, metadata) {
         attributes['langfuse.trace.tags'] = metadata.tags;
     }
     if (metadata.input !== undefined) {
-        attributes['langfuse.trace.input'] = JSON.stringify(metadata.input);
+        attributes['langfuse.trace.input'] = redactSecrets(JSON.stringify(metadata.input));
     }
     return attributes;
 }

package/dist/src/playwright-recorder.d.ts

@@ -0,0 +1,31 @@
+export interface TraceCall {
+    class: string;
+    method: string;
+    params: Record<string, any>;
+}
+export interface VerificationStep {
+    name: string;
+    args: any[];
+}
+export declare class PlaywrightRecorder {
+    context: any;
+    tracing: any;
+    active: boolean;
+    nextGroupId: number;
+    verifications: VerificationStep[];
+    recordVerification(steps: VerificationStep[]): void;
+    drainVerifications(): VerificationStep[];
+    start(browserContext: any): Promise<void>;
+    beginAction(title: string): Promise<string | null>;
+    endAction(): Promise<void>;
+    exportChunk(): Promise<Map<string, TraceCall[]>>;
+    stop(): Promise<void>;
+    isActive(): boolean;
+}
+declare function parseTrace(ndjson: string): Map<string, TraceCall[]>;
+export declare function renderCall(call: TraceCall): string;
+export declare function renderAssertion(assertion: {
+    name: string;
+    args: any[];
+}): string;
+export { parseTrace };

package/dist/src/reporter.d.ts

@@ -0,0 +1,34 @@
+import { Client } from '@testomatio/reporter';
+import type { Step } from '@testomatio/reporter/types/types.js';
+import type { ReporterConfig } from './config.js';
+import type { StateManager } from './state-manager.js';
+import { Test } from './test-plan.js';
+export type ReporterMeta = Record<string, string | undefined>;
+export interface ReporterStep {
+    title: string;
+    status: 'passed' | 'failed';
+    code: string[];
+    discovery?: string;
+}
+export declare class Reporter {
+    client: Client;
+    isRunStarted: boolean;
+    reporterEnabled: boolean;
+    stateManager?: StateManager;
+    constructor(config?: ReporterConfig, stateManager?: StateManager);
+    buildTitle(): string;
+    static resolveEnabled(config?: ReporterConfig): boolean;
+    configureHtmlPipe(): void;
+    configureMarkdownPipe(): void;
+    configureRunGroup(runGroup: string | null | undefined): void;
+    startRun(): Promise<void>;
+    reportTestStart(test: Test): Promise<void>;
+    combineStepsAndNotes(test: Test, lastScreenshotFile?: string): Step[];
+    buildVerificationStep(test: Test, lastScreenshotFile?: string): Step | undefined;
+    reportTest(test: Test, meta?: ReporterMeta): Promise<void>;
+    finishRun(): Promise<void>;
+    isEnabled(): boolean;
+    setRunDescription(text: string): Promise<void>;
+    extractLastNoteMessage(test: Test): string;
+    reportSteps(test: Test, steps: ReporterStep[]): Promise<void>;
+}

package/dist/src/state-manager.d.ts

@@ -0,0 +1,185 @@
+import { ActionResult } from './action-result.js';
+import { ExperienceTracker } from './experience-tracker.js';
+export interface Link {
+    title: string;
+    url: string;
+    visible?: boolean;
+}
+export interface WebPageState {
+    /** Unique incremental state identifier */
+    id?: number;
+    /** URL path without domain, including hash: /path/to/page#section */
+    url: string;
+    /** Page title */
+    title?: string;
+    /** HTTP status of the main document navigation */
+    httpStatus?: number;
+    /** Full URL for reference */
+    fullUrl?: string;
+    /** Timestamp when state was captured */
+    timestamp?: Date;
+    /** Hash of the state for unique identification */
+    hash?: string;
+    /** HTML file path */
+    htmlFile?: string;
+    /** Screenshot file path */
+    screenshotFile?: string;
+    /** Log file path */
+    logFile?: string;
+    /** HTML content */
+    html?: string;
+    notes?: string[];
+    /** Page headings */
+    h1?: string;
+    h2?: string;
+    h3?: string;
+    h4?: string;
+    ariaSnapshot?: string | null;
+    ariaSnapshotFile?: string;
+    links?: Link[];
+    verifications?: Record<string, boolean>;
+}
+export interface StateTransition {
+    /** Previous state (null if this is the first state) */
+    fromState: WebPageState | null;
+    /** Current state */
+    toState: WebPageState;
+    /** Code block that caused the transition */
+    codeBlock: string;
+    /** Timestamp of the transition */
+    timestamp: Date;
+    /** Any error that occurred during transition */
+    error?: string;
+    /** What triggered the state change */
+    trigger: 'manual' | 'navigation' | 'automatic';
+}
+export type StateChangeListener = (event: StateTransition) => void;
+export interface Knowledge extends WebPageState {
+    /** File path */
+    filePath: string;
+    /** Markdown content */
+    content: string;
+}
+export declare class StateManager {
+    currentState: WebPageState | null;
+    stateHistory: StateTransition[];
+    allVisitedUrls: Set<string>;
+    knowledgeCache: Knowledge[];
+    lastKnowledgeScan: Date | null;
+    stateChangeListeners: StateChangeListener[];
+    experienceTracker: ExperienceTracker;
+    knowledgeDir: string;
+    nextStateId: number;
+    constructor();
+    getExperienceTracker(): ExperienceTracker;
+    /**
+     * Subscribe to state change events
+     */
+    onStateChange(listener: StateChangeListener): () => void;
+    /**
+     * Emit state change event to all listeners
+     */
+    emitStateChange(event: StateTransition): void;
+    /**
+     * Extract state path from full URL
+     * Removes domain, port, protocol
+     * Keeps path, query, and hash: /path/to/page?tab=users#section
+     */
+    /**
+     * Update current state from ActionResult and record transition if state changed
+     */
+    updateState(actionResult: ActionResult, codeBlock?: string, trigger?: 'manual' | 'navigation' | 'automatic'): WebPageState;
+    /**
+     * Update state from basic data (for navigation events)
+     */
+    updateStateFromBasic(url: string, title?: string, trigger?: 'manual' | 'navigation' | 'automatic'): WebPageState;
+    hasDialogAppeared(previousState: WebPageState | null, newState: WebPageState): boolean;
+    /**
+     * Generate a basic hash for state comparison
+     */
+    generateBasicHash(url: string, title?: string): string;
+    /**
+     * Get current state
+     */
+    getCurrentState(): WebPageState | null;
+    /**
+     * Check if state has changed since a given state
+     */
+    hasStateChanged(previousState: WebPageState | null): boolean;
+    /**
+     * Compare two states by their hash
+     */
+    statesEqual(state1: WebPageState | null, state2: WebPageState | null): boolean;
+    /**
+     * Get state history
+     */
+    getStateHistory(): StateTransition[];
+    isInDeadLoop(): boolean;
+    /**
+     * Get the last N transitions
+     */
+    getRecentTransitions(count?: number): StateTransition[];
+    /**
+     * Scan knowledge directory for .md files and cache them
+     */
+    scanKnowledgeFiles(): void;
+    /**
+     * Get relevant knowledge files for current state
+     */
+    getRelevantKnowledge(): Knowledge[];
+    /**
+     * Get relevant experience files for current state
+     */
+    getRelevantExperience(): string[];
+    /**
+     * Get all context for current state (knowledge + experience)
+     */
+    getCurrentContext(): {
+        state: WebPageState;
+        knowledge: Knowledge[];
+        experience: string[];
+        recentTransitions: StateTransition[];
+    };
+    /**
+     * Check if we've been in this state before
+     */
+    hasVisitedState(path: string): boolean;
+    getAllVisitedUrls(): Set<string>;
+    /**
+     * Get how many times we've visited a specific path
+     */
+    getVisitCount(path: string): number;
+    /**
+     * Find the most recent transition to a specific path
+     */
+    getLastVisitToPath(path: string): StateTransition | null;
+    /**
+     * Get previous state from history for comparison.
+     * If the last transition changed URL, returns the fromState (for URL change detection).
+     * Otherwise returns the most recent toState with content (for diffing).
+     */
+    getPreviousState(): WebPageState | null;
+    /**
+     * Load HTML content from file
+     */
+    loadHtmlFromFile(htmlFile: string): string | null;
+    /**
+     * Clear state history (useful for testing or reset)
+     * Note: This preserves the current state, only clears navigation history
+     */
+    clearHistory(): void;
+    /**
+     * Get the number of active state change listeners
+     */
+    getListenerCount(): number;
+    /**
+     * Clear all state change listeners
+     */
+    clearListeners(): void;
+    /**
+     * Complete cleanup of the StateManager instance
+     * Clears all state, history, listeners, and caches
+     */
+    cleanup(): void;
+}
+export declare function normalizeUrl(url: string): string;

package/dist/src/state-manager.js

@@ -5,7 +5,6 @@ import { ActionResult } from './action-result.js';
 import { ConfigParser, outputPath } from './config.js';
 import { ExperienceTracker } from './experience-tracker.js';
 import { detectFocusArea } from './utils/aria.js';
-import { htmlTextSnapshot } from './utils/html.js';
 import { createDebug } from './utils/logger.js';
 import { extractStatePath } from './utils/url-matcher.js';
 const debugLog = createDebug('explorbot:state');
@@ -53,12 +52,6 @@ export class StateManager {
      * Emit state change event to all listeners
      */
     emitStateChange(event) {
-        // Log HTML content when state changes
-        if (event.toState.html && event.toState.html !== event.fromState?.html) {
-            let htmlContent = event?.toState?.html ?? '';
-            htmlContent = htmlTextSnapshot(htmlContent);
-            // tag('html').log(`Page HTML for ${event.toState.url}:\n${htmlContent}`);
-        }
         this.stateChangeListeners.forEach((listener) => {
             try {
                 listener(event);

package/dist/src/stats.d.ts

@@ -0,0 +1,24 @@
+interface TokenUsage {
+    input: number;
+    output: number;
+    total: number;
+    cached?: number;
+}
+export type ExplorbotMode = 'explore' | 'test' | 'freesail' | 'tui';
+export declare class Stats {
+    static startTime: number;
+    static sessionName: string;
+    static researches: number;
+    static tests: number;
+    static plans: number;
+    static mode?: ExplorbotMode;
+    static focus?: string;
+    static models: Record<string, TokenUsage>;
+    static recordTokens(_agent: string, model: string, usage: TokenUsage): void;
+    static getElapsedTime(): string;
+    static humanizeTokens(num: number): string;
+    static modelsTable(roleModels: Record<string, string>): string;
+    static hasActivity(): boolean;
+    static sessionLabel(): string;
+}
+export {};

package/dist/src/suite.d.ts

@@ -0,0 +1,24 @@
+export declare class Suite {
+    readonly url: string;
+    _automatedTests: AutomatedTest[] | null;
+    _plannedScenarios: string[] | null;
+    constructor(url: string);
+    getAutomatedTests(): AutomatedTest[];
+    getPlannedScenarios(): string[];
+    getActiveScenarioTitles(): Set<string>;
+    get automatedTestCount(): number;
+    getAutomatedTestNames(): string[];
+    getAutomatedTestFiles(): string[];
+    loadAutomatedTests(): AutomatedTest[];
+    parseTestFile(filePath: string): {
+        url: string;
+        tests: AutomatedTest[];
+    } | null;
+    loadPlannedScenarios(): string[];
+}
+interface AutomatedTest {
+    title: string;
+    pending: boolean;
+    file: string;
+}
+export {};

package/dist/src/test-plan.d.ts

@@ -0,0 +1,166 @@
+import { WebPageState } from './state-manager.js';
+export declare const TestResult: {
+    readonly PASSED: "passed";
+    readonly FAILED: "failed";
+    readonly SKIPPED: "skipped";
+};
+export type TestResultType = (typeof TestResult)[keyof typeof TestResult] | null;
+export declare const TestStatus: {
+    readonly PENDING: "pending";
+    readonly IN_PROGRESS: "in_progress";
+    readonly DONE: "done";
+};
+export type TestStatusType = (typeof TestStatus)[keyof typeof TestStatus];
+export interface Note {
+    message: string;
+    status?: TestResultType;
+    startTime: number;
+    endTime: number;
+    screenshot?: string;
+    log?: string;
+}
+export declare class ActiveNote {
+    task: Task;
+    startTime: number;
+    message: string;
+    status?: TestResultType;
+    screenshot?: string;
+    log?: string;
+    constructor(task: Task, message: string, status?: TestResultType);
+    commit(finalStatus?: TestResultType): void;
+    getMessage(): string;
+    getStartTime(): number;
+}
+export interface StepData {
+    text: string;
+    duration?: number;
+    status?: string;
+    error?: string;
+    log?: string;
+    artifacts?: string[];
+    noteStartTime?: number;
+}
+export declare class Task {
+    id: string;
+    description: string;
+    notes: Record<string, Note>;
+    steps: Record<string, StepData>;
+    states: WebPageState[];
+    startUrl: string;
+    verification?: Verification;
+    timestampCounter: number;
+    activeNote?: ActiveNote;
+    constructor(description: string, startUrl?: string);
+    startNote(message: string, status?: TestResultType): ActiveNote;
+    finishNote(activeNote: ActiveNote, endTime: number, finalStatus?: TestResultType): void;
+    getPrintableNotes(): string[];
+    notesToString(): string;
+    addNote(message: string, status?: TestResultType, screenshot?: string, log?: string): void;
+    addUrlNote(state: UrlNoteState, prevState?: {
+        title?: string;
+        h1?: string;
+        h2?: string;
+    }): void;
+    addState(state: WebPageState): void;
+    addStep(text: string, duration?: number, status?: string, error?: string, log?: string, artifacts?: string[]): void;
+    setActiveNoteScreenshot(screenshotFile?: string): void;
+    setVerification(message: string, status: TestResultType, state?: UrlNoteState): void;
+    addVerificationDetail(detail: string): void;
+    getLog(): Array<{
+        type: 'step' | 'note' | 'artifact';
+        content: string;
+        timestamp: number;
+    }>;
+    getLogString(): string;
+    getRunResult(): 'success' | 'partial' | 'failed';
+}
+export declare class Test extends Task {
+    scenario: string;
+    sessionName?: string;
+    status: TestStatusType;
+    result: TestResultType;
+    priority: 'critical' | 'important' | 'high' | 'normal' | 'low';
+    expected: string[];
+    plannedSteps: string[];
+    plan?: Plan;
+    summary: string;
+    artifacts: Record<string, string>;
+    generatedCode?: string;
+    style?: string;
+    planIteration: number;
+    enabled: boolean;
+    startTime?: number;
+    endTime?: number;
+    resetCount: number;
+    constructor(scenario: string, priority: 'critical' | 'important' | 'high' | 'normal' | 'low', expectedOutcome: string | string[], startUrl: string, plannedSteps?: string[]);
+    getVisitedUrls({ localOnly }?: {
+        localOnly?: boolean;
+    }): string[];
+    addArtifact(artifact?: string): void;
+    get hasFinished(): boolean;
+    get isSuccessful(): boolean;
+    get hasFailed(): boolean;
+    get isSkipped(): boolean;
+    getCheckedNotes(): Note[];
+    getCheckedExpectations(): string[];
+    hasAchievedAny(): boolean;
+    getRunResult(): 'success' | 'partial' | 'failed';
+    hasAchievedAll(): boolean;
+    isComplete(): boolean;
+    start(): void;
+    finish(result?: TestResultType): void;
+    getDurationMs(): number | null;
+    getRemainingExpectations(): string[];
+    getLog(): Array<{
+        type: 'step' | 'note' | 'artifact';
+        content: string;
+        timestamp: number;
+    }>;
+}
+type PlanChangeListener = (tests: Test[]) => void;
+export declare class Plan {
+    title: string;
+    tests: Test[];
+    url?: string;
+    iteration: number;
+    parentPlan?: Plan;
+    changeListeners: PlanChangeListener[];
+    constructor(title: string);
+    nextIteration(): void;
+    addTest(test: Test): void;
+    removeTest(test: Test): void;
+    onTestsChange(listener: PlanChangeListener): () => void;
+    notifyChange(): void;
+    getAllTests(): Test[];
+    listTests(): Test[];
+    getPendingTests(): Test[];
+    get isComplete(): boolean;
+    get allSuccessful(): boolean;
+    get allFailed(): boolean;
+    updateStatus(): void;
+    static fromMarkdown(filePath: string): Plan;
+    saveToMarkdown(filePath: string): void;
+    static saveMultipleToMarkdown(plans: Plan[], filePath: string): void;
+    getVisitedPages(): WebPageState[];
+    merge(otherPlan: Plan): Plan;
+    toAiContext(options?: {
+        skipSteps?: boolean;
+    }): string;
+}
+interface Verification {
+    message: string;
+    status: TestResultType;
+    screenshot?: string;
+    url?: string;
+    pageLabel?: string;
+    details: string[];
+}
+interface UrlNoteState {
+    url?: string;
+    fullUrl?: string;
+    title?: string;
+    h1?: string;
+    h2?: string;
+    screenshotFile?: string;
+}
+export {};

package/dist/src/utils/aria.d.ts

@@ -0,0 +1,20 @@
+export interface FocusAreaResult {
+    detected: boolean;
+    type: 'dialog' | 'modal' | null;
+    name: string | null;
+}
+export declare const compactAriaSnapshot: (snapshot: string | null, keepNamed?: boolean) => string;
+export declare const diffAriaSnapshots: (previous: string | null, current: string | null) => string | null;
+export declare const detectFocusArea: (snapshot: string | null) => FocusAreaResult;
+export declare const collectInteractiveNodes: (snapshot: string | null) => Array<Record<string, unknown>>;
+export interface FocusedElementInfo {
+    role: string;
+    name: string;
+    value?: string;
+    attributes?: string[];
+}
+export declare function extractFocusedElement(ariaSnapshot: string | null): FocusedElementInfo | null;
+export declare function parseAriaLocator(ariaStr: string): {
+    role: string;
+    text: string;
+} | null;

package/dist/src/utils/browser-errors.d.ts

@@ -0,0 +1,8 @@
+export declare class BrowserRecoveryError extends Error {
+    originalError: unknown;
+    recovered: boolean;
+    constructor(label: string, originalError: unknown, recovered: boolean);
+}
+export declare function isFatalBrowserError(error: unknown): boolean;
+export declare function isNavigationTransitionError(error: unknown): boolean;
+export declare function browserErrorMessage(error: unknown): string;

package/dist/src/utils/cli-name.d.ts

@@ -0,0 +1 @@
+export declare function getCliName(): string;

package/dist/src/utils/code-extractor.d.ts

@@ -0,0 +1 @@
+export declare function extractCodeBlocks(aiResponse: string): string[];