explorbot 0.1.31 → 0.2.0

package/dist/src/explorer.d.ts

@@ -0,0 +1,129 @@
+import type { BrowserContextOptions } from 'playwright';
+import { ActionResult } from './action-result.js';
+import Action from './action.js';
+import { AIProvider } from './ai/provider.js';
+import { RequestStore } from './api/request-store.js';
+import { XhrCapture } from './api/xhr-capture.js';
+import type { ExplorbotConfig } from './config.js';
+import type { ExperienceTracker } from './experience-tracker.js';
+import { KnowledgeTracker } from './knowledge-tracker.js';
+import { PlaywrightRecorder } from './playwright-recorder.js';
+import { Reporter } from './reporter.js';
+import { StateManager } from './state-manager.js';
+import { Test } from './test-plan.js';
+import { WebElement } from './utils/web-element.js';
+declare global {
+    namespace NodeJS {
+        interface Global {
+            output_dir: string;
+        }
+    }
+}
+declare namespace CodeceptJS {
+    interface I {
+        [key: string]: any;
+    }
+}
+interface TabInfo {
+    url: string;
+    title: string;
+}
+interface BrowserExecutionErrorResult {
+    action: 'continue' | 'stop';
+    message: string;
+    recovered?: boolean;
+}
+declare class Explorer {
+    aiProvider: AIProvider;
+    playwrightHelper: any;
+    isStarted: boolean;
+    isSharedBrowser: boolean;
+    actor: CodeceptJS.I;
+    stateManager: StateManager;
+    knowledgeTracker: KnowledgeTracker;
+    config: ExplorbotConfig;
+    options?: {
+        show?: boolean;
+        headless?: boolean;
+        incognito?: boolean;
+        session?: string;
+    };
+    reporter: Reporter;
+    otherTabs: TabInfo[];
+    _activeTest: Test | null;
+    xhrCapture: XhrCapture | null;
+    requestStore: RequestStore | null;
+    playwrightRecorder: PlaywrightRecorder;
+    observedTestPages: Set<any>;
+    testPageErrorHandler: ((error: Error) => void) | null;
+    testConsoleHandler: ((message: any) => void) | null;
+    testDialogHandler: ((dialog: any) => void) | null;
+    constructor(config: ExplorbotConfig, aiProvider: AIProvider, options: {
+        show?: boolean;
+        headless?: boolean;
+        incognito?: boolean;
+        session?: string;
+    } | undefined, experienceTracker: ExperienceTracker, knowledgeTracker: KnowledgeTracker);
+    initializeContainer(): void;
+    convertToCodeceptConfig(config: ExplorbotConfig): any;
+    getConfig(): ExplorbotConfig;
+    getAIProvider(): AIProvider;
+    getStateManager(): StateManager;
+    getKnowledgeTracker(): KnowledgeTracker;
+    getReporter(): Reporter;
+    getRequestStore(): RequestStore | null;
+    extractCookies(): Promise<Record<string, string>>;
+    setupXhrCapture(reuseRequestStore?: boolean): void;
+    start(): Promise<any>;
+    connectOrLaunchBrowser(): Promise<void>;
+    createBrowserContextOptions(): BrowserContextOptions;
+    createAction(): Action;
+    runWithBrowserRecovery<T>(label: string, operation: () => Promise<T>): Promise<T>;
+    capturePageState(opts?: {
+        includeScreenshot?: boolean;
+    }): Promise<ActionResult>;
+    capturePageWithScreenshot(): Promise<ActionResult>;
+    executeAction(code: string): Promise<Action>;
+    attemptAction(code: string, originalMessage?: string): Promise<boolean>;
+    getPlaywrightRecorder(): PlaywrightRecorder;
+    visit(url: string): Promise<Action>;
+    visitOnce(url: string): Promise<Action>;
+    annotateElements(): Promise<WebElement[]>;
+    visuallyAnnotateElements(opts?: {
+        containers?: Array<{
+            css: string;
+            label: string;
+        }>;
+    }): Promise<number>;
+    getEidxInContainer(containerCss: string | null): Promise<string[]>;
+    getEidxByLocator(locator: string, container?: string | null): Promise<string | null>;
+    resolveBrowserUrl(url?: string): string | null;
+    isFatalBrowserError(error: unknown): boolean;
+    recoverFromBrowserError(): Promise<boolean>;
+    restartBrowser(): Promise<boolean>;
+    switchToMainFrame(): Promise<void>;
+    waitForPageReadiness(): Promise<boolean>;
+    isInsideIframe(): Promise<boolean>;
+    getCurrentIframeInfo(): string | null;
+    hasOtherTabs(): boolean;
+    getOtherTabsInfo(): TabInfo[];
+    clearOtherTabsInfo(): void;
+    listenToStateChanged(): void;
+    stop(): Promise<void>;
+    get activeTest(): Test | null;
+    startTest(test: Test): Promise<boolean>;
+    ensurePageAvailable(): Promise<boolean>;
+    handleExecutionError(error: unknown): Promise<BrowserExecutionErrorResult>;
+    watchActiveTestPage(page?: any): void;
+    stopTest(test: Test, meta?: Record<string, string>): Promise<void>;
+    unwatchActiveTestPages(): void;
+    playwrightLocatorCount(locatorFn: (page: any) => any): Promise<number>;
+    bindFrameNavigated(page: any): void;
+    openFreshTab(): Promise<void>;
+    closeOtherTabs(): Promise<void>;
+}
+export declare function annotatePageElements(page: any): Promise<{
+    ariaSnapshot: string;
+    elements: WebElement[];
+}>;
+export default Explorer;

package/dist/src/explorer.js

@@ -12,15 +12,14 @@ import { visuallyAnnotateContainers } from "./ai/researcher/coordinates.js";
 import { RequestStore } from "./api/request-store.js";
 import { XhrCapture } from "./api/xhr-capture.js";
 import { ConfigParser } from './config.js';
-import { KnowledgeTracker } from './knowledge-tracker.js';
 import { PlaywrightRecorder } from "./playwright-recorder.js";
 import { Reporter } from "./reporter.js";
 import { StateManager } from './state-manager.js';
 import { TestResult } from "./test-plan.js";
-import { BrowserRecoveryError, isFatalBrowserError, isNavigationTransitionError } from "./utils/browser-errors.js";
+import { BrowserRecoveryError, browserErrorMessage, isFatalBrowserError, isNavigationTransitionError } from "./utils/browser-errors.js";
 import { ELEMENT_EXTRACTION_CONFIG, getElementDataExtractorSource } from "./utils/html.js";
 import { createDebug, log, tag } from './utils/logger.js';
-import { waitForPageReadiness } from "./utils/page-readiness.js";
+import { sleep, waitForPageReadiness } from "./utils/page-readiness.js";
 import { WebElement } from "./utils/web-element.js";
 const debugLog = createDebug('explorbot:explorer');
 const RECOVERABLE_NAVIGATION_ERRORS = /net::ERR_ABORTED|page\.screenshot.*Timeout|waiting for fonts to load/i;
@@ -33,7 +32,6 @@ class Explorer {
     stateManager;
     knowledgeTracker;
     config;
-    userResolveFn = null;
     options;
     reporter;
     otherTabs = [];
@@ -45,21 +43,20 @@ class Explorer {
     testPageErrorHandler = null;
     testConsoleHandler = null;
     testDialogHandler = null;
-    constructor(config, aiProvider, options) {
+    constructor(config, aiProvider, options, experienceTracker, knowledgeTracker) {
         this.config = config;
         this.aiProvider = aiProvider;
         this.options = options;
         this.initializeContainer();
-        this.stateManager = new StateManager({ incognito: this.options?.incognito });
-        this.knowledgeTracker = new KnowledgeTracker();
+        this.knowledgeTracker = knowledgeTracker;
+        this.stateManager = new StateManager(experienceTracker, knowledgeTracker);
         this.reporter = new Reporter(config.reporter, this.stateManager);
     }
     initializeContainer() {
         try {
             // Use project root for output directory, not current working directory
             const configParser = ConfigParser.getInstance();
-            const configPath = configParser.getConfigPath();
-            const projectRoot = configPath ? path.dirname(configPath) : process.cwd();
+            const projectRoot = configParser.getProjectRoot();
             global.output_dir = path.join(projectRoot, 'output', 'states');
             global.codecept_dir = projectRoot;
             configParser.validateConfig(this.config);
@@ -113,31 +110,20 @@ class Explorer {
             },
         };
         if (this.config.stepsFile) {
-            const configPath = ConfigParser.getInstance().getConfigPath();
-            const projectRoot = configPath ? path.dirname(configPath) : process.cwd();
+            const projectRoot = ConfigParser.getInstance().getProjectRoot();
             codeceptConfig.include = { I: path.resolve(projectRoot, this.config.stepsFile) };
         }
         return codeceptConfig;
     }
     getConfig() {
-        if (!this.config) {
-            throw new Error('Configuration not loaded. Call run() first.');
-        }
         return this.config;
     }
-    getConfigPath() {
-        const configParser = ConfigParser.getInstance();
-        return configParser.getConfigPath();
-    }
     getAIProvider() {
         return this.aiProvider;
     }
     getStateManager() {
         return this.stateManager;
     }
-    getCurrentUrl() {
-        return this.stateManager.getCurrentState().url || '?';
-    }
     getKnowledgeTracker() {
         return this.knowledgeTracker;
     }
@@ -175,9 +161,6 @@ class Explorer {
         if (this.isStarted) {
             return;
         }
-        if (!this.config) {
-            await this.initializeContainer();
-        }
         await codeceptjs.recorder.start();
         await codeceptjs.container.started(null);
         storeListener();
@@ -294,8 +277,8 @@ class Explorer {
     async executeAction(code) {
         return this.runWithBrowserRecovery('executeAction', () => this.createAction().execute(code));
     }
-    async attemptAction(code, originalMessage, experience = true) {
-        return this.runWithBrowserRecovery('attemptAction', () => this.createAction().attempt(code, originalMessage, experience));
+    async attemptAction(code, originalMessage) {
+        return this.runWithBrowserRecovery('attemptAction', () => this.createAction().attempt(code, originalMessage));
     }
     getPlaywrightRecorder() {
         return this.playwrightRecorder;
@@ -318,7 +301,7 @@ class Explorer {
                 await action.execute(`I.amOnPage(${serializedUrl})`);
             }
             catch (err) {
-                const msg = err instanceof Error ? err.message : String(err);
+                const msg = browserErrorMessage(err);
                 if (!RECOVERABLE_NAVIGATION_ERRORS.test(msg))
                     throw err;
                 tag('warning').log(`Navigation warning (continuing after load): ${msg.split('\n')[0]}`);
@@ -363,7 +346,7 @@ class Explorer {
         }
         catch (error) {
             if (this.isFatalBrowserError(error)) {
-                tag('warning').log(`getEidxInContainer: ${error instanceof Error ? error.message : error}`);
+                tag('warning').log(`getEidxInContainer: ${browserErrorMessage(error)}`);
                 await this.recoverFromBrowserError();
             }
             return [];
@@ -378,16 +361,12 @@ class Explorer {
         }
         catch (error) {
             if (this.isFatalBrowserError(error)) {
-                tag('warning').log(`getEidxByLocator: ${error instanceof Error ? error.message : error}`);
+                tag('warning').log(`getEidxByLocator: ${browserErrorMessage(error)}`);
                 await this.recoverFromBrowserError();
             }
             return null;
         }
     }
-    async reload() {
-        await this.closeOtherTabs();
-        await this.playwrightHelper.page.reload();
-    }
     resolveBrowserUrl(url) {
         if (!url)
             return null;
@@ -433,7 +412,7 @@ class Explorer {
             return this.waitForPageReadiness();
         }
         catch (err) {
-            tag('error').log(`Browser recovery failed: ${err instanceof Error ? err.message : err}`);
+            tag('error').log(`Browser recovery failed: ${browserErrorMessage(err)}`);
             return false;
         }
     }
@@ -471,7 +450,7 @@ class Explorer {
             return true;
         }
         catch (err) {
-            tag('error').log(`Browser restart failed: ${err instanceof Error ? err.message : err}`);
+            tag('error').log(`Browser restart failed: ${browserErrorMessage(err)}`);
             return false;
         }
     }
@@ -502,7 +481,7 @@ class Explorer {
         }
         catch (error) {
             if (this.isFatalBrowserError(error)) {
-                tag('warning').log(`isInsideIframe: ${error instanceof Error ? error.message : error}`);
+                tag('warning').log(`isInsideIframe: ${browserErrorMessage(error)}`);
                 await this.recoverFromBrowserError();
             }
             return false;
@@ -522,9 +501,6 @@ class Explorer {
     clearOtherTabsInfo() {
         this.otherTabs = [];
     }
-    setUserResolve(userResolveFn) {
-        this.userResolveFn = userResolveFn;
-    }
     listenToStateChanged() {
         if (!this.playwrightHelper) {
             debugLog('Playwright helper not available for state monitoring');
@@ -629,17 +605,21 @@ class Explorer {
             test.setActiveNoteScreenshot(lastScreenshot);
         };
         this.watchActiveTestPage();
-        codeceptjs.event.dispatcher.emit('test.before', codeceptjsTest);
-        codeceptjs.event.dispatcher.emit('test.start', codeceptjsTest);
-        codeceptjs.event.dispatcher.on('step.passed', (step) => stepHandler(step, 'passed'));
-        codeceptjs.event.dispatcher.on('step.failed', (step, error) => {
+        const onStepPassed = (step) => stepHandler(step, 'passed');
+        const onStepFailed = (step, error) => {
             stepHandler(step, 'failed', error?.message || String(error), error?.stack);
-        });
-        codeceptjs.event.dispatcher.on('test.after', () => {
-            codeceptjs.event.dispatcher.off('step.passed', stepHandler);
-            codeceptjs.event.dispatcher.off('step.failed', stepHandler);
+        };
+        const onTestAfter = () => {
+            codeceptjs.event.dispatcher.off('step.passed', onStepPassed);
+            codeceptjs.event.dispatcher.off('step.failed', onStepFailed);
+            codeceptjs.event.dispatcher.off('test.after', onTestAfter);
             this.unwatchActiveTestPages();
-        });
+        };
+        codeceptjs.event.dispatcher.emit('test.before', codeceptjsTest);
+        codeceptjs.event.dispatcher.emit('test.start', codeceptjsTest);
+        codeceptjs.event.dispatcher.on('step.passed', onStepPassed);
+        codeceptjs.event.dispatcher.on('step.failed', onStepFailed);
+        codeceptjs.event.dispatcher.on('test.after', onTestAfter);
         return true;
     }
     async ensurePageAvailable() {
@@ -654,11 +634,8 @@ class Explorer {
         this.watchActiveTestPage();
         return true;
     }
-    async ensureActiveTestPageAvailable() {
-        return this.ensurePageAvailable();
-    }
     async handleExecutionError(error) {
-        const message = error instanceof Error ? error.message : String(error);
+        const message = browserErrorMessage(error);
         tag('error').log(`Browser execution error: ${message}`);
         if (error instanceof Error && error.name === 'AbortError') {
             return {
@@ -762,29 +739,6 @@ class Explorer {
         }
         this.observedTestPages.clear();
     }
-    async hasPlaywrightLocator(locatorFn, opts = {}) {
-        try {
-            const pwLocator = locatorFn(this.playwrightHelper.page);
-            const count = await pwLocator.count();
-            if (opts.multiple ? count === 0 : count !== 1)
-                return false;
-            if (opts.contents) {
-                const html = await pwLocator.first().innerHTML();
-                if (!html?.trim())
-                    return false;
-            }
-            if (opts.success)
-                await opts.success(pwLocator);
-            return true;
-        }
-        catch (error) {
-            if (this.isFatalBrowserError(error)) {
-                tag('warning').log(`hasPlaywrightLocator: ${error instanceof Error ? error.message : error}`);
-                await this.recoverFromBrowserError();
-            }
-            return false;
-        }
-    }
     async playwrightLocatorCount(locatorFn) {
         try {
             const pwLocator = locatorFn(this.playwrightHelper.page);
@@ -792,7 +746,7 @@ class Explorer {
         }
         catch (error) {
             if (this.isFatalBrowserError(error)) {
-                tag('warning').log(`playwrightLocatorCount: ${error instanceof Error ? error.message : error}`);
+                tag('warning').log(`playwrightLocatorCount: ${browserErrorMessage(error)}`);
                 await this.recoverFromBrowserError();
             }
             throw error;
@@ -811,7 +765,7 @@ class Explorer {
                 debugLog('Failed to get page title:', error);
             }
             this.stateManager.updateStateFromBasic(newUrl, newTitle, 'navigation');
-            await new Promise((resolve) => setTimeout(resolve, 500));
+            await sleep(500);
         });
     }
     async openFreshTab() {

package/dist/src/index.d.ts

@@ -0,0 +1,6 @@
+export { ExplorBot } from './explorbot.js';
+export type { ExplorBotOptions, UserResolveFunction } from './explorbot.js';
+export { Plan, Test, TestResult, TestStatus } from './test-plan.js';
+export type { Note, StepData, TestResultType, TestStatusType } from './test-plan.js';
+export type { Link, StateTransition, WebPageState } from './state-manager.js';
+export type { AIConfig, ExplorbotConfig, PlaywrightConfig } from './config.js';

package/dist/src/index.js

@@ -1,55 +1,2 @@
-#!/usr/bin/env node
-import { Command } from 'commander';
-import { addKnowledgeCommand } from './commands/add-knowledge.js';
-import { cleanCommand } from './commands/clean.js';
-import { exploreCommand } from './commands/explore.js';
-import { initCommand } from './commands/init.js';
-const program = new Command();
-program
-    .name('explorbot')
-    .description('AI-powered web exploration tool')
-    .option('-f, --from <url>', 'Start exploration from a specific URL')
-    .option('-v, --verbose', 'Enable verbose logging')
-    .option('--debug', 'Enable debug logging (same as --verbose)')
-    .option('-c, --config <path>', 'Path to configuration file')
-    .option('-p, --path <path>', 'Working directory path')
-    .helpOption('-h, --help', 'Show this help message');
-program
-    .command('clean')
-    .description('Clean up artifacts or experience folders')
-    .option('-t, --type <type>', 'Type of cleanup (artifacts|experience|all)', 'artifacts')
-    .option('-p, --path <path>', 'Custom path to clean')
-    .action(async (options, command) => {
-    const globalOptions = command.parent.opts();
-    const allOptions = { ...globalOptions, ...options };
-    await cleanCommand(allOptions);
-});
-program
-    .command('init')
-    .description('Initialize a new project with configuration')
-    .option('-c, --config-path <path>', 'Path for the config file', './explorbot.config.js')
-    .option('-f, --force', 'Overwrite existing config file', false)
-    .option('-p, --path <path>', 'Working directory for initialization')
-    .action(async (options, command) => {
-    const globalOptions = command.parent.opts();
-    const allOptions = { ...globalOptions, ...options };
-    await initCommand(allOptions);
-});
-program
-    .command('add-knowledge')
-    .alias('learn')
-    .description('Add knowledge for specific URLs')
-    .option('-p, --path <path>', 'Knowledge directory path')
-    .action(async (options) => {
-    await addKnowledgeCommand(options);
-});
-program.parse();
-const options = program.opts();
-const command = program.args[0];
-if (command === 'clean' || command === 'init' || command === 'add-knowledge' || command === 'learn') {
-    // These commands are handled by their respective actions
-}
-else {
-    // Default to explore command
-    exploreCommand(options);
-}
+export { ExplorBot } from "./explorbot.js";
+export { Plan, Test, TestResult, TestStatus } from "./test-plan.js";

package/dist/src/knowledge-tracker.d.ts

@@ -0,0 +1,33 @@
+import { ActionResult } from './action-result.js';
+export interface Knowledge {
+    filePath: string;
+    url: string;
+    content: string;
+    [key: string]: any;
+}
+export declare class KnowledgeTracker {
+    knowledgeDir: string;
+    knowledgeFiles: Knowledge[];
+    isLoaded: boolean;
+    constructor();
+    loadKnowledgeFiles(): void;
+    getRelevantKnowledge(state: ActionResult): Knowledge[];
+    renderRelevantKnowledge(state: ActionResult): string;
+    addKnowledge(urlPattern: string, description: string): {
+        filename: string;
+        filePath: string;
+        isNewFile: boolean;
+    };
+    interpolateVars(content: string): string;
+    generateFilename(url: string): string;
+    getExistingUrls(): string[];
+    getKnowledgeForUrl(urlPattern: string): string[];
+    listAllKnowledge(): Array<{
+        url: string;
+        firstLine: string;
+        filePath: string;
+    }>;
+    getMatchingKnowledge(url: string): Knowledge[];
+    normalizeUrl(url: string): string;
+    getStateParameters(state: ActionResult, keys: string[]): Record<string, any>;
+}

package/dist/src/knowledge-tracker.js

@@ -1,10 +1,15 @@
-import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
-import { dirname, join } from 'node:path';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import dedent from 'dedent';
 import matter from 'gray-matter';
 import { ActionResult } from './action-result.js';
 import { ConfigParser } from './config.js';
 import { getCliName } from "./utils/cli-name.js";
-import { createDebug } from './utils/logger.js';
+import { createDebug, pluralize, tag } from './utils/logger.js';
+import { loadMarkdownFiles } from './utils/markdown-files.js';
+import { mdq } from './utils/markdown-query.js';
+import { isSecretName, registerSecret } from './utils/secrets.js';
+import { slugify } from './utils/strings.js';
 const debugLog = createDebug('explorbot:knowledge-tracker');
 export class KnowledgeTracker {
     knowledgeDir;
@@ -13,14 +18,7 @@ export class KnowledgeTracker {
     constructor() {
         const configParser = ConfigParser.getInstance();
         const config = configParser.getConfig();
-        const configPath = configParser.getConfigPath();
-        if (configPath) {
-            const projectRoot = dirname(configPath);
-            this.knowledgeDir = join(projectRoot, config.dirs?.knowledge || 'knowledge');
-        }
-        else {
-            this.knowledgeDir = config.dirs?.knowledge || 'knowledge';
-        }
+        this.knowledgeDir = configParser.resolveProjectDir(config.dirs?.knowledge || 'knowledge');
         if (!existsSync(this.knowledgeDir)) {
             mkdirSync(this.knowledgeDir, { recursive: true });
         }
@@ -29,27 +27,13 @@ export class KnowledgeTracker {
         if (this.isLoaded)
             return;
         this.knowledgeFiles = [];
-        if (!existsSync(this.knowledgeDir)) {
-            return;
-        }
-        const files = readdirSync(this.knowledgeDir)
-            .filter((file) => file.endsWith('.md'))
-            .map((file) => join(this.knowledgeDir, file));
-        for (const filePath of files) {
-            try {
-                const fileContent = readFileSync(filePath, 'utf8');
-                const parsed = matter(fileContent);
-                const urlPattern = parsed.data.url || parsed.data.path || '*';
-                this.knowledgeFiles.push({
-                    filePath,
-                    url: urlPattern,
-                    content: this.interpolateVars(parsed.content),
-                    ...parsed.data,
-                });
-            }
-            catch (error) {
-                // Skip invalid files
-            }
+        for (const entry of loadMarkdownFiles(this.knowledgeDir, { recursive: true })) {
+            this.knowledgeFiles.push({
+                filePath: entry.filePath,
+                url: entry.data.url || entry.data.path || '*',
+                content: this.interpolateVars(entry.content),
+                ...entry.data,
+            });
         }
         this.isLoaded = true;
     }
@@ -59,21 +43,35 @@ export class KnowledgeTracker {
             return state.isMatchedBy(knowledge);
         });
     }
+    renderRelevantKnowledge(state) {
+        const knowledgeFiles = this.getRelevantKnowledge(state);
+        if (knowledgeFiles.length === 0)
+            return '';
+        const knowledgeContent = knowledgeFiles
+            .map((k) => k.content)
+            .filter((k) => !!k)
+            .join('\n\n');
+        tag('operation').log(`Found ${knowledgeFiles.length} relevant knowledge ${pluralize(knowledgeFiles.length, 'file')}`);
+        return dedent `
+      <knowledge>
+      Here is relevant knowledge for this page:
+
+      ${knowledgeContent}
+      </knowledge>
+    `;
+    }
     addKnowledge(urlPattern, description) {
         const configParser = ConfigParser.getInstance();
-        const config = configParser.getConfig();
         const configPath = configParser.getConfigPath();
         if (!configPath) {
             throw new Error(`No explorbot configuration found. Please run "${getCliName()} init" first.`);
         }
-        const projectRoot = dirname(configPath);
-        const knowledgeDir = join(projectRoot, config.dirs?.knowledge || 'knowledge');
-        if (!existsSync(knowledgeDir)) {
-            mkdirSync(knowledgeDir, { recursive: true });
+        if (!existsSync(this.knowledgeDir)) {
+            mkdirSync(this.knowledgeDir, { recursive: true });
         }
         const normalizedUrl = this.normalizeUrl(urlPattern);
         const filename = this.generateFilename(normalizedUrl);
-        const filePath = join(knowledgeDir, filename);
+        const filePath = join(this.knowledgeDir, filename);
         const isNewFile = !existsSync(filePath);
         if (isNewFile) {
             const frontmatter = {
@@ -100,6 +98,7 @@ export class KnowledgeTracker {
             const fileContent = matter.stringify(newContent, frontmatter);
             writeFileSync(filePath, fileContent, 'utf8');
         }
+        this.isLoaded = false;
         return { filename, filePath, isNewFile };
     }
     interpolateVars(content) {
@@ -109,9 +108,15 @@ export class KnowledgeTracker {
                 return match;
             const namespace = expr.slice(0, dotIndex);
             const key = expr.slice(dotIndex + 1);
-            if (namespace === 'env')
-                return process.env[key] ?? '';
+            if (namespace === 'env') {
+                const value = process.env[key] ?? '';
+                if (isSecretName(key))
+                    registerSecret(value);
+                return value;
+            }
             if (namespace === 'config') {
+                if (isSecretName(key))
+                    return '';
                 const config = ConfigParser.getInstance().getConfig();
                 const value = key.split('.').reduce((obj, k) => obj?.[k], config);
                 if (value !== undefined && typeof value !== 'object')
@@ -122,12 +127,7 @@ export class KnowledgeTracker {
         });
     }
     generateFilename(url) {
-        let filename = url
-            .replace(/https?:\/\//g, '')
-            .replace(/[^a-zA-Z0-9_]/g, '_')
-            .replace(/_+/g, '_')
-            .replace(/^_|_$/g, '')
-            .toLowerCase();
+        let filename = slugify(url.replace(/https?:\/\//g, ''));
         if (!filename || filename === '*') {
             filename = 'general';
         }
@@ -149,7 +149,7 @@ export class KnowledgeTracker {
         this.loadKnowledgeFiles();
         return this.knowledgeFiles.map((knowledge) => {
             const content = knowledge.content.trim();
-            const firstLine = content.split('\n')[0]?.trim() || '';
+            const firstLine = mdq(content).meta()[0]?.text.split('\n')[0]?.trim() || '';
             return {
                 url: knowledge.url,
                 firstLine,

package/dist/src/observability.d.ts

@@ -0,0 +1,13 @@
+import { type Span } from '@opentelemetry/api';
+type TelemetryMetadata = Record<string, unknown>;
+export declare const Observability: {
+    run<T>(name: string, metadata: TelemetryMetadata, fn: () => Promise<T>): Promise<T>;
+    getTelemetry(): {
+        isEnabled: boolean;
+        functionId: string;
+        metadata: Record<string, unknown>;
+    };
+    isTracing(): boolean;
+    getSpan(): Span;
+};
+export {};