@promptbook/remote-server 0.105.0-21 → 0.105.0-23

package/esm/index.es.js

@@ -9,13 +9,16 @@ import http from 'http';
 import { Server } from 'socket.io';
 import swaggerUi from 'swagger-ui-express';
 import { randomBytes } from 'crypto';
-import { stat, access, constants, readFile, writeFile, readdir, mkdir, watch } from 'fs/promises';
+import { stat, access, constants, readFile, writeFile, readdir, mkdir, watch, rm } from 'fs/promises';
 import { Subject } from 'rxjs';
 import hexEncoder from 'crypto-js/enc-hex';
 import sha256 from 'crypto-js/sha256';
 import { SHA256 } from 'crypto-js';
 import { lookup, extension } from 'mime-types';
 import { parse, unparse } from 'papaparse';
+import { Readability } from '@mozilla/readability';
+import { JSDOM } from 'jsdom';
+import { Converter } from 'showdown';
 import moment from 'moment';
 import { createElement } from 'react';
 import { renderToStaticMarkup } from 'react-dom/server';
@@ -34,7 +37,7 @@ const BOOK_LANGUAGE_VERSION = '2.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-const PROMPTBOOK_ENGINE_VERSION = '0.105.0-21';
+const PROMPTBOOK_ENGINE_VERSION = '0.105.0-23';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -1918,6 +1921,7 @@ function $provideFilesystemForNode(options) {
 }
 /**
  * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ * TODO: [🏓] Unite `xxxForServer` and `xxxForNode` naming
  */
 
 /**
@@ -2104,6 +2108,7 @@ async function $provideExecutablesForNode(options) {
 /**
  * TODO: [🧠] Allow to override the executables without need to call `locatePandoc` / `locateLibreoffice` in case of provided
  * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ * TODO: [🏓] Unite `xxxForServer` and `xxxForNode` naming
  */
 
 /**
@@ -7977,6 +7982,7 @@ async function $provideScrapersForNode(tools, options) {
 }
 /**
  * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ * TODO: [🏓] Unite `xxxForServer` and `xxxForNode` naming
  */
 
 /**
@@ -8275,6 +8281,536 @@ function isValidAgentUrl(url) {
  * TODO: [🐠] Maybe more info why the URL is invalid
  */
 
+/**
+ * Retrieves an intermediate source for a scraper based on the knowledge source.
+ * Manages the caching and retrieval of intermediate scraper results for optimized performance.
+ *
+ * @private as internal utility for scrapers
+ */
+async function getScraperIntermediateSource(source, options) {
+    const { filename: sourceFilename, url } = source;
+    const { rootDirname, cacheDirname, intermediateFilesStrategy, extension, isVerbose } = options;
+    // TODO: [👬] DRY
+    const hash = SHA256(
+    //    <- TODO: [🥬] Encapsulate sha256 to some private utility function
+    hexEncoder.parse(sourceFilename || url || 'untitled'))
+        .toString( /* hex */)
+        .substring(0, 20);
+    //    <- TODO: [🥬] Make some system for hashes and ids of promptbook
+    const semanticName = normalizeToKebabCase(titleToName((sourceFilename || url || '').split('intermediate').join(''))).substring(0, 20);
+    // <- TODO: [🐱‍🐉]
+    const pieces = ['intermediate', semanticName, hash].filter((piece) => piece !== '');
+    const name = pieces.join('-').split('--').join('-');
+    const cacheFilename = join(process.cwd(), cacheDirname, ...nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */), name)
+        .split('\\')
+        .join('/') +
+        '.' +
+        extension;
+    // Note: Try to create cache directory, but don't fail if filesystem has issues
+    try {
+        await mkdir(dirname(cacheFilename), { recursive: true });
+    }
+    catch (error) {
+        // Note: If we can't create cache directory, continue without it
+        //       This handles read-only filesystems, permission issues, and missing parent directories
+        if (error instanceof Error &&
+            (error.message.includes('EROFS') ||
+                error.message.includes('read-only') ||
+                error.message.includes('EACCES') ||
+                error.message.includes('EPERM') ||
+                error.message.includes('ENOENT'))) ;
+        else {
+            // Re-throw other unexpected errors
+            throw error;
+        }
+    }
+    let isDestroyed = true;
+    const fileHandler = {
+        filename: cacheFilename,
+        get isDestroyed() {
+            return isDestroyed;
+        },
+        async destroy() {
+            if (intermediateFilesStrategy === 'HIDE_AND_CLEAN') {
+                if (isVerbose) {
+                    console.info('legacyDocumentScraper: Clening cache');
+                }
+                await rm(cacheFilename);
+                // TODO: [🐿][🧠] Maybe remove empty folders
+            }
+            isDestroyed = true;
+        },
+    };
+    return fileHandler;
+}
+/**
+ * Note: Not using `FileCacheStorage` for two reasons:
+ * 1) Need to store more than serialized JSONs
+ * 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: [😡]
+ * TODO: [🐱‍🐉][🧠] Make some smart crop
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ */
+
+/**
+ * Metadata of the scraper
+ *
+ * @private within the scraper directory
+ */
+const markdownScraperMetadata = $deepFreeze({
+    title: 'Markdown scraper',
+    packageName: '@promptbook/markdown-utils',
+    className: 'MarkdownScraper',
+    mimeTypes: ['text/markdown', 'text/plain'],
+    documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
+    isAvailableInBrowser: true,
+    // <- Note: [🌏] This is the only scraper which makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server
+    requiredExecutables: [],
+}); /* <- Note: [🤛] */
+/**
+ * Registration of known scraper metadata
+ *
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
+ *
+ * @public exported from `@promptbook/core`
+ * @public exported from `@promptbook/wizard`
+ * @public exported from `@promptbook/cli`
+ */
+$scrapersMetadataRegister.register(markdownScraperMetadata);
+/**
+ * Note: [💞] Ignore a discrepancy between file name and entity name
+ */
+
+/**
+ * Scraper for markdown files
+ *
+ * @see `documentationUrl` for more details
+ * @public exported from `@promptbook/markdown-utils`
+ */
+class MarkdownScraper {
+    /**
+     * Metadata of the scraper which includes title, mime types, etc.
+     */
+    get metadata() {
+        return markdownScraperMetadata;
+    }
+    constructor(tools, options) {
+        this.tools = tools;
+        this.options = options;
+    }
+    /**
+     * Scrapes the markdown file and returns the knowledge pieces or `null` if it can't scrape it
+     */
+    async scrape(source) {
+        const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT, isVerbose = DEFAULT_IS_VERBOSE } = this.options;
+        const { llm } = this.tools;
+        if (llm === undefined) {
+            throw new MissingToolsError('LLM tools are required for scraping external files');
+            // <- Note: This scraper is used in all other scrapers, so saying "external files" not "markdown files"
+        }
+        const llmTools = getSingleLlmExecutionTools(llm);
+        // TODO: [🌼] In future use `ptbk make` and made getPipelineCollection
+        const collection = createPipelineCollectionFromJson(...PipelineCollection);
+        const prepareKnowledgeFromMarkdownExecutor = createPipelineExecutor({
+            pipeline: await collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book'),
+            tools: {
+                llm: llm,
+            },
+        });
+        const prepareTitleExecutor = createPipelineExecutor({
+            pipeline: await collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-title.book'),
+            tools: {
+                llm: llm,
+            },
+        });
+        const prepareKeywordsExecutor = createPipelineExecutor({
+            pipeline: await collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-keywords.book'),
+            tools: {
+                llm: llm,
+            },
+        });
+        const knowledgeContent = await source.asText();
+        const result = await prepareKnowledgeFromMarkdownExecutor({ knowledgeContent }).asPromise({
+            isCrashedOnError: true,
+        });
+        const { outputParameters } = result;
+        const { knowledgePieces: knowledgePiecesRaw } = outputParameters;
+        const knowledgeTextPieces = (knowledgePiecesRaw || '').split('\n---\n');
+        //                                                               <- TODO: [main] Smarter split and filter out empty pieces
+        if (isVerbose) {
+            console.info('knowledgeTextPieces:', knowledgeTextPieces);
+        }
+        // const usage = ;
+        const knowledge = await Promise.all(
+        // TODO: [🪂] Do not send all at once but in chunks
+        knowledgeTextPieces.map(async (knowledgeTextPiece, i) => {
+            // Note: These are just default values, they will be overwritten by the actual values:
+            let name = `piece-${i}`;
+            let title = spaceTrim$2(knowledgeTextPiece.substring(0, 100));
+            const knowledgePieceContent = spaceTrim$2(knowledgeTextPiece);
+            let keywords = [];
+            const index = [];
+            /*
+          TODO: [☀] Track line and column of the source
+          const sources: KnowledgePiecePreparedJson['sources'] = [
+          ];
+          */
+            try {
+                const titleResult = await prepareTitleExecutor({ knowledgePieceContent }).asPromise({
+                    isCrashedOnError: true,
+                });
+                const { title: titleRaw = 'Untitled' } = titleResult.outputParameters;
+                title = spaceTrim$2(titleRaw) /* <- TODO: Maybe do in pipeline */;
+                name = titleToName(title);
+                // --- Keywords
+                const keywordsResult = await prepareKeywordsExecutor({ knowledgePieceContent }).asPromise({
+                    isCrashedOnError: true,
+                });
+                const { keywords: keywordsRaw = '' } = keywordsResult.outputParameters;
+                keywords = (keywordsRaw || '')
+                    .split(',')
+                    .map((keyword) => keyword.trim())
+                    .filter((keyword) => keyword !== '');
+                if (isVerbose) {
+                    console.info(`Keywords for "${title}":`, keywords);
+                }
+                // ---
+                if (!llmTools.callEmbeddingModel) {
+                    // TODO: [🟥] Detect browser / node and make it colorful
+                    console.error('No callEmbeddingModel function provided');
+                }
+                else {
+                    // TODO: [🧠][🎛] Embedding via multiple models
+                    const embeddingResult = await llmTools.callEmbeddingModel({
+                        title: `Embedding for ${title}` /* <- Note: No impact on embedding result itself, just for logging */,
+                        parameters: {},
+                        content: knowledgePieceContent,
+                        modelRequirements: {
+                            modelVariant: 'EMBEDDING',
+                        },
+                    });
+                    index.push({
+                        modelName: embeddingResult.modelName,
+                        position: [...embeddingResult.content],
+                        // <- TODO: [🪓] Here should be no need for spreading new array, just `position: embeddingResult.content`
+                    });
+                }
+            }
+            catch (error) {
+                // Note: Here is expected error:
+                //     > PipelineExecutionError: You have not provided any `LlmExecutionTools` that support model variant "EMBEDDING
+                if (!(error instanceof PipelineExecutionError)) {
+                    throw error;
+                }
+                // TODO: [🟥] Detect browser / node and make it colorful
+                console.error(error, "<- Note: This error is not critical to prepare the pipeline, just knowledge pieces won't have embeddings");
+            }
+            return {
+                name,
+                title,
+                content: knowledgePieceContent,
+                keywords,
+                index,
+                // <- TODO: [☀] sources,
+            };
+        }));
+        return knowledge;
+    }
+}
+/**
+ * TODO: [🪂] Do it in parallel 11:11
+ * Note: No need to aggregate usage here, it is done by intercepting the llmTools
+ */
+
+/**
+ * Metadata of the scraper
+ *
+ * @private within the scraper directory
+ */
+const websiteScraperMetadata = $deepFreeze({
+    title: 'Website scraper',
+    packageName: '@promptbook/website-crawler',
+    className: 'WebsiteScraper',
+    mimeTypes: ['text/html'],
+    documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
+    isAvailableInBrowser: false,
+    // <- Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server
+    requiredExecutables: [],
+}); /* <- Note: [🤛] */
+/**
+ * Registration of known scraper metadata
+ *
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
+ *
+ * @public exported from `@promptbook/core`
+ * @public exported from `@promptbook/wizard`
+ * @public exported from `@promptbook/cli`
+ */
+$scrapersMetadataRegister.register(websiteScraperMetadata);
+/**
+ * Note: [💞] Ignore a discrepancy between file name and entity name
+ */
+
+/**
+ * Create a new showdown converter instance
+ *
+ * @private utility of `WebsiteScraper`
+ */
+function createShowdownConverter() {
+    return new Converter({
+        flavor: 'github',
+        /*
+        > import showdownHighlight from 'showdown-highlight';
+        > extensions: [
+        >     showdownHighlight({
+        >         // Whether to add the classes to the <pre> tag, default is false
+        >         pre: true,
+        >         // Whether to use hljs' auto language detection, default is true
+        >         auto_detection: true,
+        >     }),
+        > ],
+        */
+    });
+}
+
+// TODO: [🏳‍🌈] Finally take pick of .json vs .ts
+/**
+ * Scraper for websites
+ *
+ * @see `documentationUrl` for more details
+ * @public exported from `@promptbook/website-crawler`
+ */
+class WebsiteScraper {
+    /**
+     * Metadata of the scraper which includes title, mime types, etc.
+     */
+    get metadata() {
+        return websiteScraperMetadata;
+    }
+    constructor(tools, options) {
+        this.tools = tools;
+        this.options = options;
+        this.markdownScraper = new MarkdownScraper(tools, options);
+        this.showdownConverter = createShowdownConverter();
+    }
+    /**
+     * Convert the website  to `.md` file and returns intermediate source
+     *
+     * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
+     */
+    async $convert(source) {
+        const { 
+        // TODO: [🧠] Maybe in node use headless browser not just JSDOM
+        rootDirname = process.cwd(), cacheDirname = DEFAULT_SCRAPE_CACHE_DIRNAME, intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY, isVerbose = DEFAULT_IS_VERBOSE, } = this.options;
+        if (source.url === null) {
+            throw new KnowledgeScrapeError('Website scraper requires URL');
+        }
+        if (this.tools.fs === undefined) {
+            throw new EnvironmentMismatchError('Can not scrape websites without filesystem tools');
+        }
+        const jsdom = new JSDOM(await source.asText(), {
+            url: source.url,
+        });
+        const reader = new Readability(jsdom.window.document);
+        const article = reader.parse();
+        // console.log(article);
+        // await forTime(10000);
+        let html = (article === null || article === void 0 ? void 0 : article.content) || (article === null || article === void 0 ? void 0 : article.textContent) || jsdom.window.document.body.innerHTML;
+        // Note: Unwrap html such as it is convertable by `markdownConverter`
+        for (let i = 0; i < 2; i++) {
+            html = html.replace(/<div\s*(?:id="readability-page-\d+"\s+class="page")?>(.*)<\/div>/is, '$1');
+        }
+        if (html.includes('<div')) {
+            html = (article === null || article === void 0 ? void 0 : article.textContent) || '';
+        }
+        const cacheFilehandler = await getScraperIntermediateSource(source, {
+            rootDirname,
+            cacheDirname,
+            intermediateFilesStrategy,
+            extension: 'html',
+            isVerbose,
+        });
+        // Note: Try to cache the scraped content, but don't fail if the filesystem is read-only
+        try {
+            await this.tools.fs.writeFile(cacheFilehandler.filename, html, 'utf-8');
+        }
+        catch (error) {
+            // Note: If we can't write to cache, we'll continue without caching
+            //       This handles read-only filesystems like Vercel
+            if (error instanceof Error &&
+                (error.message.includes('EROFS') ||
+                    error.message.includes('read-only') ||
+                    error.message.includes('EACCES') ||
+                    error.message.includes('EPERM') ||
+                    error.message.includes('ENOENT'))) ;
+            else {
+                // Re-throw other unexpected errors
+                throw error;
+            }
+        }
+        const markdown = this.showdownConverter.makeMarkdown(html, jsdom.window.document);
+        return { ...cacheFilehandler, markdown };
+    }
+    /**
+     * Scrapes the website and returns the knowledge pieces or `null` if it can't scrape it
+     */
+    async scrape(source) {
+        const cacheFilehandler = await this.$convert(source);
+        const markdownSource = {
+            source: source.source,
+            filename: cacheFilehandler.filename,
+            url: null,
+            mimeType: 'text/markdown',
+            asText() {
+                return cacheFilehandler.markdown;
+            },
+            asJson() {
+                throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
+            },
+            /*
+            TODO: [🥽]
+                > asBlob() {
+                >     throw new UnexpectedError(
+                >         'Did not expect that `markdownScraper` would need to get the content `asBlob`',
+                >     );
+                > },
+            */
+        };
+        const knowledge = this.markdownScraper.scrape(markdownSource);
+        await cacheFilehandler.destroy();
+        return knowledge;
+    }
+}
+/**
+ * TODO: [👣] Scraped website in .md can act as cache item - there is no need to run conversion each time
+ * TODO: [🪂] Do it in parallel 11:11
+ * Note: No need to aggregate usage here, it is done by intercepting the llmTools
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ */
+
+/**
+ * Fetches and scrapes content from a URL (SERVER-SIDE ONLY)
+ *
+ * This function:
+ * 1. Fetches the URL content using promptbookFetch
+ * 2. Determines the content type (HTML, PDF, etc.)
+ * 3. Uses the appropriate scraper to convert to markdown
+ * 4. Returns the scraped markdown content
+ *
+ * @param url The URL to fetch and scrape
+ * @returns Markdown content from the URL
+ *
+ * @private internal utility for USE BROWSER commitment
+ *
+ * WARNING: This function should NOT be used directly in browser environments.
+ * For browser environments, use fetchUrlContentViaBrowser which proxies through
+ * the Agents Server API endpoint at /api/scrape
+ */
+async function fetchUrlContent(url) {
+    try {
+        // Validate URL
+        let parsedUrl;
+        try {
+            parsedUrl = new URL(url);
+        }
+        catch (error) {
+            throw new Error(`Invalid URL: ${url}`);
+        }
+        // Fetch the URL content
+        const response = await promptbookFetch(url);
+        if (!response.ok) {
+            throw new Error(`Failed to fetch URL: ${response.status} ${response.statusText}`);
+        }
+        // Get content type
+        const contentType = response.headers.get('content-type') || 'text/html';
+        const content = await response.text();
+        // Determine the appropriate scraper based on content type
+        const mimeType = (contentType.split(';')[0] || 'text/html').trim();
+        // Handle PDFs
+        if (mimeType === 'application/pdf' || url.toLowerCase().endsWith('.pdf')) {
+            // TODO: [🔰] Implement PDF scraping using PdfScraper
+            // For now, return a message indicating PDF support is coming
+            return spaceTrim$1(`
+                # PDF Document: ${parsedUrl.pathname.split('/').pop()}
+
+                **Note:** PDF scraping is not yet fully implemented.
+
+                **URL:** ${url}
+
+                The PDF scraper will be integrated to provide full document content extraction.
+                For now, please try to find an HTML version of this content or use a different source.
+            `);
+        }
+        // Handle HTML/websites
+        if (mimeType === 'text/html' || mimeType === 'application/xhtml+xml' || !mimeType.includes('/')) {
+            // Create a scraper source handler
+            const sourceHandler = {
+                source: url,
+                filename: null,
+                url,
+                mimeType: 'text/html',
+                asText: () => content,
+                asJson: () => {
+                    throw new Error('JSON conversion not supported for HTML content');
+                },
+            };
+            // Use WebsiteScraper to convert HTML to markdown
+            // Note: We need minimal tools for the scraper
+            const scraper = new WebsiteScraper(// <- TODO: !!!!! Things marked with [🟢] should never be never released in packages that could be imported into browser environment
+            {
+                // Minimal tools - fs and llm are optional for basic scraping
+                fs: undefined,
+                llm: undefined,
+            }, {
+                isVerbose: false,
+            });
+            // Convert to markdown
+            const intermediateSource = await scraper.$convert(sourceHandler);
+            const markdown = intermediateSource.markdown;
+            // Clean up intermediate files
+            await intermediateSource.destroy();
+            // Add URL header to the content
+            return spaceTrim$1(`
+                # Content from: ${url}
+
+                ${markdown}
+
+                ---
+
+                *Source: ${url}*
+            `);
+        }
+        // For other content types, return the raw content with a note
+        return spaceTrim$1(`
+            # Content from: ${url}
+
+            **Content Type:** ${contentType}
+
+            ${content}
+
+            ---
+
+            *Source: ${url}*
+        `);
+    }
+    catch (error) {
+        // Handle errors gracefully
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        return spaceTrim$1(`
+            # Error fetching content from URL
+
+            **URL:** ${url}
+
+            **Error:** ${errorMessage}
+
+            Unable to fetch and scrape the content from this URL.
+            Please verify the URL is correct and accessible.
+        `);
+    }
+}
+/**
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ */
+
 /**
  * Generates a regex pattern to match a specific commitment
  *
@@ -11650,6 +12186,129 @@ function stripToolCallLines(text) {
  * Note: [💞] Ignore a discrepancy between file name and entity name
  */
 
+/**
+ * TEMPLATE commitment definition
+ *
+ * The TEMPLATE commitment enforces a specific response structure or template
+ * that the agent must follow when generating responses. This helps ensure
+ * consistent message formatting across all agent interactions.
+ *
+ * Example usage in agent source:
+ *
+ * ```book
+ * TEMPLATE Always structure your response with: 1) Summary, 2) Details, 3) Next steps
+ * TEMPLATE Use the following format: **Question:** [user question] | **Answer:** [your answer]
+ * ```
+ *
+ * When used without content, it enables template mode which instructs the agent
+ * to follow any template patterns defined in other commitments or context.
+ *
+ * @private [🪔] Maybe export the commitments through some package
+ */
+class TemplateCommitmentDefinition extends BaseCommitmentDefinition {
+    constructor(type = 'TEMPLATE') {
+        super(type);
+    }
+    /**
+     * Short one-line description of TEMPLATE.
+     */
+    get description() {
+        return 'Enforce a specific message structure or response template.';
+    }
+    /**
+     * Icon for this commitment.
+     */
+    get icon() {
+        return '📋';
+    }
+    /**
+     * Markdown documentation for TEMPLATE commitment.
+     */
+    get documentation() {
+        return spaceTrim$1(`
+            # ${this.type}
+
+            Enforces a specific response structure or template that the agent must follow when generating responses.
+
+            ## Key aspects
+
+            - Both terms work identically and can be used interchangeably.
+            - Can be used with or without content.
+            - When used without content, enables template mode for structured responses.
+            - When used with content, defines the specific template structure to follow.
+            - Multiple templates can be combined, with later ones taking precedence.
+
+            ## Examples
+
+            \`\`\`book
+            Customer Support Agent
+
+            PERSONA You are a helpful customer support representative
+            TEMPLATE Always structure your response with: 1) Acknowledgment, 2) Solution, 3) Follow-up question
+            STYLE Be professional and empathetic
+            \`\`\`
+
+            \`\`\`book
+            Technical Documentation Assistant
+
+            PERSONA You are a technical writing expert
+            TEMPLATE Use the following format: **Topic:** [topic] | **Explanation:** [details] | **Example:** [code]
+            FORMAT Use markdown with clear headings
+            \`\`\`
+
+            \`\`\`book
+            Simple Agent
+
+            PERSONA You are a virtual assistant
+            TEMPLATE
+            \`\`\`
+        `);
+    }
+    /**
+     * TEMPLATE can be used with or without content.
+     */
+    get requiresContent() {
+        return false;
+    }
+    applyToAgentModelRequirements(requirements, content) {
+        var _a;
+        const trimmedContent = content.trim();
+        // If no content is provided, enable template mode
+        if (!trimmedContent) {
+            // Store template mode flag in metadata
+            const updatedMetadata = {
+                ...requirements.metadata,
+                templateMode: true,
+            };
+            // Add a general instruction about using structured templates
+            const templateModeInstruction = spaceTrim$1(`
+                Use a clear, structured template format for your responses.
+                Maintain consistency in how you organize and present information.
+            `);
+            return {
+                ...this.appendToSystemMessage(requirements, templateModeInstruction, '\n\n'),
+                metadata: updatedMetadata,
+            };
+        }
+        // If content is provided, add the specific template instructions
+        const templateSection = `Response Template: ${trimmedContent}`;
+        // Store the template in metadata for potential programmatic access
+        const existingTemplates = ((_a = requirements.metadata) === null || _a === void 0 ? void 0 : _a.templates) || [];
+        const updatedMetadata = {
+            ...requirements.metadata,
+            templates: [...existingTemplates, trimmedContent],
+            templateMode: true,
+        };
+        return {
+            ...this.appendToSystemMessage(requirements, templateSection, '\n\n'),
+            metadata: updatedMetadata,
+        };
+    }
+}
+/**
+ * Note: [💞] Ignore a discrepancy between file name and entity name
+ */
+
 /**
  * USE commitment definition
  *
@@ -11764,12 +12423,56 @@ class UseCommitmentDefinition extends BaseCommitmentDefinition {
  * Note: [💞] Ignore a discrepancy between file name and entity name
  */
 
+/**
+ * Client-side safe wrapper for fetching URL content
+ *
+ * This function proxies requests to the Agents Server API endpoint for scraping,
+ * making it safe to use in browser environments.
+ *
+ * @param url The URL to fetch and scrape
+ * @param agentsServerUrl The base URL of the agents server (defaults to current origin)
+ * @returns Markdown content from the URL
+ *
+ * @private internal utility for USE BROWSER commitment
+ */
+async function fetchUrlContentViaBrowser(url, agentsServerUrl) {
+    try {
+        // Determine the agents server URL
+        const baseUrl = agentsServerUrl || (typeof window !== 'undefined' ? window.location.origin : '');
+        if (!baseUrl) {
+            throw new Error('Agents server URL is required in non-browser environments');
+        }
+        // Build the API endpoint URL
+        const apiUrl = new URL('/api/scrape', baseUrl);
+        apiUrl.searchParams.set('url', url);
+        // Fetch from the API endpoint
+        const response = await fetch(apiUrl.toString());
+        if (!response.ok) {
+            const errorData = await response.json().catch(() => ({ error: 'Unknown error' }));
+            throw new Error(`Failed to scrape URL: ${errorData.error || response.statusText}`);
+        }
+        const data = await response.json();
+        if (!data.success || !data.content) {
+            throw new Error(`Scraping failed: ${data.error || 'No content returned'}`);
+        }
+        return data.content;
+    }
+    catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        throw new Error(`Error fetching URL content via browser: ${errorMessage}`);
+    }
+}
+
 /**
  * USE BROWSER commitment definition
  *
- * The `USE BROWSER` commitment indicates that the agent should utilize a web browser tool
+ * The `USE BROWSER` commitment indicates that the agent should utilize browser tools
  * to access and retrieve up-to-date information from the internet when necessary.
  *
+ * This commitment provides two levels of browser access:
+ * 1. One-shot URL fetching: Simple function to fetch and scrape URL content
+ * 2. Running browser: For complex tasks like scrolling, clicking, etc. (prepared but not active yet)
+ *
  * The content following `USE BROWSER` is ignored (similar to NOTE).
  *
  * Example usage in agent source:
@@ -11795,7 +12498,7 @@ class UseBrowserCommitmentDefinition extends BaseCommitmentDefinition {
      * Short one-line description of USE BROWSER.
      */
     get description() {
-        return 'Enable the agent to use a web browser tool for accessing internet information.';
+        return 'Enable the agent to use browser tools for accessing internet information.';
     }
     /**
      * Icon for this commitment.
@@ -11810,14 +12513,18 @@ class UseBrowserCommitmentDefinition extends BaseCommitmentDefinition {
         return spaceTrim$1(`
             # USE BROWSER
 
-            Enables the agent to use a web browser tool to access and retrieve up-to-date information from the internet.
+            Enables the agent to use browser tools to access and retrieve up-to-date information from the internet.
 
             ## Key aspects
 
             - The content following \`USE BROWSER\` is ignored (similar to NOTE)
+            - Provides two levels of browser access:
+              1. **One-shot URL fetching**: Simple function to fetch and scrape URL content (active)
+              2. **Running browser**: For complex tasks like scrolling, clicking, etc. (prepared but not active yet)
             - The actual browser tool usage is handled by the agent runtime
-            - Allows the agent to fetch current information from websites
+            - Allows the agent to fetch current information from websites and documents
             - Useful for research tasks, fact-checking, and accessing dynamic content
+            - Supports various content types including HTML pages and PDF documents
 
             ## Examples
 
@@ -11853,48 +12560,306 @@ class UseBrowserCommitmentDefinition extends BaseCommitmentDefinition {
      */
     getToolTitles() {
         return {
-            web_browser: 'Web browser',
+            fetch_url_content: 'Fetch URL content',
+            run_browser: 'Run browser',
         };
     }
     applyToAgentModelRequirements(requirements, content) {
         // Get existing tools array or create new one
         const existingTools = requirements.tools || [];
-        // Add 'web_browser' to tools if not already present
-        const updatedTools = existingTools.some((tool) => tool.name === 'web_browser')
+        // Add browser tools if not already present
+        const toolsToAdd = [];
+        // Tool 1: One-shot URL content fetching
+        if (!existingTools.some((tool) => tool.name === 'fetch_url_content')) {
+            toolsToAdd.push({
+                name: 'fetch_url_content',
+                description: spaceTrim$1(`
+                    Fetches and scrapes the content from a URL (webpage or document).
+                    This tool retrieves the content of the specified URL and converts it to markdown format.
+                    Use this when you need to access information from a specific website or document.
+                    Supports various content types including HTML pages and PDF documents.
+                `),
+                parameters: {
+                    type: 'object',
+                    properties: {
+                        url: {
+                            type: 'string',
+                            description: 'The URL to fetch and scrape (e.g., "https://example.com" or "https://example.com/document.pdf")',
+                        },
+                    },
+                    required: ['url'],
+                },
+            });
+        }
+        // Tool 2: Running browser (prepared but not active yet)
+        if (!existingTools.some((tool) => tool.name === 'run_browser')) {
+            toolsToAdd.push({
+                name: 'run_browser',
+                description: spaceTrim$1(`
+                    Launches a browser session for complex interactions.
+                    This tool is for advanced browser automation tasks like scrolling, clicking, form filling, etc.
+                    Note: This tool is prepared but not yet active. It will be implemented in a future update.
+                `),
+                parameters: {
+                    type: 'object',
+                    properties: {
+                        url: {
+                            type: 'string',
+                            description: 'The initial URL to navigate to',
+                        },
+                        actions: {
+                            type: 'array',
+                            description: 'Array of actions to perform in the browser',
+                            items: {
+                                type: 'object',
+                                properties: {
+                                    type: {
+                                        type: 'string',
+                                        enum: ['navigate', 'click', 'scroll', 'type', 'wait'],
+                                    },
+                                    selector: {
+                                        type: 'string',
+                                        description: 'CSS selector for the element (for click, type actions)',
+                                    },
+                                    value: {
+                                        type: 'string',
+                                        description: 'Value to type or navigate to',
+                                    },
+                                },
+                            },
+                        },
+                    },
+                    required: ['url'],
+                },
+            });
+        }
+        const updatedTools = [...existingTools, ...toolsToAdd];
+        // Return requirements with updated tools and metadata
+        return this.appendToSystemMessage({
+            ...requirements,
+            tools: updatedTools,
+            metadata: {
+                ...requirements.metadata,
+                useBrowser: true,
+            },
+        }, spaceTrim$1(`
+                You have access to browser tools to fetch and access content from the internet.
+                - Use "fetch_url_content" to retrieve content from specific URLs (webpages or documents)
+                - Use "run_browser" for complex browser interactions (note: not yet active)
+                When you need to know information from a specific website or document, use the fetch_url_content tool.
+            `));
+    }
+    /**
+     * Gets the browser tool function implementations.
+     *
+     * This method automatically detects the environment and uses:
+     * - Server-side: Direct scraping via fetchUrlContent (Node.js)
+     * - Browser: Proxy through Agents Server API via fetchUrlContentViaBrowser
+     */
+    getToolFunctions() {
+        return {
+            /**
+             * @@@
+             *
+             * Note: [🛺] This function has implementation both for browser and node, this is the proxied one for browser
+             */
+            async fetch_url_content(args) {
+                console.log('!!!! [Tool] fetch_url_content called', { args });
+                const { url } = args;
+                return await fetchUrlContentViaBrowser(url);
+            },
+            /**
+             * @@@
+             */
+            async run_browser(args) {
+                console.log('!!!! [Tool] run_browser called', { args });
+                const { url } = args;
+                // This tool is prepared but not active yet
+                return spaceTrim$1(`
+                    # Running browser
+
+                    The running browser tool is not yet active.
+
+                    Requested URL: ${url}
+
+                    This feature will be implemented in a future update to support:
+                    - Complex browser interactions
+                    - Scrolling and navigation
+                    - Clicking and form filling
+                    - Taking screenshots
+
+                    For now, please use the "fetch_url_content" tool instead.
+                `);
+            },
+        };
+    }
+}
+/**
+ * Note: [💞] Ignore a discrepancy between file name and entity name
+ */
+
+/**
+ * @@@
+ *
+ * @private utility for commitments
+ */
+function formatOptionalInstructionBlock(label, content) {
+    const trimmedContent = spaceTrim$1(content);
+    if (!trimmedContent) {
+        return '';
+    }
+    return spaceTrim$1((block) => `
+        - ${label}:
+            ${block(trimmedContent
+        .split('\n')
+        .map((line) => `- ${line}`)
+        .join('\n'))}
+    `);
+}
+
+/**
+ * USE EMAIL commitment definition
+ *
+ * The `USE EMAIL` commitment enables the agent to send emails.
+ *
+ * Example usage in agent source:
+ *
+ * ```book
+ * USE EMAIL
+ * USE EMAIL Write always formal and polite emails, always greet.
+ * ```
+ *
+ * @private [🪔] Maybe export the commitments through some package
+ */
+class UseEmailCommitmentDefinition extends BaseCommitmentDefinition {
+    constructor() {
+        super('USE EMAIL', ['EMAIL', 'MAIL']);
+    }
+    get requiresContent() {
+        return false;
+    }
+    /**
+     * Short one-line description of USE EMAIL.
+     */
+    get description() {
+        return 'Enable the agent to send emails.';
+    }
+    /**
+     * Icon for this commitment.
+     */
+    get icon() {
+        return '📧';
+    }
+    /**
+     * Markdown documentation for USE EMAIL commitment.
+     */
+    get documentation() {
+        return spaceTrim$1(`
+            # USE EMAIL
+
+            Enables the agent to send emails through the email service.
+
+            ## Key aspects
+
+            - The agent can send emails to specified recipients.
+            - Supports multiple recipients, CC, subject, and markdown content.
+            - Emails are queued and sent through configured email providers.
+            - The content following \`USE EMAIL\` can provide additional instructions for email composition (e.g., style, tone, formatting preferences).
+
+            ## Examples
+
+            \`\`\`book
+            Email Assistant
+
+            PERSONA You are a helpful assistant who can send emails.
+            USE EMAIL
+            \`\`\`
+
+            \`\`\`book
+            Formal Email Assistant
+
+            PERSONA You help with professional communication.
+            USE EMAIL Write always formal and polite emails, always greet.
+            \`\`\`
+        `);
+    }
+    applyToAgentModelRequirements(requirements, content) {
+        const extraInstructions = formatOptionalInstructionBlock('Email instructions', content);
+        // Get existing tools array or create new one
+        const existingTools = requirements.tools || [];
+        // Add 'send_email' to tools if not already present
+        const updatedTools = existingTools.some((tool) => tool.name === 'send_email')
             ? existingTools
-            : ([
-                // TODO: [🔰] Use through proper MCP server
+            : [
                 ...existingTools,
                 {
-                    name: 'web_browser',
-                    description: spaceTrim$1(`
-                        A tool that can browse the web.
-                        Use this tool when you need to access specific websites or browse the internet.
-                    `),
+                    name: 'send_email',
+                    description: `Send an email to one or more recipients. ${!content ? '' : `Style instructions: ${content}`}`,
                     parameters: {
                         type: 'object',
                         properties: {
-                            url: {
+                            to: {
+                                type: 'array',
+                                items: { type: 'string' },
+                                description: 'Array of recipient email addresses (e.g., ["user@example.com", "Jane Doe <jane@example.com>"])',
+                            },
+                            cc: {
+                                type: 'array',
+                                items: { type: 'string' },
+                                description: 'Optional array of CC email addresses',
+                            },
+                            subject: {
+                                type: 'string',
+                                description: 'Email subject line',
+                            },
+                            body: {
                                 type: 'string',
-                                description: 'The URL to browse',
+                                description: 'Email body content in markdown format',
                             },
                         },
-                        required: ['url'],
+                        required: ['to', 'subject', 'body'],
                     },
                 },
-            ]);
+                // <- TODO: !!!! define the function in LLM tools
+            ];
         // Return requirements with updated tools and metadata
         return this.appendToSystemMessage({
             ...requirements,
             tools: updatedTools,
             metadata: {
                 ...requirements.metadata,
-                useBrowser: true,
+                useEmail: content || true,
             },
-        }, spaceTrim$1(`
-                You have access to the web browser. Use it to access specific websites or browse the internet.
-                When you need to know some information from a specific website, use the tool provided to you.
-            `));
+        }, spaceTrim$1((block) => `
+                    Email tool:
+                    - You have access to send emails via the tool "send_email".
+                    - Use it when you need to send emails to users or other recipients.
+                    - The email body should be written in markdown format.
+                    - Always ensure the email content is clear, professional, and appropriate.
+                    ${block(extraInstructions)}
+                `));
+    }
+    /**
+     * Gets human-readable titles for tool functions provided by this commitment.
+     */
+    getToolTitles() {
+        return {
+            send_email: 'Send email',
+        };
+    }
+    /**
+     * Gets the `send_email` tool function implementation.
+     * Note: This is a placeholder - the actual implementation is provided by the agent server.
+     */
+    getToolFunctions() {
+        return {
+            async send_email(args) {
+                console.log('!!!! [Tool] send_email called', { args });
+                // This is a placeholder implementation
+                // The actual implementation should be provided by the agent server
+                throw new Error('send_email tool not implemented. This commitment requires integration with an email service.');
+            },
+        };
     }
 }
 /**
@@ -12163,25 +13128,6 @@ class SerpSearchEngine {
     }
 }
 
-/**
- * @@@
- *
- * @private utility for commitments
- */
-function formatOptionalInstructionBlock(label, content) {
-    const trimmedContent = spaceTrim$1(content);
-    if (!trimmedContent) {
-        return '';
-    }
-    return spaceTrim$1((block) => `
-        - ${label}:
-            ${block(trimmedContent
-        .split('\n')
-        .map((line) => `- ${line}`)
-        .join('\n'))}
-    `);
-}
-
 /**
  * USE SEARCH ENGINE commitment definition
  *
@@ -12610,6 +13556,8 @@ const COMMITMENT_REGISTRY = [
     new SampleCommitmentDefinition('EXAMPLE'),
     new FormatCommitmentDefinition('FORMAT'),
     new FormatCommitmentDefinition('FORMATS'),
+    new TemplateCommitmentDefinition('TEMPLATE'),
+    new TemplateCommitmentDefinition('TEMPLATES'),
     new FromCommitmentDefinition('FROM'),
     new ImportCommitmentDefinition('IMPORT'),
     new ImportCommitmentDefinition('IMPORTS'),
@@ -12648,15 +13596,13 @@ const COMMITMENT_REGISTRY = [
     new UseBrowserCommitmentDefinition(),
     new UseSearchEngineCommitmentDefinition(),
     new UseTimeCommitmentDefinition(),
+    new UseEmailCommitmentDefinition(),
     new UseImageGeneratorCommitmentDefinition('USE IMAGE GENERATOR'),
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    new UseImageGeneratorCommitmentDefinition('USE IMAGE GENERATION'),
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    new UseImageGeneratorCommitmentDefinition('IMAGE GENERATOR'),
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    new UseImageGeneratorCommitmentDefinition('IMAGE GENERATION'),
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    new UseImageGeneratorCommitmentDefinition('USE IMAGE'),
+    new UseImageGeneratorCommitmentDefinition('USE IMAGE GENERATION' /* <- TODO: Remove any */),
+    new UseImageGeneratorCommitmentDefinition('IMAGE GENERATOR' /* <- TODO: Remove any */),
+    new UseImageGeneratorCommitmentDefinition('IMAGE GENERATION' /* <- TODO: Remove any */),
+    new UseImageGeneratorCommitmentDefinition('USE IMAGE' /* <- TODO: Remove any */),
+    // <- Note: [⛹️] How to deal with commitment aliases with defined functions
     new UseMcpCommitmentDefinition(),
     new UseCommitmentDefinition(),
     // Not yet implemented commitments (using placeholder)
@@ -12668,6 +13614,11 @@ const COMMITMENT_REGISTRY = [
     new NotYetImplementedCommitmentDefinition('CONTEXT'),
     // <- TODO: Prompt: Leverage aliases instead of duplicating commitment definitions
 ];
+/**
+ * TODO: [🧠] Maybe create through standardized $register
+ * Note: [💞] Ignore a discrepancy between file name and entity name
+ */
+
 /**
  * Gets all available commitment definitions
  * @returns Array of all commitment definitions
@@ -12677,24 +13628,65 @@ const COMMITMENT_REGISTRY = [
 function getAllCommitmentDefinitions() {
     return $deepFreeze([...COMMITMENT_REGISTRY]);
 }
+
 /**
  * Gets all function implementations provided by all commitments
  *
- * @public exported from `@promptbook/core`
+ * Note: This function is intended for browser use, there is also equivalent `getAllCommitmentsToolFunctionsForNode` for server use
+ *
+ * @public exported from `@promptbook/browser`
  */
-function getAllCommitmentsToolFunctions() {
+function getAllCommitmentsToolFunctionsForBrowser() {
     const allToolFunctions = {};
     for (const commitmentDefinition of getAllCommitmentDefinitions()) {
         const toolFunctions = commitmentDefinition.getToolFunctions();
         for (const [funcName, funcImpl] of Object.entries(toolFunctions)) {
+            if (allToolFunctions[funcName] !== undefined &&
+                just(false) /* <- Note: [⛹️] How to deal with commitment aliases */) {
+                throw new UnexpectedError(`Duplicate tool function name detected: \`${funcName}\` provided by commitment \`${commitmentDefinition.type}\``);
+            }
             allToolFunctions[funcName] = funcImpl;
         }
     }
     return allToolFunctions;
 }
+
 /**
- * TODO: [🧠] Maybe create through standardized $register
- * Note: [💞] Ignore a discrepancy between file name and entity name
+ * Gets all function implementations provided by all commitments
+ *
+ * Note: This function is intended for server use, there is also equivalent `getAllCommitmentsToolFunctionsForBrowser` for browser use
+ *
+ * @public exported from `@promptbook/node`
+ */
+function getAllCommitmentsToolFunctionsForNode() {
+    if (!$isRunningInNode()) {
+        throw new EnvironmentMismatchError(spaceTrim(`
+                Function getAllCommitmentsToolFunctionsForNode should be run in Node.js environment.
+
+                - In browser use getAllCommitmentsToolFunctionsForBrowser instead.
+                - This function can include server-only tools which cannot run in browser environment.
+
+            `));
+    }
+    const allToolFunctionsInBrowser = getAllCommitmentsToolFunctionsForBrowser();
+    const allToolFunctionsInNode = {
+        /**
+         * @@@
+         *
+         * Note: [🛺] This function has implementation both for browser and node, this is the full one for node
+         */
+        async fetch_url_content(args) {
+            console.log('!!!! [Tool] fetch_url_content called', { args });
+            const { url } = args;
+            return await fetchUrlContent(url);
+        },
+        // TODO: !!!! Unhardcode, make proper server function register from definitions
+    };
+    return { ...allToolFunctionsInBrowser, ...allToolFunctionsInNode };
+}
+/**
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ * TODO: [🏓] Unite `xxxForServer` and `xxxForNode` naming
  */
 
 /**
@@ -12908,13 +13900,6 @@ class JavascriptEvalExecutionTools {
         `const ${functionName} = buildinFunctions.${functionName};`)
             .join('\n');
         // TODO: DRY [🍯]
-        const commitmentsFunctions = getAllCommitmentsToolFunctions();
-        const commitmentsFunctionsStatement = Object.keys(commitmentsFunctions)
-            .map((functionName) => 
-        // Note: Custom functions are exposed to the current scope as variables
-        `const ${functionName} = commitmentsFunctions.${functionName};`)
-            .join('\n');
-        // TODO: DRY [🍯]
         const customFunctions = this.options.functions || {};
         const customFunctionsStatement = Object.keys(customFunctions)
             .map((functionName) => 
@@ -12928,10 +13913,6 @@ class JavascriptEvalExecutionTools {
                 // Build-in functions:
                 ${block(buildinFunctionsStatement)}
 
-                // Commitments functions:
-                ${block(commitmentsFunctionsStatement)}
-
-
                 // Custom functions:
                 ${block(customFunctionsStatement || '// -- No custom functions --')}
 
@@ -13029,10 +14010,11 @@ async function $provideScriptingForNode(options) {
         throw new EnvironmentMismatchError('Function `$provideScriptingForNode` works only in Node.js environment');
     }
     // TODO: [🔱] Do here auto-installation
-    return [new JavascriptExecutionTools(options)];
+    return [new JavascriptExecutionTools({ ...options, functions: { ...getAllCommitmentsToolFunctionsForNode() } })];
 }
 /**
  * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ * TODO: [🏓] Unite `xxxForServer` and `xxxForNode` naming
  */
 
 // TODO: [🥺] List running services from REMOTE_SERVER_URLS