@promptbook/markdown-utils 0.101.0-2 → 0.101.0-21

package/README.md

@@ -48,7 +48,93 @@ npm i ptbk
 npm install @promptbook/markdown-utils
 ```
 
-Utility functions used for processing markdown. Its part of the larger [`@promptbook/utils`](https://www.npmjs.com/package/@promptbook/utils) ecosystem.
+Comprehensive markdown processing utilities for the Promptbook ecosystem, providing tools for parsing, manipulating, and extracting content from markdown documents.
+
+## 🎯 Purpose and Motivation
+
+This package provides specialized markdown processing capabilities that are essential for working with promptbooks and LLM-generated content. It handles markdown parsing, content extraction, text humanization, and various markdown transformations that are commonly needed when processing AI-generated text and promptbook documentation.
+
+## 🔧 High-Level Functionality
+
+The package offers comprehensive markdown processing capabilities:
+
+-   **Content Extraction**: Extract code blocks, lists, and sections from markdown
+-   **Text Processing**: Parse and manipulate markdown sections and content
+-   **AI Text Humanization**: Transform AI-generated text to be more human-readable
+-   **Markdown Manipulation**: Add, remove, and modify markdown formatting
+-   **Table and Chart Creation**: Generate markdown tables and charts
+-   **Scraping Integration**: Markdown-specific content scraping capabilities
+
+## ✨ Key Features
+
+-   📝 **Content Extraction** - Extract blocks, lists, and sections from markdown documents
+-   🎨 **Text Humanization** - Transform AI text with proper quotes, ellipsis, and formatting
+-   🔧 **Markdown Manipulation** - Add, remove, and modify markdown formatting
+-   📊 **Table Generation** - Create formatted markdown tables and charts
+-   🧹 **Content Cleaning** - Remove comments, links, and unwanted formatting
+-   📖 **Section Parsing** - Parse and split markdown into structured sections
+-   🔍 **Block Processing** - Extract and process code blocks and content blocks
+-   🤖 **AI Integration** - Specialized tools for processing LLM-generated markdown
+
+## 📦 Exported Entities
+
+### Version Information
+
+-   `BOOK_LANGUAGE_VERSION` - Current book language version
+-   `PROMPTBOOK_ENGINE_VERSION` - Current engine version
+
+### Content Extraction
+
+-   `extractBlock` - Extract specific blocks from content
+-   `extractJsonBlock` - Extract JSON blocks from content
+-   `extractAllBlocksFromMarkdown` - Extract all code blocks from markdown
+-   `extractAllListItemsFromMarkdown` - Extract all list items from markdown
+-   `extractOneBlockFromMarkdown` - Extract single code block from markdown
+
+### Markdown Scraping
+
+-   `createMarkdownScraper` - Create markdown content scraper
+-   `MarkdownScraper` - Markdown scraper class
+-   `_MarkdownScraperRegistration` - Markdown scraper registration
+
+### Type Definitions
+
+-   `string_markdown` - Markdown string type (type)
+-   `string_markdown_section` - Markdown section string type (type)
+-   `string_markdown_section_content` - Markdown section content type (type)
+-   `string_markdown_text` - Markdown text string type (type)
+-   `string_markdown_codeblock_language` - Markdown code block language type (type)
+-   `MarkdownSection` - Markdown section structure (type)
+
+### Content Generation
+
+-   `addAutoGeneratedSection` - Add auto-generated sections to markdown
+-   `createMarkdownChart` - Create markdown charts and diagrams
+-   `createMarkdownTable` - Create formatted markdown tables
+
+### Text Processing
+
+-   `escapeMarkdownBlock` - Escape markdown block content
+-   `flattenMarkdown` - Flatten markdown structure
+-   `parseMarkdownSection` - Parse markdown into sections
+-   `splitMarkdownIntoSections` - Split markdown into structured sections
+
+### AI Text Humanization
+
+-   `humanizeAiText` - Comprehensive AI text humanization
+-   `humanizeAiTextEllipsis` - Fix ellipsis formatting in AI text
+-   `humanizeAiTextEmdashed` - Fix em-dash formatting in AI text
+-   `humanizeAiTextQuotes` - Fix quote formatting in AI text
+-   `humanizeAiTextWhitespace` - Fix whitespace in AI text
+-   `promptbookifyAiText` - Transform AI text for promptbook format
+
+### Content Cleaning
+
+-   `removeMarkdownComments` - Remove comments from markdown
+-   `removeMarkdownFormatting` - Remove markdown formatting
+-   `removeMarkdownLinks` - Remove links from markdown
+
+> 💡 This package provides markdown processing utilities for promptbook applications. For the core functionality, see [@promptbook/core](#-packages) or install all packages with `npm i ptbk`
 
 
 ---

package/esm/index.es.js

@@ -23,7 +23,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-const PROMPTBOOK_ENGINE_VERSION = '0.101.0-2';
+const PROMPTBOOK_ENGINE_VERSION = '0.101.0-21';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -2626,75 +2626,32 @@ function countUsage(llmTools) {
  */
 
 /**
- * Predefined profiles for LLM providers to maintain consistency across the application
- * These profiles represent each provider as a virtual persona in chat interfaces
+ * Takes an item or an array of items and returns an array of items
+ *
+ * 1) Any item except array and undefined returns array with that one item (also null)
+ * 2) Undefined returns empty array
+ * 3) Array returns itself
  *
- * @private !!!!
+ * @private internal utility
  */
-const LLM_PROVIDER_PROFILES = {
-    OPENAI: {
-        name: 'OPENAI',
-        fullname: 'OpenAI GPT',
-        color: '#10a37f', // OpenAI's signature green
-        // Note: avatarSrc could be added when we have provider logos available
-    },
-    ANTHROPIC: {
-        name: 'ANTHROPIC',
-        fullname: 'Anthropic Claude',
-        color: '#d97706', // Anthropic's orange/amber color
-    },
-    AZURE_OPENAI: {
-        name: 'AZURE_OPENAI',
-        fullname: 'Azure OpenAI',
-        color: '#0078d4', // Microsoft Azure blue
-    },
-    GOOGLE: {
-        name: 'GOOGLE',
-        fullname: 'Google Gemini',
-        color: '#4285f4', // Google blue
-    },
-    DEEPSEEK: {
-        name: 'DEEPSEEK',
-        fullname: 'DeepSeek',
-        color: '#7c3aed', // Purple color for DeepSeek
-    },
-    OLLAMA: {
-        name: 'OLLAMA',
-        fullname: 'Ollama',
-        color: '#059669', // Emerald green for local models
-    },
-    REMOTE: {
-        name: 'REMOTE',
-        fullname: 'Remote Server',
-        color: '#6b7280', // Gray for remote/proxy connections
-    },
-    MOCKED_ECHO: {
-        name: 'MOCKED_ECHO',
-        fullname: 'Echo (Test)',
-        color: '#8b5cf6', // Purple for test/mock tools
-    },
-    MOCKED_FAKE: {
-        name: 'MOCKED_FAKE',
-        fullname: 'Fake LLM (Test)',
-        color: '#ec4899', // Pink for fake/test tools
-    },
-    VERCEL: {
-        name: 'VERCEL',
-        fullname: 'Vercel AI',
-        color: '#000000', // Vercel's black
-    },
-    MULTIPLE: {
-        name: 'MULTIPLE',
-        fullname: 'Multiple Providers',
-        color: '#6366f1', // Indigo for combined/multiple providers
-    },
-};
+function arrayableToArray(input) {
+    if (input === undefined) {
+        return [];
+    }
+    if (input instanceof Array) {
+        return input;
+    }
+    return [input];
+}
+
 /**
- * TODO: Refactor this - each profile must be alongside the provider definition
- * TODO: [🕛] Unite `AgentBasicInformation`, `ChatParticipant`, `LlmExecutionTools` +  `LlmToolsMetadata`
- * Note: [💞] Ignore a discrepancy between file name and entity name
+ * Profile for Multiple providers aggregation
  */
-
+const MULTIPLE_PROVIDER_PROFILE = {
+    name: 'MULTIPLE',
+    fullname: 'Multiple Providers',
+    color: '#6366f1',
+};
 /**
  * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
  *
@@ -2705,12 +2662,10 @@ class MultipleLlmExecutionTools {
     /**
      * Gets array of execution tools in order of priority
      */
-    constructor(...llmExecutionTools) {
+    constructor(title, ...llmExecutionTools) {
+        this.title = title;
         this.llmExecutionTools = llmExecutionTools;
     }
-    get title() {
-        return 'Multiple LLM Providers';
-    }
     get description() {
         const innerModelsTitlesAndDescriptions = this.llmExecutionTools
             .map(({ title, description }, index) => {
@@ -2732,7 +2687,7 @@ class MultipleLlmExecutionTools {
             `);
     }
     get profile() {
-        return LLM_PROVIDER_PROFILES.MULTIPLE;
+        return MULTIPLE_PROVIDER_PROFILE;
     }
     /**
      * Check the configuration of all execution tools
@@ -2796,7 +2751,7 @@ class MultipleLlmExecutionTools {
                         return await llmExecutionTools.callEmbeddingModel(prompt);
                     // <- case [🤖]:
                     default:
-                        throw new UnexpectedError(`Unknown model variant "${prompt.modelRequirements.modelVariant}"`);
+                        throw new UnexpectedError(`Unknown model variant "${prompt.modelRequirements.modelVariant}" in ${llmExecutionTools.title}`);
                 }
             }
             catch (error) {
@@ -2817,7 +2772,7 @@ class MultipleLlmExecutionTools {
             //     2) AnthropicClaude throw PipelineExecutionError: Parameter `{knowledge}` is not defined
             //     3) ...
             spaceTrim((block) => `
-                          All execution tools failed:
+                          All execution tools of ${this.title} failed:
 
                           ${block(errors
                 .map(({ error, llmExecutionTools }, i) => `${i + 1}) **${llmExecutionTools.title}** thrown **${error.name || 'Error'}:** ${error.message}`)
@@ -2826,11 +2781,11 @@ class MultipleLlmExecutionTools {
                     `));
         }
         else if (this.llmExecutionTools.length === 0) {
-            throw new PipelineExecutionError(`You have not provided any \`LlmExecutionTools\``);
+            throw new PipelineExecutionError(`You have not provided any \`LlmExecutionTools\` into ${this.title}`);
         }
         else {
             throw new PipelineExecutionError(spaceTrim((block) => `
-                          You have not provided any \`LlmExecutionTools\` that support model variant "${prompt.modelRequirements.modelVariant}"
+                          You have not provided any \`LlmExecutionTools\` that support model variant "${prompt.modelRequirements.modelVariant}" into ${this.title}
 
                           Available \`LlmExecutionTools\`:
                           ${block(this.description)}
@@ -2860,7 +2815,7 @@ class MultipleLlmExecutionTools {
  *
  * @public exported from `@promptbook/core`
  */
-function joinLlmExecutionTools(...llmExecutionTools) {
+function joinLlmExecutionTools(title, ...llmExecutionTools) {
     if (llmExecutionTools.length === 0) {
         const warningMessage = spaceTrim(`
             You have not provided any \`LlmExecutionTools\`
@@ -2892,30 +2847,27 @@ function joinLlmExecutionTools(...llmExecutionTools) {
         };
         */
     }
-    return new MultipleLlmExecutionTools(...llmExecutionTools);
+    return new MultipleLlmExecutionTools(title || 'Multiple LLM Providers joined by `joinLlmExecutionTools`', ...llmExecutionTools);
 }
 /**
  * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
  */
 
 /**
- * Takes an item or an array of items and returns an array of items
- *
- * 1) Any item except array and undefined returns array with that one item (also null)
- * 2) Undefined returns empty array
- * 3) Array returns itself
+ * Just returns the given `LlmExecutionTools` or joins multiple into one
  *
- * @private internal utility
+ * @public exported from `@promptbook/core`
  */
-function arrayableToArray(input) {
-    if (input === undefined) {
-        return [];
-    }
-    if (input instanceof Array) {
-        return input;
-    }
-    return [input];
+function getSingleLlmExecutionTools(oneOrMoreLlmExecutionTools) {
+    const _llms = arrayableToArray(oneOrMoreLlmExecutionTools);
+    const llmTools = _llms.length === 1
+        ? _llms[0]
+        : joinLlmExecutionTools('Multiple LLM Providers joined by `getSingleLlmExecutionTools`', ..._llms);
+    return llmTools;
 }
+/**
+ * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
+ */
 
 /**
  * Prepares the persona for the pipeline
@@ -2934,8 +2886,7 @@ async function preparePersona(personaDescription, tools, options) {
         pipeline: await collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-persona.book'),
         tools,
     });
-    const _llms = arrayableToArray(tools.llm);
-    const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
+    const llmTools = getSingleLlmExecutionTools(tools.llm);
     const availableModels = (await llmTools.listModels())
         .filter(({ modelVariant }) => modelVariant === 'CHAT')
         .map(({ modelName, modelDescription }) => ({
@@ -2979,6 +2930,7 @@ async function preparePersona(personaDescription, tools, options) {
     };
 }
 /**
+ * TODO: [😩] DRY `preparePersona` and `selectBestModelFromAvailable`
  * TODO: [🔃][main] If the persona was prepared with different version or different set of models, prepare it once again
  * TODO: [🏢] Check validity of `modelName` in pipeline
  * TODO: [🏢] Check validity of `systemMessage` in pipeline
@@ -4097,9 +4049,7 @@ async function preparePipeline(pipeline, tools, options) {
     if (tools === undefined || tools.llm === undefined) {
         throw new MissingToolsError('LLM tools are required for preparing the pipeline');
     }
-    // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
-    const _llms = arrayableToArray(tools.llm);
-    const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
+    const llmTools = getSingleLlmExecutionTools(tools.llm);
     const llmToolsWithUsage = countUsage(llmTools);
     //    <- TODO: [🌯]
     /*
@@ -5140,9 +5090,7 @@ async function executeAttempts(options) {
         $scriptPipelineExecutionErrors: [],
         $failedResults: [], // Track all failed attempts
     };
-    // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
-    const _llms = arrayableToArray(tools.llm);
-    const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
+    const llmTools = getSingleLlmExecutionTools(tools.llm);
     attempts: for (let attemptIndex = -jokerParameterNames.length; attemptIndex < maxAttempts; attemptIndex++) {
         const isJokerAttempt = attemptIndex < 0;
         const jokerParameterName = jokerParameterNames[jokerParameterNames.length + attemptIndex];
@@ -5662,9 +5610,7 @@ async function getKnowledgeForTask(options) {
         return ''; // <- Note: Np knowledge present, return empty string
     }
     try {
-        // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
-        const _llms = arrayableToArray(tools.llm);
-        const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
+        const llmTools = getSingleLlmExecutionTools(tools.llm);
         const taskEmbeddingPrompt = {
             title: 'Knowledge Search',
             modelRequirements: {
@@ -6265,13 +6211,13 @@ function createPipelineExecutor(options) {
                     // Calculate and update tldr based on pipeline progress
                     const cv = newOngoingResult;
                     // Calculate progress based on parameters resolved vs total parameters
-                    const totalParameters = pipeline.parameters.filter(p => !p.isInput).length;
+                    const totalParameters = pipeline.parameters.filter((p) => !p.isInput).length;
                     let resolvedParameters = 0;
                     let currentTaskTitle = '';
                     // Get the resolved parameters from output parameters
                     if (cv === null || cv === void 0 ? void 0 : cv.outputParameters) {
                         // Count how many output parameters have non-empty values
-                        resolvedParameters = Object.values(cv.outputParameters).filter(value => value !== undefined && value !== null && String(value).trim() !== '').length;
+                        resolvedParameters = Object.values(cv.outputParameters).filter((value) => value !== undefined && value !== null && String(value).trim() !== '').length;
                     }
                     // Try to determine current task from execution report
                     if (((_a = cv === null || cv === void 0 ? void 0 : cv.executionReport) === null || _a === void 0 ? void 0 : _a.promptExecutions) && cv.executionReport.promptExecutions.length > 0) {
@@ -6381,9 +6327,7 @@ class MarkdownScraper {
             throw new MissingToolsError('LLM tools are required for scraping external files');
             // <- Note: This scraper is used in all other scrapers, so saying "external files" not "markdown files"
         }
-        // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
-        const _llms = arrayableToArray(llm);
-        const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
+        const llmTools = getSingleLlmExecutionTools(llm);
         // TODO: [🌼] In future use `ptbk make` and made getPipelineCollection
         const collection = createCollectionFromJson(...PipelineCollection);
         const prepareKnowledgeFromMarkdownExecutor = createPipelineExecutor({
@@ -6885,27 +6829,58 @@ function humanizeAiText(aiText) {
     return cleanedText;
 }
 /**
- * TODO: [🅾️] !!! Use this across the project where AI text is involved
+ * TODO: [🧠] Maybe this should be exported from `@promptbook/utils` not `@promptbook/markdown-utils`
  */
 
+/**
+ * @private
+ */
+const PROMPTBOOK_PSEUDOTOKEN_SUBSTITUTION = {
+    'a ': 'a  ',
+    'the ': 'the  ',
+    'is ': 'is  ',
+    'or ': 'or  ',
+    'be ': 'be   ',
+};
 /**
  * Function `promptbookifyAiText` will slightly modify the text so we know it was processed by Promptbook
  *
+ * Note: [🔂] This function is idempotent.
+ *
  * @public exported from `@promptbook/markdown-utils`
  */
 function promptbookifyAiText(text) {
-    // Note: Duplicating some spaces
-    const words = text.split(' '); // <- Note: [✌️] Use `splitWords` when available
-    const wordLength = words.length; // <- Note: [✌️] `countWords` should be just `splitWords(...).length`
-    for (const wordIndex of [3, 7, 11, 19].filter((i) => i < wordLength)) {
-        words[wordIndex] = ' ' + words[wordIndex];
+    const textLength = text.length;
+    let currentToken = '';
+    const textTokens = [
+    /* <- TODO: [✌️] Create `splitToPromptbookTokens` */
+    ];
+    for (let textPosition = 0; textPosition < textLength; textPosition++) {
+        const currentCharacter = text[textPosition];
+        if (currentToken.endsWith(' ') && currentCharacter !== ' ') {
+            textTokens.push(currentToken);
+            currentToken = '';
+        }
+        currentToken += currentCharacter;
+    }
+    if (currentToken.length > 0) {
+        textTokens.push(currentToken);
+    }
+    // [✌️] <- End of `splitToPromptbookTokens`
+    const promptbookifiedTextTokens = [];
+    for (let i = 0; i < textTokens.length; i++) {
+        const token = textTokens[i];
+        const tokenSubstitute = PROMPTBOOK_PSEUDOTOKEN_SUBSTITUTION[token];
+        if (tokenSubstitute !== undefined) {
+            promptbookifiedTextTokens.push(tokenSubstitute);
+        }
+        else {
+            promptbookifiedTextTokens.push(token);
+        }
     }
-    const promptbookifiedText = words.join(' ');
-    return promptbookifiedText;
+    return promptbookifiedTextTokens.join('');
 }
 /**
- * TODO: !!!!! Make the function idempotent and add "Note: [🔂] This function is idempotent."
- * TODO: [🅾️]!!! Use this across the project where AI text is involved
  * TODO: [🧠][✌️] Make some Promptbook-native token system
  */