@promptbook/markdown-utils 0.101.0-2 → 0.101.0-20

package/README.md

@@ -48,7 +48,93 @@ npm i ptbk
 npm install @promptbook/markdown-utils
 ```
 
-Utility functions used for processing markdown. Its part of the larger [`@promptbook/utils`](https://www.npmjs.com/package/@promptbook/utils) ecosystem.
+Comprehensive markdown processing utilities for the Promptbook ecosystem, providing tools for parsing, manipulating, and extracting content from markdown documents.
+
+## 🎯 Purpose and Motivation
+
+This package provides specialized markdown processing capabilities that are essential for working with promptbooks and LLM-generated content. It handles markdown parsing, content extraction, text humanization, and various markdown transformations that are commonly needed when processing AI-generated text and promptbook documentation.
+
+## 🔧 High-Level Functionality
+
+The package offers comprehensive markdown processing capabilities:
+
+-   **Content Extraction**: Extract code blocks, lists, and sections from markdown
+-   **Text Processing**: Parse and manipulate markdown sections and content
+-   **AI Text Humanization**: Transform AI-generated text to be more human-readable
+-   **Markdown Manipulation**: Add, remove, and modify markdown formatting
+-   **Table and Chart Creation**: Generate markdown tables and charts
+-   **Scraping Integration**: Markdown-specific content scraping capabilities
+
+## ✨ Key Features
+
+-   📝 **Content Extraction** - Extract blocks, lists, and sections from markdown documents
+-   🎨 **Text Humanization** - Transform AI text with proper quotes, ellipsis, and formatting
+-   🔧 **Markdown Manipulation** - Add, remove, and modify markdown formatting
+-   📊 **Table Generation** - Create formatted markdown tables and charts
+-   🧹 **Content Cleaning** - Remove comments, links, and unwanted formatting
+-   📖 **Section Parsing** - Parse and split markdown into structured sections
+-   🔍 **Block Processing** - Extract and process code blocks and content blocks
+-   🤖 **AI Integration** - Specialized tools for processing LLM-generated markdown
+
+## 📦 Exported Entities
+
+### Version Information
+
+-   `BOOK_LANGUAGE_VERSION` - Current book language version
+-   `PROMPTBOOK_ENGINE_VERSION` - Current engine version
+
+### Content Extraction
+
+-   `extractBlock` - Extract specific blocks from content
+-   `extractJsonBlock` - Extract JSON blocks from content
+-   `extractAllBlocksFromMarkdown` - Extract all code blocks from markdown
+-   `extractAllListItemsFromMarkdown` - Extract all list items from markdown
+-   `extractOneBlockFromMarkdown` - Extract single code block from markdown
+
+### Markdown Scraping
+
+-   `createMarkdownScraper` - Create markdown content scraper
+-   `MarkdownScraper` - Markdown scraper class
+-   `_MarkdownScraperRegistration` - Markdown scraper registration
+
+### Type Definitions
+
+-   `string_markdown` - Markdown string type (type)
+-   `string_markdown_section` - Markdown section string type (type)
+-   `string_markdown_section_content` - Markdown section content type (type)
+-   `string_markdown_text` - Markdown text string type (type)
+-   `string_markdown_codeblock_language` - Markdown code block language type (type)
+-   `MarkdownSection` - Markdown section structure (type)
+
+### Content Generation
+
+-   `addAutoGeneratedSection` - Add auto-generated sections to markdown
+-   `createMarkdownChart` - Create markdown charts and diagrams
+-   `createMarkdownTable` - Create formatted markdown tables
+
+### Text Processing
+
+-   `escapeMarkdownBlock` - Escape markdown block content
+-   `flattenMarkdown` - Flatten markdown structure
+-   `parseMarkdownSection` - Parse markdown into sections
+-   `splitMarkdownIntoSections` - Split markdown into structured sections
+
+### AI Text Humanization
+
+-   `humanizeAiText` - Comprehensive AI text humanization
+-   `humanizeAiTextEllipsis` - Fix ellipsis formatting in AI text
+-   `humanizeAiTextEmdashed` - Fix em-dash formatting in AI text
+-   `humanizeAiTextQuotes` - Fix quote formatting in AI text
+-   `humanizeAiTextWhitespace` - Fix whitespace in AI text
+-   `promptbookifyAiText` - Transform AI text for promptbook format
+
+### Content Cleaning
+
+-   `removeMarkdownComments` - Remove comments from markdown
+-   `removeMarkdownFormatting` - Remove markdown formatting
+-   `removeMarkdownLinks` - Remove links from markdown
+
+> 💡 This package provides markdown processing utilities for promptbook applications. For the core functionality, see [@promptbook/core](#-packages) or install all packages with `npm i ptbk`
 
 
 ---

package/esm/index.es.js

@@ -23,7 +23,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-const PROMPTBOOK_ENGINE_VERSION = '0.101.0-2';
+const PROMPTBOOK_ENGINE_VERSION = '0.101.0-20';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -2625,6 +2625,25 @@ function countUsage(llmTools) {
  * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
  */
 
+/**
+ * Takes an item or an array of items and returns an array of items
+ *
+ * 1) Any item except array and undefined returns array with that one item (also null)
+ * 2) Undefined returns empty array
+ * 3) Array returns itself
+ *
+ * @private internal utility
+ */
+function arrayableToArray(input) {
+    if (input === undefined) {
+        return [];
+    }
+    if (input instanceof Array) {
+        return input;
+    }
+    return [input];
+}
+
 /**
  * Predefined profiles for LLM providers to maintain consistency across the application
  * These profiles represent each provider as a virtual persona in chat interfaces
@@ -2705,12 +2724,10 @@ class MultipleLlmExecutionTools {
     /**
      * Gets array of execution tools in order of priority
      */
-    constructor(...llmExecutionTools) {
+    constructor(title, ...llmExecutionTools) {
+        this.title = title;
         this.llmExecutionTools = llmExecutionTools;
     }
-    get title() {
-        return 'Multiple LLM Providers';
-    }
     get description() {
         const innerModelsTitlesAndDescriptions = this.llmExecutionTools
             .map(({ title, description }, index) => {
@@ -2796,7 +2813,7 @@ class MultipleLlmExecutionTools {
                         return await llmExecutionTools.callEmbeddingModel(prompt);
                     // <- case [🤖]:
                     default:
-                        throw new UnexpectedError(`Unknown model variant "${prompt.modelRequirements.modelVariant}"`);
+                        throw new UnexpectedError(`Unknown model variant "${prompt.modelRequirements.modelVariant}" in ${llmExecutionTools.title}`);
                 }
             }
             catch (error) {
@@ -2817,7 +2834,7 @@ class MultipleLlmExecutionTools {
             //     2) AnthropicClaude throw PipelineExecutionError: Parameter `{knowledge}` is not defined
             //     3) ...
             spaceTrim((block) => `
-                          All execution tools failed:
+                          All execution tools of ${this.title} failed:
 
                           ${block(errors
                 .map(({ error, llmExecutionTools }, i) => `${i + 1}) **${llmExecutionTools.title}** thrown **${error.name || 'Error'}:** ${error.message}`)
@@ -2826,11 +2843,11 @@ class MultipleLlmExecutionTools {
                     `));
         }
         else if (this.llmExecutionTools.length === 0) {
-            throw new PipelineExecutionError(`You have not provided any \`LlmExecutionTools\``);
+            throw new PipelineExecutionError(`You have not provided any \`LlmExecutionTools\` into ${this.title}`);
         }
         else {
             throw new PipelineExecutionError(spaceTrim((block) => `
-                          You have not provided any \`LlmExecutionTools\` that support model variant "${prompt.modelRequirements.modelVariant}"
+                          You have not provided any \`LlmExecutionTools\` that support model variant "${prompt.modelRequirements.modelVariant}" into ${this.title}
 
                           Available \`LlmExecutionTools\`:
                           ${block(this.description)}
@@ -2860,7 +2877,7 @@ class MultipleLlmExecutionTools {
  *
  * @public exported from `@promptbook/core`
  */
-function joinLlmExecutionTools(...llmExecutionTools) {
+function joinLlmExecutionTools(title, ...llmExecutionTools) {
     if (llmExecutionTools.length === 0) {
         const warningMessage = spaceTrim(`
             You have not provided any \`LlmExecutionTools\`
@@ -2892,30 +2909,27 @@ function joinLlmExecutionTools(...llmExecutionTools) {
         };
         */
     }
-    return new MultipleLlmExecutionTools(...llmExecutionTools);
+    return new MultipleLlmExecutionTools(title || 'Multiple LLM Providers joined by `joinLlmExecutionTools`', ...llmExecutionTools);
 }
 /**
  * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
  */
 
 /**
- * Takes an item or an array of items and returns an array of items
- *
- * 1) Any item except array and undefined returns array with that one item (also null)
- * 2) Undefined returns empty array
- * 3) Array returns itself
+ * Just returns the given `LlmExecutionTools` or joins multiple into one
  *
- * @private internal utility
+ * @public exported from `@promptbook/core`
  */
-function arrayableToArray(input) {
-    if (input === undefined) {
-        return [];
-    }
-    if (input instanceof Array) {
-        return input;
-    }
-    return [input];
+function getSingleLlmExecutionTools(oneOrMoreLlmExecutionTools) {
+    const _llms = arrayableToArray(oneOrMoreLlmExecutionTools);
+    const llmTools = _llms.length === 1
+        ? _llms[0]
+        : joinLlmExecutionTools('Multiple LLM Providers joined by `getSingleLlmExecutionTools`', ..._llms);
+    return llmTools;
 }
+/**
+ * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
+ */
 
 /**
  * Prepares the persona for the pipeline
@@ -2934,8 +2948,7 @@ async function preparePersona(personaDescription, tools, options) {
         pipeline: await collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-persona.book'),
         tools,
     });
-    const _llms = arrayableToArray(tools.llm);
-    const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
+    const llmTools = getSingleLlmExecutionTools(tools.llm);
     const availableModels = (await llmTools.listModels())
         .filter(({ modelVariant }) => modelVariant === 'CHAT')
         .map(({ modelName, modelDescription }) => ({
@@ -2979,6 +2992,7 @@ async function preparePersona(personaDescription, tools, options) {
     };
 }
 /**
+ * TODO: [😩] DRY `preparePersona` and `selectBestModelFromAvailable`
  * TODO: [🔃][main] If the persona was prepared with different version or different set of models, prepare it once again
  * TODO: [🏢] Check validity of `modelName` in pipeline
  * TODO: [🏢] Check validity of `systemMessage` in pipeline
@@ -4097,9 +4111,7 @@ async function preparePipeline(pipeline, tools, options) {
     if (tools === undefined || tools.llm === undefined) {
         throw new MissingToolsError('LLM tools are required for preparing the pipeline');
     }
-    // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
-    const _llms = arrayableToArray(tools.llm);
-    const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
+    const llmTools = getSingleLlmExecutionTools(tools.llm);
     const llmToolsWithUsage = countUsage(llmTools);
     //    <- TODO: [🌯]
     /*
@@ -5140,9 +5152,7 @@ async function executeAttempts(options) {
         $scriptPipelineExecutionErrors: [],
         $failedResults: [], // Track all failed attempts
     };
-    // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
-    const _llms = arrayableToArray(tools.llm);
-    const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
+    const llmTools = getSingleLlmExecutionTools(tools.llm);
     attempts: for (let attemptIndex = -jokerParameterNames.length; attemptIndex < maxAttempts; attemptIndex++) {
         const isJokerAttempt = attemptIndex < 0;
         const jokerParameterName = jokerParameterNames[jokerParameterNames.length + attemptIndex];
@@ -5662,9 +5672,7 @@ async function getKnowledgeForTask(options) {
         return ''; // <- Note: Np knowledge present, return empty string
     }
     try {
-        // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
-        const _llms = arrayableToArray(tools.llm);
-        const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
+        const llmTools = getSingleLlmExecutionTools(tools.llm);
         const taskEmbeddingPrompt = {
             title: 'Knowledge Search',
             modelRequirements: {
@@ -6265,13 +6273,13 @@ function createPipelineExecutor(options) {
                     // Calculate and update tldr based on pipeline progress
                     const cv = newOngoingResult;
                     // Calculate progress based on parameters resolved vs total parameters
-                    const totalParameters = pipeline.parameters.filter(p => !p.isInput).length;
+                    const totalParameters = pipeline.parameters.filter((p) => !p.isInput).length;
                     let resolvedParameters = 0;
                     let currentTaskTitle = '';
                     // Get the resolved parameters from output parameters
                     if (cv === null || cv === void 0 ? void 0 : cv.outputParameters) {
                         // Count how many output parameters have non-empty values
-                        resolvedParameters = Object.values(cv.outputParameters).filter(value => value !== undefined && value !== null && String(value).trim() !== '').length;
+                        resolvedParameters = Object.values(cv.outputParameters).filter((value) => value !== undefined && value !== null && String(value).trim() !== '').length;
                     }
                     // Try to determine current task from execution report
                     if (((_a = cv === null || cv === void 0 ? void 0 : cv.executionReport) === null || _a === void 0 ? void 0 : _a.promptExecutions) && cv.executionReport.promptExecutions.length > 0) {
@@ -6381,9 +6389,7 @@ class MarkdownScraper {
             throw new MissingToolsError('LLM tools are required for scraping external files');
             // <- Note: This scraper is used in all other scrapers, so saying "external files" not "markdown files"
         }
-        // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
-        const _llms = arrayableToArray(llm);
-        const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
+        const llmTools = getSingleLlmExecutionTools(llm);
         // TODO: [🌼] In future use `ptbk make` and made getPipelineCollection
         const collection = createCollectionFromJson(...PipelineCollection);
         const prepareKnowledgeFromMarkdownExecutor = createPipelineExecutor({
@@ -6885,6 +6891,7 @@ function humanizeAiText(aiText) {
     return cleanedText;
 }
 /**
+ * TODO: [🧠] Maybe this should be exported from `@promptbook/utils` not `@promptbook/markdown-utils`
  * TODO: [🅾️] !!! Use this across the project where AI text is involved
  */