@promptbook/core 0.83.0 → 0.84.0-10

package/README.md

@@ -23,6 +23,10 @@
 
 
 
+<blockquote style="color: #ff8811">
+    <b>⚠ Warning:</b> This is a pre-release version of the library. It is not yet ready for production use. Please look at <a href="https://www.npmjs.com/package/@promptbook/core?activeTab=versions">latest stable release</a>.
+</blockquote>
+
 ## 📦 Package `@promptbook/core`
 
 - Promptbooks are [divided into several](#-packages) packages, all are published from [single monorepo](https://github.com/webgptorg/promptbook).
@@ -269,6 +273,7 @@ Or you can install them separately:
 -   **[@promptbook/remote-client](https://www.npmjs.com/package/@promptbook/remote-client)** - Remote client for remote execution of promptbooks
 -   **[@promptbook/remote-server](https://www.npmjs.com/package/@promptbook/remote-server)** - Remote server for remote execution of promptbooks
 -   **[@promptbook/pdf](https://www.npmjs.com/package/@promptbook/pdf)** - Read knowledge from `.pdf` documents
+-   **[@promptbook/documents](https://www.npmjs.com/package/@promptbook/markitdown)** - Integration of [Markitdown by Microsoft](https://github.com/microsoft/markitdown)
 -   **[@promptbook/documents](https://www.npmjs.com/package/@promptbook/documents)** - Read knowledge from documents like `.docx`, `.odt`,…
 -   **[@promptbook/legacy-documents](https://www.npmjs.com/package/@promptbook/legacy-documents)** - Read knowledge from legacy documents like `.doc`, `.rtf`,…
 -   **[@promptbook/website-crawler](https://www.npmjs.com/package/@promptbook/website-crawler)** - Crawl knowledge from the web

package/esm/index.es.js

@@ -25,7 +25,7 @@ var BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-var PROMPTBOOK_ENGINE_VERSION = '0.82.0';
+var PROMPTBOOK_ENGINE_VERSION = '0.84.0-9';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -526,7 +526,7 @@ var NAME = "Promptbook";
  *
  * @public exported from `@promptbook/core`
  */
-var ADMIN_EMAIL = 'me@pavolhejny.com';
+var ADMIN_EMAIL = 'pavol@ptbk.io';
 /**
  * Name of the responsible person for the Promptbook on GitHub
  *
@@ -1037,7 +1037,7 @@ var ORDER_OF_PIPELINE_JSON = [
  *
  * @private within the repository
  */
-var REPLACING_NONCE = 'u$k42k%!V2zo34w7Fu#@QUHYPW';
+var REPLACING_NONCE = 'ptbkauk42kV2dzao34faw7FudQUHYPtW';
 /**
  * @@@
  *
@@ -2084,7 +2084,7 @@ var LimitReachedError = /** @class */ (function (_super) {
 var NotYetImplementedError = /** @class */ (function (_super) {
     __extends(NotYetImplementedError, _super);
     function NotYetImplementedError(message) {
-        var _this = _super.call(this, spaceTrim$1(function (block) { return "\n                    ".concat(block(message), "\n\n                    Note: This feature is not implemented yet but it will be soon.\n\n                    If you want speed up the implementation or just read more, look here:\n                    https://github.com/webgptorg/promptbook\n\n                    Or contact us on me@pavolhejny.com\n\n                "); })) || this;
+        var _this = _super.call(this, spaceTrim$1(function (block) { return "\n                    ".concat(block(message), "\n\n                    Note: This feature is not implemented yet but it will be soon.\n\n                    If you want speed up the implementation or just read more, look here:\n                    https://github.com/webgptorg/promptbook\n\n                    Or contact us on pavol@ptbk.io\n\n                "); })) || this;
         _this.name = 'NotYetImplementedError';
         Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
         return _this;
@@ -8971,6 +8971,55 @@ var HIGH_LEVEL_ABSTRACTIONS = [
 var SUPPORTED_SCRIPT_LANGUAGES = ['javascript', 'typescript', 'python'];
 //          <- TODO: [🏥] DRY
 
+/**
+ * Removes Markdown (or HTML) comments
+ *
+ * @param {string} content - The string to remove comments from.
+ * @returns {string} The input string with all comments removed.
+ * @public exported from `@promptbook/markdown-utils`
+ */
+function removeMarkdownComments(content) {
+    return spaceTrim$1(content.replace(/<!--(.*?)-->/gs, ''));
+}
+
+/**
+ * @@@
+ *
+ * @public exported from `@promptbook/editable`
+ */
+function isFlatPipeline(pipelineString) {
+    pipelineString = removeMarkdownComments(pipelineString);
+    pipelineString = spaceTrim(pipelineString);
+    var isMarkdownBeginningWithHeadline = pipelineString.startsWith('# ');
+    var isLastLineReturnStatement = pipelineString.split('\n').pop().split('`').join('').startsWith('->');
+    // TODO: Also (double)check
+    // > const usedCommands
+    // > const isBlocksUsed
+    // > const returnStatementCount
+    var isFlat = !isMarkdownBeginningWithHeadline && isLastLineReturnStatement;
+    return isFlat;
+}
+
+/**
+ * @@@
+ *
+ * @public exported from `@promptbook/editable`
+ */
+function deflatePipeline(pipelineString) {
+    if (!isFlatPipeline(pipelineString)) {
+        return pipelineString;
+    }
+    var pipelineStringLines = pipelineString.split('\n');
+    var returnStatement = pipelineStringLines.pop();
+    var prompt = spaceTrim(pipelineStringLines.join('\n'));
+    pipelineString = validatePipelineString(spaceTrim(function (block) { return "\n                # ".concat(DEFAULT_BOOK_TITLE, "\n\n                ## Prompt\n\n                ```\n                ").concat(block(prompt), "\n                ```\n\n                ").concat(returnStatement, "\n            "); }));
+    // <- TODO: Maybe use book` notation
+    return pipelineString;
+}
+/**
+ * TODO: Unit test
+ */
+
 /**
  * Utility function to extract all list items from markdown
  *
@@ -9186,17 +9235,6 @@ function flattenMarkdown(markdown) {
  *       NOW we are working just with markdown string and its good enough
  */
 
-/**
- * Removes Markdown (or HTML) comments
- *
- * @param {string} content - The string to remove comments from.
- * @returns {string} The input string with all comments removed.
- * @public exported from `@promptbook/markdown-utils`
- */
-function removeMarkdownComments(content) {
-    return spaceTrim$1(content.replace(/<!--(.*?)-->/gs, ''));
-}
-
 /**
  * @@@
  *
@@ -9281,26 +9319,9 @@ function parsePipeline(pipelineString) {
     pipelineString = removeMarkdownComments(pipelineString);
     pipelineString = spaceTrim$1(pipelineString);
     // <- TODO: [😧] `spaceTrim` should preserve discriminated type *(or at lease `PipelineString`)*
+    pipelineString = deflatePipeline(pipelineString);
     // ==============
-    // Note: 1️⃣◽2️⃣ Process flat pipeline
-    var isMarkdownBeginningWithHeadline = pipelineString.startsWith('# ');
-    var isLastLineReturnStatement = pipelineString.split('\n').pop().split('`').join('').startsWith('->');
-    // TODO: Also (double)check
-    // > const usedCommands
-    // > const isBlocksUsed
-    // > const returnStatementCount
-    var isFlatPipeline = !isMarkdownBeginningWithHeadline && isLastLineReturnStatement;
-    // console.log({ isMarkdownBeginningWithHeadline, isLastLineReturnStatement, isFlatPipeline });
-    if (isFlatPipeline) {
-        var pipelineStringLines = pipelineString.split('\n');
-        var returnStatement_1 = pipelineStringLines.pop();
-        var prompt_1 = spaceTrim$1(pipelineStringLines.join('\n'));
-        pipelineString = validatePipelineString(spaceTrim$1(function (block) { return "\n                    # ".concat(DEFAULT_BOOK_TITLE, "\n\n                    ## Prompt\n\n                    ```\n                    ").concat(block(prompt_1), "\n                    ```\n\n                    ").concat(returnStatement_1, "\n                "); }));
-        // <- TODO: Maybe use book` notation
-        // console.log(pipelineString);
-    }
-    // ==============
-    // Note: 1️⃣◽3️⃣ Parse the markdown
+    // Note: 1️⃣◽2️⃣ Parse the markdown
     pipelineString = flattenMarkdown(pipelineString) /* <- Note: [🥞] */;
     pipelineString = pipelineString.replaceAll(/`\{(?<parameterName>[a-z0-9_]+)\}`/gi, '{$<parameterName>}');
     pipelineString = pipelineString.replaceAll(/`->\s+\{(?<parameterName>[a-z0-9_]+)\}`/gi, '-> {$<parameterName>}');
@@ -11051,9 +11072,59 @@ function isValidPipelineString(pipelineString) {
  * TODO: [🧠][🈴] Where is the best location for this file
  */
 
+/**
+ * Tag function for notating a prompt as template literal
+ *
+ * Note: There are 2 similar functions:
+ * 1) `prompt` for notating single prompt exported from `@promptbook/utils`
+ * 1) `book` for notating and validating entire books exported from `@promptbook/utils`
+ *
+ * @param strings @@@
+ * @param values @@@
+ * @returns the pipeline string
+ * @public exported from `@promptbook/utils`
+ */
+function prompt(strings) {
+    var values = [];
+    for (var _i = 1; _i < arguments.length; _i++) {
+        values[_i - 1] = arguments[_i];
+    }
+    if (values.length === 0) {
+        return spaceTrim(strings.join(''));
+    }
+    var placeholderParameterNames = values.map(function (value, i) { return "".concat(REPLACING_NONCE).concat(i); });
+    var parameters = Object.fromEntries(values.map(function (value, i) { return [placeholderParameterNames[i], value]; }));
+    // Combine strings and values
+    var pipelineString = strings.reduce(function (result, stringsItem, i) {
+        return placeholderParameterNames[i] === undefined
+            ? "".concat(result).concat(stringsItem)
+            : "".concat(result).concat(stringsItem, "{").concat(placeholderParameterNames[i], "}");
+    }, '');
+    pipelineString = spaceTrim(pipelineString);
+    try {
+        pipelineString = templateParameters(pipelineString, parameters);
+    }
+    catch (error) {
+        if (!(error instanceof PipelineExecutionError)) {
+            throw error;
+        }
+        console.error({ pipelineString: pipelineString, parameters: parameters, placeholderParameterNames: placeholderParameterNames, error: error });
+        throw new UnexpectedError(spaceTrim(function (block) { return "\n                    Internal error in prompt template literal\n                  \n                    ".concat(block(JSON.stringify({ strings: strings, values: values }, null, 4)), "}\n                  \n                "); }));
+    }
+    return pipelineString;
+}
+/**
+ * TODO: [🧠][🈴] Where is the best location for this file
+ * Note: [💞] Ignore a discrepancy between file name and entity name
+ */
+
 /**
  * Tag function for notating a pipeline with a book\`...\ notation as template literal
  *
+ * Note: There are 2 similar functions:
+ * 1) `prompt` for notating single prompt exported from `@promptbook/utils`
+ * 1) `book` for notating and validating entire books exported from `@promptbook/utils`
+ *
  * @param strings @@@
  * @param values @@@
  * @returns the pipeline string
@@ -11064,11 +11135,7 @@ function book(strings) {
     for (var _i = 1; _i < arguments.length; _i++) {
         values[_i - 1] = arguments[_i];
     }
-    if (strings.length !== 1 && values.length !== 0) {
-        throw new NotYetImplementedError("Only one string without interpolated value is supported for now in book`...` notation");
-    }
-    var pipelineString = strings[0];
-    pipelineString = spaceTrim(pipelineString);
+    var pipelineString = prompt.apply(void 0, __spreadArray([strings], __read(values), false));
     if (!isValidPipelineString(pipelineString)) {
         // TODO: Make the CustomError for this
         throw new Error(spaceTrim("\n                The string is not a valid pipeline string\n\n                book`\n                    ".concat(pipelineString, "\n                `\n            ")));
@@ -11080,6 +11147,40 @@ function book(strings) {
  * Note: [💞] Ignore a discrepancy between file name and entity name
  */
 
+/**
+ * Metadata of the scraper
+ *
+ * @private within the scraper directory
+ */
+var boilerplateScraperMetadata = $deepFreeze({
+    title: 'Boilerplate scraper',
+    packageName: '@promptbook/boilerplate',
+    className: 'BoilerplateScraper',
+    mimeTypes: [
+        '@@@/@@@',
+        // <- TODO: @@@ Add compatible mime types with Boilerplate scraper
+    ],
+    documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@@',
+    isAvilableInBrowser: false,
+    // <- Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server
+    requiredExecutables: [
+    /* @@@ 'Pandoc' */
+    ],
+}); /* <- Note: [🤛] */
+/**
+ * Registration of known scraper metadata
+ *
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
+ *
+ * @public exported from `@promptbook/core`
+ * @public exported from `@promptbook/wizzard`
+ * @public exported from `@promptbook/cli`
+ */
+var _BoilerplateScraperMetadataRegistration = $scrapersMetadataRegister.register(boilerplateScraperMetadata);
+/**
+ * Note: [💞] Ignore a discrepancy between file name and entity name
+ */
+
 /**
  * Metadata of the scraper
  *
@@ -11171,6 +11272,40 @@ var _MarkdownScraperMetadataRegistration = $scrapersMetadataRegister.register(ma
  * Note: [💞] Ignore a discrepancy between file name and entity name
  */
 
+/**
+ * Metadata of the scraper
+ *
+ * @private within the scraper directory
+ */
+var markitdownScraperMetadata = $deepFreeze({
+    title: 'Markitdown scraper',
+    packageName: '@promptbook/markitdown',
+    className: 'MarkitdownScraper',
+    mimeTypes: [
+        'application/pdf',
+        // TODO: Make priority for scrapers and than allow all mime types here:
+        // 'text/html',
+        // 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+    ],
+    documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
+    isAvilableInBrowser: false,
+    // <- Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server
+    requiredExecutables: [],
+}); /* <- Note: [🤛] */
+/**
+ * Registration of known scraper metadata
+ *
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
+ *
+ * @public exported from `@promptbook/core`
+ * @public exported from `@promptbook/wizzard`
+ * @public exported from `@promptbook/cli`
+ */
+var _MarkitdownScraperMetadataRegistration = $scrapersMetadataRegister.register(markitdownScraperMetadata);
+/**
+ * Note: [💞] Ignore a discrepancy between file name and entity name
+ */
+
 /**
  * Metadata of the scraper
  *
@@ -11180,7 +11315,7 @@ var pdfScraperMetadata = $deepFreeze({
     title: 'Pdf scraper',
     packageName: '@promptbook/pdf',
     className: 'PdfScraper',
-    mimeTypes: ['application/pdf'],
+    mimeTypes: ['application/pdf-DISABLED'],
     documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
     isAvilableInBrowser: false,
     // <- Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server
@@ -11310,5 +11445,5 @@ var PrefixStorage = /** @class */ (function () {
     return PrefixStorage;
 }());
 
-export { $llmToolsMetadataRegister, $llmToolsRegister, $scrapersMetadataRegister, $scrapersRegister, ADMIN_EMAIL, ADMIN_GITHUB_NAME, AbstractFormatError, BOOK_LANGUAGE_VERSION, BlackholeStorage, BoilerplateError, BoilerplateFormfactorDefinition, CLAIM, CallbackInterfaceTools, ChatbotFormfactorDefinition, CollectionError, CsvFormatDefinition, CsvFormatError, DEFAULT_BOOKS_DIRNAME, DEFAULT_BOOK_TITLE, DEFAULT_CSV_SETTINGS, DEFAULT_EXECUTIONS_CACHE_DIRNAME, DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME, DEFAULT_INTERMEDIATE_FILES_STRATEGY, DEFAULT_IS_AUTO_INSTALLED, DEFAULT_IS_VERBOSE, DEFAULT_MAX_EXECUTION_ATTEMPTS, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL, DEFAULT_MAX_PARALLEL_COUNT, DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, DEFAULT_REMOTE_URL, DEFAULT_REMOTE_URL_PATH, DEFAULT_SCRAPE_CACHE_DIRNAME, DEFAULT_TASK_TITLE, EXPECTATION_UNITS, EnvironmentMismatchError, ExecutionReportStringOptionsDefaults, ExpectError, FORMFACTOR_DEFINITIONS, GENERIC_PIPELINE_INTERFACE, GeneratorFormfactorDefinition, GenericFormfactorDefinition, ImageGeneratorFormfactorDefinition, KnowledgeScrapeError, LOGO_DARK_SRC, LOGO_LIGHT_SRC, LimitReachedError, MANDATORY_CSV_SETTINGS, MAX_FILENAME_LENGTH, MODEL_VARIANTS, MatcherFormfactorDefinition, MemoryStorage, MissingToolsError, MultipleLlmExecutionTools, NAME, NonTaskSectionTypes, NotFoundError, NotYetImplementedError, ORDER_OF_PIPELINE_JSON, PROMPTBOOK_ENGINE_VERSION, PROMPTBOOK_ERRORS, ParseError, PipelineExecutionError, PipelineLogicError, PipelineUrlError, PrefixStorage, RESERVED_PARAMETER_NAMES, SET_IS_VERBOSE, SectionTypes, SheetsFormfactorDefinition, TaskTypes, TextFormatDefinition, TranslatorFormfactorDefinition, UNCERTAIN_USAGE, UnexpectedError, ZERO_USAGE, _AnthropicClaudeMetadataRegistration, _AzureOpenAiMetadataRegistration, _DocumentScraperMetadataRegistration, _GoogleMetadataRegistration, _LegacyDocumentScraperMetadataRegistration, _MarkdownScraperMetadataRegistration, _OpenAiAssistantMetadataRegistration, _OpenAiMetadataRegistration, _PdfScraperMetadataRegistration, _WebsiteScraperMetadataRegistration, addUsage, assertsExecutionSuccessful, book, cacheLlmTools, collectionToJson, compilePipeline, countTotalUsage, createCollectionFromJson, createCollectionFromPromise, createCollectionFromUrl, createLlmToolsFromConfiguration, createPipelineExecutor, createSubcollection, embeddingVectorToString, executionReportJsonToString, extractParameterNamesFromTask, getPipelineInterface, isPassingExpectations, isPipelineImplementingInterface, isPipelineInterfacesEqual, isPipelinePrepared, isValidPipelineString, joinLlmExecutionTools, limitTotalUsage, makeKnowledgeSourceHandler, parsePipeline, pipelineJsonToString, prepareKnowledgePieces, preparePersona, preparePipeline, prepareTasks, prettifyPipelineString, unpreparePipeline, usageToHuman, usageToWorktime, validatePipeline, validatePipelineString };
+export { $llmToolsMetadataRegister, $llmToolsRegister, $scrapersMetadataRegister, $scrapersRegister, ADMIN_EMAIL, ADMIN_GITHUB_NAME, AbstractFormatError, BOOK_LANGUAGE_VERSION, BlackholeStorage, BoilerplateError, BoilerplateFormfactorDefinition, CLAIM, CallbackInterfaceTools, ChatbotFormfactorDefinition, CollectionError, CsvFormatDefinition, CsvFormatError, DEFAULT_BOOKS_DIRNAME, DEFAULT_BOOK_TITLE, DEFAULT_CSV_SETTINGS, DEFAULT_EXECUTIONS_CACHE_DIRNAME, DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME, DEFAULT_INTERMEDIATE_FILES_STRATEGY, DEFAULT_IS_AUTO_INSTALLED, DEFAULT_IS_VERBOSE, DEFAULT_MAX_EXECUTION_ATTEMPTS, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL, DEFAULT_MAX_PARALLEL_COUNT, DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, DEFAULT_REMOTE_URL, DEFAULT_REMOTE_URL_PATH, DEFAULT_SCRAPE_CACHE_DIRNAME, DEFAULT_TASK_TITLE, EXPECTATION_UNITS, EnvironmentMismatchError, ExecutionReportStringOptionsDefaults, ExpectError, FORMFACTOR_DEFINITIONS, GENERIC_PIPELINE_INTERFACE, GeneratorFormfactorDefinition, GenericFormfactorDefinition, ImageGeneratorFormfactorDefinition, KnowledgeScrapeError, LOGO_DARK_SRC, LOGO_LIGHT_SRC, LimitReachedError, MANDATORY_CSV_SETTINGS, MAX_FILENAME_LENGTH, MODEL_VARIANTS, MatcherFormfactorDefinition, MemoryStorage, MissingToolsError, MultipleLlmExecutionTools, NAME, NonTaskSectionTypes, NotFoundError, NotYetImplementedError, ORDER_OF_PIPELINE_JSON, PROMPTBOOK_ENGINE_VERSION, PROMPTBOOK_ERRORS, ParseError, PipelineExecutionError, PipelineLogicError, PipelineUrlError, PrefixStorage, RESERVED_PARAMETER_NAMES, SET_IS_VERBOSE, SectionTypes, SheetsFormfactorDefinition, TaskTypes, TextFormatDefinition, TranslatorFormfactorDefinition, UNCERTAIN_USAGE, UnexpectedError, ZERO_USAGE, _AnthropicClaudeMetadataRegistration, _AzureOpenAiMetadataRegistration, _BoilerplateScraperMetadataRegistration, _DocumentScraperMetadataRegistration, _GoogleMetadataRegistration, _LegacyDocumentScraperMetadataRegistration, _MarkdownScraperMetadataRegistration, _MarkitdownScraperMetadataRegistration, _OpenAiAssistantMetadataRegistration, _OpenAiMetadataRegistration, _PdfScraperMetadataRegistration, _WebsiteScraperMetadataRegistration, addUsage, assertsExecutionSuccessful, book, cacheLlmTools, collectionToJson, compilePipeline, countTotalUsage, createCollectionFromJson, createCollectionFromPromise, createCollectionFromUrl, createLlmToolsFromConfiguration, createPipelineExecutor, createSubcollection, embeddingVectorToString, executionReportJsonToString, extractParameterNamesFromTask, getPipelineInterface, isPassingExpectations, isPipelineImplementingInterface, isPipelineInterfacesEqual, isPipelinePrepared, isValidPipelineString, joinLlmExecutionTools, limitTotalUsage, makeKnowledgeSourceHandler, parsePipeline, pipelineJsonToString, prepareKnowledgePieces, preparePersona, preparePipeline, prepareTasks, prettifyPipelineString, unpreparePipeline, usageToHuman, usageToWorktime, validatePipeline, validatePipelineString };
 //# sourceMappingURL=index.es.js.map