@promptbook/core 0.84.0-12 → 0.84.0-14

package/esm/index.es.js

@@ -7,7 +7,7 @@ import hexEncoder from 'crypto-js/enc-hex';
 import sha256 from 'crypto-js/sha256';
 import { basename, join, dirname } from 'path';
 import { SHA256 } from 'crypto-js';
-import { lookup } from 'mime-types';
+import { lookup, extension } from 'mime-types';
 import moment from 'moment';
 import colors from 'colors';
 
@@ -25,7 +25,7 @@ var BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-var PROMPTBOOK_ENGINE_VERSION = '0.84.0-11';
+var PROMPTBOOK_ENGINE_VERSION = '0.84.0-13';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -672,6 +672,18 @@ var DEFAULT_BOOK_TITLE = "\u2728 Untitled Book";
  * @public exported from `@promptbook/core`
  */
 var DEFAULT_TASK_TITLE = "Task";
+/**
+ * When the title of the prompt task is not provided, the default title is used
+ *
+ * @public exported from `@promptbook/core`
+ */
+var DEFAULT_PROMPT_TASK_TITLE = "Prompt";
+/**
+ * When the pipeline is flat and no name of return parameter is provided, this name is used
+ *
+ * @public exported from `@promptbook/core`
+ */
+var DEFAULT_BOOK_OUTPUT_PARAMETER_NAME = 'result';
 // <- TODO: [🧠] Better system for generator warnings - not always "code" and "by `@promptbook/cli`"
 /**
  * The maximum number of iterations for a loops
@@ -5674,22 +5686,6 @@ function $registeredScrapersMessage(availableScrapers) {
  * TODO: [®] DRY Register logic
  */
 
-/**
- * Removes emojis from a string and fix whitespaces
- *
- * @param text with emojis
- * @returns text without emojis
- * @public exported from `@promptbook/utils`
- */
-function removeEmojis(text) {
-    // Replace emojis (and also ZWJ sequence) with hyphens
-    text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
-    text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
-    text = text.replace(/(\p{Extended_Pictographic})(\u{200D}\p{Extended_Pictographic})*/gu, '$1');
-    text = text.replace(/\p{Extended_Pictographic}/gu, '');
-    return text;
-}
-
 /**
  * @@@
  *
@@ -5752,30 +5748,6 @@ function normalizeToKebabCase(text) {
  * Note: [💞] Ignore a discrepancy between file name and entity name
  */
 
-/**
- * @@@
- *
- * @param value @@@
- * @returns @@@
- * @example @@@
- * @public exported from `@promptbook/utils`
- */
-function titleToName(value) {
-    if (isValidUrl(value)) {
-        value = value.replace(/^https?:\/\//, '');
-        value = value.replace(/\.html$/, '');
-    }
-    else if (isValidFilePath(value)) {
-        value = basename(value);
-        // Note: Keeping extension in the name
-    }
-    value = value.split('/').join('-');
-    value = removeEmojis(value);
-    value = normalizeToKebabCase(value);
-    // TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
-    return value;
-}
-
 /**
  * Creates unique name for the source
  *
@@ -5861,6 +5833,57 @@ function isFileExisting(filename, fs) {
  * TODO: [🖇] What about symlinks?
  */
 
+/**
+ * Convert mime type to file extension
+ *
+ * Note: If the mime type is invalid, `null` is returned
+ *
+ * @private within the repository
+ */
+function mimeTypeToExtension(value) {
+    return extension(value) || null;
+}
+
+/**
+ * Removes emojis from a string and fix whitespaces
+ *
+ * @param text with emojis
+ * @returns text without emojis
+ * @public exported from `@promptbook/utils`
+ */
+function removeEmojis(text) {
+    // Replace emojis (and also ZWJ sequence) with hyphens
+    text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
+    text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
+    text = text.replace(/(\p{Extended_Pictographic})(\u{200D}\p{Extended_Pictographic})*/gu, '$1');
+    text = text.replace(/\p{Extended_Pictographic}/gu, '');
+    return text;
+}
+
+/**
+ * @@@
+ *
+ * @param value @@@
+ * @returns @@@
+ * @example @@@
+ * @public exported from `@promptbook/utils`
+ */
+function titleToName(value) {
+    if (isValidUrl(value)) {
+        value = value.replace(/^https?:\/\//, '');
+        value = value.replace(/\.html$/, '');
+    }
+    else if (isValidFilePath(value)) {
+        value = basename(value);
+        // Note: Keeping extension in the name
+    }
+    value = value.split('/').join('-');
+    value = removeEmojis(value);
+    value = normalizeToKebabCase(value);
+    // TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
+    return value;
+}
+
 /**
  * The built-in `fetch' function with a lightweight error handling wrapper as default fetch function used in Promptbook scrapers
  *
@@ -5896,7 +5919,7 @@ var scraperFetch = function (url, init) { return __awaiter(void 0, void 0, void
 function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
     var _a;
     return __awaiter(this, void 0, void 0, function () {
-        var _b, fetch, knowledgeSourceContent, name, _c, _d, rootDirname, url, response, mimeType, filename, hash, rootDirname_1, filepath, _f, _g, _h, _j, _k, filename_1, fileExtension, mimeType;
+        var _b, fetch, knowledgeSourceContent, name, _c, _d, rootDirname, url, response_1, mimeType, basename, hash, rootDirname_1, filepath, _f, _g, _h, _j, _k, filename_1, fileExtension, mimeType;
         return __generator(this, function (_l) {
             switch (_l.label) {
                 case 0:
@@ -5912,25 +5935,67 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
                     url = knowledgeSourceContent;
                     return [4 /*yield*/, fetch(url)];
                 case 1:
-                    response = _l.sent();
-                    mimeType = ((_a = response.headers.get('content-type')) === null || _a === void 0 ? void 0 : _a.split(';')[0]) || 'text/html';
-                    filename = url.split('/').pop() || titleToName(url);
+                    response_1 = _l.sent();
+                    mimeType = ((_a = response_1.headers.get('content-type')) === null || _a === void 0 ? void 0 : _a.split(';')[0]) || 'text/html';
+                    if (tools.fs === undefined || !url.endsWith('.pdf' /* <- TODO: [💵] */)) {
+                        return [2 /*return*/, {
+                                source: name,
+                                filename: null,
+                                url: url,
+                                mimeType: mimeType,
+                                /*
+                              TODO: [🥽]
+                                  > async asBlob() {
+                                  >     // TODO: [👨🏻‍🤝‍👨🏻] This can be called multiple times BUT when called second time, response in already consumed
+                                  >     const content = await response.blob();
+                                  >     return content;
+                                  > },
+                              */
+                                asJson: function () {
+                                    return __awaiter(this, void 0, void 0, function () {
+                                        var content;
+                                        return __generator(this, function (_a) {
+                                            switch (_a.label) {
+                                                case 0: return [4 /*yield*/, response_1.json()];
+                                                case 1:
+                                                    content = _a.sent();
+                                                    return [2 /*return*/, content];
+                                            }
+                                        });
+                                    });
+                                },
+                                asText: function () {
+                                    return __awaiter(this, void 0, void 0, function () {
+                                        var content;
+                                        return __generator(this, function (_a) {
+                                            switch (_a.label) {
+                                                case 0: return [4 /*yield*/, response_1.text()];
+                                                case 1:
+                                                    content = _a.sent();
+                                                    return [2 /*return*/, content];
+                                            }
+                                        });
+                                    });
+                                },
+                            }];
+                    }
+                    basename = url.split('/').pop() || titleToName(url);
                     hash = sha256(hexEncoder.parse(url)).toString( /* hex */);
                     rootDirname_1 = join(process.cwd(), DEFAULT_DOWNLOAD_CACHE_DIRNAME);
-                    filepath = join.apply(void 0, __spreadArray(__spreadArray([], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), ["".concat(filename.substring(0, MAX_FILENAME_LENGTH), ".pdf")], false));
+                    filepath = join.apply(void 0, __spreadArray(__spreadArray([], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), ["".concat(basename.substring(0, MAX_FILENAME_LENGTH), ".").concat(mimeTypeToExtension(mimeType))], false));
                     return [4 /*yield*/, tools.fs.mkdir(dirname(join(rootDirname_1, filepath)), { recursive: true })];
                 case 2:
                     _l.sent();
                     _g = (_f = tools.fs).writeFile;
                     _h = [join(rootDirname_1, filepath)];
                     _k = (_j = Buffer).from;
-                    return [4 /*yield*/, response.arrayBuffer()];
+                    return [4 /*yield*/, response_1.arrayBuffer()];
                 case 3: return [4 /*yield*/, _g.apply(_f, _h.concat([_k.apply(_j, [_l.sent()])]))];
                 case 4:
                     _l.sent();
-                    // TODO: !!!!!!!! Check the file security
+                    // TODO: [💵] Check the file security
                     // TODO: !!!!!!!! Check the file size (if it is not too big)
-                    // TODO: !!!!!!!! Delete the file
+                    // TODO: !!!!!!!! Delete the file after the scraping is done
                     return [2 /*return*/, makeKnowledgeSourceHandler({ name: name, knowledgeSourceContent: filepath }, tools, __assign(__assign({}, options), { rootDirname: rootDirname_1 }))];
                 case 5:
                     if (!isValidFilePath(knowledgeSourceContent)) return [3 /*break*/, 7];
@@ -9051,12 +9116,15 @@ function isFlatPipeline(pipelineString) {
     pipelineString = removeMarkdownComments(pipelineString);
     pipelineString = spaceTrim(pipelineString);
     var isMarkdownBeginningWithHeadline = pipelineString.startsWith('# ');
-    var isLastLineReturnStatement = pipelineString.split('\n').pop().split('`').join('').startsWith('->');
-    // TODO: Also (double)check
+    //const isLastLineReturnStatement = pipelineString.split('\n').pop()!.split('`').join('').startsWith('->');
+    var isBacktickBlockUsed = pipelineString.includes('```');
+    var isQuoteBlocksUsed = /^>\s+/m.test(pipelineString);
+    var isBlocksUsed = isBacktickBlockUsed || isQuoteBlocksUsed;
+    // TODO: [🧉] Also (double)check
     // > const usedCommands
     // > const isBlocksUsed
     // > const returnStatementCount
-    var isFlat = !isMarkdownBeginningWithHeadline && isLastLineReturnStatement;
+    var isFlat = !isMarkdownBeginningWithHeadline && !isBlocksUsed; /* && isLastLineReturnStatement */
     return isFlat;
 }
 
@@ -9070,9 +9138,26 @@ function deflatePipeline(pipelineString) {
         return pipelineString;
     }
     var pipelineStringLines = pipelineString.split('\n');
-    var returnStatement = pipelineStringLines.pop();
+    var potentialReturnStatement = pipelineStringLines.pop();
+    var returnStatement;
+    if (/(-|=)>\s*\{.*\}/.test(potentialReturnStatement)) {
+        // Note: Last line is return statement
+        returnStatement = potentialReturnStatement;
+    }
+    else {
+        // Note: Last line is not a return statement
+        returnStatement = "-> {".concat(DEFAULT_BOOK_OUTPUT_PARAMETER_NAME, "}");
+        pipelineStringLines.push(potentialReturnStatement);
+    }
     var prompt = spaceTrim(pipelineStringLines.join('\n'));
-    pipelineString = validatePipelineString(spaceTrim(function (block) { return "\n                # ".concat(DEFAULT_BOOK_TITLE, "\n\n                ## Prompt\n\n                ```\n                ").concat(block(prompt), "\n                ```\n\n                ").concat(returnStatement, "\n            "); }));
+    var quotedPrompt;
+    if (prompt.split('\n').length <= 1) {
+        quotedPrompt = "> ".concat(prompt);
+    }
+    else {
+        quotedPrompt = spaceTrim(function (block) { return "\n                ```\n                ".concat(block(prompt.split('`').join('\\`')), "\n                ```\n            "); });
+    }
+    pipelineString = validatePipelineString(spaceTrim(function (block) { return "\n                # ".concat(DEFAULT_BOOK_TITLE, "\n\n                ## Prompt\n\n                ").concat(block(quotedPrompt), "\n\n                ").concat(returnStatement, "\n            "); }));
     // <- TODO: Maybe use book` notation
     return pipelineString;
 }
@@ -11331,7 +11416,7 @@ var markitdownScraperMetadata = $deepFreeze({
     className: 'MarkitdownScraper',
     mimeTypes: [
         'application/pdf',
-        // TODO: Make priority for scrapers and than allow all mime types here:
+        // TODO: [💵] Make priority for scrapers and than analyze which mime-types can Markitdown scrape and allow all mime types here:
         // 'text/html',
         // 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
     ],
@@ -11493,5 +11578,5 @@ var PrefixStorage = /** @class */ (function () {
     return PrefixStorage;
 }());
 
-export { $llmToolsMetadataRegister, $llmToolsRegister, $scrapersMetadataRegister, $scrapersRegister, ADMIN_EMAIL, ADMIN_GITHUB_NAME, AbstractFormatError, BOOK_LANGUAGE_VERSION, BlackholeStorage, BoilerplateError, BoilerplateFormfactorDefinition, CLAIM, CallbackInterfaceTools, ChatbotFormfactorDefinition, CollectionError, CsvFormatDefinition, CsvFormatError, DEFAULT_BOOKS_DIRNAME, DEFAULT_BOOK_TITLE, DEFAULT_CSV_SETTINGS, DEFAULT_DOWNLOAD_CACHE_DIRNAME, DEFAULT_EXECUTION_CACHE_DIRNAME, DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME, DEFAULT_INTERMEDIATE_FILES_STRATEGY, DEFAULT_IS_AUTO_INSTALLED, DEFAULT_IS_VERBOSE, DEFAULT_MAX_EXECUTION_ATTEMPTS, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL, DEFAULT_MAX_PARALLEL_COUNT, DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, DEFAULT_REMOTE_URL, DEFAULT_REMOTE_URL_PATH, DEFAULT_SCRAPE_CACHE_DIRNAME, DEFAULT_TASK_TITLE, EXPECTATION_UNITS, EnvironmentMismatchError, ExecutionReportStringOptionsDefaults, ExpectError, FORMFACTOR_DEFINITIONS, GENERIC_PIPELINE_INTERFACE, GeneratorFormfactorDefinition, GenericFormfactorDefinition, ImageGeneratorFormfactorDefinition, KnowledgeScrapeError, LOGO_DARK_SRC, LOGO_LIGHT_SRC, LimitReachedError, MANDATORY_CSV_SETTINGS, MAX_FILENAME_LENGTH, MODEL_VARIANTS, MatcherFormfactorDefinition, MemoryStorage, MissingToolsError, MultipleLlmExecutionTools, NAME, NonTaskSectionTypes, NotFoundError, NotYetImplementedError, ORDER_OF_PIPELINE_JSON, PROMPTBOOK_ENGINE_VERSION, PROMPTBOOK_ERRORS, ParseError, PipelineExecutionError, PipelineLogicError, PipelineUrlError, PrefixStorage, RESERVED_PARAMETER_NAMES, SET_IS_VERBOSE, SectionTypes, SheetsFormfactorDefinition, TaskTypes, TextFormatDefinition, TranslatorFormfactorDefinition, UNCERTAIN_USAGE, UnexpectedError, ZERO_USAGE, _AnthropicClaudeMetadataRegistration, _AzureOpenAiMetadataRegistration, _BoilerplateScraperMetadataRegistration, _DocumentScraperMetadataRegistration, _GoogleMetadataRegistration, _LegacyDocumentScraperMetadataRegistration, _MarkdownScraperMetadataRegistration, _MarkitdownScraperMetadataRegistration, _OpenAiAssistantMetadataRegistration, _OpenAiMetadataRegistration, _PdfScraperMetadataRegistration, _WebsiteScraperMetadataRegistration, addUsage, assertsExecutionSuccessful, book, cacheLlmTools, collectionToJson, compilePipeline, countTotalUsage, createCollectionFromJson, createCollectionFromPromise, createCollectionFromUrl, createLlmToolsFromConfiguration, createPipelineExecutor, createSubcollection, embeddingVectorToString, executionReportJsonToString, extractParameterNamesFromTask, getPipelineInterface, isPassingExpectations, isPipelineImplementingInterface, isPipelineInterfacesEqual, isPipelinePrepared, isValidPipelineString, joinLlmExecutionTools, limitTotalUsage, makeKnowledgeSourceHandler, parsePipeline, pipelineJsonToString, prepareKnowledgePieces, preparePersona, preparePipeline, prepareTasks, prettifyPipelineString, unpreparePipeline, usageToHuman, usageToWorktime, validatePipeline, validatePipelineString };
+export { $llmToolsMetadataRegister, $llmToolsRegister, $scrapersMetadataRegister, $scrapersRegister, ADMIN_EMAIL, ADMIN_GITHUB_NAME, AbstractFormatError, BOOK_LANGUAGE_VERSION, BlackholeStorage, BoilerplateError, BoilerplateFormfactorDefinition, CLAIM, CallbackInterfaceTools, ChatbotFormfactorDefinition, CollectionError, CsvFormatDefinition, CsvFormatError, DEFAULT_BOOKS_DIRNAME, DEFAULT_BOOK_OUTPUT_PARAMETER_NAME, DEFAULT_BOOK_TITLE, DEFAULT_CSV_SETTINGS, DEFAULT_DOWNLOAD_CACHE_DIRNAME, DEFAULT_EXECUTION_CACHE_DIRNAME, DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME, DEFAULT_INTERMEDIATE_FILES_STRATEGY, DEFAULT_IS_AUTO_INSTALLED, DEFAULT_IS_VERBOSE, DEFAULT_MAX_EXECUTION_ATTEMPTS, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL, DEFAULT_MAX_PARALLEL_COUNT, DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, DEFAULT_PROMPT_TASK_TITLE, DEFAULT_REMOTE_URL, DEFAULT_REMOTE_URL_PATH, DEFAULT_SCRAPE_CACHE_DIRNAME, DEFAULT_TASK_TITLE, EXPECTATION_UNITS, EnvironmentMismatchError, ExecutionReportStringOptionsDefaults, ExpectError, FORMFACTOR_DEFINITIONS, GENERIC_PIPELINE_INTERFACE, GeneratorFormfactorDefinition, GenericFormfactorDefinition, ImageGeneratorFormfactorDefinition, KnowledgeScrapeError, LOGO_DARK_SRC, LOGO_LIGHT_SRC, LimitReachedError, MANDATORY_CSV_SETTINGS, MAX_FILENAME_LENGTH, MODEL_VARIANTS, MatcherFormfactorDefinition, MemoryStorage, MissingToolsError, MultipleLlmExecutionTools, NAME, NonTaskSectionTypes, NotFoundError, NotYetImplementedError, ORDER_OF_PIPELINE_JSON, PROMPTBOOK_ENGINE_VERSION, PROMPTBOOK_ERRORS, ParseError, PipelineExecutionError, PipelineLogicError, PipelineUrlError, PrefixStorage, RESERVED_PARAMETER_NAMES, SET_IS_VERBOSE, SectionTypes, SheetsFormfactorDefinition, TaskTypes, TextFormatDefinition, TranslatorFormfactorDefinition, UNCERTAIN_USAGE, UnexpectedError, ZERO_USAGE, _AnthropicClaudeMetadataRegistration, _AzureOpenAiMetadataRegistration, _BoilerplateScraperMetadataRegistration, _DocumentScraperMetadataRegistration, _GoogleMetadataRegistration, _LegacyDocumentScraperMetadataRegistration, _MarkdownScraperMetadataRegistration, _MarkitdownScraperMetadataRegistration, _OpenAiAssistantMetadataRegistration, _OpenAiMetadataRegistration, _PdfScraperMetadataRegistration, _WebsiteScraperMetadataRegistration, addUsage, assertsExecutionSuccessful, book, cacheLlmTools, collectionToJson, compilePipeline, countTotalUsage, createCollectionFromJson, createCollectionFromPromise, createCollectionFromUrl, createLlmToolsFromConfiguration, createPipelineExecutor, createSubcollection, embeddingVectorToString, executionReportJsonToString, extractParameterNamesFromTask, getPipelineInterface, isPassingExpectations, isPipelineImplementingInterface, isPipelineInterfacesEqual, isPipelinePrepared, isValidPipelineString, joinLlmExecutionTools, limitTotalUsage, makeKnowledgeSourceHandler, parsePipeline, pipelineJsonToString, prepareKnowledgePieces, preparePersona, preparePipeline, prepareTasks, prettifyPipelineString, unpreparePipeline, usageToHuman, usageToWorktime, validatePipeline, validatePipelineString };
 //# sourceMappingURL=index.es.js.map