@promptbook/markdown-utils 0.84.0-12 → 0.84.0-14

package/esm/typings/src/_packages/core.index.d.ts

@@ -12,6 +12,8 @@ import { LOGO_LIGHT_SRC } from '../config';
 import { LOGO_DARK_SRC } from '../config';
 import { DEFAULT_BOOK_TITLE } from '../config';
 import { DEFAULT_TASK_TITLE } from '../config';
+import { DEFAULT_PROMPT_TASK_TITLE } from '../config';
+import { DEFAULT_BOOK_OUTPUT_PARAMETER_NAME } from '../config';
 import { MAX_FILENAME_LENGTH } from '../config';
 import { DEFAULT_INTERMEDIATE_FILES_STRATEGY } from '../config';
 import { DEFAULT_MAX_PARALLEL_COUNT } from '../config';
@@ -139,6 +141,8 @@ export { LOGO_LIGHT_SRC };
 export { LOGO_DARK_SRC };
 export { DEFAULT_BOOK_TITLE };
 export { DEFAULT_TASK_TITLE };
+export { DEFAULT_PROMPT_TASK_TITLE };
+export { DEFAULT_BOOK_OUTPUT_PARAMETER_NAME };
 export { MAX_FILENAME_LENGTH };
 export { DEFAULT_INTERMEDIATE_FILES_STRATEGY };
 export { DEFAULT_MAX_PARALLEL_COUNT };

package/esm/typings/src/config.d.ts

@@ -65,6 +65,18 @@ export declare const DEFAULT_BOOK_TITLE = "\u2728 Untitled Book";
  * @public exported from `@promptbook/core`
  */
 export declare const DEFAULT_TASK_TITLE = "Task";
+/**
+ * When the title of the prompt task is not provided, the default title is used
+ *
+ * @public exported from `@promptbook/core`
+ */
+export declare const DEFAULT_PROMPT_TASK_TITLE = "Prompt";
+/**
+ * When the pipeline is flat and no name of return parameter is provided, this name is used
+ *
+ * @public exported from `@promptbook/core`
+ */
+export declare const DEFAULT_BOOK_OUTPUT_PARAMETER_NAME = "result";
 /**
  * Warning message for the generated sections and files files
  *

package/esm/typings/src/utils/editable/edit-pipeline-string/deflatePipeline.test.d.ts

@@ -0,0 +1 @@
+export {};

package/esm/typings/src/utils/editable/utils/isFlatPipeline.test.d.ts

@@ -0,0 +1 @@
+export {};

package/esm/typings/src/utils/files/mimeTypeToExtension.d.ts

@@ -0,0 +1,10 @@
+import type { string_file_extension } from '../../types/typeAliases';
+import type { string_mime_type } from '../../types/typeAliases';
+/**
+ * Convert mime type to file extension
+ *
+ * Note: If the mime type is invalid, `null` is returned
+ *
+ * @private within the repository
+ */
+export declare function mimeTypeToExtension(value: string_mime_type): string_file_extension | null;

package/esm/typings/src/utils/files/mimeTypeToExtension.test.d.ts

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@promptbook/markdown-utils",
-    "version": "0.84.0-12",
+    "version": "0.84.0-14",
     "description": "It's time for a paradigm shift. The future of software in plain English, French or Latin",
     "--note-0": "    <- [🐊]",
     "private": false,

package/umd/index.umd.js

@@ -25,7 +25,7 @@
      * @generated
      * @see https://github.com/webgptorg/promptbook
      */
-    var PROMPTBOOK_ENGINE_VERSION = '0.84.0-11';
+    var PROMPTBOOK_ENGINE_VERSION = '0.84.0-13';
     /**
      * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
      * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -3208,22 +3208,6 @@
      * TODO: [®] DRY Register logic
      */
 
-    /**
-     * Removes emojis from a string and fix whitespaces
-     *
-     * @param text with emojis
-     * @returns text without emojis
-     * @public exported from `@promptbook/utils`
-     */
-    function removeEmojis(text) {
-        // Replace emojis (and also ZWJ sequence) with hyphens
-        text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
-        text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
-        text = text.replace(/(\p{Extended_Pictographic})(\u{200D}\p{Extended_Pictographic})*/gu, '$1');
-        text = text.replace(/\p{Extended_Pictographic}/gu, '');
-        return text;
-    }
-
     var defaultDiacriticsRemovalMap = [
         {
             base: 'A',
@@ -3547,30 +3531,6 @@
      * Note: [💞] Ignore a discrepancy between file name and entity name
      */
 
-    /**
-     * @@@
-     *
-     * @param value @@@
-     * @returns @@@
-     * @example @@@
-     * @public exported from `@promptbook/utils`
-     */
-    function titleToName(value) {
-        if (isValidUrl(value)) {
-            value = value.replace(/^https?:\/\//, '');
-            value = value.replace(/\.html$/, '');
-        }
-        else if (isValidFilePath(value)) {
-            value = path.basename(value);
-            // Note: Keeping extension in the name
-        }
-        value = value.split('/').join('-');
-        value = removeEmojis(value);
-        value = normalizeToKebabCase(value);
-        // TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
-        return value;
-    }
-
     /**
      * Creates unique name for the source
      *
@@ -3656,6 +3616,57 @@
      * TODO: [🖇] What about symlinks?
      */
 
+    /**
+     * Convert mime type to file extension
+     *
+     * Note: If the mime type is invalid, `null` is returned
+     *
+     * @private within the repository
+     */
+    function mimeTypeToExtension(value) {
+        return mimeTypes.extension(value) || null;
+    }
+
+    /**
+     * Removes emojis from a string and fix whitespaces
+     *
+     * @param text with emojis
+     * @returns text without emojis
+     * @public exported from `@promptbook/utils`
+     */
+    function removeEmojis(text) {
+        // Replace emojis (and also ZWJ sequence) with hyphens
+        text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
+        text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
+        text = text.replace(/(\p{Extended_Pictographic})(\u{200D}\p{Extended_Pictographic})*/gu, '$1');
+        text = text.replace(/\p{Extended_Pictographic}/gu, '');
+        return text;
+    }
+
+    /**
+     * @@@
+     *
+     * @param value @@@
+     * @returns @@@
+     * @example @@@
+     * @public exported from `@promptbook/utils`
+     */
+    function titleToName(value) {
+        if (isValidUrl(value)) {
+            value = value.replace(/^https?:\/\//, '');
+            value = value.replace(/\.html$/, '');
+        }
+        else if (isValidFilePath(value)) {
+            value = path.basename(value);
+            // Note: Keeping extension in the name
+        }
+        value = value.split('/').join('-');
+        value = removeEmojis(value);
+        value = normalizeToKebabCase(value);
+        // TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
+        return value;
+    }
+
     /**
      * The built-in `fetch' function with a lightweight error handling wrapper as default fetch function used in Promptbook scrapers
      *
@@ -3691,7 +3702,7 @@
     function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
         var _a;
         return __awaiter(this, void 0, void 0, function () {
-            var _b, fetch, knowledgeSourceContent, name, _c, _d, rootDirname, url, response, mimeType, filename, hash, rootDirname_1, filepath, _f, _g, _h, _j, _k, filename_1, fileExtension, mimeType;
+            var _b, fetch, knowledgeSourceContent, name, _c, _d, rootDirname, url, response_1, mimeType, basename, hash, rootDirname_1, filepath, _f, _g, _h, _j, _k, filename_1, fileExtension, mimeType;
             return __generator(this, function (_l) {
                 switch (_l.label) {
                     case 0:
@@ -3707,25 +3718,67 @@
                         url = knowledgeSourceContent;
                         return [4 /*yield*/, fetch(url)];
                     case 1:
-                        response = _l.sent();
-                        mimeType = ((_a = response.headers.get('content-type')) === null || _a === void 0 ? void 0 : _a.split(';')[0]) || 'text/html';
-                        filename = url.split('/').pop() || titleToName(url);
+                        response_1 = _l.sent();
+                        mimeType = ((_a = response_1.headers.get('content-type')) === null || _a === void 0 ? void 0 : _a.split(';')[0]) || 'text/html';
+                        if (tools.fs === undefined || !url.endsWith('.pdf' /* <- TODO: [💵] */)) {
+                            return [2 /*return*/, {
+                                    source: name,
+                                    filename: null,
+                                    url: url,
+                                    mimeType: mimeType,
+                                    /*
+                                  TODO: [🥽]
+                                      > async asBlob() {
+                                      >     // TODO: [👨🏻‍🤝‍👨🏻] This can be called multiple times BUT when called second time, response in already consumed
+                                      >     const content = await response.blob();
+                                      >     return content;
+                                      > },
+                                  */
+                                    asJson: function () {
+                                        return __awaiter(this, void 0, void 0, function () {
+                                            var content;
+                                            return __generator(this, function (_a) {
+                                                switch (_a.label) {
+                                                    case 0: return [4 /*yield*/, response_1.json()];
+                                                    case 1:
+                                                        content = _a.sent();
+                                                        return [2 /*return*/, content];
+                                                }
+                                            });
+                                        });
+                                    },
+                                    asText: function () {
+                                        return __awaiter(this, void 0, void 0, function () {
+                                            var content;
+                                            return __generator(this, function (_a) {
+                                                switch (_a.label) {
+                                                    case 0: return [4 /*yield*/, response_1.text()];
+                                                    case 1:
+                                                        content = _a.sent();
+                                                        return [2 /*return*/, content];
+                                                }
+                                            });
+                                        });
+                                    },
+                                }];
+                        }
+                        basename = url.split('/').pop() || titleToName(url);
                         hash = sha256__default["default"](hexEncoder__default["default"].parse(url)).toString( /* hex */);
                         rootDirname_1 = path.join(process.cwd(), DEFAULT_DOWNLOAD_CACHE_DIRNAME);
-                        filepath = path.join.apply(void 0, __spreadArray(__spreadArray([], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), ["".concat(filename.substring(0, MAX_FILENAME_LENGTH), ".pdf")], false));
+                        filepath = path.join.apply(void 0, __spreadArray(__spreadArray([], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), ["".concat(basename.substring(0, MAX_FILENAME_LENGTH), ".").concat(mimeTypeToExtension(mimeType))], false));
                         return [4 /*yield*/, tools.fs.mkdir(path.dirname(path.join(rootDirname_1, filepath)), { recursive: true })];
                     case 2:
                         _l.sent();
                         _g = (_f = tools.fs).writeFile;
                         _h = [path.join(rootDirname_1, filepath)];
                         _k = (_j = Buffer).from;
-                        return [4 /*yield*/, response.arrayBuffer()];
+                        return [4 /*yield*/, response_1.arrayBuffer()];
                     case 3: return [4 /*yield*/, _g.apply(_f, _h.concat([_k.apply(_j, [_l.sent()])]))];
                     case 4:
                         _l.sent();
-                        // TODO: !!!!!!!! Check the file security
+                        // TODO: [💵] Check the file security
                         // TODO: !!!!!!!! Check the file size (if it is not too big)
-                        // TODO: !!!!!!!! Delete the file
+                        // TODO: !!!!!!!! Delete the file after the scraping is done
                         return [2 /*return*/, makeKnowledgeSourceHandler({ name: name, knowledgeSourceContent: filepath }, tools, __assign(__assign({}, options), { rootDirname: rootDirname_1 }))];
                     case 5:
                         if (!isValidFilePath(knowledgeSourceContent)) return [3 /*break*/, 7];