@promptbook/markdown-utils 0.84.0-12 → 0.84.0-13

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@promptbook/markdown-utils",
-    "version": "0.84.0-12",
+    "version": "0.84.0-13",
     "description": "It's time for a paradigm shift. The future of software in plain English, French or Latin",
     "--note-0": "    <- [🐊]",
     "private": false,

package/umd/index.umd.js

@@ -25,7 +25,7 @@
      * @generated
      * @see https://github.com/webgptorg/promptbook
      */
-    var PROMPTBOOK_ENGINE_VERSION = '0.84.0-11';
+    var PROMPTBOOK_ENGINE_VERSION = '0.84.0-12';
     /**
      * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
      * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -3208,22 +3208,6 @@
      * TODO: [®] DRY Register logic
      */
 
-    /**
-     * Removes emojis from a string and fix whitespaces
-     *
-     * @param text with emojis
-     * @returns text without emojis
-     * @public exported from `@promptbook/utils`
-     */
-    function removeEmojis(text) {
-        // Replace emojis (and also ZWJ sequence) with hyphens
-        text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
-        text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
-        text = text.replace(/(\p{Extended_Pictographic})(\u{200D}\p{Extended_Pictographic})*/gu, '$1');
-        text = text.replace(/\p{Extended_Pictographic}/gu, '');
-        return text;
-    }
-
     var defaultDiacriticsRemovalMap = [
         {
             base: 'A',
@@ -3547,30 +3531,6 @@
      * Note: [💞] Ignore a discrepancy between file name and entity name
      */
 
-    /**
-     * @@@
-     *
-     * @param value @@@
-     * @returns @@@
-     * @example @@@
-     * @public exported from `@promptbook/utils`
-     */
-    function titleToName(value) {
-        if (isValidUrl(value)) {
-            value = value.replace(/^https?:\/\//, '');
-            value = value.replace(/\.html$/, '');
-        }
-        else if (isValidFilePath(value)) {
-            value = path.basename(value);
-            // Note: Keeping extension in the name
-        }
-        value = value.split('/').join('-');
-        value = removeEmojis(value);
-        value = normalizeToKebabCase(value);
-        // TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
-        return value;
-    }
-
     /**
      * Creates unique name for the source
      *
@@ -3656,6 +3616,46 @@
      * TODO: [🖇] What about symlinks?
      */
 
+    /**
+     * Removes emojis from a string and fix whitespaces
+     *
+     * @param text with emojis
+     * @returns text without emojis
+     * @public exported from `@promptbook/utils`
+     */
+    function removeEmojis(text) {
+        // Replace emojis (and also ZWJ sequence) with hyphens
+        text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
+        text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
+        text = text.replace(/(\p{Extended_Pictographic})(\u{200D}\p{Extended_Pictographic})*/gu, '$1');
+        text = text.replace(/\p{Extended_Pictographic}/gu, '');
+        return text;
+    }
+
+    /**
+     * @@@
+     *
+     * @param value @@@
+     * @returns @@@
+     * @example @@@
+     * @public exported from `@promptbook/utils`
+     */
+    function titleToName(value) {
+        if (isValidUrl(value)) {
+            value = value.replace(/^https?:\/\//, '');
+            value = value.replace(/\.html$/, '');
+        }
+        else if (isValidFilePath(value)) {
+            value = path.basename(value);
+            // Note: Keeping extension in the name
+        }
+        value = value.split('/').join('-');
+        value = removeEmojis(value);
+        value = normalizeToKebabCase(value);
+        // TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
+        return value;
+    }
+
     /**
      * The built-in `fetch' function with a lightweight error handling wrapper as default fetch function used in Promptbook scrapers
      *
@@ -3691,7 +3691,7 @@
     function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
         var _a;
         return __awaiter(this, void 0, void 0, function () {
-            var _b, fetch, knowledgeSourceContent, name, _c, _d, rootDirname, url, response, mimeType, filename, hash, rootDirname_1, filepath, _f, _g, _h, _j, _k, filename_1, fileExtension, mimeType;
+            var _b, fetch, knowledgeSourceContent, name, _c, _d, rootDirname, url, response_1, mimeType, basename, hash, rootDirname_1, filepath, _f, _g, _h, _j, _k, filename_1, fileExtension, mimeType;
             return __generator(this, function (_l) {
                 switch (_l.label) {
                     case 0:
@@ -3707,19 +3707,61 @@
                         url = knowledgeSourceContent;
                         return [4 /*yield*/, fetch(url)];
                     case 1:
-                        response = _l.sent();
-                        mimeType = ((_a = response.headers.get('content-type')) === null || _a === void 0 ? void 0 : _a.split(';')[0]) || 'text/html';
-                        filename = url.split('/').pop() || titleToName(url);
+                        response_1 = _l.sent();
+                        mimeType = ((_a = response_1.headers.get('content-type')) === null || _a === void 0 ? void 0 : _a.split(';')[0]) || 'text/html';
+                        if (tools.fs === undefined || !url.endsWith('.pdf')) {
+                            return [2 /*return*/, {
+                                    source: name,
+                                    filename: null,
+                                    url: url,
+                                    mimeType: mimeType,
+                                    /*
+                                  TODO: [🥽]
+                                      > async asBlob() {
+                                      >     // TODO: [👨🏻‍🤝‍👨🏻] This can be called multiple times BUT when called second time, response in already consumed
+                                      >     const content = await response.blob();
+                                      >     return content;
+                                      > },
+                                  */
+                                    asJson: function () {
+                                        return __awaiter(this, void 0, void 0, function () {
+                                            var content;
+                                            return __generator(this, function (_a) {
+                                                switch (_a.label) {
+                                                    case 0: return [4 /*yield*/, response_1.json()];
+                                                    case 1:
+                                                        content = _a.sent();
+                                                        return [2 /*return*/, content];
+                                                }
+                                            });
+                                        });
+                                    },
+                                    asText: function () {
+                                        return __awaiter(this, void 0, void 0, function () {
+                                            var content;
+                                            return __generator(this, function (_a) {
+                                                switch (_a.label) {
+                                                    case 0: return [4 /*yield*/, response_1.text()];
+                                                    case 1:
+                                                        content = _a.sent();
+                                                        return [2 /*return*/, content];
+                                                }
+                                            });
+                                        });
+                                    },
+                                }];
+                        }
+                        basename = url.split('/').pop() || titleToName(url);
                         hash = sha256__default["default"](hexEncoder__default["default"].parse(url)).toString( /* hex */);
                         rootDirname_1 = path.join(process.cwd(), DEFAULT_DOWNLOAD_CACHE_DIRNAME);
-                        filepath = path.join.apply(void 0, __spreadArray(__spreadArray([], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), ["".concat(filename.substring(0, MAX_FILENAME_LENGTH), ".pdf")], false));
+                        filepath = path.join.apply(void 0, __spreadArray(__spreadArray([], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), ["".concat(basename.substring(0, MAX_FILENAME_LENGTH), ".pdf")], false));
                         return [4 /*yield*/, tools.fs.mkdir(path.dirname(path.join(rootDirname_1, filepath)), { recursive: true })];
                     case 2:
                         _l.sent();
                         _g = (_f = tools.fs).writeFile;
                         _h = [path.join(rootDirname_1, filepath)];
                         _k = (_j = Buffer).from;
-                        return [4 /*yield*/, response.arrayBuffer()];
+                        return [4 /*yield*/, response_1.arrayBuffer()];
                     case 3: return [4 /*yield*/, _g.apply(_f, _h.concat([_k.apply(_j, [_l.sent()])]))];
                     case 4:
                         _l.sent();