@promptbook/website-crawler 0.84.0-11 → 0.84.0-13

package/esm/index.es.js

@@ -8,6 +8,7 @@ import { basename, join, dirname } from 'path';
 import { format } from 'prettier';
 import parserHtml from 'prettier/parser-html';
 import { forTime } from 'waitasecond';
+import sha256 from 'crypto-js/sha256';
 import { lookup } from 'mime-types';
 import { unparse, parse } from 'papaparse';
 import { Converter } from 'showdown';
@@ -26,7 +27,7 @@ var BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-var PROMPTBOOK_ENGINE_VERSION = '0.84.0-10';
+var PROMPTBOOK_ENGINE_VERSION = '0.84.0-12';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -268,6 +269,12 @@ var SMALL_NUMBER = 0.001;
  * @private within the repository - too low-level in comparison with other `MAX_...`
  */
 var IMMEDIATE_TIME = 10;
+/**
+ * The maximum length of the (generated) filename
+ *
+ * @public exported from `@promptbook/core`
+ */
+var MAX_FILENAME_LENGTH = 30;
 /**
  * Strategy for caching the intermediate results for knowledge sources
  *
@@ -287,6 +294,15 @@ var DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
  * @public exported from `@promptbook/core`
  */
 var DEFAULT_MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹‍♂️]
+// <- TODO: [🕝] Make also `BOOKS_DIRNAME_ALTERNATIVES`
+/**
+ * Where to store the temporary downloads
+ *
+ * Note: When the folder does not exist, it is created recursively
+ *
+ * @public exported from `@promptbook/core`
+ */
+var DEFAULT_DOWNLOAD_CACHE_DIRNAME = './.promptbook/download-cache';
 /**
  * Where to store the scrape cache
  *
@@ -3609,10 +3625,11 @@ var scraperFetch = function (url, init) { return __awaiter(void 0, void 0, void
 function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
     var _a;
     return __awaiter(this, void 0, void 0, function () {
-        var _b, fetch, knowledgeSourceContent, name, _c, _d, rootDirname, url, response_1, mimeType, filename_1, fileExtension, mimeType;
-        return __generator(this, function (_f) {
-            switch (_f.label) {
+        var _b, fetch, knowledgeSourceContent, name, _c, _d, rootDirname, url, response_1, mimeType, basename, hash, rootDirname_1, filepath, _f, _g, _h, _j, _k, filename_1, fileExtension, mimeType;
+        return __generator(this, function (_l) {
+            switch (_l.label) {
                 case 0:
+                    console.log('!!! makeKnowledgeSourceHandler', knowledgeSource);
                     _b = tools.fetch, fetch = _b === void 0 ? scraperFetch : _b;
                     knowledgeSourceContent = knowledgeSource.knowledgeSourceContent;
                     name = knowledgeSource.name;
@@ -3620,54 +3637,74 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
                     if (!name) {
                         name = knowledgeSourceContentToName(knowledgeSourceContent);
                     }
-                    if (!isValidUrl(knowledgeSourceContent)) return [3 /*break*/, 2];
+                    if (!isValidUrl(knowledgeSourceContent)) return [3 /*break*/, 5];
                     url = knowledgeSourceContent;
                     return [4 /*yield*/, fetch(url)];
                 case 1:
-                    response_1 = _f.sent();
+                    response_1 = _l.sent();
                     mimeType = ((_a = response_1.headers.get('content-type')) === null || _a === void 0 ? void 0 : _a.split(';')[0]) || 'text/html';
-                    return [2 /*return*/, {
-                            source: name,
-                            filename: null,
-                            url: url,
-                            mimeType: mimeType,
-                            /*
-                            TODO: [🥽]
-                                > async asBlob() {
-                                >     // TODO: [👨🏻‍🤝‍👨🏻] This can be called multiple times BUT when called second time, response in already consumed
-                                >     const content = await response.blob();
-                                >     return content;
-                                > },
-                            */
-                            asJson: function () {
-                                return __awaiter(this, void 0, void 0, function () {
-                                    var content;
-                                    return __generator(this, function (_a) {
-                                        switch (_a.label) {
-                                            case 0: return [4 /*yield*/, response_1.json()];
-                                            case 1:
-                                                content = _a.sent();
-                                                return [2 /*return*/, content];
-                                        }
+                    if (tools.fs === undefined || !url.endsWith('.pdf')) {
+                        return [2 /*return*/, {
+                                source: name,
+                                filename: null,
+                                url: url,
+                                mimeType: mimeType,
+                                /*
+                              TODO: [🥽]
+                                  > async asBlob() {
+                                  >     // TODO: [👨🏻‍🤝‍👨🏻] This can be called multiple times BUT when called second time, response in already consumed
+                                  >     const content = await response.blob();
+                                  >     return content;
+                                  > },
+                              */
+                                asJson: function () {
+                                    return __awaiter(this, void 0, void 0, function () {
+                                        var content;
+                                        return __generator(this, function (_a) {
+                                            switch (_a.label) {
+                                                case 0: return [4 /*yield*/, response_1.json()];
+                                                case 1:
+                                                    content = _a.sent();
+                                                    return [2 /*return*/, content];
+                                            }
+                                        });
                                     });
-                                });
-                            },
-                            asText: function () {
-                                return __awaiter(this, void 0, void 0, function () {
-                                    var content;
-                                    return __generator(this, function (_a) {
-                                        switch (_a.label) {
-                                            case 0: return [4 /*yield*/, response_1.text()];
-                                            case 1:
-                                                content = _a.sent();
-                                                return [2 /*return*/, content];
-                                        }
+                                },
+                                asText: function () {
+                                    return __awaiter(this, void 0, void 0, function () {
+                                        var content;
+                                        return __generator(this, function (_a) {
+                                            switch (_a.label) {
+                                                case 0: return [4 /*yield*/, response_1.text()];
+                                                case 1:
+                                                    content = _a.sent();
+                                                    return [2 /*return*/, content];
+                                            }
+                                        });
                                     });
-                                });
-                            },
-                        }];
+                                },
+                            }];
+                    }
+                    basename = url.split('/').pop() || titleToName(url);
+                    hash = sha256(hexEncoder.parse(url)).toString( /* hex */);
+                    rootDirname_1 = join(process.cwd(), DEFAULT_DOWNLOAD_CACHE_DIRNAME);
+                    filepath = join.apply(void 0, __spreadArray(__spreadArray([], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), ["".concat(basename.substring(0, MAX_FILENAME_LENGTH), ".pdf")], false));
+                    return [4 /*yield*/, tools.fs.mkdir(dirname(join(rootDirname_1, filepath)), { recursive: true })];
                 case 2:
-                    if (!isValidFilePath(knowledgeSourceContent)) return [3 /*break*/, 4];
+                    _l.sent();
+                    _g = (_f = tools.fs).writeFile;
+                    _h = [join(rootDirname_1, filepath)];
+                    _k = (_j = Buffer).from;
+                    return [4 /*yield*/, response_1.arrayBuffer()];
+                case 3: return [4 /*yield*/, _g.apply(_f, _h.concat([_k.apply(_j, [_l.sent()])]))];
+                case 4:
+                    _l.sent();
+                    // TODO: !!!!!!!! Check the file security
+                    // TODO: !!!!!!!! Check the file size (if it is not too big)
+                    // TODO: !!!!!!!! Delete the file
+                    return [2 /*return*/, makeKnowledgeSourceHandler({ name: name, knowledgeSourceContent: filepath }, tools, __assign(__assign({}, options), { rootDirname: rootDirname_1 }))];
+                case 5:
+                    if (!isValidFilePath(knowledgeSourceContent)) return [3 /*break*/, 7];
                     if (tools.fs === undefined) {
                         throw new EnvironmentMismatchError('Can not import file knowledge without filesystem tools');
                         //          <- TODO: [🧠] What is the best error type here`
@@ -3680,8 +3717,8 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
                     fileExtension = getFileExtension(filename_1);
                     mimeType = extensionToMimeType(fileExtension || '');
                     return [4 /*yield*/, isFileExisting(filename_1, tools.fs)];
-                case 3:
-                    if (!(_f.sent())) {
+                case 6:
+                    if (!(_l.sent())) {
                         throw new NotFoundError(spaceTrim$1(function (block) { return "\n                          Can not make source handler for file which does not exist:\n\n                          File:\n                          ".concat(block(knowledgeSourceContent), "\n\n                          Full file path:\n                          ").concat(block(filename_1), "\n                      "); }));
                     }
                     // TODO: [🧠][😿] Test security file - file is scoped to the project (BUT maybe do this in `filesystemTools`)
@@ -3727,7 +3764,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
                                 });
                             },
                         }];
-                case 4: return [2 /*return*/, {
+                case 7: return [2 /*return*/, {
                         source: name,
                         filename: null,
                         url: null,