@promptbook/remote-server 0.84.0-13 → 0.84.0-14

package/esm/index.es.js

@@ -12,7 +12,7 @@ import { unparse, parse } from 'papaparse';
 import hexEncoder from 'crypto-js/enc-hex';
 import sha256 from 'crypto-js/sha256';
 import { SHA256 } from 'crypto-js';
-import { lookup } from 'mime-types';
+import { lookup, extension } from 'mime-types';
 
 // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten
 /**
@@ -28,7 +28,7 @@ var BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-var PROMPTBOOK_ENGINE_VERSION = '0.84.0-12';
+var PROMPTBOOK_ENGINE_VERSION = '0.84.0-13';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -6146,6 +6146,17 @@ function isFileExisting(filename, fs) {
  * TODO: [🖇] What about symlinks?
  */
 
+/**
+ * Convert mime type to file extension
+ *
+ * Note: If the mime type is invalid, `null` is returned
+ *
+ * @private within the repository
+ */
+function mimeTypeToExtension(value) {
+    return extension(value) || null;
+}
+
 /**
  * Removes emojis from a string and fix whitespaces
  *
@@ -6239,7 +6250,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
                 case 1:
                     response_1 = _l.sent();
                     mimeType = ((_a = response_1.headers.get('content-type')) === null || _a === void 0 ? void 0 : _a.split(';')[0]) || 'text/html';
-                    if (tools.fs === undefined || !url.endsWith('.pdf')) {
+                    if (tools.fs === undefined || !url.endsWith('.pdf' /* <- TODO: [💵] */)) {
                         return [2 /*return*/, {
                                 source: name,
                                 filename: null,
@@ -6284,7 +6295,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
                     basename = url.split('/').pop() || titleToName(url);
                     hash = sha256(hexEncoder.parse(url)).toString( /* hex */);
                     rootDirname_1 = join(process.cwd(), DEFAULT_DOWNLOAD_CACHE_DIRNAME);
-                    filepath = join.apply(void 0, __spreadArray(__spreadArray([], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), ["".concat(basename.substring(0, MAX_FILENAME_LENGTH), ".pdf")], false));
+                    filepath = join.apply(void 0, __spreadArray(__spreadArray([], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), ["".concat(basename.substring(0, MAX_FILENAME_LENGTH), ".").concat(mimeTypeToExtension(mimeType))], false));
                     return [4 /*yield*/, tools.fs.mkdir(dirname(join(rootDirname_1, filepath)), { recursive: true })];
                 case 2:
                     _l.sent();
@@ -6295,9 +6306,9 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
                 case 3: return [4 /*yield*/, _g.apply(_f, _h.concat([_k.apply(_j, [_l.sent()])]))];
                 case 4:
                     _l.sent();
-                    // TODO: !!!!!!!! Check the file security
+                    // TODO: [💵] Check the file security
                     // TODO: !!!!!!!! Check the file size (if it is not too big)
-                    // TODO: !!!!!!!! Delete the file
+                    // TODO: !!!!!!!! Delete the file after the scraping is done
                     return [2 /*return*/, makeKnowledgeSourceHandler({ name: name, knowledgeSourceContent: filepath }, tools, __assign(__assign({}, options), { rootDirname: rootDirname_1 }))];
                 case 5:
                     if (!isValidFilePath(knowledgeSourceContent)) return [3 /*break*/, 7];