@promptbook/website-crawler 0.84.0-16 → 0.84.0-18

package/README.md

@@ -16,7 +16,8 @@
 
 ## 🌟 New Features
 
--   🐋 **Support of [DeepSeek models](https://www.deepseek.com/)**
+-   💫 Support of [`o3-mini` model by OpenAI](https://openai.com/index/openai-o3-mini/)
+-   🐋 **Support of [DeepSeek models](https://www.npmjs.com/package/@promptbook/deepseek)**
 -   💙 Working [the **Book** language v1.0.0](https://github.com/webgptorg/book)
 -   🖤 Run books from CLI - `npx ptbk run path/to/your/book`
 -   📚 Support of `.docx`, `.doc` and `.pdf` documents as knowledge
@@ -435,6 +436,10 @@ See [TODO.md](./TODO.md)
 
 
 
+## 🤝 Partners
+
+[![Technology Incubation](./other/partners/CI-Technology-Incubation.png)](https://technologickainkubace.org/en/about-technology-incubation/about-the-project/)
+
 ## 🖋️ Contributing
 
 I am open to pull requests, feedback, and suggestions. Or if you like this utility, you can [☕ buy me a coffee](https://www.buymeacoffee.com/hejny) or [donate via cryptocurrencies](https://github.com/hejny/hejny/blob/main/documents/crypto.md).

package/esm/index.es.js

@@ -27,7 +27,7 @@ var BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-var PROMPTBOOK_ENGINE_VERSION = '0.84.0-15';
+var PROMPTBOOK_ENGINE_VERSION = '0.84.0-17';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -236,6 +236,12 @@ var ADMIN_GITHUB_NAME = 'hejny';
  * @public exported from `@promptbook/core`
  */
 var DEFAULT_BOOK_TITLE = "\u2728 Untitled Book";
+/**
+ * Maximum file size limit
+ *
+ * @public exported from `@promptbook/core`
+ */
+var DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
 // <- TODO: [🧠] Better system for generator warnings - not always "code" and "by `@promptbook/cli`"
 /**
  * The maximum number of iterations for a loops
@@ -3636,11 +3642,10 @@ var scraperFetch = function (url, init) { return __awaiter(void 0, void 0, void
 function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
     var _a;
     return __awaiter(this, void 0, void 0, function () {
-        var _b, fetch, knowledgeSourceContent, name, _c, _d, rootDirname, url, response_1, mimeType, basename, hash, rootDirname_1, filepath, _f, _g, _h, _j, _k, filename_1, fileExtension, mimeType;
-        return __generator(this, function (_l) {
-            switch (_l.label) {
+        var _b, fetch, knowledgeSourceContent, name, _c, _d, rootDirname, url, response_1, mimeType, basename, hash, rootDirname_1, filepath, fileContent, _f, _g, filename_1, fileExtension, mimeType;
+        return __generator(this, function (_h) {
+            switch (_h.label) {
                 case 0:
-                    console.log('!!! makeKnowledgeSourceHandler', knowledgeSource);
                     _b = tools.fetch, fetch = _b === void 0 ? scraperFetch : _b;
                     knowledgeSourceContent = knowledgeSource.knowledgeSourceContent;
                     name = knowledgeSource.name;
@@ -3652,7 +3657,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
                     url = knowledgeSourceContent;
                     return [4 /*yield*/, fetch(url)];
                 case 1:
-                    response_1 = _l.sent();
+                    response_1 = _h.sent();
                     mimeType = ((_a = response_1.headers.get('content-type')) === null || _a === void 0 ? void 0 : _a.split(';')[0]) || 'text/html';
                     if (tools.fs === undefined || !url.endsWith('.pdf' /* <- TODO: [💵] */)) {
                         return [2 /*return*/, {
@@ -3702,17 +3707,19 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
                     filepath = join.apply(void 0, __spreadArray(__spreadArray([], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), ["".concat(basename.substring(0, MAX_FILENAME_LENGTH), ".").concat(mimeTypeToExtension(mimeType))], false));
                     return [4 /*yield*/, tools.fs.mkdir(dirname(join(rootDirname_1, filepath)), { recursive: true })];
                 case 2:
-                    _l.sent();
-                    _g = (_f = tools.fs).writeFile;
-                    _h = [join(rootDirname_1, filepath)];
-                    _k = (_j = Buffer).from;
+                    _h.sent();
+                    _g = (_f = Buffer).from;
                     return [4 /*yield*/, response_1.arrayBuffer()];
-                case 3: return [4 /*yield*/, _g.apply(_f, _h.concat([_k.apply(_j, [_l.sent()])]))];
+                case 3:
+                    fileContent = _g.apply(_f, [_h.sent()]);
+                    if (fileContent.length > DEFAULT_MAX_FILE_SIZE /* <- TODO: Allow to pass different value to remote server */) {
+                        throw new LimitReachedError("File is too large (".concat(Math.round(fileContent.length / 1024 / 1024), "MB). Maximum allowed size is ").concat(Math.round(DEFAULT_MAX_FILE_SIZE / 1024 / 1024), "MB."));
+                    }
+                    return [4 /*yield*/, tools.fs.writeFile(join(rootDirname_1, filepath), fileContent)];
                 case 4:
-                    _l.sent();
+                    _h.sent();
                     // TODO: [💵] Check the file security
-                    // TODO: !!!!!!!! Check the file size (if it is not too big)
-                    // TODO: !!!!!!!! Delete the file after the scraping is done
+                    // TODO: [🧹][🧠] Delete the file after the scraping is done
                     return [2 /*return*/, makeKnowledgeSourceHandler({ name: name, knowledgeSourceContent: filepath }, tools, __assign(__assign({}, options), { rootDirname: rootDirname_1 }))];
                 case 5:
                     if (!isValidFilePath(knowledgeSourceContent)) return [3 /*break*/, 7];
@@ -3729,7 +3736,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
                     mimeType = extensionToMimeType(fileExtension || '');
                     return [4 /*yield*/, isFileExisting(filename_1, tools.fs)];
                 case 6:
-                    if (!(_l.sent())) {
+                    if (!(_h.sent())) {
                         throw new NotFoundError(spaceTrim$1(function (block) { return "\n                          Can not make source handler for file which does not exist:\n\n                          File:\n                          ".concat(block(knowledgeSourceContent), "\n\n                          Full file path:\n                          ").concat(block(filename_1), "\n                      "); }));
                     }
                     // TODO: [🧠][😿] Test security file - file is scoped to the project (BUT maybe do this in `filesystemTools`)