@promptbook/website-crawler 0.84.0-11 → 0.84.0-13

package/esm/typings/src/_packages/core.index.d.ts

@@ -19,7 +19,8 @@ import { DEFAULT_MAX_EXECUTION_ATTEMPTS } from '../config';
 import { DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH } from '../config';
 import { DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL } from '../config';
 import { DEFAULT_BOOKS_DIRNAME } from '../config';
-import { DEFAULT_EXECUTIONS_CACHE_DIRNAME } from '../config';
+import { DEFAULT_DOWNLOAD_CACHE_DIRNAME } from '../config';
+import { DEFAULT_EXECUTION_CACHE_DIRNAME } from '../config';
 import { DEFAULT_SCRAPE_CACHE_DIRNAME } from '../config';
 import { DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME } from '../config';
 import { DEFAULT_REMOTE_URL } from '../config';
@@ -145,7 +146,8 @@ export { DEFAULT_MAX_EXECUTION_ATTEMPTS };
 export { DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH };
 export { DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL };
 export { DEFAULT_BOOKS_DIRNAME };
-export { DEFAULT_EXECUTIONS_CACHE_DIRNAME };
+export { DEFAULT_DOWNLOAD_CACHE_DIRNAME };
+export { DEFAULT_EXECUTION_CACHE_DIRNAME };
 export { DEFAULT_SCRAPE_CACHE_DIRNAME };
 export { DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME };
 export { DEFAULT_REMOTE_URL };

package/esm/typings/src/config.d.ts

@@ -166,6 +166,14 @@ export declare const DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL = 200;
  * @public exported from `@promptbook/core`
  */
 export declare const DEFAULT_BOOKS_DIRNAME = "./books";
+/**
+ * Where to store the temporary downloads
+ *
+ * Note: When the folder does not exist, it is created recursively
+ *
+ * @public exported from `@promptbook/core`
+ */
+export declare const DEFAULT_DOWNLOAD_CACHE_DIRNAME = "./.promptbook/download-cache";
 /**
  * Where to store the cache of executions for promptbook CLI
  *
@@ -173,7 +181,7 @@ export declare const DEFAULT_BOOKS_DIRNAME = "./books";
  *
  * @public exported from `@promptbook/core`
  */
-export declare const DEFAULT_EXECUTIONS_CACHE_DIRNAME = "./.promptbook/executions-cache";
+export declare const DEFAULT_EXECUTION_CACHE_DIRNAME = "./.promptbook/execution-cache";
 /**
  * Where to store the scrape cache
  *

package/esm/typings/src/execution/FilesystemTools.d.ts

@@ -3,7 +3,7 @@ import type fs from 'fs/promises';
 /**
  * Container for all the tools needed to manipulate with filesystem
  */
-export type FilesystemTools = Pick<typeof fs, 'access' | 'constants' | 'readFile' | 'writeFile' | 'stat' | 'readdir'>;
+export type FilesystemTools = Pick<typeof fs, 'access' | 'constants' | 'readFile' | 'writeFile' | 'stat' | 'readdir' | 'mkdir'>;
 /**
  * TODO: Implement destroyable pattern to free resources
  */

package/esm/typings/src/wizzard/wizzard.d.ts

@@ -6,6 +6,7 @@ import type { PipelineString } from '../pipeline/PipelineString';
 import type { TaskProgress } from '../types/TaskProgress';
 import type { InputParameters } from '../types/typeAliases';
 import type { string_filename } from '../types/typeAliases';
+import type { string_parameter_value } from '../types/typeAliases';
 import type { string_pipeline_url } from '../types/typeAliases';
 /**
  * Wizzard for simple usage of the Promptbook
@@ -27,7 +28,12 @@ declare class Wizzard {
      *
      * Note: This works simmilar to the `ptbk run` command
      */
-    execute(book: string_pipeline_url | string_filename | PipelineString, inputParameters: InputParameters, onProgress?: (taskProgress: TaskProgress) => Promisable<void>): Promise<PipelineExecutorResult>;
+    execute(book: string_pipeline_url | string_filename | PipelineString, inputParameters: InputParameters, onProgress?: (taskProgress: TaskProgress) => Promisable<void>): Promise<{
+        /**
+         * Simple result of the execution
+         */
+        result: string_parameter_value;
+    } & PipelineExecutorResult>;
     private executionTools;
     /**
      * Provides the tools automatically for the Node.js environment

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@promptbook/website-crawler",
-    "version": "0.84.0-11",
+    "version": "0.84.0-13",
     "description": "It's time for a paradigm shift. The future of software in plain English, French or Latin",
     "--note-0": "    <- [🐊]",
     "private": false,
@@ -54,7 +54,7 @@
     "module": "./esm/index.es.js",
     "typings": "./esm/typings/src/_packages/website-crawler.index.d.ts",
     "peerDependencies": {
-        "@promptbook/core": "0.84.0-11"
+        "@promptbook/core": "0.84.0-13"
     },
     "dependencies": {
         "@mozilla/readability": "0.5.0",

package/umd/index.umd.js

@@ -1,14 +1,15 @@
 (function (global, factory) {
-    typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('spacetrim'), require('@mozilla/readability'), require('jsdom'), require('crypto-js'), require('crypto-js/enc-hex'), require('fs/promises'), require('path'), require('prettier'), require('prettier/parser-html'), require('waitasecond'), require('mime-types'), require('papaparse'), require('showdown')) :
-    typeof define === 'function' && define.amd ? define(['exports', 'spacetrim', '@mozilla/readability', 'jsdom', 'crypto-js', 'crypto-js/enc-hex', 'fs/promises', 'path', 'prettier', 'prettier/parser-html', 'waitasecond', 'mime-types', 'papaparse', 'showdown'], factory) :
-    (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-website-crawler"] = {}, global.spaceTrim, global.readability, global.jsdom, global.cryptoJs, global.hexEncoder, global.promises, global.path, global.prettier, global.parserHtml, global.waitasecond, global.mimeTypes, global.papaparse, global.showdown));
-})(this, (function (exports, spaceTrim, readability, jsdom, cryptoJs, hexEncoder, promises, path, prettier, parserHtml, waitasecond, mimeTypes, papaparse, showdown) { 'use strict';
+    typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('spacetrim'), require('@mozilla/readability'), require('jsdom'), require('crypto-js'), require('crypto-js/enc-hex'), require('fs/promises'), require('path'), require('prettier'), require('prettier/parser-html'), require('waitasecond'), require('crypto-js/sha256'), require('mime-types'), require('papaparse'), require('showdown')) :
+    typeof define === 'function' && define.amd ? define(['exports', 'spacetrim', '@mozilla/readability', 'jsdom', 'crypto-js', 'crypto-js/enc-hex', 'fs/promises', 'path', 'prettier', 'prettier/parser-html', 'waitasecond', 'crypto-js/sha256', 'mime-types', 'papaparse', 'showdown'], factory) :
+    (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-website-crawler"] = {}, global.spaceTrim, global.readability, global.jsdom, global.cryptoJs, global.hexEncoder, global.promises, global.path, global.prettier, global.parserHtml, global.waitasecond, global.sha256, global.mimeTypes, global.papaparse, global.showdown));
+})(this, (function (exports, spaceTrim, readability, jsdom, cryptoJs, hexEncoder, promises, path, prettier, parserHtml, waitasecond, sha256, mimeTypes, papaparse, showdown) { 'use strict';
 
     function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
 
     var spaceTrim__default = /*#__PURE__*/_interopDefaultLegacy(spaceTrim);
     var hexEncoder__default = /*#__PURE__*/_interopDefaultLegacy(hexEncoder);
     var parserHtml__default = /*#__PURE__*/_interopDefaultLegacy(parserHtml);
+    var sha256__default = /*#__PURE__*/_interopDefaultLegacy(sha256);
 
     // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten
     /**
@@ -24,7 +25,7 @@
      * @generated
      * @see https://github.com/webgptorg/promptbook
      */
-    var PROMPTBOOK_ENGINE_VERSION = '0.84.0-10';
+    var PROMPTBOOK_ENGINE_VERSION = '0.84.0-12';
     /**
      * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
      * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -266,6 +267,12 @@
      * @private within the repository - too low-level in comparison with other `MAX_...`
      */
     var IMMEDIATE_TIME = 10;
+    /**
+     * The maximum length of the (generated) filename
+     *
+     * @public exported from `@promptbook/core`
+     */
+    var MAX_FILENAME_LENGTH = 30;
     /**
      * Strategy for caching the intermediate results for knowledge sources
      *
@@ -285,6 +292,15 @@
      * @public exported from `@promptbook/core`
      */
     var DEFAULT_MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹‍♂️]
+    // <- TODO: [🕝] Make also `BOOKS_DIRNAME_ALTERNATIVES`
+    /**
+     * Where to store the temporary downloads
+     *
+     * Note: When the folder does not exist, it is created recursively
+     *
+     * @public exported from `@promptbook/core`
+     */
+    var DEFAULT_DOWNLOAD_CACHE_DIRNAME = './.promptbook/download-cache';
     /**
      * Where to store the scrape cache
      *
@@ -3607,10 +3623,11 @@
     function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
         var _a;
         return __awaiter(this, void 0, void 0, function () {
-            var _b, fetch, knowledgeSourceContent, name, _c, _d, rootDirname, url, response_1, mimeType, filename_1, fileExtension, mimeType;
-            return __generator(this, function (_f) {
-                switch (_f.label) {
+            var _b, fetch, knowledgeSourceContent, name, _c, _d, rootDirname, url, response_1, mimeType, basename, hash, rootDirname_1, filepath, _f, _g, _h, _j, _k, filename_1, fileExtension, mimeType;
+            return __generator(this, function (_l) {
+                switch (_l.label) {
                     case 0:
+                        console.log('!!! makeKnowledgeSourceHandler', knowledgeSource);
                         _b = tools.fetch, fetch = _b === void 0 ? scraperFetch : _b;
                         knowledgeSourceContent = knowledgeSource.knowledgeSourceContent;
                         name = knowledgeSource.name;
@@ -3618,54 +3635,74 @@
                         if (!name) {
                             name = knowledgeSourceContentToName(knowledgeSourceContent);
                         }
-                        if (!isValidUrl(knowledgeSourceContent)) return [3 /*break*/, 2];
+                        if (!isValidUrl(knowledgeSourceContent)) return [3 /*break*/, 5];
                         url = knowledgeSourceContent;
                         return [4 /*yield*/, fetch(url)];
                     case 1:
-                        response_1 = _f.sent();
+                        response_1 = _l.sent();
                         mimeType = ((_a = response_1.headers.get('content-type')) === null || _a === void 0 ? void 0 : _a.split(';')[0]) || 'text/html';
-                        return [2 /*return*/, {
-                                source: name,
-                                filename: null,
-                                url: url,
-                                mimeType: mimeType,
-                                /*
-                                TODO: [🥽]
-                                    > async asBlob() {
-                                    >     // TODO: [👨🏻‍🤝‍👨🏻] This can be called multiple times BUT when called second time, response in already consumed
-                                    >     const content = await response.blob();
-                                    >     return content;
-                                    > },
-                                */
-                                asJson: function () {
-                                    return __awaiter(this, void 0, void 0, function () {
-                                        var content;
-                                        return __generator(this, function (_a) {
-                                            switch (_a.label) {
-                                                case 0: return [4 /*yield*/, response_1.json()];
-                                                case 1:
-                                                    content = _a.sent();
-                                                    return [2 /*return*/, content];
-                                            }
+                        if (tools.fs === undefined || !url.endsWith('.pdf')) {
+                            return [2 /*return*/, {
+                                    source: name,
+                                    filename: null,
+                                    url: url,
+                                    mimeType: mimeType,
+                                    /*
+                                  TODO: [🥽]
+                                      > async asBlob() {
+                                      >     // TODO: [👨🏻‍🤝‍👨🏻] This can be called multiple times BUT when called second time, response in already consumed
+                                      >     const content = await response.blob();
+                                      >     return content;
+                                      > },
+                                  */
+                                    asJson: function () {
+                                        return __awaiter(this, void 0, void 0, function () {
+                                            var content;
+                                            return __generator(this, function (_a) {
+                                                switch (_a.label) {
+                                                    case 0: return [4 /*yield*/, response_1.json()];
+                                                    case 1:
+                                                        content = _a.sent();
+                                                        return [2 /*return*/, content];
+                                                }
+                                            });
                                         });
-                                    });
-                                },
-                                asText: function () {
-                                    return __awaiter(this, void 0, void 0, function () {
-                                        var content;
-                                        return __generator(this, function (_a) {
-                                            switch (_a.label) {
-                                                case 0: return [4 /*yield*/, response_1.text()];
-                                                case 1:
-                                                    content = _a.sent();
-                                                    return [2 /*return*/, content];
-                                            }
+                                    },
+                                    asText: function () {
+                                        return __awaiter(this, void 0, void 0, function () {
+                                            var content;
+                                            return __generator(this, function (_a) {
+                                                switch (_a.label) {
+                                                    case 0: return [4 /*yield*/, response_1.text()];
+                                                    case 1:
+                                                        content = _a.sent();
+                                                        return [2 /*return*/, content];
+                                                }
+                                            });
                                         });
-                                    });
-                                },
-                            }];
+                                    },
+                                }];
+                        }
+                        basename = url.split('/').pop() || titleToName(url);
+                        hash = sha256__default["default"](hexEncoder__default["default"].parse(url)).toString( /* hex */);
+                        rootDirname_1 = path.join(process.cwd(), DEFAULT_DOWNLOAD_CACHE_DIRNAME);
+                        filepath = path.join.apply(void 0, __spreadArray(__spreadArray([], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), ["".concat(basename.substring(0, MAX_FILENAME_LENGTH), ".pdf")], false));
+                        return [4 /*yield*/, tools.fs.mkdir(path.dirname(path.join(rootDirname_1, filepath)), { recursive: true })];
                     case 2:
-                        if (!isValidFilePath(knowledgeSourceContent)) return [3 /*break*/, 4];
+                        _l.sent();
+                        _g = (_f = tools.fs).writeFile;
+                        _h = [path.join(rootDirname_1, filepath)];
+                        _k = (_j = Buffer).from;
+                        return [4 /*yield*/, response_1.arrayBuffer()];
+                    case 3: return [4 /*yield*/, _g.apply(_f, _h.concat([_k.apply(_j, [_l.sent()])]))];
+                    case 4:
+                        _l.sent();
+                        // TODO: !!!!!!!! Check the file security
+                        // TODO: !!!!!!!! Check the file size (if it is not too big)
+                        // TODO: !!!!!!!! Delete the file
+                        return [2 /*return*/, makeKnowledgeSourceHandler({ name: name, knowledgeSourceContent: filepath }, tools, __assign(__assign({}, options), { rootDirname: rootDirname_1 }))];
+                    case 5:
+                        if (!isValidFilePath(knowledgeSourceContent)) return [3 /*break*/, 7];
                         if (tools.fs === undefined) {
                             throw new EnvironmentMismatchError('Can not import file knowledge without filesystem tools');
                             //          <- TODO: [🧠] What is the best error type here`
@@ -3678,8 +3715,8 @@
                         fileExtension = getFileExtension(filename_1);
                         mimeType = extensionToMimeType(fileExtension || '');
                         return [4 /*yield*/, isFileExisting(filename_1, tools.fs)];
-                    case 3:
-                        if (!(_f.sent())) {
+                    case 6:
+                        if (!(_l.sent())) {
                             throw new NotFoundError(spaceTrim__default["default"](function (block) { return "\n                          Can not make source handler for file which does not exist:\n\n                          File:\n                          ".concat(block(knowledgeSourceContent), "\n\n                          Full file path:\n                          ").concat(block(filename_1), "\n                      "); }));
                         }
                         // TODO: [🧠][😿] Test security file - file is scoped to the project (BUT maybe do this in `filesystemTools`)
@@ -3725,7 +3762,7 @@
                                     });
                                 },
                             }];
-                    case 4: return [2 /*return*/, {
+                    case 7: return [2 /*return*/, {
                             source: name,
                             filename: null,
                             url: null,