@promptbook/remote-server 0.85.0-8 → 0.85.0

package/README.md

@@ -24,10 +24,6 @@
 
 
 
-<blockquote style="color: #ff8811">
-    <b>⚠ Warning:</b> This is a pre-release version of the library. It is not yet ready for production use. Please look at <a href="https://www.npmjs.com/package/@promptbook/core?activeTab=versions">latest stable release</a>.
-</blockquote>
-
 ## 📦 Package `@promptbook/remote-server`
 
 - Promptbooks are [divided into several](#-packages) packages, all are published from [single monorepo](https://github.com/webgptorg/promptbook).

package/esm/index.es.js

@@ -31,7 +31,7 @@ var BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-var PROMPTBOOK_ENGINE_VERSION = '0.85.0-7';
+var PROMPTBOOK_ENGINE_VERSION = '0.85.0-16';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -1491,57 +1491,6 @@ function isValidPromptbookVersion(version) {
     return true;
 }
 
-/**
- * Checks if an URL is reserved for private networks or localhost.
- *
- * Note: There are two simmilar functions:
- * - `isUrlOnPrivateNetwork` which tests full URL
- * - `isHostnameOnPrivateNetwork` *(this one)* which tests just hostname
- *
- * @public exported from `@promptbook/utils`
- */
-function isHostnameOnPrivateNetwork(hostname) {
-    if (hostname === 'example.com' ||
-        hostname === 'localhost' ||
-        hostname.endsWith('.localhost') ||
-        hostname.endsWith('.local') ||
-        hostname.endsWith('.test') ||
-        hostname === '127.0.0.1' ||
-        hostname === '::1') {
-        return true;
-    }
-    if (hostname.includes(':')) {
-        // IPv6
-        var ipParts = hostname.split(':');
-        return ipParts[0] === 'fc00' || ipParts[0] === 'fd00' || ipParts[0] === 'fe80';
-    }
-    else {
-        // IPv4
-        var ipParts = hostname.split('.').map(function (part) { return Number.parseInt(part, 10); });
-        return (ipParts[0] === 10 ||
-            (ipParts[0] === 172 && ipParts[1] >= 16 && ipParts[1] <= 31) ||
-            (ipParts[0] === 192 && ipParts[1] === 168));
-    }
-}
-
-/**
- * Checks if an IP address or hostname is reserved for private networks or localhost.
- *
- * Note: There are two simmilar functions:
- * - `isUrlOnPrivateNetwork` *(this one)* which tests full URL
- * - `isHostnameOnPrivateNetwork` which tests just hostname
- *
- * @param {string} ipAddress - The IP address to check.
- * @returns {boolean} Returns true if the IP address is reserved for private networks or localhost, otherwise false.
- * @public exported from `@promptbook/utils`
- */
-function isUrlOnPrivateNetwork(url) {
-    if (typeof url === 'string') {
-        url = new URL(url);
-    }
-    return isHostnameOnPrivateNetwork(url.hostname);
-}
-
 /**
  * Tests if given string is valid URL.
  *
@@ -1584,16 +1533,19 @@ function isValidPipelineUrl(url) {
     if (!isValidUrl(url)) {
         return false;
     }
-    if (!url.startsWith('https://')) {
+    if (!url.startsWith('https://') && !url.startsWith('http://') /* <- Note: [👣] */) {
         return false;
     }
     if (url.includes('#')) {
         // TODO: [🐠]
         return false;
     }
+    /*
+    Note: [👣][🧠] Is it secure to allow pipeline URLs on private and unsecured networks?
     if (isUrlOnPrivateNetwork(url)) {
         return false;
     }
+    */
     return true;
 }
 /**
@@ -2009,18 +1961,16 @@ function assertsTaskSuccessful(executionResult) {
  */
 function createTask(options) {
     var taskType = options.taskType, taskProcessCallback = options.taskProcessCallback;
-    var taskId = "".concat(taskType.toLowerCase().substring(0, 4), "-").concat($randomToken(8 /* <- TODO: !!! To global config + Use Base58 to avoid simmilar char conflicts   */));
+    var taskId = "".concat(taskType.toLowerCase().substring(0, 4), "-").concat($randomToken(8 /* <- TODO: To global config + Use Base58 to avoid simmilar char conflicts   */));
     var partialResultSubject = new BehaviorSubject({});
     var finalResultPromise = /* not await */ taskProcessCallback(function (newOngoingResult) {
         partialResultSubject.next(newOngoingResult);
     });
     finalResultPromise
         .catch(function (error) {
-        // console.error('!!!!! Task failed:', error);
         partialResultSubject.error(error);
     })
         .then(function (value) {
-        // console.error('!!!!! Task finished:', value);
         if (value) {
             try {
                 assertsTaskSuccessful(value);
@@ -2042,9 +1992,7 @@ function createTask(options) {
                         return [4 /*yield*/, finalResultPromise];
                     case 1:
                         finalResult = _b.sent();
-                        console.error('!!!!! finalResult:', finalResult);
                         if (isCrashedOnError) {
-                            console.error('!!!!! isCrashedOnError:', finalResult);
                             assertsTaskSuccessful(finalResult);
                         }
                         return [2 /*return*/, finalResult];
@@ -6897,6 +6845,57 @@ function startRemoteServer(options) {
             .filter(function (part) { return part !== ''; })
             .join('/');
     var startupDate = new Date();
+    function getExecutionToolsFromIdentification(identification) {
+        return __awaiter(this, void 0, void 0, function () {
+            var isAnonymous, llm, llmToolsConfiguration, appId, userId, customOptions, fs, executables, tools;
+            var _a;
+            return __generator(this, function (_b) {
+                switch (_b.label) {
+                    case 0:
+                        if (identification === null || identification === undefined) {
+                            throw new Error("Identification is not provided");
+                        }
+                        isAnonymous = identification.isAnonymous;
+                        if (isAnonymous === true && !isAnonymousModeAllowed) {
+                            throw new PipelineExecutionError("Anonymous mode is not allowed"); // <- TODO: [main] !!3 Test
+                        }
+                        if (isAnonymous === false && !isApplicationModeAllowed) {
+                            throw new PipelineExecutionError("Application mode is not allowed"); // <- TODO: [main] !!3 Test
+                        }
+                        if (!(isAnonymous === true)) return [3 /*break*/, 1];
+                        llmToolsConfiguration = identification.llmToolsConfiguration;
+                        llm = createLlmToolsFromConfiguration(llmToolsConfiguration, { isVerbose: isVerbose });
+                        return [3 /*break*/, 4];
+                    case 1:
+                        if (!(isAnonymous === false && createLlmExecutionTools !== null)) return [3 /*break*/, 3];
+                        appId = identification.appId, userId = identification.userId, customOptions = identification.customOptions;
+                        return [4 /*yield*/, createLlmExecutionTools({
+                                appId: appId,
+                                userId: userId,
+                                customOptions: customOptions,
+                            })];
+                    case 2:
+                        llm = _b.sent();
+                        return [3 /*break*/, 4];
+                    case 3: throw new PipelineExecutionError("You must provide either llmToolsConfiguration or non-anonymous mode must be propperly configured");
+                    case 4:
+                        fs = $provideFilesystemForNode();
+                        return [4 /*yield*/, $provideExecutablesForNode()];
+                    case 5:
+                        executables = _b.sent();
+                        _a = {
+                            llm: llm,
+                            fs: fs
+                        };
+                        return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables })];
+                    case 6:
+                        tools = (_a.scrapers = _b.sent(),
+                            _a);
+                        return [2 /*return*/, tools];
+                }
+            });
+        });
+    }
     var app = express();
     app.use(express.json());
     app.use(function (request, response, next) {
@@ -6970,6 +6969,46 @@ function startRemoteServer(options) {
             }
         });
     }); });
+    // TODO: [🧠] Is it secure / good idea to expose source codes of hosted books
+    app.get("".concat(rootPath, "/books/*"), function (request, response) { return __awaiter(_this, void 0, void 0, function () {
+        var pipelines, fullUrl, pipelineUrl, pipeline, source, error_1;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    _a.trys.push([0, 3, , 4]);
+                    if (collection === null) {
+                        response.status(500).send('No collection nor books available');
+                        return [2 /*return*/];
+                    }
+                    return [4 /*yield*/, collection.listPipelines()];
+                case 1:
+                    pipelines = _a.sent();
+                    fullUrl = request.protocol + '://' + request.get('host') + request.originalUrl;
+                    pipelineUrl = pipelines.find(function (pipelineUrl) { return pipelineUrl.endsWith(request.originalUrl); }) || fullUrl;
+                    return [4 /*yield*/, collection.getPipelineByUrl(pipelineUrl)];
+                case 2:
+                    pipeline = _a.sent();
+                    source = pipeline.sources[0];
+                    if (source === undefined || source.type !== 'BOOK') {
+                        throw new Error('Pipeline source is not a book');
+                    }
+                    response
+                        .type('text/markdown')
+                        .send(source.content);
+                    return [3 /*break*/, 4];
+                case 3:
+                    error_1 = _a.sent();
+                    if (!(error_1 instanceof Error)) {
+                        throw error_1;
+                    }
+                    response
+                        .status(404)
+                        .send({ error: serializeError(error_1) });
+                    return [3 /*break*/, 4];
+                case 4: return [2 /*return*/];
+            }
+        });
+    }); });
     app.get("".concat(rootPath, "/executions"), function (request, response) { return __awaiter(_this, void 0, void 0, function () {
         return __generator(this, function (_a) {
             response.send(runningExecutionTasks);
@@ -6982,7 +7021,9 @@ function startRemoteServer(options) {
             taskId = request.params.taskId;
             execution = runningExecutionTasks.find(function (executionTask) { return executionTask.taskId === taskId; });
             if (execution === undefined) {
-                response.status(404).send("Execution \"".concat(taskId, "\" not found"));
+                response
+                    .status(404)
+                    .send("Execution \"".concat(taskId, "\" not found"));
                 return [2 /*return*/];
             }
             response.send(execution.currentValue);
@@ -6990,13 +7031,12 @@ function startRemoteServer(options) {
         });
     }); });
     app.post("".concat(rootPath, "/executions/new"), function (request, response) { return __awaiter(_this, void 0, void 0, function () {
-        var inputParameters, pipelineUrl, pipeline, llm, fs, executables, tools, pipelineExecutor, executionTask, error_1;
-        var _a;
+        var _a, inputParameters, identification, pipelineUrl, pipeline, tools, pipelineExecutor, executionTask, error_2;
         return __generator(this, function (_b) {
             switch (_b.label) {
                 case 0:
-                    _b.trys.push([0, 6, , 7]);
-                    inputParameters = request.body.inputParameters;
+                    _b.trys.push([0, 4, , 5]);
+                    _a = request.body, inputParameters = _a.inputParameters, identification = _a.identification;
                     pipelineUrl = request.body.pipelineUrl || request.body.book;
                     return [4 /*yield*/, (collection === null || collection === void 0 ? void 0 : collection.getPipelineByUrl(pipelineUrl))];
                 case 1:
@@ -7005,43 +7045,27 @@ function startRemoteServer(options) {
                         response.status(404).send("Pipeline \"".concat(pipelineUrl, "\" not found"));
                         return [2 /*return*/];
                     }
-                    return [4 /*yield*/, createLlmExecutionTools({
-                            appId: '!!!!',
-                            userId: '!!!!',
-                            customOptions: {},
-                        })];
+                    return [4 /*yield*/, getExecutionToolsFromIdentification(identification)];
                 case 2:
-                    llm = _b.sent();
-                    fs = $provideFilesystemForNode();
-                    return [4 /*yield*/, $provideExecutablesForNode()];
-                case 3:
-                    executables = _b.sent();
-                    _a = {
-                        llm: llm,
-                        fs: fs
-                    };
-                    return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables })];
-                case 4:
-                    tools = (_a.scrapers = _b.sent(),
-                        _a);
+                    tools = _b.sent();
                     pipelineExecutor = createPipelineExecutor(__assign({ pipeline: pipeline, tools: tools }, options));
                     executionTask = pipelineExecutor(inputParameters);
                     runningExecutionTasks.push(executionTask);
                     return [4 /*yield*/, forTime(10)];
-                case 5:
+                case 3:
                     _b.sent();
                     // <- Note: Wait for a while to wait for quick responses or sudden but asynchronous errors
                     // <- TODO: Put this into configuration
                     response.send(executionTask);
-                    return [3 /*break*/, 7];
-                case 6:
-                    error_1 = _b.sent();
-                    if (!(error_1 instanceof Error)) {
-                        throw error_1;
+                    return [3 /*break*/, 5];
+                case 4:
+                    error_2 = _b.sent();
+                    if (!(error_2 instanceof Error)) {
+                        throw error_2;
                     }
-                    response.status(400).send({ error: serializeError(error_1) });
-                    return [3 /*break*/, 7];
-                case 7: return [2 /*return*/];
+                    response.status(400).send({ error: serializeError(error_2) });
+                    return [3 /*break*/, 5];
+                case 5: return [2 /*return*/];
             }
         });
     }); });
@@ -7058,55 +7082,9 @@ function startRemoteServer(options) {
         if (isVerbose) {
             console.info(colors.gray("Client connected"), socket.id);
         }
-        var getExecutionToolsFromIdentification = function (identification) { return __awaiter(_this, void 0, void 0, function () {
-            var isAnonymous, llm, llmToolsConfiguration, appId, userId, customOptions, fs, executables, tools;
-            var _a;
-            return __generator(this, function (_b) {
-                switch (_b.label) {
-                    case 0:
-                        isAnonymous = identification.isAnonymous;
-                        if (isAnonymous === true && !isAnonymousModeAllowed) {
-                            throw new PipelineExecutionError("Anonymous mode is not allowed"); // <- TODO: [main] !!3 Test
-                        }
-                        if (isAnonymous === false && !isApplicationModeAllowed) {
-                            throw new PipelineExecutionError("Application mode is not allowed"); // <- TODO: [main] !!3 Test
-                        }
-                        if (!(isAnonymous === true)) return [3 /*break*/, 1];
-                        llmToolsConfiguration = identification.llmToolsConfiguration;
-                        llm = createLlmToolsFromConfiguration(llmToolsConfiguration, { isVerbose: isVerbose });
-                        return [3 /*break*/, 4];
-                    case 1:
-                        if (!(isAnonymous === false && createLlmExecutionTools !== null)) return [3 /*break*/, 3];
-                        appId = identification.appId, userId = identification.userId, customOptions = identification.customOptions;
-                        return [4 /*yield*/, createLlmExecutionTools({
-                                appId: appId,
-                                userId: userId,
-                                customOptions: customOptions,
-                            })];
-                    case 2:
-                        llm = _b.sent();
-                        return [3 /*break*/, 4];
-                    case 3: throw new PipelineExecutionError("You must provide either llmToolsConfiguration or non-anonymous mode must be propperly configured");
-                    case 4:
-                        fs = $provideFilesystemForNode();
-                        return [4 /*yield*/, $provideExecutablesForNode()];
-                    case 5:
-                        executables = _b.sent();
-                        _a = {
-                            llm: llm,
-                            fs: fs
-                        };
-                        return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables })];
-                    case 6:
-                        tools = (_a.scrapers = _b.sent(),
-                            _a);
-                        return [2 /*return*/, tools];
-                }
-            });
-        }); };
         // -----------
         socket.on('prompt-request', function (request) { return __awaiter(_this, void 0, void 0, function () {
-            var identification, prompt, executionTools, llm, _a, promptResult, _b, error_2;
+            var identification, prompt, tools, llm, _a, promptResult, _b, error_3;
             return __generator(this, function (_c) {
                 switch (_c.label) {
                     case 0:
@@ -7119,8 +7097,8 @@ function startRemoteServer(options) {
                         _c.trys.push([1, 13, 14, 15]);
                         return [4 /*yield*/, getExecutionToolsFromIdentification(identification)];
                     case 2:
-                        executionTools = _c.sent();
-                        llm = executionTools.llm;
+                        tools = _c.sent();
+                        llm = tools.llm;
                         _a = identification.isAnonymous === false &&
                             collection !== null;
                         if (!_a) return [3 /*break*/, 4];
@@ -7175,11 +7153,11 @@ function startRemoteServer(options) {
                         socket.emit('prompt-response', { promptResult: promptResult } /* <- Note: [🤛] */);
                         return [3 /*break*/, 15];
                     case 13:
-                        error_2 = _c.sent();
-                        if (!(error_2 instanceof Error)) {
-                            throw error_2;
+                        error_3 = _c.sent();
+                        if (!(error_3 instanceof Error)) {
+                            throw error_3;
                         }
-                        socket.emit('error', serializeError(error_2) /* <- Note: [🤛] */);
+                        socket.emit('error', serializeError(error_3) /* <- Note: [🤛] */);
                         return [3 /*break*/, 15];
                     case 14:
                         socket.disconnect();
@@ -7191,7 +7169,7 @@ function startRemoteServer(options) {
         // -----------
         // TODO: [👒] Listing models (and checking configuration) probbably should go through REST API not Socket.io
         socket.on('listModels-request', function (request) { return __awaiter(_this, void 0, void 0, function () {
-            var identification, executionTools, llm, models, error_3;
+            var identification, tools, llm, models, error_4;
             return __generator(this, function (_a) {
                 switch (_a.label) {
                     case 0:
@@ -7204,19 +7182,19 @@ function startRemoteServer(options) {
                         _a.trys.push([1, 4, 5, 6]);
                         return [4 /*yield*/, getExecutionToolsFromIdentification(identification)];
                     case 2:
-                        executionTools = _a.sent();
-                        llm = executionTools.llm;
+                        tools = _a.sent();
+                        llm = tools.llm;
                         return [4 /*yield*/, llm.listModels()];
                     case 3:
                         models = _a.sent();
                         socket.emit('listModels-response', { models: models } /* <- Note: [🤛] */);
                         return [3 /*break*/, 6];
                     case 4:
-                        error_3 = _a.sent();
-                        if (!(error_3 instanceof Error)) {
-                            throw error_3;
+                        error_4 = _a.sent();
+                        if (!(error_4 instanceof Error)) {
+                            throw error_4;
                         }
-                        socket.emit('error', serializeError(error_3));
+                        socket.emit('error', serializeError(error_4));
                         return [3 /*break*/, 6];
                     case 5:
                         socket.disconnect();
@@ -7228,7 +7206,7 @@ function startRemoteServer(options) {
         // -----------
         // TODO: [👒] Listing models (and checking configuration) probbably should go through REST API not Socket.io
         socket.on('preparePipeline-request', function (request) { return __awaiter(_this, void 0, void 0, function () {
-            var identification, pipeline, executionTools, preparedPipeline, error_4;
+            var identification, pipeline, tools, preparedPipeline, error_5;
             return __generator(this, function (_a) {
                 switch (_a.label) {
                     case 0:
@@ -7241,18 +7219,18 @@ function startRemoteServer(options) {
                         _a.trys.push([1, 4, 5, 6]);
                         return [4 /*yield*/, getExecutionToolsFromIdentification(identification)];
                     case 2:
-                        executionTools = _a.sent();
-                        return [4 /*yield*/, preparePipeline(pipeline, executionTools, options)];
+                        tools = _a.sent();
+                        return [4 /*yield*/, preparePipeline(pipeline, tools, options)];
                     case 3:
                         preparedPipeline = _a.sent();
                         socket.emit('preparePipeline-response', { preparedPipeline: preparedPipeline } /* <- Note: [🤛] */);
                         return [3 /*break*/, 6];
                     case 4:
-                        error_4 = _a.sent();
-                        if (!(error_4 instanceof Error)) {
-                            throw error_4;
+                        error_5 = _a.sent();
+                        if (!(error_5 instanceof Error)) {
+                            throw error_5;
                         }
-                        socket.emit('error', serializeError(error_4));
+                        socket.emit('error', serializeError(error_5));
                         return [3 /*break*/, 6];
                     case 5:
                         socket.disconnect();
@@ -7291,8 +7269,7 @@ function startRemoteServer(options) {
     };
 }
 /**
- * TODO: !!!!!!! CORS and security
- * TODO: !!!!!!! Allow to pass tokem here
+ * TODO: !! Add CORS and security - probbably via `helmet`
  * TODO: [👩🏾‍🤝‍🧑🏾] Allow to pass custom fetch function here - PromptbookFetch
  * TODO: Split this file into multiple functions - handler for each request
  * TODO: Maybe use `$exportJson`