@promptbook/markdown-utils 0.92.0-5 → 0.92.0-7

package/esm/typings/src/remote-server/openapi.d.ts

@@ -16,6 +16,13 @@ export declare const openapiJson: {
                 responses: {
                     '200': {
                         description: string;
+                        content: {
+                            'text/markdown': {
+                                schema: {
+                                    type: string;
+                                };
+                            };
+                        };
                     };
                 };
             };
@@ -46,13 +53,22 @@ export declare const openapiJson: {
                     };
                 };
                 responses: {
-                    '200': {
+                    '201': {
                         description: string;
                         content: {
                             'application/json': {
                                 schema: {
                                     type: string;
                                     properties: {
+                                        isSuccess: {
+                                            type: string;
+                                        };
+                                        message: {
+                                            type: string;
+                                        };
+                                        error: {
+                                            type: string;
+                                        };
                                         identification: {
                                             type: string;
                                         };
@@ -61,6 +77,43 @@ export declare const openapiJson: {
                             };
                         };
                     };
+                    '400': {
+                        description: string;
+                        content: {
+                            'application/json': {
+                                schema: {
+                                    type: string;
+                                    properties: {
+                                        error: {
+                                            type: string;
+                                        };
+                                    };
+                                };
+                            };
+                        };
+                    };
+                    '401': {
+                        description: string;
+                        content: {
+                            'application/json': {
+                                schema: {
+                                    type: string;
+                                    properties: {
+                                        isSuccess: {
+                                            type: string;
+                                            enum: boolean[];
+                                        };
+                                        message: {
+                                            type: string;
+                                        };
+                                        error: {
+                                            type: string;
+                                        };
+                                    };
+                                };
+                            };
+                        };
+                    };
                 };
             };
         };
@@ -82,6 +135,16 @@ export declare const openapiJson: {
                             };
                         };
                     };
+                    '500': {
+                        description: string;
+                        content: {
+                            'text/plain': {
+                                schema: {
+                                    type: string;
+                                };
+                            };
+                        };
+                    };
                 };
             };
         };
@@ -111,6 +174,28 @@ export declare const openapiJson: {
                     };
                     '404': {
                         description: string;
+                        content: {
+                            'application/json': {
+                                schema: {
+                                    type: string;
+                                    properties: {
+                                        error: {
+                                            type: string;
+                                        };
+                                    };
+                                };
+                            };
+                        };
+                    };
+                    '500': {
+                        description: string;
+                        content: {
+                            'text/plain': {
+                                schema: {
+                                    type: string;
+                                };
+                            };
+                        };
                     };
                 };
             };
@@ -128,11 +213,172 @@ export declare const openapiJson: {
                                     type: string;
                                     items: {
                                         type: string;
+                                        properties: {
+                                            nonce: {
+                                                type: string;
+                                            };
+                                            taskId: {
+                                                type: string;
+                                            };
+                                            taskType: {
+                                                type: string;
+                                            };
+                                            status: {
+                                                type: string;
+                                            };
+                                            createdAt: {
+                                                type: string;
+                                                format: string;
+                                            };
+                                            updatedAt: {
+                                                type: string;
+                                                format: string;
+                                            };
+                                        };
+                                    };
+                                };
+                            };
+                        };
+                    };
+                };
+            };
+        };
+        '/executions/last': {
+            get: {
+                summary: string;
+                description: string;
+                responses: {
+                    '200': {
+                        description: string;
+                        content: {
+                            'application/json': {
+                                schema: {
+                                    type: string;
+                                    properties: {
+                                        nonce: {
+                                            type: string;
+                                        };
+                                        taskId: {
+                                            type: string;
+                                        };
+                                        taskType: {
+                                            type: string;
+                                        };
+                                        status: {
+                                            type: string;
+                                        };
+                                        errors: {
+                                            type: string;
+                                            items: {
+                                                type: string;
+                                            };
+                                        };
+                                        warnings: {
+                                            type: string;
+                                            items: {
+                                                type: string;
+                                            };
+                                        };
+                                        createdAt: {
+                                            type: string;
+                                            format: string;
+                                        };
+                                        updatedAt: {
+                                            type: string;
+                                            format: string;
+                                        };
+                                        currentValue: {
+                                            type: string;
+                                        };
+                                    };
+                                };
+                            };
+                        };
+                    };
+                    '404': {
+                        description: string;
+                        content: {
+                            'text/plain': {
+                                schema: {
+                                    type: string;
+                                };
+                            };
+                        };
+                    };
+                };
+            };
+        };
+        '/executions/{taskId}': {
+            get: {
+                summary: string;
+                description: string;
+                parameters: {
+                    in: string;
+                    name: string;
+                    required: boolean;
+                    schema: {
+                        type: string;
+                    };
+                    description: string;
+                }[];
+                responses: {
+                    '200': {
+                        description: string;
+                        content: {
+                            'application/json': {
+                                schema: {
+                                    type: string;
+                                    properties: {
+                                        nonce: {
+                                            type: string;
+                                        };
+                                        taskId: {
+                                            type: string;
+                                        };
+                                        taskType: {
+                                            type: string;
+                                        };
+                                        status: {
+                                            type: string;
+                                        };
+                                        errors: {
+                                            type: string;
+                                            items: {
+                                                type: string;
+                                            };
+                                        };
+                                        warnings: {
+                                            type: string;
+                                            items: {
+                                                type: string;
+                                            };
+                                        };
+                                        createdAt: {
+                                            type: string;
+                                            format: string;
+                                        };
+                                        updatedAt: {
+                                            type: string;
+                                            format: string;
+                                        };
+                                        currentValue: {
+                                            type: string;
+                                        };
                                     };
                                 };
                             };
                         };
                     };
+                    '404': {
+                        description: string;
+                        content: {
+                            'text/plain': {
+                                schema: {
+                                    type: string;
+                                };
+                            };
+                        };
+                    };
                 };
             };
         };
@@ -149,12 +395,19 @@ export declare const openapiJson: {
                                 properties: {
                                     pipelineUrl: {
                                         type: string;
+                                        description: string;
+                                    };
+                                    book: {
+                                        type: string;
+                                        description: string;
                                     };
                                     inputParameters: {
                                         type: string;
+                                        description: string;
                                     };
                                     identification: {
                                         type: string;
+                                        description: string;
                                     };
                                 };
                             };
@@ -174,13 +427,154 @@ export declare const openapiJson: {
                     };
                     '400': {
                         description: string;
+                        content: {
+                            'application/json': {
+                                schema: {
+                                    type: string;
+                                    properties: {
+                                        error: {
+                                            type: string;
+                                        };
+                                    };
+                                };
+                            };
+                        };
+                    };
+                    '404': {
+                        description: string;
+                        content: {
+                            'text/plain': {
+                                schema: {
+                                    type: string;
+                                };
+                            };
+                        };
+                    };
+                };
+            };
+        };
+        '/api-docs': {
+            get: {
+                summary: string;
+                description: string;
+                responses: {
+                    '200': {
+                        description: string;
+                    };
+                };
+            };
+        };
+        '/swagger': {
+            get: {
+                summary: string;
+                description: string;
+                responses: {
+                    '200': {
+                        description: string;
+                    };
+                };
+            };
+        };
+        '/openapi': {
+            get: {
+                summary: string;
+                description: string;
+                responses: {
+                    '200': {
+                        description: string;
+                        content: {
+                            'application/json': {
+                                schema: {
+                                    type: string;
+                                };
+                            };
+                        };
                     };
                 };
             };
         };
     };
-    components: {};
-    tags: never[];
+    components: {
+        schemas: {
+            Error: {
+                type: string;
+                properties: {
+                    error: {
+                        type: string;
+                    };
+                };
+            };
+            ExecutionTaskSummary: {
+                type: string;
+                properties: {
+                    nonce: {
+                        type: string;
+                    };
+                    taskId: {
+                        type: string;
+                    };
+                    taskType: {
+                        type: string;
+                    };
+                    status: {
+                        type: string;
+                    };
+                    createdAt: {
+                        type: string;
+                        format: string;
+                    };
+                    updatedAt: {
+                        type: string;
+                        format: string;
+                    };
+                };
+            };
+            ExecutionTaskFull: {
+                type: string;
+                properties: {
+                    nonce: {
+                        type: string;
+                    };
+                    taskId: {
+                        type: string;
+                    };
+                    taskType: {
+                        type: string;
+                    };
+                    status: {
+                        type: string;
+                    };
+                    errors: {
+                        type: string;
+                        items: {
+                            type: string;
+                        };
+                    };
+                    warnings: {
+                        type: string;
+                        items: {
+                            type: string;
+                        };
+                    };
+                    createdAt: {
+                        type: string;
+                        format: string;
+                    };
+                    updatedAt: {
+                        type: string;
+                        format: string;
+                    };
+                    currentValue: {
+                        type: string;
+                    };
+                };
+            };
+        };
+    };
+    tags: {
+        name: string;
+        description: string;
+    }[];
 };
 /**
  * Note: [💞] Ignore a discrepancy between file name and entity name

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@promptbook/markdown-utils",
-    "version": "0.92.0-5",
+    "version": "0.92.0-7",
     "description": "It's time for a paradigm shift. The future of software in plain English, French or Latin",
     "private": false,
     "sideEffects": false,

package/umd/index.umd.js

@@ -25,7 +25,7 @@
      * @generated
      * @see https://github.com/webgptorg/promptbook
      */
-    const PROMPTBOOK_ENGINE_VERSION = '0.92.0-5';
+    const PROMPTBOOK_ENGINE_VERSION = '0.92.0-7';
     /**
      * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
      * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -1797,6 +1797,45 @@
      *     - [♨] Are tasks prepared
      */
 
+    /**
+     * Converts a JavaScript Object Notation (JSON) string into an object.
+     *
+     * Note: This is wrapper around `JSON.parse()` with better error and type handling
+     *
+     * @public exported from `@promptbook/utils`
+     */
+    function jsonParse(value) {
+        if (value === undefined) {
+            throw new Error(`Can not parse JSON from undefined value.`);
+        }
+        else if (typeof value !== 'string') {
+            console.error('Can not parse JSON from non-string value.', { text: value });
+            throw new Error(spaceTrim__default["default"](`
+                    Can not parse JSON from non-string value.
+
+                    The value type: ${typeof value}
+                    See more in console.
+                `));
+        }
+        try {
+            return JSON.parse(value);
+        }
+        catch (error) {
+            if (!(error instanceof Error)) {
+                throw error;
+            }
+            throw new Error(spaceTrim__default["default"]((block) => `
+                    ${block(error.message)}
+            
+                    The JSON text:
+                    ${block(value)}
+                `));
+        }
+    }
+    /**
+     * TODO: !!!! Use in Promptbook.studio
+     */
+
     /**
      * Recursively converts JSON strings to JSON objects
 
@@ -1815,7 +1854,7 @@
         const newObject = { ...object };
         for (const [key, value] of Object.entries(object)) {
             if (typeof value === 'string' && isValidJsonString(value)) {
-                newObject[key] = JSON.parse(value);
+                newObject[key] = jsonParse(value);
             }
             else {
                 newObject[key] = jsonStringsToJsons(value);
@@ -2688,18 +2727,26 @@
         }).asPromise();
         const { outputParameters } = result;
         const { modelsRequirements: modelsRequirementsJson } = outputParameters;
-        const modelsRequirementsUnchecked = JSON.parse(modelsRequirementsJson);
+        let modelsRequirementsUnchecked = jsonParse(modelsRequirementsJson);
         if (isVerbose) {
             console.info(`PERSONA ${personaDescription}`, modelsRequirementsUnchecked);
         }
         if (!Array.isArray(modelsRequirementsUnchecked)) {
-            throw new UnexpectedError(spaceTrim__default["default"]((block) => `
-                    Invalid \`modelsRequirements\`:
+            // <- TODO: Book should have syntax and system to enforce shape of JSON
+            modelsRequirementsUnchecked = [modelsRequirementsUnchecked];
+            /*
+            throw new UnexpectedError(
+                spaceTrim(
+                    (block) => `
+                        Invalid \`modelsRequirements\`:
 
-                    \`\`\`json
-                    ${block(JSON.stringify(modelsRequirementsUnchecked, null, 4))}
-                    \`\`\`
-                `));
+                        \`\`\`json
+                        ${block(JSON.stringify(modelsRequirementsUnchecked, null, 4))}
+                        \`\`\`
+                    `,
+                ),
+            );
+            */
         }
         const modelsRequirements = modelsRequirementsUnchecked.map((modelRequirements) => ({
             modelVariant: 'CHAT',
@@ -3530,7 +3577,7 @@
                     > },
                 */
                 async asJson() {
-                    return JSON.parse(await tools.fs.readFile(filename, 'utf-8'));
+                    return jsonParse(await tools.fs.readFile(filename, 'utf-8'));
                 },
                 async asText() {
                     return await tools.fs.readFile(filename, 'utf-8');
@@ -5115,13 +5162,79 @@
     /**
      * @@@
      *
+     * Here is the place where RAG (retrieval-augmented generation) happens
+     *
      * @private internal utility of `createPipelineExecutor`
      */
     async function getKnowledgeForTask(options) {
-        const { preparedPipeline, task } = options;
-        return preparedPipeline.knowledgePieces.map(({ content }) => `- ${content}`).join('\n');
+        const { tools, preparedPipeline, task } = options;
+        const firstKnowlegePiece = preparedPipeline.knowledgePieces[0];
+        const firstKnowlegeIndex = firstKnowlegePiece === null || firstKnowlegePiece === void 0 ? void 0 : firstKnowlegePiece.index[0];
+        // <- TODO: Do not use just first knowledge piece and first index to determine embedding model, use also keyword search
+        if (firstKnowlegePiece === undefined || firstKnowlegeIndex === undefined) {
+            return 'No knowledge pieces found';
+        }
+        // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
+        const _llms = arrayableToArray(tools.llm);
+        const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
+        const taskEmbeddingPrompt = {
+            title: 'Knowledge Search',
+            modelRequirements: {
+                modelVariant: 'EMBEDDING',
+                modelName: firstKnowlegeIndex.modelName,
+            },
+            content: task.content,
+            parameters: {
+            /* !!!!!!!! */
+            },
+        };
+        const taskEmbeddingResult = await llmTools.callEmbeddingModel(taskEmbeddingPrompt);
+        const knowledgePiecesWithRelevance = preparedPipeline.knowledgePieces.map((knowledgePiece) => {
+            const { index } = knowledgePiece;
+            const knowledgePieceIndex = index.find((i) => i.modelName === firstKnowlegeIndex.modelName);
+            // <- TODO: Do not use just first knowledge piece and first index to determine embedding model
+            if (knowledgePieceIndex === undefined) {
+                return {
+                    content: knowledgePiece.content,
+                    relevance: 0,
+                };
+            }
+            const relevance = computeCosineSimilarity(knowledgePieceIndex.position, taskEmbeddingResult.content);
+            return {
+                content: knowledgePiece.content,
+                relevance,
+            };
+        });
+        const knowledgePiecesSorted = knowledgePiecesWithRelevance.sort((a, b) => a.relevance - b.relevance);
+        const knowledgePiecesLimited = knowledgePiecesSorted.slice(0, 5);
+        console.log('!!! Embedding', {
+            task,
+            taskEmbeddingPrompt,
+            taskEmbeddingResult,
+            firstKnowlegePiece,
+            firstKnowlegeIndex,
+            knowledgePiecesWithRelevance,
+            knowledgePiecesSorted,
+            knowledgePiecesLimited,
+        });
+        return knowledgePiecesLimited.map(({ content }) => `- ${content}`).join('\n');
         //                                                      <- TODO: [🧠] Some smart aggregation of knowledge pieces, single-line vs multi-line vs mixed
     }
+    // TODO: !!!!!! Annotate + to new file
+    function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
+        if (embeddingVector1.length !== embeddingVector2.length) {
+            throw new TypeError('Embedding vectors must have the same length');
+        }
+        const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
+        const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
+        const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
+        return 1 - dotProduct / (magnitude1 * magnitude2);
+    }
+    /**
+     * TODO: !!!! Verify if this is working
+     * TODO: [♨] Implement Better - use keyword search
+     * TODO: [♨] Examples of values
+     */
 
     /**
      * @@@
@@ -5129,9 +5242,9 @@
      * @private internal utility of `createPipelineExecutor`
      */
     async function getReservedParametersForTask(options) {
-        const { preparedPipeline, task, pipelineIdentification } = options;
+        const { tools, preparedPipeline, task, pipelineIdentification } = options;
         const context = await getContextForTask(); // <- [🏍]
-        const knowledge = await getKnowledgeForTask({ preparedPipeline, task });
+        const knowledge = await getKnowledgeForTask({ tools, preparedPipeline, task });
         const examples = await getExamplesForTask();
         const currentDate = new Date().toISOString(); // <- TODO: [🧠][💩] Better
         const modelName = RESERVED_PARAMETER_MISSING_VALUE;
@@ -5193,6 +5306,7 @@
         }
         const definedParameters = Object.freeze({
             ...(await getReservedParametersForTask({
+                tools,
                 preparedPipeline,
                 task: currentTask,
                 pipelineIdentification,