@promptbook/documents 0.89.0-1 → 0.89.0-11

package/esm/typings/src/remote-server/types/RemoteServerOptions.d.ts

@@ -1,10 +1,15 @@
+import type { Request, Response } from 'express';
 import type { Promisable } from 'type-fest';
 import type { PipelineCollection } from '../../collection/PipelineCollection';
+import { AuthenticationError } from '../../errors/AuthenticationError';
 import type { CommonToolsOptions } from '../../execution/CommonToolsOptions';
 import type { LlmExecutionTools } from '../../execution/LlmExecutionTools';
 import type { string_app_id } from '../../types/typeAliases';
+import type { string_email } from '../../types/typeAliases';
+import type { string_password } from '../../types/typeAliases';
 import type { string_uri } from '../../types/typeAliases';
 import type { string_user_id } from '../../types/typeAliases';
+import type { Identification } from '../socket-types/_subtypes/Identification';
 /**
  * @@@
  *
@@ -16,7 +21,6 @@ import type { string_user_id } from '../../types/typeAliases';
  *
  * You can enable both modes at the same time.
  *
- * @public exported from `@promptbook/remote-client`
  * @public exported from `@promptbook/remote-server`
  */
 export type RemoteServerOptions<TCustomOptions> = CommonToolsOptions & {
@@ -32,8 +36,12 @@ export type RemoteServerOptions<TCustomOptions> = CommonToolsOptions & {
      * @default '/'
      * @example '/api/promptbook/'
      */
-    readonly rootPath: string_uri;
-} & (AnonymousRemoteServerOptions | ApplicationRemoteServerOptions<TCustomOptions> | (AnonymousRemoteServerOptions & ApplicationRemoteServerOptions<TCustomOptions>));
+    readonly rootPath?: string_uri;
+} & ((AnonymousRemoteServerOptions & {
+    readonly isApplicationModeAllowed?: false;
+}) | ({
+    readonly isAnonymousModeAllowed?: false;
+} & ApplicationRemoteServerOptions<TCustomOptions>) | (AnonymousRemoteServerOptions & ApplicationRemoteServerOptions<TCustomOptions>));
 export type AnonymousRemoteServerOptions = {
     /**
      * Enable anonymous mode
@@ -51,6 +59,18 @@ export type ApplicationRemoteServerOptions<TCustomOptions> = {
      * This is used to check validity of the prompt to prevent misuse
      */
     readonly collection: PipelineCollection;
+    /**
+     * User tries to login to the server, this function will be called verify the user and return the identification or throw an error
+     * This can be also doubled as a function to register the user
+     *
+     * Note: In most cases, you will return `PromptbookServer_ApplicationIdentification`
+     *       `PromptbookServer_AnonymousIdentification` is useful only in scenarios when user stores its own api keys on the application server and
+     *       server acts only as a api key provider
+     *
+     * Note: In most cases DO NOT THROW `AuthenticationError` but return `isSuccess: false` with message
+     * @throws `AuthenticationError`  if the user is not allowed to login for example because of invalid credentials
+     */
+    login(loginRequest: LoginRequest): Promise<LoginResponse<TCustomOptions>>;
     /**
      * Creates llm execution tools for each client
      */
@@ -82,6 +102,56 @@ export type ApplicationRemoteServerClientOptions<TCustomOptions> = {
      */
     readonly customOptions?: TCustomOptions;
 };
+/**
+ * Login request for the application mode
+ */
+export type LoginRequest = {
+    /**
+     * Identifier of the application you are using
+     *
+     * Note: This is usefull when you use Promptbook remote server for multiple apps/frontends, if its used just for single app, use here just "app" or "your-app-name"
+     */
+    readonly appId: string_app_id | null;
+    /**
+     * Username (for example email) of the user
+     */
+    readonly username: string_email | string;
+    /**
+     * Password of the user
+     */
+    readonly password: string_password;
+    /**
+     * Request object from express if you want to access some request data for example headers, IP address, etc.
+     */
+    readonly rawRequest: Request;
+    /**
+     * Response object from express if you want to add some custom headers.
+     *
+     * Note: It is not recommended to use this object to send body of the response because it can confuse the client
+     */
+    readonly rawResponse: Response;
+};
+/**
+ * Login response for the application mode
+ */
+export type LoginResponse<TCustomOptions> = {
+    /**
+     * Was the login successful
+     */
+    readonly isSuccess: boolean;
+    /**
+     * Message to display to the user, this message is always displayed
+     */
+    readonly message?: string;
+    /**
+     * Optional error if the login was not successful
+     */
+    readonly error?: AuthenticationError;
+    /**
+     * Identification of the user to be used in the future requests
+     */
+    readonly identification?: Identification<TCustomOptions>;
+};
 /**
  * TODO: Constrain anonymous mode for specific models / providers
  */

package/esm/typings/src/scrapers/_common/utils/{scraperFetch.d.ts → promptbookFetch.d.ts}

@@ -2,9 +2,9 @@ import type { PromptbookFetch } from '../../../execution/PromptbookFetch';
 /**
  * The built-in `fetch' function with a lightweight error handling wrapper as default fetch function used in Promptbook scrapers
  *
- * @private as default `fetch` function used in Promptbook scrapers
+ * @public exported from `@promptbook/core`
  */
-export declare const scraperFetch: PromptbookFetch;
+export declare const promptbookFetch: PromptbookFetch;
 /**
  * TODO: [🧠] Maybe rename because it is not used only for scrapers but also in `$getCompiledBook`
  */

package/esm/typings/src/storage/env-storage/$EnvStorage.d.ts

@@ -0,0 +1,37 @@
+import type { PromptbookStorage } from '../_common/PromptbookStorage';
+/**
+ * Stores data in .env variables (Remove !!! nonce 1)
+ *
+ * Note: `$` is used to indicate that this function is not a pure function - it uses filesystem to access `.env` file and also writes to `process.env`
+ *
+ * @private within the repository - for CLI utils
+ */
+export declare class $EnvStorage<TItem> implements PromptbookStorage<TItem> {
+    private envFilename;
+    private $provideOrCreateEnvFile;
+    private transformKey;
+    /**
+     * Returns the number of key/value pairs currently present in the list associated with the object.
+     */
+    get length(): number;
+    /**
+     * Empties the list associated with the object of all key/value pairs, if there are any.
+     */
+    clear(): void;
+    /**
+     * Returns the current value associated with the given key, or null if the given key does not exist in the list associated with the object.
+     */
+    getItem(key: string): Promise<TItem | null>;
+    /**
+     * Returns the name of the nth key in the list, or null if n is greater than or equal to the number of key/value pairs in the object.
+     */
+    key(index: number): string | null;
+    /**
+     * Sets the value of the pair identified by key to value, creating a new key/value pair if none existed for key previously.
+     */
+    setItem(key: string, value: TItem): Promise<void>;
+    /**
+     * Removes the key/value pair with the given key from the list associated with the object, if a key/value pair with the given key exists.
+     */
+    removeItem(key: string): void;
+}

package/esm/typings/src/types/typeAliases.d.ts

@@ -242,6 +242,12 @@ export type string_promptbook_documentation_url = `https://github.com/webgptorg/
  * For example `"towns.cz"`
  */
 export type string_domain = string;
+/**
+ * Semantic helper
+ *
+ * For example `"https://*.pavolhejny.com/*"`
+ */
+export type string_origin = string;
 /**
  * Semantic helper
  *

package/esm/typings/src/utils/organization/TODO_narrow.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Organizational helper to better mark the place where the more specific type is missing
+ *
+ * @private within the repository
+ */
+export type TODO_narrow<T> = T;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@promptbook/documents",
-    "version": "0.89.0-1",
+    "version": "0.89.0-11",
     "description": "It's time for a paradigm shift. The future of software in plain English, French or Latin",
     "private": false,
     "sideEffects": false,
@@ -47,7 +47,7 @@
     "module": "./esm/index.es.js",
     "typings": "./esm/typings/src/_packages/documents.index.d.ts",
     "peerDependencies": {
-        "@promptbook/core": "0.89.0-1"
+        "@promptbook/core": "0.89.0-11"
     },
     "dependencies": {
         "colors": "1.4.0",

package/umd/index.umd.js

@@ -26,7 +26,7 @@
      * @generated
      * @see https://github.com/webgptorg/promptbook
      */
-    const PROMPTBOOK_ENGINE_VERSION = '0.89.0-1';
+    const PROMPTBOOK_ENGINE_VERSION = '0.89.0-11';
     /**
      * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
      * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -160,6 +160,7 @@
      */
     const DEFAULT_MAX_EXECUTION_ATTEMPTS = 10; // <- TODO: [🤹‍♂️]
     // <- TODO: [🕝] Make also `BOOKS_DIRNAME_ALTERNATIVES`
+    // TODO: !!!!!! Just .promptbook dir, hardocode others
     /**
      * Where to store the temporary downloads
      *
@@ -211,7 +212,7 @@
     true);
     /**
      * Note: [💞] Ignore a discrepancy between file name and entity name
-     * TODO: [🧠][🧜‍♂️] Maybe join remoteUrl and path into single value
+     * TODO: [🧠][🧜‍♂️] Maybe join remoteServerUrl and path into single value
      */
 
     /**
@@ -430,6 +431,7 @@
                     }
                     else {
                         console.warn(`Command "${humanReadableCommand}" exceeded time limit of ${timeout}ms but continues running`);
+                        // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
                         resolve('Command exceeded time limit');
                     }
                 });
@@ -455,6 +457,7 @@
                     output.push(stderr.toString());
                     if (isVerbose && stderr.toString().trim()) {
                         console.warn(stderr.toString());
+                        // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
                     }
                 });
                 const finishWithCode = (code) => {
@@ -466,6 +469,7 @@
                         else {
                             if (isVerbose) {
                                 console.warn(`Command "${humanReadableCommand}" exited with code ${code}`);
+                                // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
                             }
                             resolve(spaceTrim.spaceTrim(output.join('\n')));
                         }
@@ -487,6 +491,7 @@
                     else {
                         if (isVerbose) {
                             console.warn(error);
+                            // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
                         }
                         resolve(spaceTrim.spaceTrim(output.join('\n')));
                     }
@@ -2302,6 +2307,19 @@
         }
     }
 
+    /**
+     * AuthenticationError is thrown from login function which is dependency of remote server
+     *
+     * @public exported from `@promptbook/core`
+     */
+    class AuthenticationError extends Error {
+        constructor(message) {
+            super(message);
+            this.name = 'AuthenticationError';
+            Object.setPrototypeOf(this, AuthenticationError.prototype);
+        }
+    }
+
     /**
      * This error indicates that the pipeline collection cannot be propperly loaded
      *
@@ -2367,6 +2385,19 @@
         }
     }
 
+    /**
+     * Error thrown when a fetch request fails
+     *
+     * @public exported from `@promptbook/core`
+     */
+    class PromptbookFetchError extends Error {
+        constructor(message) {
+            super(message);
+            this.name = 'PromptbookFetchError';
+            Object.setPrototypeOf(this, PromptbookFetchError.prototype);
+        }
+    }
+
     /**
      * Index of all custom errors
      *
@@ -2404,6 +2435,8 @@
         TypeError,
         URIError,
         AggregateError,
+        AuthenticationError,
+        PromptbookFetchError,
         /*
       Note: Not widely supported
       > InternalError,
@@ -2465,6 +2498,7 @@
         const { isSuccessful, errors, warnings } = executionResult;
         for (const warning of warnings) {
             console.warn(warning.message);
+            // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
         }
         if (isSuccessful === true) {
             return;
@@ -2602,6 +2636,10 @@
 
                     Cannot serialize error with name "${name}"
 
+                    Authors of Promptbook probably forgot to add this error into the list of errors:
+                    https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
+
+
                     ${block(stack || message)}
 
                 `));
@@ -2643,30 +2681,42 @@
         await Promise.all(tasks);
     }
 
+    /**
+     * Represents the uncertain value
+     *
+     * @public exported from `@promptbook/core`
+     */
+    const ZERO_VALUE = $deepFreeze({ value: 0 });
+    /**
+     * Represents the uncertain value
+     *
+     * @public exported from `@promptbook/core`
+     */
+    const UNCERTAIN_ZERO_VALUE = $deepFreeze({ value: 0, isUncertain: true });
     /**
      * Represents the usage with no resources consumed
      *
      * @public exported from `@promptbook/core`
      */
     const ZERO_USAGE = $deepFreeze({
-        price: { value: 0 },
+        price: ZERO_VALUE,
         input: {
-            tokensCount: { value: 0 },
-            charactersCount: { value: 0 },
-            wordsCount: { value: 0 },
-            sentencesCount: { value: 0 },
-            linesCount: { value: 0 },
-            paragraphsCount: { value: 0 },
-            pagesCount: { value: 0 },
+            tokensCount: ZERO_VALUE,
+            charactersCount: ZERO_VALUE,
+            wordsCount: ZERO_VALUE,
+            sentencesCount: ZERO_VALUE,
+            linesCount: ZERO_VALUE,
+            paragraphsCount: ZERO_VALUE,
+            pagesCount: ZERO_VALUE,
         },
         output: {
-            tokensCount: { value: 0 },
-            charactersCount: { value: 0 },
-            wordsCount: { value: 0 },
-            sentencesCount: { value: 0 },
-            linesCount: { value: 0 },
-            paragraphsCount: { value: 0 },
-            pagesCount: { value: 0 },
+            tokensCount: ZERO_VALUE,
+            charactersCount: ZERO_VALUE,
+            wordsCount: ZERO_VALUE,
+            sentencesCount: ZERO_VALUE,
+            linesCount: ZERO_VALUE,
+            paragraphsCount: ZERO_VALUE,
+            pagesCount: ZERO_VALUE,
         },
     });
     /**
@@ -2675,24 +2725,24 @@
      * @public exported from `@promptbook/core`
      */
     $deepFreeze({
-        price: { value: 0, isUncertain: true },
+        price: UNCERTAIN_ZERO_VALUE,
         input: {
-            tokensCount: { value: 0, isUncertain: true },
-            charactersCount: { value: 0, isUncertain: true },
-            wordsCount: { value: 0, isUncertain: true },
-            sentencesCount: { value: 0, isUncertain: true },
-            linesCount: { value: 0, isUncertain: true },
-            paragraphsCount: { value: 0, isUncertain: true },
-            pagesCount: { value: 0, isUncertain: true },
+            tokensCount: UNCERTAIN_ZERO_VALUE,
+            charactersCount: UNCERTAIN_ZERO_VALUE,
+            wordsCount: UNCERTAIN_ZERO_VALUE,
+            sentencesCount: UNCERTAIN_ZERO_VALUE,
+            linesCount: UNCERTAIN_ZERO_VALUE,
+            paragraphsCount: UNCERTAIN_ZERO_VALUE,
+            pagesCount: UNCERTAIN_ZERO_VALUE,
         },
         output: {
-            tokensCount: { value: 0, isUncertain: true },
-            charactersCount: { value: 0, isUncertain: true },
-            wordsCount: { value: 0, isUncertain: true },
-            sentencesCount: { value: 0, isUncertain: true },
-            linesCount: { value: 0, isUncertain: true },
-            paragraphsCount: { value: 0, isUncertain: true },
-            pagesCount: { value: 0, isUncertain: true },
+            tokensCount: UNCERTAIN_ZERO_VALUE,
+            charactersCount: UNCERTAIN_ZERO_VALUE,
+            wordsCount: UNCERTAIN_ZERO_VALUE,
+            sentencesCount: UNCERTAIN_ZERO_VALUE,
+            linesCount: UNCERTAIN_ZERO_VALUE,
+            paragraphsCount: UNCERTAIN_ZERO_VALUE,
+            pagesCount: UNCERTAIN_ZERO_VALUE,
         },
     });
     /**
@@ -2980,6 +3030,7 @@
         `);
             // TODO: [🟥] Detect browser / node and make it colorfull
             console.warn(warningMessage);
+            // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
             /*
             return {
                 async listModels() {
@@ -3355,17 +3406,24 @@
     /**
      * The built-in `fetch' function with a lightweight error handling wrapper as default fetch function used in Promptbook scrapers
      *
-     * @private as default `fetch` function used in Promptbook scrapers
+     * @public exported from `@promptbook/core`
      */
-    const scraperFetch = async (url, init) => {
+    const promptbookFetch = async (urlOrRequest, init) => {
         try {
-            return await fetch(url, init);
+            return await fetch(urlOrRequest, init);
         }
         catch (error) {
             if (!(error instanceof Error)) {
                 throw error;
             }
-            throw new KnowledgeScrapeError(spaceTrim__default["default"]((block) => `
+            let url;
+            if (typeof urlOrRequest === 'string') {
+                url = urlOrRequest;
+            }
+            else if (urlOrRequest instanceof Request) {
+                url = urlOrRequest.url;
+            }
+            throw new PromptbookFetchError(spaceTrim__default["default"]((block) => `
                     Can not fetch "${url}"
 
                     Fetch error:
@@ -3386,7 +3444,7 @@
     async function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
         // console.log('!! makeKnowledgeSourceHandler', knowledgeSource);
         var _a;
-        const { fetch = scraperFetch } = tools;
+        const { fetch = promptbookFetch } = tools;
         const { knowledgeSourceContent } = knowledgeSource;
         let { name } = knowledgeSource;
         const { rootDirname = null, 
@@ -3527,63 +3585,73 @@
         const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT, rootDirname, isVerbose = DEFAULT_IS_VERBOSE } = options;
         const knowledgePreparedUnflatten = new Array(knowledgeSources.length);
         await forEachAsync(knowledgeSources, { maxParallelCount }, async (knowledgeSource, index) => {
-            let partialPieces = null;
-            const sourceHandler = await makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname, isVerbose });
-            const scrapers = arrayableToArray(tools.scrapers);
-            for (const scraper of scrapers) {
-                if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
-                // <- TODO: [🦔] Implement mime-type wildcards
-                ) {
-                    continue;
-                }
-                const partialPiecesUnchecked = await scraper.scrape(sourceHandler);
-                if (partialPiecesUnchecked !== null) {
-                    partialPieces = [...partialPiecesUnchecked];
-                    // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
-                    break;
-                }
-                console.warn(spaceTrim__default["default"]((block) => `
-                        Cannot scrape knowledge from source despite the scraper \`${scraper.metadata.className}\` supports the mime type "${sourceHandler.mimeType}".
+            try {
+                let partialPieces = null;
+                const sourceHandler = await makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname, isVerbose });
+                const scrapers = arrayableToArray(tools.scrapers);
+                for (const scraper of scrapers) {
+                    if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
+                    // <- TODO: [🦔] Implement mime-type wildcards
+                    ) {
+                        continue;
+                    }
+                    const partialPiecesUnchecked = await scraper.scrape(sourceHandler);
+                    if (partialPiecesUnchecked !== null) {
+                        partialPieces = [...partialPiecesUnchecked];
+                        // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
+                        break;
+                    }
+                    console.warn(spaceTrim__default["default"]((block) => `
+                            Cannot scrape knowledge from source despite the scraper \`${scraper.metadata.className}\` supports the mime type "${sourceHandler.mimeType}".
 
-                        The source:
-                        ${block(knowledgeSource.knowledgeSourceContent
-                .split('\n')
-                .map((line) => `> ${line}`)
-                .join('\n'))}
+                            The source:
+                            ${block(knowledgeSource.knowledgeSourceContent
+                    .split('\n')
+                    .map((line) => `> ${line}`)
+                    .join('\n'))}
 
-                        ${block($registeredScrapersMessage(scrapers))}
+                            ${block($registeredScrapersMessage(scrapers))}
 
 
-                    `));
-            }
-            if (partialPieces === null) {
-                throw new KnowledgeScrapeError(spaceTrim__default["default"]((block) => `
-                        Cannot scrape knowledge
+                        `));
+                    // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
+                }
+                if (partialPieces === null) {
+                    throw new KnowledgeScrapeError(spaceTrim__default["default"]((block) => `
+                            Cannot scrape knowledge
 
-                        The source:
-                        > ${block(knowledgeSource.knowledgeSourceContent
-                .split('\n')
-                .map((line) => `> ${line}`)
-                .join('\n'))}
+                            The source:
+                            > ${block(knowledgeSource.knowledgeSourceContent
+                    .split('\n')
+                    .map((line) => `> ${line}`)
+                    .join('\n'))}
 
-                        No scraper found for the mime type "${sourceHandler.mimeType}"
+                            No scraper found for the mime type "${sourceHandler.mimeType}"
 
-                        ${block($registeredScrapersMessage(scrapers))}
+                            ${block($registeredScrapersMessage(scrapers))}
 
 
-                    `));
+                        `));
+                }
+                const pieces = partialPieces.map((partialPiece) => ({
+                    ...partialPiece,
+                    sources: [
+                        {
+                            name: knowledgeSource.name,
+                            // line, column <- TODO: [☀]
+                            // <- TODO: [❎]
+                        },
+                    ],
+                }));
+                knowledgePreparedUnflatten[index] = pieces;
+            }
+            catch (error) {
+                if (!(error instanceof Error)) {
+                    throw error;
+                }
+                console.warn(error);
+                // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
             }
-            const pieces = partialPieces.map((partialPiece) => ({
-                ...partialPiece,
-                sources: [
-                    {
-                        name: knowledgeSource.name,
-                        // line, column <- TODO: [☀]
-                        // <- TODO: [❎]
-                    },
-                ],
-            }));
-            knowledgePreparedUnflatten[index] = pieces;
         });
         const knowledgePrepared = knowledgePreparedUnflatten.flat();
         return knowledgePrepared;
@@ -4001,7 +4069,7 @@
                 if (parameterNames.has(subparameterName)) {
                     parameterNames.delete(subparameterName);
                     parameterNames.add(foreach.parameterName);
-                    // <- TODO: [🚎] Warn/logic error when `subparameterName` not used
+                    // <- TODO: [🏮] Warn/logic error when `subparameterName` not used
                 }
             }
         }
@@ -5597,6 +5665,7 @@
 
                     @see more at https://ptbk.io/prepare-pipeline
                 `));
+            // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
         }
         let runCount = 0;
         const pipelineExecutorWithCallback = async (inputParameters, onProgress) => {