@promptbook/core 0.72.0-0 → 0.72.0-10

package/esm/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts

@@ -1,5 +1,5 @@
 import type { AvailableModel } from '../../execution/AvailableModel';
-import type { CommonExecutionToolsOptions } from '../../execution/CommonExecutionToolsOptions';
+import type { CommonToolsOptions } from '../../execution/CommonToolsOptions';
 import type { LlmExecutionTools } from '../../execution/LlmExecutionTools';
 import type { ChatPromptResult } from '../../execution/PromptResult';
 import type { CompletionPromptResult } from '../../execution/PromptResult';
@@ -13,8 +13,8 @@ import type { string_title } from '../../types/typeAliases';
  * @public exported from `@promptbook/fake-llm`
  */
 export declare class MockedEchoLlmExecutionTools implements LlmExecutionTools {
-    protected readonly options: CommonExecutionToolsOptions;
-    constructor(options?: CommonExecutionToolsOptions);
+    protected readonly options: CommonToolsOptions;
+    constructor(options?: CommonToolsOptions);
     get title(): string_title & string_markdown_text;
     get description(): string_markdown;
     /**

package/esm/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts

@@ -1,5 +1,5 @@
 import type { AvailableModel } from '../../execution/AvailableModel';
-import type { CommonExecutionToolsOptions } from '../../execution/CommonExecutionToolsOptions';
+import type { CommonToolsOptions } from '../../execution/CommonToolsOptions';
 import type { LlmExecutionTools } from '../../execution/LlmExecutionTools';
 import type { ChatPromptResult } from '../../execution/PromptResult';
 import type { CompletionPromptResult } from '../../execution/PromptResult';
@@ -14,8 +14,8 @@ import type { string_title } from '../../types/typeAliases';
  * @public exported from `@promptbook/fake-llm`
  */
 export declare class MockedFackedLlmExecutionTools implements LlmExecutionTools {
-    protected readonly options: CommonExecutionToolsOptions;
-    constructor(options?: CommonExecutionToolsOptions);
+    protected readonly options: CommonToolsOptions;
+    constructor(options?: CommonToolsOptions);
     get title(): string_title & string_markdown_text;
     get description(): string_markdown;
     /**

package/esm/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts

@@ -16,6 +16,7 @@ import type { string_title } from '../../types/typeAliases';
  *
  * Note: Internal utility of `joinLlmExecutionTools` but exposed type
  * @public exported from `@promptbook/types`
+ * TODO: !!!!!! Export as runtime class not just type
  */
 export declare class MultipleLlmExecutionTools implements LlmExecutionTools {
     /**

package/esm/typings/src/llm-providers/multiple/playground/playground.d.ts

@@ -1,2 +1,5 @@
 #!/usr/bin/env ts-node
 export {};
+/**
+ * Note: [⚫] Code in this file should never be published in any package
+ */

package/esm/typings/src/llm-providers/openai/OpenAiAssistantExecutionTools.d.ts

@@ -14,13 +14,13 @@ import { OpenAiExecutionTools } from './OpenAiExecutionTools';
  * @public exported from `@promptbook/openai`
  */
 export declare class OpenAiAssistantExecutionTools extends OpenAiExecutionTools implements LlmExecutionTools {
-    private readonly assistantId?;
+    private readonly assistantId;
     /**
      * Creates OpenAI Execution Tools.
      *
      * @param options which are relevant are directly passed to the OpenAI client
      */
-    constructor(options?: OpenAiAssistantExecutionToolsOptions);
+    constructor(options: OpenAiAssistantExecutionToolsOptions);
     get title(): string_title & string_markdown_text;
     get description(): string_markdown;
     /**

package/esm/typings/src/llm-providers/openai/OpenAiAssistantExecutionToolsOptions.d.ts

@@ -10,5 +10,5 @@ export type OpenAiAssistantExecutionToolsOptions = OpenAiExecutionToolsOptions &
     /**
      * Which assistant to use
      */
-    assistantId?: string_token;
+    assistantId: string_token;
 };

package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts

@@ -8,6 +8,8 @@ import type { Prompt } from '../../types/Prompt';
 import type { string_markdown } from '../../types/typeAliases';
 import type { string_markdown_text } from '../../types/typeAliases';
 import type { string_title } from '../../types/typeAliases';
+import type { string_token } from '../../types/typeAliases';
+import { OpenAiAssistantExecutionTools } from './OpenAiAssistantExecutionTools';
 import type { OpenAiExecutionToolsOptions } from './OpenAiExecutionToolsOptions';
 /**
  * Execution Tools for calling OpenAI API
@@ -25,10 +27,17 @@ export declare class OpenAiExecutionTools implements LlmExecutionTools {
      *
      * @param options which are relevant are directly passed to the OpenAI client
      */
-    constructor(options?: OpenAiExecutionToolsOptions);
+    constructor(options: OpenAiExecutionToolsOptions);
     get title(): string_title & string_markdown_text;
     get description(): string_markdown;
     getClient(): Promise<OpenAI>;
+    /**
+     * Create (sub)tools for calling OpenAI API Assistants
+     *
+     * @param assistantId Which assistant to use
+     * @returns Tools for calling OpenAI API Assistants with same token
+     */
+    createAssistantSubtools(assistantId: string_token): OpenAiAssistantExecutionTools;
     /**
      * Check the `options` passed to `constructor`
      */

package/esm/typings/src/llm-providers/openai/OpenAiExecutionToolsOptions.d.ts

@@ -1,6 +1,6 @@
 import type { ClientOptions } from 'openai';
-import type { CommonExecutionToolsOptions } from '../../execution/CommonExecutionToolsOptions';
-import type { string_token } from '../../types/typeAliases';
+import type { CommonToolsOptions } from '../../execution/CommonToolsOptions';
+import type { string_user_id } from '../../types/typeAliases';
 /**
  * Options for `OpenAiExecutionTools`
  *
@@ -9,12 +9,12 @@ import type { string_token } from '../../types/typeAliases';
  *
  * @public exported from `@promptbook/openai`
  */
-export type OpenAiExecutionToolsOptions = CommonExecutionToolsOptions & ClientOptions & {
+export type OpenAiExecutionToolsOptions = CommonToolsOptions & ClientOptions & {
     /**
      * A unique identifier representing your end-user, which can help OpenAI to monitor
      * and detect abuse.
      *
      * @see https://platform.openai.com/docs/guides/safety-best-practices/end-user-ids
      */
-    user?: string_token;
+    userId: string_user_id | null;
 };

package/esm/typings/src/llm-providers/openai/createOpenAiAssistantExecutionTools.d.ts

@@ -0,0 +1,15 @@
+import { OpenAiAssistantExecutionTools } from './OpenAiAssistantExecutionTools';
+import type { OpenAiAssistantExecutionToolsOptions } from './OpenAiAssistantExecutionToolsOptions';
+/**
+ * Execution Tools for calling OpenAI API
+ *
+ * @public exported from `@promptbook/openai`
+ */
+export declare const createOpenAiAssistantExecutionTools: ((options: OpenAiAssistantExecutionToolsOptions) => OpenAiAssistantExecutionTools) & {
+    packageName: string;
+    className: string;
+};
+/**
+ * TODO: [🦺] Is there some way how to put `packageName` and `className` on top and function definition on bottom?
+ * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
+ */

package/esm/typings/src/llm-providers/openai/playground/playground.d.ts

@@ -1,5 +1,6 @@
 #!/usr/bin/env ts-node
 export {};
 /**
- * TODO:[main] !!! Test here that `systemMessage`, `temperature` and `seed` are working correctly
+ * TODO: [main] !!! Test here that `systemMessage`, `temperature` and `seed` are working correctly
+ * Note: [⚫] Code in this file should never be published in any package
  */

package/esm/typings/src/llm-providers/openai/register-configuration.d.ts

@@ -1,3 +1,13 @@
+import type { Registration } from '../../utils/$Register';
+/**
+ * Registration of LLM provider metadata
+ *
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available LLM tools
+ *
+ * @public exported from `@promptbook/core`
+ * @public exported from `@promptbook/cli`
+ */
+export declare const _OpenAiMetadataRegistration: Registration;
 /**
  * @@@ registration1 of default configuration for Open AI
  *
@@ -6,4 +16,4 @@
  * @public exported from `@promptbook/core`
  * @public exported from `@promptbook/cli`
  */
-export declare const _OpenAiMetadataRegistration: void;
+export declare const _OpenAiAssistantMetadataRegistration: Registration;

package/esm/typings/src/llm-providers/openai/register-constructor.d.ts

@@ -1,3 +1,13 @@
+import type { Registration } from '../../utils/$Register';
+/**
+ * Registration of LLM provider
+ *
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available LLM tools
+ *
+ * @public exported from `@promptbook/openai`
+ * @public exported from `@promptbook/cli`
+ */
+export declare const _OpenAiRegistration: Registration;
 /**
  * @@@ registration2
  *
@@ -6,7 +16,7 @@
  * @public exported from `@promptbook/openai`
  * @public exported from `@promptbook/cli`
  */
-export declare const _OpenAiRegistration: void;
+export declare const _OpenAiAssistantRegistration: Registration;
 /**
  * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
  */

package/esm/typings/src/llm-providers/remote/RemoteLlmExecutionTools.d.ts

@@ -19,9 +19,9 @@ import type { RemoteLlmExecutionToolsOptions } from './interfaces/RemoteLlmExecu
  * @see https://github.com/webgptorg/promptbook#remote-server
  * @public exported from `@promptbook/remote-client`
  */
-export declare class RemoteLlmExecutionTools implements LlmExecutionTools {
-    protected readonly options: RemoteLlmExecutionToolsOptions;
-    constructor(options: RemoteLlmExecutionToolsOptions);
+export declare class RemoteLlmExecutionTools<TCustomOptions = undefined> implements LlmExecutionTools {
+    protected readonly options: RemoteLlmExecutionToolsOptions<TCustomOptions>;
+    constructor(options: RemoteLlmExecutionToolsOptions<TCustomOptions>);
     get title(): string_title & string_markdown_text;
     get description(): string_markdown;
     /**

package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Request.d.ts

@@ -1,21 +1,30 @@
-import type { LlmToolsConfiguration } from '../../_common/LlmToolsConfiguration';
+import type { string_user_id } from '../../../types/typeAliases';
+import type { LlmToolsConfiguration } from '../../_common/register/LlmToolsConfiguration';
+import type { CollectionRemoteServerClientOptions } from './RemoteServerOptions';
 /**
  * Socket.io progress for remote text generation
  *
  * This is a request from client to server
  */
-export type PromptbookServer_ListModels_Request = PromptbookServer_ListModels_CollectionRequest | PromptbookServer_ListModels_AnonymousRequest;
-export type PromptbookServer_ListModels_CollectionRequest = {
+export type PromptbookServer_ListModels_Request<TCustomOptions> = PromptbookServer_ListModels_CollectionRequest<TCustomOptions> | PromptbookServer_ListModels_AnonymousRequest;
+export type PromptbookServer_ListModels_CollectionRequest<TCustomOptions> = CollectionRemoteServerClientOptions<TCustomOptions> & {
     /**
-     * Collection mode
+     * Application mode
      */
-    isAnonymous: false;
+    readonly isAnonymous: false;
 };
 export type PromptbookServer_ListModels_AnonymousRequest = {
     /**
      * Anonymous mode
      */
-    isAnonymous: true;
+    readonly isAnonymous: true;
+    /**
+     * Identifier of the end user
+     *
+     * Note: this is passed to the certain model providers to identify misuse
+     * Note: In anonymous mode, there is no need to identify yourself, nor does it change the actual configuration of LLM Tools (unlike in application mode)
+     */
+    readonly userId: string_user_id | null;
     /**
      * Configuration for the LLM tools
      */
@@ -24,6 +33,6 @@ export type PromptbookServer_ListModels_AnonymousRequest = {
 /**
  * TODO: [👡] DRY `PromptbookServer_Prompt_Request` and `PromptbookServer_ListModels_Request`
  * TODO: [🧠][🛍] Maybe not `isAnonymous: boolean` BUT `mode: 'ANONYMOUS'|'COLLECTION'`
- * TODO: [🧠][🤺] Pass `userId` in `PromptbookServer_ListModels_Request`
+ * TODO: [🧠][🤺] Maybe allow overriding of `userId` for each prompt - Pass `userId` in `PromptbookServer_ListModels_Request`
  * TODO: [👒] Listing models (and checking configuration) probbably should go through REST API not Socket.io
  */

package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Response.d.ts

@@ -8,8 +8,8 @@ export interface PromptbookServer_ListModels_Response {
     /**
      * Available models that can be used
      */
-    models: Array<AvailableModel>;
+    readonly models: Array<AvailableModel>;
 }
 /**
  * TODO: [👒] Listing models (and checking configuration) probbably should go through REST API not Socket.io
- */ 
+ */

package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Request.d.ts

@@ -1,24 +1,18 @@
 import type { Prompt } from '../../../types/Prompt';
 import type { string_user_id } from '../../../types/typeAliases';
-import type { LlmToolsConfiguration } from '../../_common/LlmToolsConfiguration';
+import type { LlmToolsConfiguration } from '../../_common/register/LlmToolsConfiguration';
+import type { CollectionRemoteServerClientOptions } from './RemoteServerOptions';
 /**
  * Socket.io progress for remote text generation
  *
  * This is a request from client to server
  */
-export type PromptbookServer_Prompt_Request = PromptbookServer_Prompt_CollectionRequest | PromptbookServer_Prompt_AnonymousRequest;
-export type PromptbookServer_Prompt_CollectionRequest = {
+export type PromptbookServer_Prompt_Request<TCustomOptions> = PromptbookServer_Prompt_CollectionRequest<TCustomOptions> | PromptbookServer_Prompt_AnonymousRequest;
+export type PromptbookServer_Prompt_CollectionRequest<TCustomOptions> = CollectionRemoteServerClientOptions<TCustomOptions> & {
     /**
-     * Collection mode
+     * Application mode
      */
     readonly isAnonymous: false;
-    /**
-     * Identifier of the end user
-     *
-     * Note: this is passed to the certain model providers to identify misuse
-     * Note: In anonymous mode it is not required to identify
-     */
-    readonly userId: string_user_id;
     /**
      * The Prompt to execute
      */
@@ -33,9 +27,9 @@ export type PromptbookServer_Prompt_AnonymousRequest = {
      * Identifier of the end user
      *
      * Note: this is passed to the certain model providers to identify misuse
-     * Note: In anonymous mode it is not required to identify
+     * Note: In anonymous mode, there is no need to identify yourself, nor does it change the actual configuration of LLM Tools (unlike in application mode)
      */
-    readonly userId?: string_user_id;
+    readonly userId: string_user_id | null;
     /**
      * Configuration for the LLM tools
      */

package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Response.d.ts

@@ -8,5 +8,5 @@ export interface PromptbookServer_Prompt_Response {
     /**
      * The result of the prompt
      */
-    promptResult: PromptResult;
+    readonly promptResult: PromptResult;
 }

package/esm/typings/src/llm-providers/remote/interfaces/RemoteLlmExecutionToolsOptions.d.ts

@@ -1,14 +1,15 @@
-import type { CommonExecutionToolsOptions } from '../../../execution/CommonExecutionToolsOptions';
+import type { CommonToolsOptions } from '../../../execution/CommonToolsOptions';
 import type { string_base_url } from '../../../types/typeAliases';
 import type { string_uri } from '../../../types/typeAliases';
 import type { string_user_id } from '../../../types/typeAliases';
-import type { LlmToolsConfiguration } from '../../_common/LlmToolsConfiguration';
+import type { LlmToolsConfiguration } from '../../_common/register/LlmToolsConfiguration';
+import type { CollectionRemoteServerClientOptions } from './RemoteServerOptions';
 /**
  * Options for `RemoteLlmExecutionTools`
  *
  * @public exported from `@promptbook/remote-client`
  */
-export type RemoteLlmExecutionToolsOptions = CommonExecutionToolsOptions & {
+export type RemoteLlmExecutionToolsOptions<TCustomOptions> = CommonToolsOptions & {
     /**
      * URL of the remote PROMPTBOOK server
      * On this server will be connected to the socket.io server
@@ -29,7 +30,7 @@ export type RemoteLlmExecutionToolsOptions = CommonExecutionToolsOptions & {
     /**
      * Use anonymous server with anonymous mode
      */
-    isAnonymous: true;
+    readonly isAnonymous: true;
     /**
      * Configuration for the LLM tools
      */
@@ -37,22 +38,16 @@ export type RemoteLlmExecutionToolsOptions = CommonExecutionToolsOptions & {
     /**
      * Identifier of the end user
      *
-     * Note: this is passed to the certain model providers to identify misuse
-     * Note: In anonymous mode it is not required to identify
+     * Note: This is passed to the certain model providers to identify misuse
+     * Note: In anonymous mode, there is no need to identify yourself, nor does it change the actual configuration of LLM Tools (unlike in application mode).
      */
-    readonly userId?: string_user_id;
-} | {
+    readonly userId: string_user_id | null;
+} | ({
     /**
      * Use anonymous server with client identification and fixed collection
      */
-    isAnonymous: false;
-    /**
-     * Identifier of the end user
-     *
-     * Note: this is passed to the certain model providers to identify misuse
-     */
-    readonly userId: string_user_id;
-});
+    readonly isAnonymous: false;
+} & CollectionRemoteServerClientOptions<TCustomOptions>));
 /**
  * TODO: [🧠][🛍] Maybe not `isAnonymous: boolean` BUT `mode: 'ANONYMOUS'|'COLLECTION'`
  * TODO: [🧠][🧜‍♂️] Maybe join remoteUrl and path into single value

package/esm/typings/src/llm-providers/remote/interfaces/RemoteServerOptions.d.ts

@@ -1,6 +1,7 @@
 import type { PipelineCollection } from '../../../collection/PipelineCollection';
-import type { CommonExecutionToolsOptions } from '../../../execution/CommonExecutionToolsOptions';
+import type { CommonToolsOptions } from '../../../execution/CommonToolsOptions';
 import type { LlmExecutionTools } from '../../../execution/LlmExecutionTools';
+import type { string_app_id } from '../../../types/typeAliases';
 import type { string_uri } from '../../../types/typeAliases';
 import type { string_user_id } from '../../../types/typeAliases';
 /**
@@ -8,7 +9,7 @@ import type { string_user_id } from '../../../types/typeAliases';
  *
  * There are two modes of remote server:
  *
- * 1) **Collection mode** Server will recieve `collection` and execute prompts only from this collection
+ * 1) **Application mode** Server will recieve `collection` and execute prompts only from this collection
  * 2) **Anonymous mode** Server will recieve full `LlmToolsConfiguration` (with api keys) and just acts as a proxy
  *    In anonymous mode, `collection` will be ignored and any prompt will be executed
  *
@@ -17,7 +18,7 @@ import type { string_user_id } from '../../../types/typeAliases';
  * @public exported from `@promptbook/remote-client`
  * @public exported from `@promptbook/remote-server`
  */
-export type RemoteServerOptions = CommonExecutionToolsOptions & {
+export type RemoteServerOptions<TCustomOptions> = CommonToolsOptions & {
     /**
      * Port on which the server will listen
      */
@@ -29,18 +30,18 @@ export type RemoteServerOptions = CommonExecutionToolsOptions & {
      * @example '/promptbook/socket.io'
      */
     readonly path: string_uri;
-} & (AnonymousRemoteServerOptions | CollectionRemoteServerOptions | (AnonymousRemoteServerOptions & CollectionRemoteServerOptions));
+} & (AnonymousRemoteServerOptions | CollectionRemoteServerOptions<TCustomOptions> | (AnonymousRemoteServerOptions & CollectionRemoteServerOptions<TCustomOptions>));
 export type AnonymousRemoteServerOptions = {
     /**
      * Enable anonymous mode
      */
     readonly isAnonymousModeAllowed: true;
 };
-export type CollectionRemoteServerOptions = {
+export type CollectionRemoteServerOptions<TCustomOptions> = {
     /**
-     * Enable collection mode
+     * Enable application mode
      */
-    readonly isCollectionModeAllowed: true;
+    readonly isApplicationModeAllowed: true;
     /**
      * Promptbook collection to use
      *
@@ -50,9 +51,22 @@ export type CollectionRemoteServerOptions = {
     /**
      * Creates llm execution tools for each client
      */
-    createLlmExecutionTools(userId: string_user_id | undefined): LlmExecutionTools;
+    createLlmExecutionTools(options: CollectionRemoteServerClientOptions<TCustomOptions>): LlmExecutionTools;
+};
+export type CollectionRemoteServerClientOptions<TCustomOptions> = {
+    /**
+     * @@@
+     */
+    readonly appId: string_app_id | null;
+    /**
+     * @@@
+     */
+    readonly userId: string_user_id | null;
+    /**
+     * @@@
+     */
+    readonly customOptions?: TCustomOptions;
 };
 /**
  * TODO: Constrain anonymous mode for specific models / providers
- * TODO: [🧠][🤺] Remove `createLlmExecutionTools`, pass just `llmExecutionTools`
  */

package/esm/typings/src/llm-providers/remote/playground/playground.d.ts

@@ -1,2 +1,5 @@
 #!/usr/bin/env ts-node
 import '../../openai/register-constructor';
+/**
+ * Note: [⚫] Code in this file should never be published in any package
+ */

package/esm/typings/src/llm-providers/remote/startRemoteServer.d.ts

@@ -9,7 +9,7 @@ import type { RemoteServerOptions } from './interfaces/RemoteServerOptions';
  * @see https://github.com/webgptorg/promptbook#remote-server
  * @public exported from `@promptbook/remote-server`
  */
-export declare function startRemoteServer(options: RemoteServerOptions): IDestroyable;
+export declare function startRemoteServer<TCustomOptions = undefined>(options: RemoteServerOptions<TCustomOptions>): IDestroyable;
 /**
  * TODO: Maybe use `$asDeeplyFrozenSerializableJson`
  * TODO: [🧠][🛍] Maybe not `isAnonymous: boolean` BUT `mode: 'ANONYMOUS'|'COLLECTION'`

package/esm/typings/src/personas/preparePersona.d.ts

@@ -1,4 +1,5 @@
-import type { PrepareOptions } from '../prepare/PrepareOptions';
+import type { ExecutionTools } from '../execution/ExecutionTools';
+import type { PrepareAndScrapeOptions } from '../prepare/PrepareAndScrapeOptions';
 import type { PersonaPreparedJson } from '../types/PipelineJson/PersonaJson';
 import type { string_persona_description } from '../types/typeAliases';
 /**
@@ -7,7 +8,7 @@ import type { string_persona_description } from '../types/typeAliases';
  * @see https://github.com/webgptorg/promptbook/discussions/22
  * @public exported from `@promptbook/core`
  */
-export declare function preparePersona(personaDescription: string_persona_description, options: PrepareOptions): Promise<PersonaPreparedJson['modelRequirements']>;
+export declare function preparePersona(personaDescription: string_persona_description, tools: Pick<ExecutionTools, 'llm'>, options: PrepareAndScrapeOptions): Promise<PersonaPreparedJson['modelRequirements']>;
 /**
  * TODO: [🔃][main] !!!!! If the persona was prepared with different version or different set of models, prepare it once again
  * TODO: [🏢] !! Check validity of `modelName` in pipeline

package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts

@@ -0,0 +1,68 @@
+import type { string_dirname } from '../types/typeAliases';
+/**
+ * Options for preparation of the pipeline
+ */
+export type PrepareAndScrapeOptions = {
+    /**
+     * Path to the root folder of the pipeline
+     *
+     * Note: When the pipeline is not created from files, it is `null`
+     * Note: This folder must exist (=it is not created recursively)
+     *
+     * @default process.cwd()
+     */
+    readonly rootDirname?: string_dirname | null;
+    /**
+     * Path to the cache folder
+     *
+     * Note: When the folder does not exist, it is created recursively
+     *
+     * @default SCRAPE_CACHE_DIRNAME
+     */
+    readonly cacheDirname?: string_dirname;
+    /**
+     * If true, the cache is cleaned after the scraping
+     *
+     *
+     * @default false // <- TODO: !!!!!! Change to `cacheStrategy`/`intermediateFiles`, Put to global config, change to `true` and explicitly set to `false` in all playgrounds
+     */
+    readonly isCacheCleaned?: boolean;
+    /**
+     * Maximum number of tasks running in parallel
+     *
+     * @default MAX_PARALLEL_COUNT
+     */
+    readonly maxParallelCount?: number;
+    /**
+     * Path to the external programs executables
+     *
+     * TODO: !!!!!! Transform to scrapers and make them Classy
+     */
+    readonly externalProgramsPaths?: {
+        /**
+         * Path to the `pandoc` executable
+         *
+         * @example 'C:/Users/me/AppData/Local/Pandoc/pandoc.exe'
+         */
+        readonly pandocPath?: string;
+        /**
+         * Path to the LibreOffice executable
+         *
+         * @example 'C:/Program Files/LibreOffice/program/swriter.exe'
+         */
+        readonly libreOfficePath?: string;
+    };
+    /**
+     * If true, the missing software is automatically installed
+     */
+    readonly isAutoInstalled?: boolean;
+    /**
+     * If true, the preparation logs additional information
+     *
+     * @default false
+     */
+    readonly isVerbose?: boolean;
+};
+/**
+ * TODO: [🧠] Maybe split `PrepareAndScrapeOptions` and `ScrapeOptions` (`ScrapeOptions` should be extended from `PrepareAndScrapeOptions`)
+ */

package/esm/typings/src/prepare/preparePipeline.d.ts

@@ -1,5 +1,6 @@
+import type { ExecutionTools } from '../execution/ExecutionTools';
 import type { PipelineJson } from '../types/PipelineJson/PipelineJson';
-import type { PrepareOptions } from './PrepareOptions';
+import type { PrepareAndScrapeOptions } from './PrepareAndScrapeOptions';
 /**
  * Prepare pipeline from string (markdown) format to JSON format
  *
@@ -8,7 +9,7 @@ import type { PrepareOptions } from './PrepareOptions';
  * Note: When the pipeline is already prepared, it returns the same pipeline
  * @public exported from `@promptbook/core`
  */
-export declare function preparePipeline(pipeline: PipelineJson, options: PrepareOptions): Promise<PipelineJson>;
+export declare function preparePipeline(pipeline: PipelineJson, tools: Pick<ExecutionTools, 'llm' | 'fs' | 'scrapers'>, options: PrepareAndScrapeOptions): Promise<PipelineJson>;
 /**
  * TODO: Write tests for `preparePipeline`
  * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch

package/esm/typings/src/prepare/prepareTemplates.d.ts

@@ -1,6 +1,7 @@
+import type { ExecutionTools } from '../execution/ExecutionTools';
 import type { PipelineJson } from '../types/PipelineJson/PipelineJson';
 import type { TemplateJson } from '../types/PipelineJson/TemplateJson';
-import type { PrepareOptions } from './PrepareOptions';
+import type { PrepareAndScrapeOptions } from './PrepareAndScrapeOptions';
 type PrepareTemplateInput = Pick<PipelineJson, 'templates' | 'parameters'> & {
     /**
      * @@@
@@ -18,7 +19,7 @@ type PreparedTemplates = {
  *
  * @public exported from `@promptbook/core`
  */
-export declare function prepareTemplates(pipeline: PrepareTemplateInput, options: PrepareOptions): Promise<PreparedTemplates>;
+export declare function prepareTemplates(pipeline: PrepareTemplateInput, tools: Pick<ExecutionTools, 'llm' | 'fs' | 'scrapers'>, options: PrepareAndScrapeOptions): Promise<PreparedTemplates>;
 export {};
 /**
  * TODO: [🧠] Add context to each template (if missing)

package/esm/typings/src/scrapers/_common/Converter.d.ts

@@ -0,0 +1,22 @@
+import type { ScraperAndConverterMetadata } from './register/ScraperAndConverterMetadata';
+import type { ScraperSourceHandler } from './Scraper';
+import type { ScraperIntermediateSource } from './ScraperIntermediateSource';
+/**
+ * @@@
+ *
+ */
+export type Converter = {
+    /**
+     * Metadata of the converter which includes title, mime types, etc.
+     */
+    readonly metadata: ScraperAndConverterMetadata;
+    /**
+     * Convert the the file and returns intermediate source or `null` if it can't convert it
+     *
+     * For example, convert a `.docx` to `.doc` file
+     * Or convert a `.pdf` to `.md` file
+     *
+     * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
+     */
+    $convert(source: ScraperSourceHandler): Promise<ScraperIntermediateSource>;
+};