@promptbook/openai 0.72.0-7 → 0.72.0-9

package/esm/typings/src/_packages/cli.index.d.ts

@@ -8,6 +8,16 @@ import { _OpenAiMetadataRegistration } from '../llm-providers/openai/register-co
 import { _OpenAiAssistantMetadataRegistration } from '../llm-providers/openai/register-configuration';
 import { _OpenAiRegistration } from '../llm-providers/openai/register-constructor';
 import { _OpenAiAssistantRegistration } from '../llm-providers/openai/register-constructor';
+import { _LegacyDocumentScraperRegistration } from '../scrapers/document-legacy/register-constructor';
+import { _LegacyDocumentScraperMetadataRegistration } from '../scrapers/document-legacy/register-metadata';
+import { _DocumentScraperRegistration } from '../scrapers/document/register-constructor';
+import { _DocumentScraperMetadataRegistration } from '../scrapers/document/register-metadata';
+import { _MarkdownScraperRegistration } from '../scrapers/markdown/register-constructor';
+import { _MarkdownScraperMetadataRegistration } from '../scrapers/markdown/register-metadata';
+import { _PdfScraperRegistration } from '../scrapers/pdf/register-constructor';
+import { _PdfScraperMetadataRegistration } from '../scrapers/pdf/register-metadata';
+import { _WebsiteScraperRegistration } from '../scrapers/website/register-constructor';
+import { _WebsiteScraperMetadataRegistration } from '../scrapers/website/register-metadata';
 export { PROMPTBOOK_VERSION };
 export { _CLI };
 export { _AnthropicClaudeMetadataRegistration };
@@ -18,3 +28,13 @@ export { _OpenAiMetadataRegistration };
 export { _OpenAiAssistantMetadataRegistration };
 export { _OpenAiRegistration };
 export { _OpenAiAssistantRegistration };
+export { _LegacyDocumentScraperRegistration };
+export { _LegacyDocumentScraperMetadataRegistration };
+export { _DocumentScraperRegistration };
+export { _DocumentScraperMetadataRegistration };
+export { _MarkdownScraperRegistration };
+export { _MarkdownScraperMetadataRegistration };
+export { _PdfScraperRegistration };
+export { _PdfScraperMetadataRegistration };
+export { _WebsiteScraperRegistration };
+export { _WebsiteScraperMetadataRegistration };

package/esm/typings/src/_packages/core.index.d.ts

@@ -20,6 +20,7 @@ import { DEFAULT_REMOTE_URL } from '../config';
 import { DEFAULT_REMOTE_URL_PATH } from '../config';
 import { DEFAULT_CSV_SETTINGS } from '../config';
 import { IS_VERBOSE } from '../config';
+import { IS_AUTO_INSTALLED } from '../config';
 import { pipelineJsonToString } from '../conversion/pipelineJsonToString';
 import { pipelineStringToJson } from '../conversion/pipelineStringToJson';
 import { pipelineStringToJsonSync } from '../conversion/pipelineStringToJsonSync';
@@ -56,9 +57,9 @@ import { CsvFormatDefinition } from '../formats/csv/CsvFormatDefinition';
 import { CsvFormatError } from '../formats/csv/CsvFormatError';
 import { MANDATORY_CSV_SETTINGS } from '../formats/csv/CsvSettings';
 import { TextFormatDefinition } from '../formats/text/TextFormatDefinition';
-import { $llmToolsMetadataRegister } from '../llm-providers/_common/$llmToolsMetadataRegister';
-import { $llmToolsRegister } from '../llm-providers/_common/$llmToolsRegister';
-import { createLlmToolsFromConfiguration } from '../llm-providers/_common/createLlmToolsFromConfiguration';
+import { $llmToolsMetadataRegister } from '../llm-providers/_common/register/$llmToolsMetadataRegister';
+import { $llmToolsRegister } from '../llm-providers/_common/register/$llmToolsRegister';
+import { createLlmToolsFromConfiguration } from '../llm-providers/_common/register/createLlmToolsFromConfiguration';
 import { cacheLlmTools } from '../llm-providers/_common/utils/cache/cacheLlmTools';
 import { countTotalUsage } from '../llm-providers/_common/utils/count-total-usage/countTotalUsage';
 import { limitTotalUsage } from '../llm-providers/_common/utils/count-total-usage/limitTotalUsage';
@@ -73,14 +74,15 @@ import { preparePipeline } from '../prepare/preparePipeline';
 import { prepareTemplates } from '../prepare/prepareTemplates';
 import { unpreparePipeline } from '../prepare/unpreparePipeline';
 import { prepareKnowledgePieces } from '../scrapers/_common/prepareKnowledgePieces';
-import { legacyDocumentScraper } from '../scrapers/document-legacy/legacyDocumentScraper';
-import { documentScraper } from '../scrapers/document/documentScraper';
-import { SCRAPERS } from '../scrapers/index';
-import { markdownScraper } from '../scrapers/markdown/markdownScraper';
-import { pdfScraper } from '../scrapers/pdf/pdfScraper';
-import { websiteScraper } from '../scrapers/website/websiteScraper';
+import { $scrapersMetadataRegister } from '../scrapers/_common/register/$scrapersMetadataRegister';
+import { $scrapersRegister } from '../scrapers/_common/register/$scrapersRegister';
+import { _LegacyDocumentScraperMetadataRegistration } from '../scrapers/document-legacy/register-metadata';
+import { _DocumentScraperMetadataRegistration } from '../scrapers/document/register-metadata';
+import { _MarkdownScraperMetadataRegistration } from '../scrapers/markdown/register-metadata';
+import { _PdfScraperMetadataRegistration } from '../scrapers/pdf/register-metadata';
+import { _WebsiteScraperMetadataRegistration } from '../scrapers/website/register-metadata';
 import { MemoryStorage } from '../storage/memory/MemoryStorage';
-import { PrefixStorage } from '../storage/utils/PrefixStorage';
+import { PrefixStorage } from '../storage/memory/utils/PrefixStorage';
 import { executionReportJsonToString } from '../types/execution-report/executionReportJsonToString';
 import type { ExecutionReportStringOptions } from '../types/execution-report/ExecutionReportStringOptions';
 import { ExecutionReportStringOptionsDefaults } from '../types/execution-report/ExecutionReportStringOptions';
@@ -108,6 +110,7 @@ export { DEFAULT_REMOTE_URL };
 export { DEFAULT_REMOTE_URL_PATH };
 export { DEFAULT_CSV_SETTINGS };
 export { IS_VERBOSE };
+export { IS_AUTO_INSTALLED };
 export { pipelineJsonToString };
 export { pipelineStringToJson };
 export { pipelineStringToJsonSync };
@@ -161,12 +164,13 @@ export { preparePipeline };
 export { prepareTemplates };
 export { unpreparePipeline };
 export { prepareKnowledgePieces };
-export { legacyDocumentScraper };
-export { documentScraper };
-export { SCRAPERS };
-export { markdownScraper };
-export { pdfScraper };
-export { websiteScraper };
+export { $scrapersMetadataRegister };
+export { $scrapersRegister };
+export { _LegacyDocumentScraperMetadataRegistration };
+export { _DocumentScraperMetadataRegistration };
+export { _MarkdownScraperMetadataRegistration };
+export { _PdfScraperMetadataRegistration };
+export { _WebsiteScraperMetadataRegistration };
 export { MemoryStorage };
 export { PrefixStorage };
 export { executionReportJsonToString };

package/esm/typings/src/_packages/documents.index.d.ts

@@ -0,0 +1,8 @@
+import { PROMPTBOOK_VERSION } from '../version';
+import { createDocumentScraper } from '../scrapers/document/createDocumentScraper';
+import { DocumentScraper } from '../scrapers/document/DocumentScraper';
+import { _DocumentScraperRegistration } from '../scrapers/document/register-constructor';
+export { PROMPTBOOK_VERSION };
+export { createDocumentScraper };
+export { DocumentScraper };
+export { _DocumentScraperRegistration };

package/esm/typings/src/_packages/legacy-documents.index.d.ts

@@ -0,0 +1,8 @@
+import { PROMPTBOOK_VERSION } from '../version';
+import { createLegacyDocumentScraper } from '../scrapers/document-legacy/createLegacyDocumentScraper';
+import { LegacyDocumentScraper } from '../scrapers/document-legacy/LegacyDocumentScraper';
+import { _LegacyDocumentScraperRegistration } from '../scrapers/document-legacy/register-constructor';
+export { PROMPTBOOK_VERSION };
+export { createLegacyDocumentScraper };
+export { LegacyDocumentScraper };
+export { _LegacyDocumentScraperRegistration };

package/esm/typings/src/_packages/markdown-utils.index.d.ts

@@ -1,6 +1,9 @@
 import { PROMPTBOOK_VERSION } from '../version';
 import { extractBlock } from '../postprocessing/utils/extractBlock';
 import { extractJsonBlock } from '../postprocessing/utils/extractJsonBlock';
+import { createMarkdownScraper } from '../scrapers/markdown/createMarkdownScraper';
+import { MarkdownScraper } from '../scrapers/markdown/MarkdownScraper';
+import { _MarkdownScraperRegistration } from '../scrapers/markdown/register-constructor';
 import type { string_markdown } from '../types/typeAliases';
 import type { string_markdown_section } from '../types/typeAliases';
 import type { string_markdown_section_content } from '../types/typeAliases';
@@ -21,6 +24,9 @@ import { splitMarkdownIntoSections } from '../utils/markdown/splitMarkdownIntoSe
 export { PROMPTBOOK_VERSION };
 export { extractBlock };
 export { extractJsonBlock };
+export { createMarkdownScraper };
+export { MarkdownScraper };
+export { _MarkdownScraperRegistration };
 export type { string_markdown };
 export type { string_markdown_section };
 export type { string_markdown_section_content };

package/esm/typings/src/_packages/node.index.d.ts

@@ -1,14 +1,20 @@
 import { PROMPTBOOK_VERSION } from '../version';
 import { createCollectionFromDirectory } from '../collection/constructors/createCollectionFromDirectory';
-import { createLlmToolsFromConfigurationFromEnv } from '../llm-providers/_common/createLlmToolsFromConfigurationFromEnv';
-import { createLlmToolsFromEnv } from '../llm-providers/_common/createLlmToolsFromEnv';
+import { $provideExecutionToolsForNode } from '../execution/utils/$provideExecutionToolsForNode';
+import { $provideLlmToolsConfigurationFromEnv } from '../llm-providers/_common/register/$provideLlmToolsConfigurationFromEnv';
+import { $provideLlmToolsFromEnv } from '../llm-providers/_common/register/$provideLlmToolsFromEnv';
+import { $provideFilesystemForNode } from '../scrapers/_common/register/$provideFilesystemForNode';
+import { $provideScrapersForNode } from '../scrapers/_common/register/$provideScrapersForNode';
 import { FileCacheStorage } from '../storage/file-cache-storage/FileCacheStorage';
 import { $execCommand } from '../utils/execCommand/$execCommand';
 import { $execCommands } from '../utils/execCommand/$execCommands';
 export { PROMPTBOOK_VERSION };
 export { createCollectionFromDirectory };
-export { createLlmToolsFromConfigurationFromEnv };
-export { createLlmToolsFromEnv };
+export { $provideExecutionToolsForNode };
+export { $provideLlmToolsConfigurationFromEnv };
+export { $provideLlmToolsFromEnv };
+export { $provideFilesystemForNode };
+export { $provideScrapersForNode };
 export { FileCacheStorage };
 export { $execCommand };
 export { $execCommands };

package/esm/typings/src/_packages/pdf.index.d.ts

@@ -0,0 +1,8 @@
+import { PROMPTBOOK_VERSION } from '../version';
+import { createPdfScraper } from '../scrapers/pdf/createPdfScraper';
+import { PdfScraper } from '../scrapers/pdf/PdfScraper';
+import { _PdfScraperRegistration } from '../scrapers/pdf/register-constructor';
+export { PROMPTBOOK_VERSION };
+export { createPdfScraper };
+export { PdfScraper };
+export { _PdfScraperRegistration };

package/esm/typings/src/_packages/types.index.d.ts

@@ -15,11 +15,12 @@ import type { renderPipelineMermaidOptions } from '../conversion/prettify/render
 import type { CallbackInterfaceToolsOptions } from '../dialogs/callback/CallbackInterfaceToolsOptions';
 import type { ErrorJson } from '../errors/utils/ErrorJson';
 import type { AvailableModel } from '../execution/AvailableModel';
-import type { CommonExecutionToolsOptions } from '../execution/CommonExecutionToolsOptions';
+import type { CommonToolsOptions } from '../execution/CommonToolsOptions';
 import type { CreatePipelineExecutorOptions } from '../execution/createPipelineExecutor/00-CreatePipelineExecutorOptions';
 import type { CreatePipelineExecutorSettings } from '../execution/createPipelineExecutor/00-CreatePipelineExecutorSettings';
 import type { EmbeddingVector } from '../execution/EmbeddingVector';
 import type { ExecutionTools } from '../execution/ExecutionTools';
+import type { FilesystemTools } from '../execution/FilesystemTools';
 import type { LlmExecutionTools } from '../execution/LlmExecutionTools';
 import type { LlmExecutionToolsConstructor } from '../execution/LlmExecutionToolsConstructor';
 import type { PipelineExecutor } from '../execution/PipelineExecutor';
@@ -37,9 +38,9 @@ import type { UserInterfaceTools } from '../execution/UserInterfaceTools';
 import type { UserInterfaceToolsPromptDialogOptions } from '../execution/UserInterfaceTools';
 import type { FormatSubvalueDefinition } from '../formats/_common/FormatSubvalueDefinition';
 import type { CsvSettings } from '../formats/csv/CsvSettings';
-import type { LlmToolsConfiguration } from '../llm-providers/_common/LlmToolsConfiguration';
-import type { LlmToolsMetadata } from '../llm-providers/_common/LlmToolsMetadata';
-import type { LlmToolsOptions } from '../llm-providers/_common/LlmToolsOptions';
+import type { LlmToolsConfiguration } from '../llm-providers/_common/register/LlmToolsConfiguration';
+import type { LlmToolsMetadata } from '../llm-providers/_common/register/LlmToolsMetadata';
+import type { LlmToolsOptions } from '../llm-providers/_common/register/LlmToolsOptions';
 import type { CacheItem } from '../llm-providers/_common/utils/cache/CacheItem';
 import type { CacheLlmToolsOptions } from '../llm-providers/_common/utils/cache/CacheLlmToolsOptions';
 import type { LlmExecutionToolsWithTotalUsage } from '../llm-providers/_common/utils/count-total-usage/LlmExecutionToolsWithTotalUsage';
@@ -67,6 +68,8 @@ import type { AnonymousRemoteServerOptions } from '../llm-providers/remote/inter
 import type { CollectionRemoteServerOptions } from '../llm-providers/remote/interfaces/RemoteServerOptions';
 import type { PrepareAndScrapeOptions } from '../prepare/PrepareAndScrapeOptions';
 import type { Converter } from '../scrapers/_common/Converter';
+import type { ScraperAndConverterMetadata } from '../scrapers/_common/register/ScraperAndConverterMetadata';
+import type { ScraperConstructor } from '../scrapers/_common/register/ScraperConstructor';
 import type { Scraper } from '../scrapers/_common/Scraper';
 import type { ScraperSourceHandler } from '../scrapers/_common/Scraper';
 import type { ScraperIntermediateSource } from '../scrapers/_common/ScraperIntermediateSource';
@@ -156,6 +159,7 @@ import type { string_data_url } from '../types/typeAliases';
 import type { string_base64 } from '../types/typeAliases';
 import type { string_href } from '../types/typeAliases';
 import type { string_url_image } from '../types/typeAliases';
+import type { string_executable_path } from '../types/typeAliases';
 import type { string_uri } from '../types/typeAliases';
 import type { string_uri_part } from '../types/typeAliases';
 import type { string_uriid } from '../types/typeAliases';
@@ -218,6 +222,7 @@ import type { number_megabytes } from '../types/typeAliases';
 import type { number_gigabytes } from '../types/typeAliases';
 import type { number_terabytes } from '../types/typeAliases';
 import type { Registered } from '../utils/$Register';
+import type { Registration } from '../utils/$Register';
 import type { IExecCommandOptions } from '../utils/execCommand/IExecCommandOptions';
 import type { IExecCommandOptionsAdvanced } from '../utils/execCommand/IExecCommandOptions';
 import type { FromtoItems } from '../utils/FromtoItems';
@@ -251,11 +256,12 @@ export type { renderPipelineMermaidOptions };
 export type { CallbackInterfaceToolsOptions };
 export type { ErrorJson };
 export type { AvailableModel };
-export type { CommonExecutionToolsOptions };
+export type { CommonToolsOptions };
 export type { CreatePipelineExecutorOptions };
 export type { CreatePipelineExecutorSettings };
 export type { EmbeddingVector };
 export type { ExecutionTools };
+export type { FilesystemTools };
 export type { LlmExecutionTools };
 export type { LlmExecutionToolsConstructor };
 export type { PipelineExecutor };
@@ -303,6 +309,8 @@ export type { AnonymousRemoteServerOptions };
 export type { CollectionRemoteServerOptions };
 export type { PrepareAndScrapeOptions };
 export type { Converter };
+export type { ScraperAndConverterMetadata };
+export type { ScraperConstructor };
 export type { Scraper };
 export type { ScraperSourceHandler };
 export type { ScraperIntermediateSource };
@@ -392,6 +400,7 @@ export type { string_data_url };
 export type { string_base64 };
 export type { string_href };
 export type { string_url_image };
+export type { string_executable_path };
 export type { string_uri };
 export type { string_uri_part };
 export type { string_uriid };
@@ -454,6 +463,7 @@ export type { number_megabytes };
 export type { number_gigabytes };
 export type { number_terabytes };
 export type { Registered };
+export type { Registration };
 export type { IExecCommandOptions };
 export type { IExecCommandOptionsAdvanced };
 export type { FromtoItems };

package/esm/typings/src/_packages/website-crawler.index.d.ts

@@ -0,0 +1,8 @@
+import { PROMPTBOOK_VERSION } from '../version';
+import { createWebsiteScraper } from '../scrapers/website/createWebsiteScraper';
+import { _WebsiteScraperRegistration } from '../scrapers/website/register-constructor';
+import { WebsiteScraper } from '../scrapers/website/WebsiteScraper';
+export { PROMPTBOOK_VERSION };
+export { createWebsiteScraper };
+export { _WebsiteScraperRegistration };
+export { WebsiteScraper };

package/esm/typings/src/collection/constructors/createCollectionFromDirectory.d.ts

@@ -1,3 +1,4 @@
+import type { ExecutionTools } from '../../execution/ExecutionTools';
 import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
 import type { string_dirname } from '../../types/typeAliases';
 import type { PipelineCollection } from '../PipelineCollection';
@@ -40,13 +41,14 @@ type CreatePipelineCollectionFromDirectoryOptions = Omit<PrepareAndScrapeOptions
  * Note: Works only in Node.js environment because it reads the file system
  *
  * @param path - path to the directory with pipelines
- * @param options - Misc options for the collection
+ * @param tools - Execution tools to be used for pipeline preparation if needed - If not provided, `$provideExecutionToolsForNode` will be used
+ * @param options - Options for the collection creation
  * @returns PipelineCollection
  * @public exported from `@promptbook/node`
  */
-export declare function createCollectionFromDirectory(path: string_dirname, options?: CreatePipelineCollectionFromDirectoryOptions): Promise<PipelineCollection>;
+export declare function createCollectionFromDirectory(path: string_dirname, tools?: Pick<ExecutionTools, 'llm' | 'fs' | 'scrapers'>, options?: CreatePipelineCollectionFromDirectoryOptions): Promise<PipelineCollection>;
 export {};
 /**
- * Note: [🟢] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
  * TODO: [🖇] What about symlinks? Maybe option isSymlinksFollowed
+ * TODO: Maybe move from `@promptbook/node` to `@promptbook/core` as we removes direct dependency on `fs`
  */

package/esm/typings/src/config.d.ts

@@ -160,6 +160,12 @@ export declare const DEFAULT_CSV_SETTINGS: CsvSettings;
  * @public exported from `@promptbook/core`
  */
 export declare const IS_VERBOSE = false;
+/**
+ * @@@
+ *
+ * @public exported from `@promptbook/core`
+ */
+export declare const IS_AUTO_INSTALLED = false;
 /**
  * @@@
  *

package/esm/typings/src/conversion/pipelineStringToJson.d.ts

@@ -1,3 +1,4 @@
+import type { ExecutionTools } from '../execution/ExecutionTools';
 import type { PrepareAndScrapeOptions } from '../prepare/PrepareAndScrapeOptions';
 import type { PipelineJson } from '../types/PipelineJson/PipelineJson';
 import type { PipelineString } from '../types/PipelineString';
@@ -13,12 +14,13 @@ import type { PipelineString } from '../types/PipelineString';
  * Note: This function acts as compilation process
  *
  * @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
+ * @param tools - Tools for the preparation and scraping - if not provided together with `llm`, the preparation will be skipped
  * @param options - Options and tools for the compilation
  * @returns {Promptbook} compiled in JSON format (.ptbk.json)
  * @throws {ParseError} if the promptbook string is not valid
  * @public exported from `@promptbook/core`
  */
-export declare function pipelineStringToJson(pipelineString: PipelineString, options?: PrepareAndScrapeOptions): Promise<PipelineJson>;
+export declare function pipelineStringToJson(pipelineString: PipelineString, tools?: Pick<ExecutionTools, 'llm' | 'fs' | 'scrapers'>, options?: PrepareAndScrapeOptions): Promise<PipelineJson>;
 /**
  * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
  * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools

package/esm/typings/src/dialogs/callback/CallbackInterfaceToolsOptions.d.ts

@@ -1,12 +1,12 @@
 import type { Promisable } from 'type-fest';
-import type { CommonExecutionToolsOptions } from '../../execution/CommonExecutionToolsOptions';
+import type { CommonToolsOptions } from '../../execution/CommonToolsOptions';
 import type { UserInterfaceToolsPromptDialogOptions } from '../../execution/UserInterfaceTools';
 /**
  * Options for `CallbackInterfaceTools`
  *
  * @public exported from `@promptbook/core`
  */
-export type CallbackInterfaceToolsOptions = CommonExecutionToolsOptions & {
+export type CallbackInterfaceToolsOptions = CommonToolsOptions & {
     /**
      * The callback function to be called when promptDialog is called
      */

package/esm/typings/src/dialogs/simple-prompt/SimplePromptInterfaceTools.d.ts

@@ -1,4 +1,4 @@
-import type { CommonExecutionToolsOptions } from '../../execution/CommonExecutionToolsOptions';
+import type { CommonToolsOptions } from '../../execution/CommonToolsOptions';
 import type { UserInterfaceTools } from '../../execution/UserInterfaceTools';
 import type { UserInterfaceToolsPromptDialogOptions } from '../../execution/UserInterfaceTools';
 /**
@@ -10,8 +10,8 @@ import type { UserInterfaceToolsPromptDialogOptions } from '../../execution/User
  * @public exported from `@promptbook/browser`
  */
 export declare class SimplePromptInterfaceTools implements UserInterfaceTools {
-    protected readonly options: CommonExecutionToolsOptions;
-    constructor(options?: CommonExecutionToolsOptions);
+    protected readonly options: CommonToolsOptions;
+    constructor(options?: CommonToolsOptions);
     /**
      * Trigger window.DIALOG TEMPLATE
      */

package/esm/typings/src/execution/{CommonExecutionToolsOptions.d.ts → CommonToolsOptions.d.ts}

@@ -3,7 +3,7 @@
  *
  * Note: Keep it public to allow people to make their own execution tools
  */
-export type CommonExecutionToolsOptions = {
+export type CommonToolsOptions = {
     /**
      * If true, the internal executions will be logged
      */

package/esm/typings/src/execution/ExecutionTools.d.ts

@@ -1,4 +1,6 @@
+import type { Scraper } from '../scrapers/_common/Scraper';
 import type { Arrayable } from '../types/Arrayable';
+import type { FilesystemTools } from './FilesystemTools';
 import type { LlmExecutionTools } from './LlmExecutionTools';
 import type { ScriptExecutionTools } from './ScriptExecutionTools';
 import type { UserInterfaceTools } from './UserInterfaceTools';
@@ -9,20 +11,36 @@ import type { UserInterfaceTools } from './UserInterfaceTools';
  */
 export type ExecutionTools = {
     /**
-     * Tools for executing prompts to large language models like GPT-4
+     * Tools for executing prompts in large language models
      *
-     * Tip: Use `createLlmToolsFromEnv()` to use all available LLM providers you configured
-     * @see https://github.com/webgptorg/promptbook/?tab=readme-ov-file#llm-execution-tools
+     * Tip: Use `$provideLlmToolsFromEnv` to use all available LLM providers you configured
+     *
+     * @default [] - If not provided, no LLM execution will be possible - it does not make sense in most cases
      */
     readonly llm?: Arrayable<LlmExecutionTools>;
+    /**
+     * Tools for filesystem operations
+     *
+     * Tip: Use `$provideFilesystemForNode` to use filesystem in Node.js
+     *
+     * @default undefined - If not provided, no filesystem operations will be possible
+     */
+    readonly fs?: FilesystemTools;
+    /**
+     * Scrapers for extracting knowledge from external sources
+     *
+     * Tip: Use `$provideScrapersForNode` OR `$provideScrapersForBrowser` to use all available scrapers
+     *
+     * @default [] - If not provided, no external knowledge extraction will be possible
+     */
+    readonly scrapers?: Arrayable<Scraper>;
     /**
      * Tools for executing scripts
      *
      * Note: You can pass multiple ScriptExecutionTools, they will be tried one by one until one of them supports the script
      *       If none of them supports the script, an error is thrown
-     * Tip: Use here `new JavascriptExecutionTools()`
+     * Tip: Use here `new JavascriptExecutionTools`
      *
-     * @see https://github.com/webgptorg/promptbook/?tab=readme-ov-file#script-execution-tools
      * @default [] - If not provided, no script execution will be possible
      */
     readonly script?: Arrayable<ScriptExecutionTools>;
@@ -30,7 +48,9 @@ export type ExecutionTools = {
      * Tools for interacting with the user
      *
      * Note: When undefined, the user interface is disabled and promptbook which requires user interaction will fail
-     * @see https://github.com/webgptorg/promptbook/?tab=readme-ov-file#user-interface-tools
      */
     readonly userInterface?: UserInterfaceTools;
 };
+/**
+ * TODO: !!!!!! Move here also the executables and make $provideXxxxForNode
+ */

package/esm/typings/src/execution/FilesystemTools.d.ts

@@ -0,0 +1,9 @@
+/// <reference types="node" />
+import type fs from 'fs/promises';
+/**
+ * Container for all the tools needed to manipulate with filesystem
+ */
+export type FilesystemTools = Pick<typeof fs, 'access' | 'constants' | 'readFile' | 'stat' | 'readdir'>;
+/**
+ * TODO: Implement destroyable pattern to free resources
+ */

package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts

@@ -8,7 +8,7 @@ import type { CreatePipelineExecutorSettings } from './00-CreatePipelineExecutor
 /**
  * @@@
  *
- * @private internal type of `executePipelinex`
+ * @private internal type of `executePipeline`
  */
 type ExecutePipelineOptions = {
     /**

package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts

@@ -1,5 +1,4 @@
 import { Promisable, ReadonlyDeep } from 'type-fest';
-import { MultipleLlmExecutionTools } from '../../llm-providers/multiple/MultipleLlmExecutionTools';
 import type { ExecutionReportJson } from '../../types/execution-report/ExecutionReportJson';
 import type { PipelineJson } from '../../types/PipelineJson/PipelineJson';
 import type { TemplateJson } from '../../types/PipelineJson/TemplateJson';
@@ -28,11 +27,7 @@ type executeSingleTemplateOptions = {
     /**
      * @@@
      */
-    readonly tools: Omit<ExecutionTools, 'llm'>;
-    /**
-     * @@@
-     */
-    readonly llmTools: MultipleLlmExecutionTools;
+    readonly tools: ExecutionTools;
     /**
      * @@@
      */

package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts

@@ -1,5 +1,4 @@
 import type { ReadonlyDeep } from 'type-fest';
-import { MultipleLlmExecutionTools } from '../../llm-providers/multiple/MultipleLlmExecutionTools';
 import type { ExecutionReportJson } from '../../types/execution-report/ExecutionReportJson';
 import type { PipelineJson } from '../../types/PipelineJson/PipelineJson';
 import type { TemplateJson } from '../../types/PipelineJson/TemplateJson';
@@ -45,11 +44,7 @@ export type ExecuteAttemptsOptions = {
     /**
      * @@@
      */
-    readonly tools: Omit<ExecutionTools, 'llm'>;
-    /**
-     * @@@
-     */
-    readonly llmTools: MultipleLlmExecutionTools;
+    readonly tools: ExecutionTools;
     /**
      * Settings for the pipeline executor
      */

package/esm/typings/src/execution/translation/automatic-translate/translateMessages.d.ts

@@ -8,4 +8,5 @@ export declare function translateMessages({ automaticTranslator, from, to, }: {
 } & TranslatorOptions): Promise<void>;
 /**
  * TODO: [😶] Unite floder listing
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
  */

package/esm/typings/src/execution/utils/$provideExecutionToolsForNode.d.ts

@@ -0,0 +1,13 @@
+import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
+import type { ExecutionTools } from '../ExecutionTools';
+/**
+ * Note: There is unfortunately no equivalent for this function in the browser environment
+ *       because it is not possible automatically detect configured LLM providers
+ *       you need to provide them manually BUT you can help by utilities like `$provideScrapersForBrowser`
+ *
+ * @public exported from `@promptbook/node`
+ */
+export declare function $provideExecutionToolsForNode(options?: PrepareAndScrapeOptions): Promise<ExecutionTools>;
+/**
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ */

package/esm/typings/src/llm-providers/_common/{$llmToolsMetadataRegister.d.ts → register/$llmToolsMetadataRegister.d.ts}

@@ -1,4 +1,4 @@
-import { $Register } from '../../utils/$Register';
+import { $Register } from '../../../utils/$Register';
 import type { LlmToolsMetadata } from './LlmToolsMetadata';
 /**
  * @@@
@@ -8,3 +8,6 @@ import type { LlmToolsMetadata } from './LlmToolsMetadata';
  * @public exported from `@promptbook/core`
  */
 export declare const $llmToolsMetadataRegister: $Register<LlmToolsMetadata>;
+/**
+ * TODO: [®] DRY Register logic
+ */ 

package/esm/typings/src/llm-providers/_common/{$llmToolsRegister.d.ts → register/$llmToolsRegister.d.ts}

@@ -1,5 +1,5 @@
-import type { LlmExecutionToolsConstructor } from '../../execution/LlmExecutionToolsConstructor';
-import { $Register } from '../../utils/$Register';
+import type { LlmExecutionToolsConstructor } from '../../../execution/LlmExecutionToolsConstructor';
+import { $Register } from '../../../utils/$Register';
 /**
  * @@@
  *
@@ -8,3 +8,6 @@ import { $Register } from '../../utils/$Register';
  * @public exported from `@promptbook/core`
  */
 export declare const $llmToolsRegister: $Register<LlmExecutionToolsConstructor>;
+/**
+ * TODO: [®] DRY Register logic
+ */ 

package/esm/typings/src/llm-providers/_common/{createLlmToolsFromConfigurationFromEnv.d.ts → register/$provideLlmToolsConfigurationFromEnv.d.ts}

@@ -11,14 +11,14 @@ import type { LlmToolsConfiguration } from './LlmToolsConfiguration';
  * @returns @@@
  * @public exported from `@promptbook/node`
  */
-export declare function createLlmToolsFromConfigurationFromEnv(): LlmToolsConfiguration;
+export declare function $provideLlmToolsConfigurationFromEnv(): LlmToolsConfiguration;
 /**
  * TODO: [🧠][🪁] Maybe do allow to do auto-install if package not registered and not found
  * TODO: Add Azure OpenAI
  * TODO: [🧠][🍛]
  * TODO: [🧠] Is there some meaningfull way how to test this util
- * Note: [🟢] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
  * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
  * TODO: This should be maybe not under `_common` but under `utils`
  * TODO: [🧠] Maybe pass env as argument
- */
+ * TODO: [®] DRY Register logic */

package/esm/typings/src/llm-providers/_common/{getLlmToolsForCli.d.ts → register/$provideLlmToolsForCli.d.ts}

@@ -1,22 +1,15 @@
-import type { LlmExecutionToolsWithTotalUsage } from './utils/count-total-usage/LlmExecutionToolsWithTotalUsage';
-type GetLlmToolsForCliOptions = {
-    /**
-     * @@@
-     *
-     * @default false
-     */
-    isCacheReloaded?: boolean;
-};
+import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
+import type { LlmExecutionToolsWithTotalUsage } from '../utils/count-total-usage/LlmExecutionToolsWithTotalUsage';
 /**
  * Returns LLM tools for CLI
  *
  * @private within the repository - for CLI utils
  */
-export declare function getLlmToolsForCli(options?: GetLlmToolsForCliOptions): LlmExecutionToolsWithTotalUsage;
-export {};
+export declare function $provideLlmToolsForCli(options?: Pick<PrepareAndScrapeOptions, 'isCacheCleaned'>): LlmExecutionToolsWithTotalUsage;
 /**
  * Note: [🟡] Code in this file should never be published outside of `@promptbook/cli`
  * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
  * TODO: [🥃] Allow `ptbk make` without llm tools
  * TODO: This should be maybe not under `_common` but under `utils-internal` / `utils/internal`
+ * TODO: [®] DRY Register logic
  */

package/esm/typings/src/llm-providers/_common/{getLlmToolsForTestingAndScriptsAndPlayground.d.ts → register/$provideLlmToolsForTestingAndScriptsAndPlayground.d.ts}

@@ -1,6 +1,6 @@
-import '../../_packages/cli.index';
+import '../../../_packages/cli.index';
+import type { LlmExecutionToolsWithTotalUsage } from '../utils/count-total-usage/LlmExecutionToolsWithTotalUsage';
 import type { CreateLlmToolsFromConfigurationOptions } from './createLlmToolsFromConfiguration';
-import type { LlmExecutionToolsWithTotalUsage } from './utils/count-total-usage/LlmExecutionToolsWithTotalUsage';
 type GetLlmToolsForTestingAndScriptsAndPlaygroundOptions = CreateLlmToolsFromConfigurationOptions & {
     /**
      * @@@
@@ -14,10 +14,11 @@ type GetLlmToolsForTestingAndScriptsAndPlaygroundOptions = CreateLlmToolsFromCon
  *
  * @private within the repository - JUST FOR TESTS, SCRIPTS AND PLAYGROUND
  */
-export declare function getLlmToolsForTestingAndScriptsAndPlayground(options?: GetLlmToolsForTestingAndScriptsAndPlaygroundOptions): LlmExecutionToolsWithTotalUsage;
+export declare function $provideLlmToolsForTestingAndScriptsAndPlayground(options?: GetLlmToolsForTestingAndScriptsAndPlaygroundOptions): LlmExecutionToolsWithTotalUsage;
 export {};
 /**
  * Note: [⚪] This should never be in any released package
  * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
  * TODO: This should be maybe not under `_common` but under `utils-internal` / `utils/internal`
+ * TODO: [®] DRY Register logic
  */