@promptbook/remote-server 0.72.0-8 → 0.72.0

package/esm/typings/src/llm-providers/remote/interfaces/RemoteLlmExecutionToolsOptions.d.ts

@@ -1,14 +1,15 @@
-import type { CommonExecutionToolsOptions } from '../../../execution/CommonExecutionToolsOptions';
+import type { CommonToolsOptions } from '../../../execution/CommonToolsOptions';
 import type { string_base_url } from '../../../types/typeAliases';
 import type { string_uri } from '../../../types/typeAliases';
 import type { string_user_id } from '../../../types/typeAliases';
-import type { LlmToolsConfiguration } from '../../_common/LlmToolsConfiguration';
+import type { LlmToolsConfiguration } from '../../_common/register/LlmToolsConfiguration';
+import type { CollectionRemoteServerClientOptions } from './RemoteServerOptions';
 /**
  * Options for `RemoteLlmExecutionTools`
  *
  * @public exported from `@promptbook/remote-client`
  */
-export type RemoteLlmExecutionToolsOptions = CommonExecutionToolsOptions & {
+export type RemoteLlmExecutionToolsOptions<TCustomOptions> = CommonToolsOptions & {
     /**
      * URL of the remote PROMPTBOOK server
      * On this server will be connected to the socket.io server
@@ -29,7 +30,7 @@ export type RemoteLlmExecutionToolsOptions = CommonExecutionToolsOptions & {
     /**
      * Use anonymous server with anonymous mode
      */
-    isAnonymous: true;
+    readonly isAnonymous: true;
     /**
      * Configuration for the LLM tools
      */
@@ -37,22 +38,16 @@ export type RemoteLlmExecutionToolsOptions = CommonExecutionToolsOptions & {
     /**
      * Identifier of the end user
      *
-     * Note: this is passed to the certain model providers to identify misuse
-     * Note: In anonymous mode it is not required to identify
+     * Note: This is passed to the certain model providers to identify misuse
+     * Note: In anonymous mode, there is no need to identify yourself, nor does it change the actual configuration of LLM Tools (unlike in application mode).
      */
-    readonly userId?: string_user_id;
-} | {
+    readonly userId: string_user_id | null;
+} | ({
     /**
      * Use anonymous server with client identification and fixed collection
      */
-    isAnonymous: false;
-    /**
-     * Identifier of the end user
-     *
-     * Note: this is passed to the certain model providers to identify misuse
-     */
-    readonly userId: string_user_id;
-});
+    readonly isAnonymous: false;
+} & CollectionRemoteServerClientOptions<TCustomOptions>));
 /**
  * TODO: [🧠][🛍] Maybe not `isAnonymous: boolean` BUT `mode: 'ANONYMOUS'|'COLLECTION'`
  * TODO: [🧠][🧜‍♂️] Maybe join remoteUrl and path into single value

package/esm/typings/src/llm-providers/remote/interfaces/RemoteServerOptions.d.ts

@@ -1,6 +1,8 @@
+import type { Promisable } from 'type-fest';
 import type { PipelineCollection } from '../../../collection/PipelineCollection';
-import type { CommonExecutionToolsOptions } from '../../../execution/CommonExecutionToolsOptions';
+import type { CommonToolsOptions } from '../../../execution/CommonToolsOptions';
 import type { LlmExecutionTools } from '../../../execution/LlmExecutionTools';
+import type { string_app_id } from '../../../types/typeAliases';
 import type { string_uri } from '../../../types/typeAliases';
 import type { string_user_id } from '../../../types/typeAliases';
 /**
@@ -8,7 +10,7 @@ import type { string_user_id } from '../../../types/typeAliases';
  *
  * There are two modes of remote server:
  *
- * 1) **Collection mode** Server will recieve `collection` and execute prompts only from this collection
+ * 1) **Application mode** Server will recieve `collection` and execute prompts only from this collection
  * 2) **Anonymous mode** Server will recieve full `LlmToolsConfiguration` (with api keys) and just acts as a proxy
  *    In anonymous mode, `collection` will be ignored and any prompt will be executed
  *
@@ -17,7 +19,7 @@ import type { string_user_id } from '../../../types/typeAliases';
  * @public exported from `@promptbook/remote-client`
  * @public exported from `@promptbook/remote-server`
  */
-export type RemoteServerOptions = CommonExecutionToolsOptions & {
+export type RemoteServerOptions<TCustomOptions> = CommonToolsOptions & {
     /**
      * Port on which the server will listen
      */
@@ -29,18 +31,18 @@ export type RemoteServerOptions = CommonExecutionToolsOptions & {
      * @example '/promptbook/socket.io'
      */
     readonly path: string_uri;
-} & (AnonymousRemoteServerOptions | CollectionRemoteServerOptions | (AnonymousRemoteServerOptions & CollectionRemoteServerOptions));
+} & (AnonymousRemoteServerOptions | CollectionRemoteServerOptions<TCustomOptions> | (AnonymousRemoteServerOptions & CollectionRemoteServerOptions<TCustomOptions>));
 export type AnonymousRemoteServerOptions = {
     /**
      * Enable anonymous mode
      */
     readonly isAnonymousModeAllowed: true;
 };
-export type CollectionRemoteServerOptions = {
+export type CollectionRemoteServerOptions<TCustomOptions> = {
     /**
-     * Enable collection mode
+     * Enable application mode
      */
-    readonly isCollectionModeAllowed: true;
+    readonly isApplicationModeAllowed: true;
     /**
      * Promptbook collection to use
      *
@@ -50,9 +52,22 @@ export type CollectionRemoteServerOptions = {
     /**
      * Creates llm execution tools for each client
      */
-    createLlmExecutionTools(userId: string_user_id | undefined): LlmExecutionTools;
+    createLlmExecutionTools(options: CollectionRemoteServerClientOptions<TCustomOptions>): Promisable<LlmExecutionTools>;
+};
+export type CollectionRemoteServerClientOptions<TCustomOptions> = {
+    /**
+     * @@@
+     */
+    readonly appId: string_app_id | null;
+    /**
+     * @@@
+     */
+    readonly userId: string_user_id | null;
+    /**
+     * @@@
+     */
+    readonly customOptions?: TCustomOptions;
 };
 /**
  * TODO: Constrain anonymous mode for specific models / providers
- * TODO: [🧠][🤺] Remove `createLlmExecutionTools`, pass just `llmExecutionTools`
  */

package/esm/typings/src/llm-providers/remote/startRemoteServer.d.ts

@@ -9,7 +9,7 @@ import type { RemoteServerOptions } from './interfaces/RemoteServerOptions';
  * @see https://github.com/webgptorg/promptbook#remote-server
  * @public exported from `@promptbook/remote-server`
  */
-export declare function startRemoteServer(options: RemoteServerOptions): IDestroyable;
+export declare function startRemoteServer<TCustomOptions = undefined>(options: RemoteServerOptions<TCustomOptions>): IDestroyable;
 /**
  * TODO: Maybe use `$asDeeplyFrozenSerializableJson`
  * TODO: [🧠][🛍] Maybe not `isAnonymous: boolean` BUT `mode: 'ANONYMOUS'|'COLLECTION'`

package/esm/typings/src/personas/preparePersona.d.ts

@@ -1,3 +1,4 @@
+import type { ExecutionTools } from '../execution/ExecutionTools';
 import type { PrepareAndScrapeOptions } from '../prepare/PrepareAndScrapeOptions';
 import type { PersonaPreparedJson } from '../types/PipelineJson/PersonaJson';
 import type { string_persona_description } from '../types/typeAliases';
@@ -7,7 +8,7 @@ import type { string_persona_description } from '../types/typeAliases';
  * @see https://github.com/webgptorg/promptbook/discussions/22
  * @public exported from `@promptbook/core`
  */
-export declare function preparePersona(personaDescription: string_persona_description, options: PrepareAndScrapeOptions): Promise<PersonaPreparedJson['modelRequirements']>;
+export declare function preparePersona(personaDescription: string_persona_description, tools: Pick<ExecutionTools, 'llm'>, options: PrepareAndScrapeOptions): Promise<PersonaPreparedJson['modelRequirements']>;
 /**
  * TODO: [🔃][main] !!!!! If the persona was prepared with different version or different set of models, prepare it once again
  * TODO: [🏢] !! Check validity of `modelName` in pipeline

package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts

@@ -1,20 +1,18 @@
-import type { LlmExecutionTools } from '../execution/LlmExecutionTools';
+import type { IntermediateFilesStrategy } from '../types/IntermediateFilesStrategy';
 import type { string_dirname } from '../types/typeAliases';
 /**
  * Options for preparation of the pipeline
  */
 export type PrepareAndScrapeOptions = {
-    /**
-     * LLM tools
-     */
-    readonly llmTools?: LlmExecutionTools;
     /**
      * Path to the root folder of the pipeline
      *
      * Note: When the pipeline is not created from files, it is `null`
-     * Note: This folder must exist
+     * Note: This folder must exist (=it is not created recursively)
+     *
+     * @default process.cwd()
      */
-    readonly rootDirname: string_dirname | null;
+    readonly rootDirname?: string_dirname | null;
     /**
      * Path to the cache folder
      *
@@ -24,12 +22,11 @@ export type PrepareAndScrapeOptions = {
      */
     readonly cacheDirname?: string_dirname;
     /**
-     * If true, the cache is cleaned after the scraping
+     * Strategy for caching the intermediate results for knowledge source
      *
-     *
-     * @default false // <- TODO: !!!!!! Change to `cacheStrategy`/`intermediateFiles`, Put to global config, change to `true` and explicitly set to `false` in all playgrounds
+     * @default DEFAULT_INTERMEDIATE_FILES_STRATEGY
      */
-    readonly isCacheCleaned?: boolean;
+    readonly intermediateFilesStrategy?: IntermediateFilesStrategy;
     /**
      * Maximum number of tasks running in parallel
      *
@@ -37,24 +34,9 @@ export type PrepareAndScrapeOptions = {
      */
     readonly maxParallelCount?: number;
     /**
-     * Path to the external programs executables
-     *
-     * TODO: !!!!!! Transform to scrapers and make them Classy
+     * If true, the missing software is automatically installed
      */
-    readonly externalProgramsPaths?: {
-        /**
-         * Path to the `pandoc` executable
-         *
-         * @example 'C:/Users/me/AppData/Local/Pandoc/pandoc.exe'
-         */
-        readonly pandocPath?: string;
-        /**
-         * Path to the LibreOffice executable
-         *
-         * @example 'C:/Program Files/LibreOffice/program/swriter.exe'
-         */
-        readonly libreOfficePath?: string;
-    };
+    readonly isAutoInstalled?: boolean;
     /**
      * If true, the preparation logs additional information
      *

package/esm/typings/src/prepare/preparePipeline.d.ts

@@ -1,3 +1,4 @@
+import type { ExecutionTools } from '../execution/ExecutionTools';
 import type { PipelineJson } from '../types/PipelineJson/PipelineJson';
 import type { PrepareAndScrapeOptions } from './PrepareAndScrapeOptions';
 /**
@@ -8,7 +9,7 @@ import type { PrepareAndScrapeOptions } from './PrepareAndScrapeOptions';
  * Note: When the pipeline is already prepared, it returns the same pipeline
  * @public exported from `@promptbook/core`
  */
-export declare function preparePipeline(pipeline: PipelineJson, options: PrepareAndScrapeOptions): Promise<PipelineJson>;
+export declare function preparePipeline(pipeline: PipelineJson, tools: Pick<ExecutionTools, 'llm' | 'fs' | 'scrapers'>, options: PrepareAndScrapeOptions): Promise<PipelineJson>;
 /**
  * TODO: Write tests for `preparePipeline`
  * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch

package/esm/typings/src/prepare/prepareTemplates.d.ts

@@ -1,3 +1,4 @@
+import type { ExecutionTools } from '../execution/ExecutionTools';
 import type { PipelineJson } from '../types/PipelineJson/PipelineJson';
 import type { TemplateJson } from '../types/PipelineJson/TemplateJson';
 import type { PrepareAndScrapeOptions } from './PrepareAndScrapeOptions';
@@ -11,14 +12,14 @@ type PreparedTemplates = {
     /**
      * @@@ Sequence of templates that are chained together to form a pipeline
      */
-    readonly templatesPrepared: Array<TemplateJson>;
+    readonly templatesPrepared: ReadonlyArray<TemplateJson>;
 };
 /**
  * @@@
  *
  * @public exported from `@promptbook/core`
  */
-export declare function prepareTemplates(pipeline: PrepareTemplateInput, options: PrepareAndScrapeOptions): Promise<PreparedTemplates>;
+export declare function prepareTemplates(pipeline: PrepareTemplateInput, tools: Pick<ExecutionTools, 'llm' | 'fs' | 'scrapers'>, options: PrepareAndScrapeOptions): Promise<PreparedTemplates>;
 export {};
 /**
  * TODO: [🧠] Add context to each template (if missing)

package/esm/typings/src/scrapers/_common/Converter.d.ts

@@ -1,6 +1,4 @@
-import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
-import type { string_mime_type } from '../../types/typeAliases';
-import type { string_promptbook_documentation_url } from '../../types/typeAliases';
+import type { ScraperAndConverterMetadata } from './register/ScraperAndConverterMetadata';
 import type { ScraperSourceHandler } from './Scraper';
 import type { ScraperIntermediateSource } from './ScraperIntermediateSource';
 /**
@@ -9,13 +7,9 @@ import type { ScraperIntermediateSource } from './ScraperIntermediateSource';
  */
 export type Converter = {
     /**
-     * Mime types that this scraper can handle
+     * Metadata of the converter which includes title, mime types, etc.
      */
-    readonly mimeTypes: Array<string_mime_type>;
-    /**
-     * Link to documentation
-     */
-    readonly documentationUrl: string_promptbook_documentation_url;
+    readonly metadata: ScraperAndConverterMetadata;
     /**
      * Convert the the file and returns intermediate source or `null` if it can't convert it
      *
@@ -24,5 +18,5 @@ export type Converter = {
      *
      * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
      */
-    $convert(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<ScraperIntermediateSource>;
+    $convert(source: ScraperSourceHandler): Promise<ScraperIntermediateSource>;
 };

package/esm/typings/src/scrapers/_common/Scraper.d.ts

@@ -1,28 +1,23 @@
 import type { Promisable } from 'type-fest';
-import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
 import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
 import type { string_filename } from '../../types/typeAliases';
 import type { string_knowledge_source_link } from '../../types/typeAliases';
 import type { string_mime_type } from '../../types/typeAliases';
-import type { string_promptbook_documentation_url } from '../../types/typeAliases';
 import type { string_url } from '../../types/typeAliases';
+import type { ScraperAndConverterMetadata } from './register/ScraperAndConverterMetadata';
 /**
  * @@@
  *
  */
 export type Scraper = {
     /**
-     * Mime types that this scraper can handle
+     * Metadata of the scraper which includes title, mime types, etc.
      */
-    readonly mimeTypes: Array<string_mime_type>;
-    /**
-     * Link to documentation
-     */
-    readonly documentationUrl: string_promptbook_documentation_url;
+    readonly metadata: ScraperAndConverterMetadata;
     /**
      * Scrapes the markdown file and returns the knowledge pieces or `null` if it can't scrape it
      */
-    scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promisable<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
+    scrape(source: ScraperSourceHandler): Promisable<ReadonlyArray<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
 };
 /**
  * @@@
@@ -56,16 +51,9 @@ export type ScraperSourceHandler = {
      * Get the content as a utf-8 string
      */
     asText(): Promisable<string>;
-    /**
-     * Get the content as a blob
-     */
-    asBlob(): Promisable<Blob>;
 };
 /**
+ * TODO: [🥽] Add ` asBlob(): Promisable<Blob>;` or asFile
  * TODO: [🐝] @@@ Annotate all
  * TODO: [🔼] Export via types
  */
-/**
- * TODO: !!!!!! Test that this is catched
- * Note: [⚫] Code in this file should never be published in any package
- */

package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts

@@ -1,13 +1,14 @@
 import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
 import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
 import type { KnowledgeSourceJson } from '../../types/PipelineJson/KnowledgeSourceJson';
+import type { ExecutionTools } from '../../execution/ExecutionTools';
 /**
  * Prepares the knowle
  *
  * @see https://github.com/webgptorg/promptbook/discussions/41
  * @public exported from `@promptbook/core`
  */
-export declare function prepareKnowledgePieces(knowledgeSources: Array<KnowledgeSourceJson>, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'preparationIds'>>>;
+export declare function prepareKnowledgePieces(knowledgeSources: ReadonlyArray<KnowledgeSourceJson>, tools: Pick<ExecutionTools, 'llm' | 'fs' | 'scrapers'>, options: PrepareAndScrapeOptions): Promise<ReadonlyArray<Omit<KnowledgePiecePreparedJson, 'preparationIds'>>>;
 /**
  * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
  *       Put `knowledgePieces` into `PrepareKnowledgeOptions`

package/esm/typings/src/scrapers/_common/register/$provideFilesystemForNode.d.ts

@@ -0,0 +1,11 @@
+import type { FilesystemTools } from '../../../execution/FilesystemTools';
+import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
+/**
+ * @@@
+ *
+ * @public exported from `@promptbook/node`
+ */
+export declare function $provideFilesystemForNode(options?: Pick<PrepareAndScrapeOptions, 'isVerbose'>): FilesystemTools;
+/**
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ */

package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts

@@ -0,0 +1,12 @@
+import type { ExecutionTools } from '../../../execution/ExecutionTools';
+import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
+import type { Scraper } from '../Scraper';
+/**
+ * @@@
+ *
+ * 1) @@@
+ * 2) @@@
+ *
+ * @public exported from `@promptbook/browser`
+ */
+export declare function $provideScrapersForBrowser(tools: Pick<ExecutionTools, 'llm'>, options?: PrepareAndScrapeOptions): Promise<ReadonlyArray<Scraper>>;

package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts

@@ -0,0 +1,15 @@
+import type { ExecutionTools } from '../../../execution/ExecutionTools';
+import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
+import type { Scraper } from '../Scraper';
+/**
+ * @@@
+ *
+ * 1) @@@
+ * 2) @@@
+ *
+ * @public exported from `@promptbook/node`
+ */
+export declare function $provideScrapersForNode(tools: Pick<ExecutionTools, 'fs' | 'llm' | 'executables'>, options?: PrepareAndScrapeOptions): Promise<ReadonlyArray<Scraper>>;
+/**
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ */

package/esm/typings/src/scrapers/_common/register/$registeredScrapersMessage.d.ts

@@ -0,0 +1,12 @@
+import type { string_markdown } from '../../../types/typeAliases';
+/**
+ * Creates a message with all registered scrapers
+ *
+ * Note: This function is used to create a (error) message when there is no scraper for particular mime type
+ *
+ * @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv`
+ */
+export declare function $registeredScrapersMessage(): string_markdown;
+/**
+ * TODO: [®] DRY Register logic
+ */

package/esm/typings/src/scrapers/_common/register/$scrapersMetadataRegister.d.ts

@@ -0,0 +1,13 @@
+import { $Register } from '../../../utils/$Register';
+import type { ScraperAndConverterMetadata } from './ScraperAndConverterMetadata';
+/**
+ * @@@
+ *
+ * Note: `$` is used to indicate that this interacts with the global scope
+ * @singleton Only one instance of each register is created per build, but thare can be more @@@
+ * @public exported from `@promptbook/core`
+ */
+export declare const $scrapersMetadataRegister: $Register<ScraperAndConverterMetadata>;
+/**
+ * TODO: [®] DRY Register logic
+ */

package/esm/typings/src/scrapers/_common/register/$scrapersRegister.d.ts

@@ -0,0 +1,13 @@
+import { $Register } from '../../../utils/$Register';
+import type { ScraperConstructor } from './ScraperConstructor';
+/**
+ * @@@
+ *
+ * Note: `$` is used to indicate that this interacts with the global scope
+ * @singleton Only one instance of each register is created per build, but thare can be more @@@
+ * @public exported from `@promptbook/core`
+ */
+export declare const $scrapersRegister: $Register<ScraperConstructor>;
+/**
+ * TODO: [®] DRY Register logic
+ */

package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts

@@ -0,0 +1,41 @@
+import type { Executables } from '../../../execution/Executables';
+import type { string_mime_type } from '../../../types/typeAliases';
+import type { string_promptbook_documentation_url } from '../../../types/typeAliases';
+import type { string_title } from '../../../types/typeAliases';
+import type { Registered } from '../../../utils/$Register';
+/**
+ * @@@
+ *
+ * @@@
+ * x) `Scraper`
+ * x) `Converter`
+ * x) `ScraperConstructor`
+ * x) `Registered`
+ * x) `ExecutionTools`
+ * x) `ScraperAndConverterMetadata`
+ * x) `PrepareAndScrapeOptions`
+ * x) `ScraperConfiguration`
+ * x) `ScraperOptions`
+ */
+export type ScraperAndConverterMetadata = Registered & {
+    /**
+     * @@@
+     */
+    readonly title: string_title;
+    /**
+     * Mime types that this scraper can handle
+     */
+    readonly mimeTypes: ReadonlyArray<string_mime_type>;
+    /**
+     * @@@
+     */
+    readonly isAvilableInBrowser: boolean;
+    /**
+     * @@@
+     */
+    readonly requiredExecutables: ReadonlyArray<Capitalize<keyof Executables extends `${infer N}Path` ? N : never>>;
+    /**
+     * Link to documentation
+     */
+    readonly documentationUrl: string_promptbook_documentation_url;
+};

package/esm/typings/src/scrapers/_common/register/ScraperConstructor.d.ts

@@ -0,0 +1,12 @@
+import type { ExecutionTools } from '../../../execution/ExecutionTools';
+import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
+import type { Registered } from '../../../utils/$Register';
+import type { Scraper } from '../Scraper';
+import type { ScraperAndConverterMetadata } from './ScraperAndConverterMetadata';
+/**
+ * @@@
+ */
+export type ScraperConstructor = Registered & ScraperAndConverterMetadata & ((tools: Pick<ExecutionTools, 'llm'>, options: PrepareAndScrapeOptions) => Scraper);
+/**
+ * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
+ */

package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts

@@ -13,7 +13,7 @@ type GetScraperIntermediateSourceSource = Pick<ScraperSourceHandler, 'filename'
  *
  * @private internal utility of `getScraperIntermediateSource`
  */
-type GetScraperIntermediateSourceHandler = Required<Pick<PrepareAndScrapeOptions, 'rootDirname' | 'cacheDirname' | 'isCacheCleaned' | 'isVerbose'>> & {
+type GetScraperIntermediateSourceHandler = Required<Pick<PrepareAndScrapeOptions, 'rootDirname' | 'cacheDirname' | 'intermediateFilesStrategy' | 'isVerbose'>> & {
     readonly extension: string_file_extension;
 };
 /**
@@ -28,6 +28,7 @@ export {};
 /**
  * Note: Not using `FileCacheStorage` for two reasons:
  * 1) Need to store more than serialized JSONs
- * 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: !!!!
+ * 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: [😡]
  * TODO: [🐱‍🐉][🧠] Make some smart crop
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
  */

package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts

@@ -1,10 +1,11 @@
 import type { SetOptional } from 'type-fest';
+import type { ExecutionTools } from '../../../execution/ExecutionTools';
 import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
 import type { KnowledgeSourceJson } from '../../../types/PipelineJson/KnowledgeSourceJson';
 import type { ScraperSourceHandler } from '../Scraper';
 /**
  * @@@
  *
- * @private for scraper utilities
+ * @public exported from `@promptbook/core`
  */
-export declare function makeKnowledgeSourceHandler(knowledgeSource: SetOptional<KnowledgeSourceJson, 'name'>, options?: Pick<PrepareAndScrapeOptions, 'rootDirname' | 'isVerbose'>): Promise<ScraperSourceHandler>;
+export declare function makeKnowledgeSourceHandler(knowledgeSource: SetOptional<KnowledgeSourceJson, 'name'>, tools: Pick<ExecutionTools, 'fs'>, options?: Pick<PrepareAndScrapeOptions, 'rootDirname' | 'isVerbose'>): Promise<ScraperSourceHandler>;

package/esm/typings/src/scrapers/document/{documentScraper.d.ts → DocumentScraper.d.ts}

@@ -1,37 +1,43 @@
-import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
 import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
+import type { ExecutionTools } from '../../execution/ExecutionTools';
+import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
+import type { Converter } from '../_common/Converter';
+import type { ScraperAndConverterMetadata } from '../_common/register/ScraperAndConverterMetadata';
+import type { Scraper } from '../_common/Scraper';
 import type { ScraperSourceHandler } from '../_common/Scraper';
 import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource';
 /**
  * Scraper of .docx and .odt files
  *
  * @see `documentationUrl` for more details
- * @public exported from `@promptbook/core`
+ * @public exported from `@promptbook/documents`
  */
-export declare const documentScraper: {
+export declare class DocumentScraper implements Converter, Scraper {
+    private readonly tools;
+    private readonly options;
     /**
-     * Mime types that this scraper can handle
+     * Metadata of the scraper which includes title, mime types, etc.
      */
-    mimeTypes: string[];
+    get metadata(): ScraperAndConverterMetadata;
     /**
-     * Link to documentation
+     * Markdown scraper is used internally
      */
-    documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
+    private readonly markdownScraper;
+    constructor(tools: Pick<ExecutionTools, 'fs' | 'llm' | 'executables'>, options: PrepareAndScrapeOptions);
     /**
      * Convert the `.docx` or `.odt`  to `.md` file and returns intermediate source
      *
      * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
      */
-    $convert(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<ScraperIntermediateSource>;
+    $convert(source: ScraperSourceHandler): Promise<ScraperIntermediateSource>;
     /**
      * Scrapes the docx file and returns the knowledge pieces or `null` if it can't scrape it
      */
-    scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
-};
+    scrape(source: ScraperSourceHandler): Promise<ReadonlyArray<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
+}
 /**
  * TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
- * TODO: [🦖] Make some system for putting scrapers to separete packages
  * TODO: [🪂] Do it in parallel 11:11
- * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
  * Note: No need to aggregate usage here, it is done by intercepting the llmTools
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
  */

package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts

@@ -0,0 +1,20 @@
+import type { ExecutionTools } from '../../execution/ExecutionTools';
+import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
+import { DocumentScraper } from './DocumentScraper';
+/**
+ * @@@
+ *
+ * @public exported from `@promptbook/documents`
+ */
+export declare const createDocumentScraper: ((tools: Pick<ExecutionTools, 'llm'>, options: PrepareAndScrapeOptions) => DocumentScraper) & import("type-fest/source/readonly-deep").ReadonlyObjectDeep<{
+    title: string;
+    packageName: string;
+    className: string;
+    mimeTypes: string[];
+    documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
+    isAvilableInBrowser: false;
+    requiredExecutables: "Pandoc"[];
+}>;
+/**
+ * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
+ */

package/esm/typings/src/scrapers/document/register-constructor.d.ts

@@ -0,0 +1,13 @@
+import type { Registration } from '../../utils/$Register';
+/**
+ * Registration of known scraper
+ *
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
+ *
+ * @public exported from `@promptbook/documents`
+ * @public exported from `@promptbook/cli`
+ */
+export declare const _DocumentScraperRegistration: Registration;
+/**
+ * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
+ */