@promptbook/cli 0.72.0-5 → 0.72.0-7

package/esm/typings/src/{knowledge/dialogs → dialogs}/callback/CallbackInterfaceToolsOptions.d.ts

@@ -1,6 +1,6 @@
 import type { Promisable } from 'type-fest';
-import type { CommonExecutionToolsOptions } from '../../../execution/CommonExecutionToolsOptions';
-import type { UserInterfaceToolsPromptDialogOptions } from '../../../execution/UserInterfaceTools';
+import type { CommonExecutionToolsOptions } from '../../execution/CommonExecutionToolsOptions';
+import type { UserInterfaceToolsPromptDialogOptions } from '../../execution/UserInterfaceTools';
 /**
  * Options for `CallbackInterfaceTools`
  *

package/esm/typings/src/{knowledge/dialogs → dialogs}/simple-prompt/SimplePromptInterfaceTools.d.ts

@@ -1,6 +1,6 @@
-import type { CommonExecutionToolsOptions } from '../../../execution/CommonExecutionToolsOptions';
-import type { UserInterfaceTools } from '../../../execution/UserInterfaceTools';
-import type { UserInterfaceToolsPromptDialogOptions } from '../../../execution/UserInterfaceTools';
+import type { CommonExecutionToolsOptions } from '../../execution/CommonExecutionToolsOptions';
+import type { UserInterfaceTools } from '../../execution/UserInterfaceTools';
+import type { UserInterfaceToolsPromptDialogOptions } from '../../execution/UserInterfaceTools';
 /**
  * Wrapper around `window.prompt` synchronous function that interacts with the user via browser prompt
  *
@@ -18,5 +18,5 @@ export declare class SimplePromptInterfaceTools implements UserInterfaceTools {
     promptDialog(options: UserInterfaceToolsPromptDialogOptions): Promise<string>;
 }
 /**
- * Note: [🔵] This code should never be published outside of `@promptbook/browser`
+ * Note: [🔵] Code in this file should never be published outside of `@promptbook/browser`
  */

package/esm/typings/src/errors/KnowledgeScrapeError.d.ts

@@ -0,0 +1,9 @@
+/**
+ * This error indicates that the promptbook can not retrieve knowledge from external sources
+ *
+ * @public exported from `@promptbook/core`
+ */
+export declare class KnowledgeScrapeError extends Error {
+    readonly name = "KnowledgeScrapeError";
+    constructor(message: string);
+}

package/esm/typings/src/errors/MissingToolsError.d.ts

@@ -0,0 +1,9 @@
+/**
+ * This error type indicates that some tools are missing for pipeline execution or preparation
+ *
+ * @public exported from `@promptbook/core`
+ */
+export declare class MissingToolsError extends Error {
+    readonly name = "MissingToolsError";
+    constructor(message: string);
+}

package/esm/typings/src/execution/ExecutionTools.d.ts

@@ -14,7 +14,7 @@ export type ExecutionTools = {
      * Tip: Use `createLlmToolsFromEnv()` to use all available LLM providers you configured
      * @see https://github.com/webgptorg/promptbook/?tab=readme-ov-file#llm-execution-tools
      */
-    llm?: Arrayable<LlmExecutionTools>;
+    readonly llm?: Arrayable<LlmExecutionTools>;
     /**
      * Tools for executing scripts
      *
@@ -25,12 +25,12 @@ export type ExecutionTools = {
      * @see https://github.com/webgptorg/promptbook/?tab=readme-ov-file#script-execution-tools
      * @default [] - If not provided, no script execution will be possible
      */
-    script?: Arrayable<ScriptExecutionTools>;
+    readonly script?: Arrayable<ScriptExecutionTools>;
     /**
      * Tools for interacting with the user
      *
      * Note: When undefined, the user interface is disabled and promptbook which requires user interaction will fail
      * @see https://github.com/webgptorg/promptbook/?tab=readme-ov-file#user-interface-tools
      */
-    userInterface?: UserInterfaceTools;
+    readonly userInterface?: UserInterfaceTools;
 };

package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.d.ts

@@ -4,7 +4,7 @@ import type { CreatePipelineExecutorSettings } from './00-CreatePipelineExecutor
 /**
  * Options for `createPipelineExecutor`
  */
-export interface CreatePipelineExecutorOptions {
+export type CreatePipelineExecutorOptions = {
     /**
      * The pipeline to be executed
      */
@@ -17,4 +17,7 @@ export interface CreatePipelineExecutorOptions {
      * Optional settings for the pipeline executor
      */
     readonly settings?: Partial<CreatePipelineExecutorSettings>;
-}
+};
+/**
+ * TODO: !!!!!! Unite `CreatePipelineExecutorOptions` and `CreatePipelineExecutorSettings` OR describe the difference
+ */ 

package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts

@@ -1,23 +1,12 @@
 import type { CsvSettings } from '../../formats/csv/CsvSettings';
-export type CreatePipelineExecutorSettings = {
+import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
+export type CreatePipelineExecutorSettings = Omit<PrepareAndScrapeOptions, 'llmTools'> & {
     /**
      * When executor does not satisfy expectations it will be retried this amount of times
      *
      * @default MAX_EXECUTION_ATTEMPTS
      */
     readonly maxExecutionAttempts: number;
-    /**
-     * Maximum number of tasks running in parallel
-     *
-     * @default MAX_PARALLEL_COUNT
-     */
-    readonly maxParallelCount: number;
-    /**
-     * If true, the preparation logs additional information
-     *
-     * @default false
-     */
-    readonly isVerbose: boolean;
     /**
      * Settings for CSV format
      *

package/esm/typings/src/execution/createPipelineExecutor/00-createPipelineExecutor.d.ts

@@ -10,4 +10,4 @@ import type { CreatePipelineExecutorOptions } from './00-CreatePipelineExecutorO
 export declare function createPipelineExecutor(options: CreatePipelineExecutorOptions): PipelineExecutor;
 /**
  * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
- */ 
+ */

package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts

@@ -55,4 +55,4 @@ export declare function executePipeline(options: ExecutePipelineOptions): Promis
 export {};
 /**
  * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
- */ 
+ */

package/esm/typings/src/execution/translation/automatic-translate/translateMessages.d.ts

@@ -6,3 +6,6 @@ import type { TranslatorOptions } from './automatic-translators/TranslatorOption
 export declare function translateMessages({ automaticTranslator, from, to, }: {
     automaticTranslator: AutomaticTranslator;
 } & TranslatorOptions): Promise<void>;
+/**
+ * TODO: [😶] Unite floder listing
+ */

package/esm/typings/src/execution/utils/forEachAsync.d.ts

@@ -14,6 +14,7 @@ type ForEachAsyncOptions = {
  * @param options - Options for the function
  * @param callbackfunction - Function to call for each item
  * @public exported from `@promptbook/utils`
+ * @deprecated [🪂] Use queues instead
  */
 export declare function forEachAsync<TItem>(array: Array<TItem>, options: ForEachAsyncOptions, callbackfunction: (value: TItem, index: number, array: Array<TItem>) => Promisable<void>): Promise<void>;
 export {};

package/esm/typings/src/llm-providers/_common/createLlmToolsFromConfigurationFromEnv.d.ts

@@ -17,7 +17,7 @@ export declare function createLlmToolsFromConfigurationFromEnv(): LlmToolsConfig
  * TODO: Add Azure OpenAI
  * TODO: [🧠][🍛]
  * TODO: [🧠] Is there some meaningfull way how to test this util
- * Note: [🟢] This code should never be published outside of `@promptbook/node` and `@promptbook/cli` and `@promptbook/cli`
+ * Note: [🟢] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
  * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
  * TODO: This should be maybe not under `_common` but under `utils`
  * TODO: [🧠] Maybe pass env as argument

package/esm/typings/src/llm-providers/_common/createLlmToolsFromEnv.d.ts

@@ -19,7 +19,7 @@ export declare function createLlmToolsFromEnv(options?: CreateLlmToolsFromConfig
  * TODO: @@@ write `createLlmToolsFromEnv` vs `createLlmToolsFromConfigurationFromEnv` vs `createLlmToolsFromConfiguration`
  * TODO: [🧠][🍛] Which name is better `createLlmToolsFromEnv` or `createLlmToolsFromEnvironment`?
  * TODO: [🧠] Is there some meaningfull way how to test this util
- * Note: [🟢] This code should never be published outside of `@promptbook/node` and `@promptbook/cli` and `@promptbook/cli`
+ * Note: [🟢] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
  * TODO: [🥃] Allow `ptbk make` without llm tools
  * TODO: This should be maybe not under `_common` but under `utils`
  */

package/esm/typings/src/llm-providers/_common/getLlmToolsForCli.d.ts

@@ -15,7 +15,7 @@ type GetLlmToolsForCliOptions = {
 export declare function getLlmToolsForCli(options?: GetLlmToolsForCliOptions): LlmExecutionToolsWithTotalUsage;
 export {};
 /**
- * Note: [🟡] This code should never be published outside of `@promptbook/cli`
+ * Note: [🟡] Code in this file should never be published outside of `@promptbook/cli`
  * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
  * TODO: [🥃] Allow `ptbk make` without llm tools
  * TODO: This should be maybe not under `_common` but under `utils-internal` / `utils/internal`

package/esm/typings/src/llm-providers/anthropic-claude/playground/playground.d.ts

@@ -3,4 +3,5 @@ export {};
 /**
  * TODO: [main] !!! Playground with WebGPT / Promptbook.studio anonymous server
  * TODO: [main] !!! Test here that `systemMessage`, `temperature` and `seed` are working correctly
+ * Note: [⚫] Code in this file should never be published in any package
  */

package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts

@@ -44,6 +44,12 @@ export declare class AzureOpenAiExecutionTools implements LlmExecutionTools {
      * Calls Azure OpenAI API to use a complete model.
      */
     callCompletionModel(prompt: Pick<Prompt, 'content' | 'parameters' | 'modelRequirements'>): Promise<CompletionPromptResult>;
+    /**
+     * Library `@azure/openai` has bug/weird behavior that it does not throw error but hangs forever
+     *
+     * This method wraps the promise with timeout
+     */
+    private withTimeout;
     /**
      * Changes Azure error (which is not propper Error but object) to propper Error
      */

package/esm/typings/src/llm-providers/azure-openai/playground/playground.d.ts

@@ -2,4 +2,5 @@
 export {};
 /**
  * TODO: Test here that `systemMessage`, `temperature` and `seed` are working correctly
+ * Note: [⚫] Code in this file should never be published in any package
  */

package/esm/typings/src/llm-providers/langtail/playground/playground.d.ts

@@ -1,2 +1,5 @@
 #!/usr/bin/env ts-node
 export {};
+/**
+ * Note: [⚫] Code in this file should never be published in any package
+ */

package/esm/typings/src/llm-providers/multiple/playground/playground.d.ts

@@ -1,2 +1,5 @@
 #!/usr/bin/env ts-node
 export {};
+/**
+ * Note: [⚫] Code in this file should never be published in any package
+ */

package/esm/typings/src/llm-providers/openai/playground/playground.d.ts

@@ -2,4 +2,5 @@
 export {};
 /**
  * TODO: [main] !!! Test here that `systemMessage`, `temperature` and `seed` are working correctly
+ * Note: [⚫] Code in this file should never be published in any package
  */

package/esm/typings/src/llm-providers/remote/playground/playground.d.ts

@@ -1,2 +1,5 @@
 #!/usr/bin/env ts-node
 import '../../openai/register-constructor';
+/**
+ * Note: [⚫] Code in this file should never be published in any package
+ */

package/esm/typings/src/personas/preparePersona.d.ts

@@ -1,4 +1,4 @@
-import type { PrepareOptions } from '../prepare/PrepareOptions';
+import type { PrepareAndScrapeOptions } from '../prepare/PrepareAndScrapeOptions';
 import type { PersonaPreparedJson } from '../types/PipelineJson/PersonaJson';
 import type { string_persona_description } from '../types/typeAliases';
 /**
@@ -7,7 +7,7 @@ import type { string_persona_description } from '../types/typeAliases';
  * @see https://github.com/webgptorg/promptbook/discussions/22
  * @public exported from `@promptbook/core`
  */
-export declare function preparePersona(personaDescription: string_persona_description, options: PrepareOptions): Promise<PersonaPreparedJson['modelRequirements']>;
+export declare function preparePersona(personaDescription: string_persona_description, options: PrepareAndScrapeOptions): Promise<PersonaPreparedJson['modelRequirements']>;
 /**
  * TODO: [🔃][main] !!!!! If the persona was prepared with different version or different set of models, prepare it once again
  * TODO: [🏢] !! Check validity of `modelName` in pipeline

package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts

@@ -0,0 +1,67 @@
+import type { LlmExecutionTools } from '../execution/LlmExecutionTools';
+import type { string_dirname } from '../types/typeAliases';
+/**
+ * Options for preparation of the pipeline
+ */
+export type PrepareAndScrapeOptions = {
+    /**
+     * LLM tools
+     */
+    readonly llmTools?: LlmExecutionTools;
+    /**
+     * Path to the root folder of the pipeline
+     *
+     * Note: When the pipeline is not created from files, it is `null`
+     * Note: This folder must exist
+     */
+    readonly rootDirname: string_dirname | null;
+    /**
+     * Path to the cache folder
+     *
+     * Note: When the folder does not exist, it is created recursively
+     *
+     * @default SCRAPE_CACHE_DIRNAME
+     */
+    readonly cacheDirname?: string_dirname;
+    /**
+     * If true, the cache is cleaned after the scraping
+     *
+     *
+     * @default false // <- TODO: !!!!!! Change to `cacheStrategy`/`intermediateFiles`, Put to global config, change to `true` and explicitly set to `false` in all playgrounds
+     */
+    readonly isCacheCleaned?: boolean;
+    /**
+     * Maximum number of tasks running in parallel
+     *
+     * @default MAX_PARALLEL_COUNT
+     */
+    readonly maxParallelCount?: number;
+    /**
+     * Path to the external programs executables
+     *
+     * TODO: !!!!!! Transform to scrapers and make them Classy
+     */
+    readonly externalProgramsPaths?: {
+        /**
+         * Path to the `pandoc` executable
+         *
+         * @example 'C:/Users/me/AppData/Local/Pandoc/pandoc.exe'
+         */
+        readonly pandocPath?: string;
+        /**
+         * Path to the LibreOffice executable
+         *
+         * @example 'C:/Program Files/LibreOffice/program/swriter.exe'
+         */
+        readonly libreOfficePath?: string;
+    };
+    /**
+     * If true, the preparation logs additional information
+     *
+     * @default false
+     */
+    readonly isVerbose?: boolean;
+};
+/**
+ * TODO: [🧠] Maybe split `PrepareAndScrapeOptions` and `ScrapeOptions` (`ScrapeOptions` should be extended from `PrepareAndScrapeOptions`)
+ */

package/esm/typings/src/prepare/preparePipeline.d.ts

@@ -1,5 +1,5 @@
 import type { PipelineJson } from '../types/PipelineJson/PipelineJson';
-import type { PrepareOptions } from './PrepareOptions';
+import type { PrepareAndScrapeOptions } from './PrepareAndScrapeOptions';
 /**
  * Prepare pipeline from string (markdown) format to JSON format
  *
@@ -8,7 +8,7 @@ import type { PrepareOptions } from './PrepareOptions';
  * Note: When the pipeline is already prepared, it returns the same pipeline
  * @public exported from `@promptbook/core`
  */
-export declare function preparePipeline(pipeline: PipelineJson, options: PrepareOptions): Promise<PipelineJson>;
+export declare function preparePipeline(pipeline: PipelineJson, options: PrepareAndScrapeOptions): Promise<PipelineJson>;
 /**
  * TODO: Write tests for `preparePipeline`
  * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch

package/esm/typings/src/prepare/prepareTemplates.d.ts

@@ -1,6 +1,6 @@
 import type { PipelineJson } from '../types/PipelineJson/PipelineJson';
 import type { TemplateJson } from '../types/PipelineJson/TemplateJson';
-import type { PrepareOptions } from './PrepareOptions';
+import type { PrepareAndScrapeOptions } from './PrepareAndScrapeOptions';
 type PrepareTemplateInput = Pick<PipelineJson, 'templates' | 'parameters'> & {
     /**
      * @@@
@@ -18,7 +18,7 @@ type PreparedTemplates = {
  *
  * @public exported from `@promptbook/core`
  */
-export declare function prepareTemplates(pipeline: PrepareTemplateInput, options: PrepareOptions): Promise<PreparedTemplates>;
+export declare function prepareTemplates(pipeline: PrepareTemplateInput, options: PrepareAndScrapeOptions): Promise<PreparedTemplates>;
 export {};
 /**
  * TODO: [🧠] Add context to each template (if missing)

package/esm/typings/src/scrapers/_common/Converter.d.ts

@@ -0,0 +1,28 @@
+import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
+import type { string_mime_type } from '../../types/typeAliases';
+import type { string_promptbook_documentation_url } from '../../types/typeAliases';
+import type { ScraperSourceHandler } from './Scraper';
+import type { ScraperIntermediateSource } from './ScraperIntermediateSource';
+/**
+ * @@@
+ *
+ */
+export type Converter = {
+    /**
+     * Mime types that this scraper can handle
+     */
+    readonly mimeTypes: Array<string_mime_type>;
+    /**
+     * Link to documentation
+     */
+    readonly documentationUrl: string_promptbook_documentation_url;
+    /**
+     * Convert the the file and returns intermediate source or `null` if it can't convert it
+     *
+     * For example, convert a `.docx` to `.doc` file
+     * Or convert a `.pdf` to `.md` file
+     *
+     * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
+     */
+    $convert(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<ScraperIntermediateSource>;
+};

package/esm/typings/src/scrapers/_common/Scraper.d.ts

@@ -0,0 +1,71 @@
+import type { Promisable } from 'type-fest';
+import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
+import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
+import type { string_filename } from '../../types/typeAliases';
+import type { string_knowledge_source_link } from '../../types/typeAliases';
+import type { string_mime_type } from '../../types/typeAliases';
+import type { string_promptbook_documentation_url } from '../../types/typeAliases';
+import type { string_url } from '../../types/typeAliases';
+/**
+ * @@@
+ *
+ */
+export type Scraper = {
+    /**
+     * Mime types that this scraper can handle
+     */
+    readonly mimeTypes: Array<string_mime_type>;
+    /**
+     * Link to documentation
+     */
+    readonly documentationUrl: string_promptbook_documentation_url;
+    /**
+     * Scrapes the markdown file and returns the knowledge pieces or `null` if it can't scrape it
+     */
+    scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promisable<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
+};
+/**
+ * @@@
+ */
+export type ScraperSourceHandler = {
+    /**
+     * The source of the knowledge
+     */
+    readonly source: string_knowledge_source_link;
+    /**
+     * The path to the file, if it is a file
+     *
+     * Note: Typically one of the `filename` or `url` is set and the other is `null`
+     */
+    readonly filename: string_filename | null;
+    /**
+     * The URL, if it is online
+     *
+     * Note: Typically one of the `filename` or `url` is set and the other is `null`
+     */
+    readonly url: string_url | null;
+    /**
+     * Mime type of the source
+     */
+    readonly mimeType: string_mime_type;
+    /**
+     * Get the content as parsed JSON
+     */
+    asJson(): Promisable<unknown>;
+    /**
+     * Get the content as a utf-8 string
+     */
+    asText(): Promisable<string>;
+    /**
+     * Get the content as a blob
+     */
+    asBlob(): Promisable<Blob>;
+};
+/**
+ * TODO: [🐝] @@@ Annotate all
+ * TODO: [🔼] Export via types
+ */
+/**
+ * TODO: !!!!!! Test that this is catched
+ * Note: [⚫] Code in this file should never be published in any package
+ */

package/esm/typings/src/scrapers/_common/ScraperIntermediateSource.d.ts

@@ -0,0 +1,11 @@
+import type { IDestroyable } from 'destroyable';
+import type { string_absolute_filename } from '../../types/typeAliases';
+/**
+ * @@@
+ */
+export type ScraperIntermediateSource = IDestroyable & {
+    /**
+     * @@@
+     */
+    readonly filename: string_absolute_filename;
+};

package/esm/typings/src/{knowledge/prepare-knowledge → scrapers}/_common/prepareKnowledgePieces.d.ts

@@ -1,13 +1,13 @@
-import type { PrepareOptions } from '../../../prepare/PrepareOptions';
-import type { KnowledgePiecePreparedJson } from '../../../types/PipelineJson/KnowledgePieceJson';
-import type { KnowledgeSourceJson } from '../../../types/PipelineJson/KnowledgeSourceJson';
+import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
+import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
+import type { KnowledgeSourceJson } from '../../types/PipelineJson/KnowledgeSourceJson';
 /**
  * Prepares the knowle
  *
  * @see https://github.com/webgptorg/promptbook/discussions/41
  * @public exported from `@promptbook/core`
  */
-export declare function prepareKnowledgePieces(knowledgeSources: Array<KnowledgeSourceJson>, options: PrepareOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'preparationIds'>>>;
+export declare function prepareKnowledgePieces(knowledgeSources: Array<KnowledgeSourceJson>, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'preparationIds'>>>;
 /**
  * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
  *       Put `knowledgePieces` into `PrepareKnowledgeOptions`

package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts

@@ -0,0 +1,33 @@
+import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
+import type { string_file_extension } from '../../../types/typeAliases';
+import type { ScraperSourceHandler } from '../Scraper';
+import type { ScraperIntermediateSource } from '../ScraperIntermediateSource';
+/**
+ * @@@
+ *
+ * @private internal utility of `getScraperIntermediateSource`
+ */
+type GetScraperIntermediateSourceSource = Pick<ScraperSourceHandler, 'filename' | 'url'>;
+/**
+ * @@@
+ *
+ * @private internal utility of `getScraperIntermediateSource`
+ */
+type GetScraperIntermediateSourceHandler = Required<Pick<PrepareAndScrapeOptions, 'rootDirname' | 'cacheDirname' | 'isCacheCleaned' | 'isVerbose'>> & {
+    readonly extension: string_file_extension;
+};
+/**
+ * Create a filename for intermediate cache for scrapers
+ *
+ * Note: It also checks if directory exists and creates it if not
+ *
+ * @private as internal utility for scrapers
+ */
+export declare function getScraperIntermediateSource(source: GetScraperIntermediateSourceSource, options: GetScraperIntermediateSourceHandler): Promise<ScraperIntermediateSource>;
+export {};
+/**
+ * Note: Not using `FileCacheStorage` for two reasons:
+ * 1) Need to store more than serialized JSONs
+ * 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: !!!!
+ * TODO: [🐱‍🐉][🧠] Make some smart crop
+ */

package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.test.d.ts

@@ -0,0 +1,4 @@
+export {};
+/**
+ * TODO: [🐱‍🐉][🧠] Make some smart crop
+ */

package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts

@@ -0,0 +1,10 @@
+import type { SetOptional } from 'type-fest';
+import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
+import type { KnowledgeSourceJson } from '../../../types/PipelineJson/KnowledgeSourceJson';
+import type { ScraperSourceHandler } from '../Scraper';
+/**
+ * @@@
+ *
+ * @private for scraper utilities
+ */
+export declare function makeKnowledgeSourceHandler(knowledgeSource: SetOptional<KnowledgeSourceJson, 'name'>, options?: Pick<PrepareAndScrapeOptions, 'rootDirname' | 'isVerbose'>): Promise<ScraperSourceHandler>;

package/esm/typings/src/scrapers/document/documentScraper.d.ts

@@ -0,0 +1,37 @@
+import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
+import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
+import type { ScraperSourceHandler } from '../_common/Scraper';
+import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource';
+/**
+ * Scraper of .docx and .odt files
+ *
+ * @see `documentationUrl` for more details
+ * @public exported from `@promptbook/core`
+ */
+export declare const documentScraper: {
+    /**
+     * Mime types that this scraper can handle
+     */
+    mimeTypes: string[];
+    /**
+     * Link to documentation
+     */
+    documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
+    /**
+     * Convert the `.docx` or `.odt`  to `.md` file and returns intermediate source
+     *
+     * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
+     */
+    $convert(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<ScraperIntermediateSource>;
+    /**
+     * Scrapes the docx file and returns the knowledge pieces or `null` if it can't scrape it
+     */
+    scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
+};
+/**
+ * TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
+ * TODO: [🦖] Make some system for putting scrapers to separete packages
+ * TODO: [🪂] Do it in parallel 11:11
+ * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
+ * Note: No need to aggregate usage here, it is done by intercepting the llmTools
+ */

package/esm/typings/src/scrapers/document/documentScraper.test.d.ts

@@ -0,0 +1,4 @@
+export {};
+/**
+ * TODO: [📓] Maybe test all file in samples (not just 10-simple.docx)
+ */

package/esm/typings/src/scrapers/document/playground/document-scraper-playground.d.ts

@@ -0,0 +1,5 @@
+#!/usr/bin/env ts-node
+export {};
+/**
+ * Note: [⚫] Code in this file should never be published in any package
+ */

package/esm/typings/src/scrapers/document-legacy/legacyDocumentScraper.d.ts

@@ -0,0 +1,37 @@
+import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
+import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
+import type { ScraperSourceHandler } from '../_common/Scraper';
+import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource';
+/**
+ * Scraper for .docx files
+ *
+ * @see `documentationUrl` for more details
+ * @public exported from `@promptbook/core`
+ */
+export declare const legacyDocumentScraper: {
+    /**
+     * Mime types that this scraper can handle
+     */
+    mimeTypes: string[];
+    /**
+     * Link to documentation
+     */
+    documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
+    /**
+     * Convert the `.doc` or `.rtf`  to `.doc` file and returns intermediate source
+     *
+     * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
+     */
+    $convert(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<ScraperIntermediateSource>;
+    /**
+     * Scrapes the `.doc` or `.rtf` file and returns the knowledge pieces or `null` if it can't scrape it
+     */
+    scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
+};
+/**
+ * TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
+ * TODO: [🦖] Make some system for putting scrapers to separete packages
+ * TODO: [🪂] Do it in parallel 11:11
+ * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
+ * Note: No need to aggregate usage here, it is done by intercepting the llmTools
+ */

package/esm/typings/src/scrapers/document-legacy/legacyDocumentScraper.test.d.ts

@@ -0,0 +1,4 @@
+export {};
+/**
+ * TODO: [📓] Maybe test all file in samples (not just 10-simple.doc)
+ */