@promptbook/vercel 0.76.0

package/esm/typings/src/pipeline/PipelineJson/ParameterJson.d.ts

@@ -0,0 +1,98 @@
+import type { string_markdown_text } from '../../types/typeAliases';
+import type { string_parameter_name } from '../../types/typeAliases';
+import type { string_parameter_value } from '../../types/typeAliases';
+/**
+ * Describes one parameter of the pipeline
+ *
+ * Note: [🚉] This is fully serializable as JSON
+ */
+export type ParameterJson = InputParameterJson | IntermediateParameterJson | OutputParameterJson;
+/**
+ * Describes input parameter of the pipeline
+ *
+ * 🔴 -> ⚪ -> ⚪
+ *
+ * Note: [🚉] This is fully serializable as JSON
+ */
+export type InputParameterJson = CommonParameterJson & {
+    /**
+     * The parameter is input of the pipeline
+     */
+    readonly isInput: true;
+    /**
+     * The parameter is NOT output of the pipeline
+     */
+    readonly isOutput: false;
+};
+/**
+ * Describes intermediate parameter of the pipeline
+ *
+ * ⚪ -> 🔴 -> ⚪
+ *
+ * Note: [🚉] This is fully serializable as JSON
+ */
+export type IntermediateParameterJson = CommonParameterJson & {
+    /**
+     * The parameter is NOT input of the pipeline
+     */
+    readonly isInput: false;
+    /**
+     * The parameter is NOT output of the pipeline
+     */
+    readonly isOutput: false;
+};
+/**
+ * Describes output parameter of the pipeline
+ *
+ * ⚪ -> ⚪ -> 🔴
+ *
+ * Note: [🚉] This is fully serializable as JSON
+ */
+export type OutputParameterJson = CommonParameterJson & {
+    /**
+     * The parameter is NOT input of the pipeline
+     */
+    readonly isInput: false;
+    /**
+     * The parameter is output of the pipeline
+     */
+    readonly isOutput: true;
+};
+/**
+ * Describes commons of one parameter of the pipeline
+ *
+ * Note: [🚉] This is fully serializable as JSON
+ */
+export type CommonParameterJson = {
+    /**
+     * Name of the parameter
+     * - It must be unique across the pipeline
+     * - It should start lowercase and contain letters and numbers
+     */
+    readonly name: string_parameter_name;
+    /**
+     * The parameter is input of the pipeline
+     */
+    readonly isInput: boolean;
+    /**
+     * The parameter is output of the pipeline
+     */
+    readonly isOutput: boolean;
+    /**
+     * Description of the parameter
+     * - It can use simple markdown formatting like **bold**, *italic*, [link](https://example.com), ... BUT not code blocks and structure
+     */
+    readonly description?: string_markdown_text;
+    /**
+     * Example values of the parameter
+     * Note: This values won't be actually used as some default values, but they are just for better understanding of the parameter
+     */
+    readonly exampleValues?: Array<string_parameter_value>;
+};
+/**
+ * TODO: [🧠][🛴][♈] Maybe add type + expectations into the intefrace, like "a person name"
+ *       [🛴] @see https://github.com/webgptorg/promptbook/discussions/53
+ * TODO: [🧠] Should be here registered subparameters from foreach or not?
+ * TODO: [♈] Probbably move expectations from tasks to parameters
+ * TODO: [🍙] Make some standard order of json properties
+ */

package/esm/typings/src/pipeline/PipelineJson/PersonaJson.d.ts

@@ -0,0 +1,45 @@
+import type { ChatModelRequirements } from '../../types/ModelRequirements';
+import type { number_id } from '../../types/typeAliases';
+import type { string_name } from '../../types/typeAliases';
+import type { string_persona_description } from '../../types/typeAliases';
+/**
+ * Defines a persona in the pipeline
+ *
+ * Note: [🚉] This is fully serializable as JSON
+ *
+ * @see https://github.com/webgptorg/promptbook/discussions/22
+ */
+export type PersonaJson = {
+    /**
+     * Name of the persona
+     * - It must be unique across the pipeline
+     * - It should start uppercase and can contain letters and numbers
+     */
+    readonly name: string_name;
+    /**
+     * Description of persona
+     */
+    readonly description: string_persona_description;
+};
+/**
+ * Defines a persona in the pipeline after it has been prepared
+ *
+ * Note: [🚉] This is fully serializable as JSON
+ *
+ * @see https://github.com/webgptorg/promptbook/discussions/22
+ */
+export type PersonaPreparedJson = PersonaJson & {
+    /**
+     * Model requirements for the persona
+     *
+     * Note: The model must be CHAT variant to be usable through persona
+     */
+    readonly modelRequirements: ChatModelRequirements;
+    /**
+     * List of preparation ids that were used to prepare this persona
+     */
+    readonly preparationIds: ReadonlyArray<number_id>;
+};
+/**
+ * TODO: [🍙] Make some standard order of json properties
+ */

package/esm/typings/src/pipeline/PipelineJson/PipelineJson.d.ts

@@ -0,0 +1,102 @@
+import type { string_formfactor_name } from '../../formfactors/_common/string_formfactor_name';
+import type { ModelRequirements } from '../../types/ModelRequirements';
+import type { string_filename } from '../../types/typeAliases';
+import type { string_markdown_text } from '../../types/typeAliases';
+import type { string_pipeline_url } from '../../types/typeAliases';
+import type { string_semantic_version } from '../../types/typeAliases';
+import type { KnowledgePiecePreparedJson } from './KnowledgePieceJson';
+import type { KnowledgeSourceJson } from './KnowledgeSourceJson';
+import type { KnowledgeSourcePreparedJson } from './KnowledgeSourceJson';
+import type { ParameterJson } from './ParameterJson';
+import type { PersonaJson } from './PersonaJson';
+import type { PersonaPreparedJson } from './PersonaJson';
+import type { PreparationJson } from './PreparationJson';
+import type { TaskJson } from './TaskJson';
+/**
+ * Promptbook is the **core concept of this package**.
+ * It represents a series of tasks chained together to form a pipeline / one big task with input and result parameters.
+ *
+ * Note: [🚉] This is fully serializable as JSON
+ *
+ * @see @@@ https://github.com/webgptorg/promptbook#promptbook
+ */
+export type PipelineJson = {
+    /**
+     * Unique identifier of the pipeline
+     *
+     * Note: It must be unique across all pipeline collections
+     * Note: It must use HTTPs URL
+     * Tip: You can do versioning in the URL
+     *      For example: https://promptbook.studio/webgpt/write-website-content-cs.book.md@1.0.0
+     * Warning: Do not hash part of the URL, hash part is used for identification of the task in the pipeline
+     */
+    readonly pipelineUrl?: string_pipeline_url;
+    /**
+     * Internal helper for tracking the source `.book.md` file of the pipeline
+     */
+    readonly sourceFile?: string_filename;
+    /**
+     * Title of the promptbook
+     * -It can use simple markdown formatting like **bold**, *italic*, [link](https://example.com), ... BUT not code blocks and structure
+     */
+    readonly title: string_markdown_text;
+    /**
+     * Version of the Book language
+     *
+     * @see https://github.com/webgptorg/book
+     */
+    readonly bookVersion?: string_semantic_version;
+    /**
+     * Description of the promptbook
+     * It can use multiple paragraphs of simple markdown formatting like **bold**, *italic*, [link](https://example.com), ... BUT not code blocks and structure
+     */
+    readonly description?: string_markdown_text;
+    /**
+     * Name of the formfactor that the pipeline is shaped for
+     */
+    readonly formfactorName: string_formfactor_name;
+    /**
+     * Set of variables that are used across the pipeline
+     */
+    readonly parameters: Array<ParameterJson>;
+    /**
+     * Default model requirements for the model for all `tasks`
+     */
+    readonly defaultModelRequirements?: Partial<ModelRequirements>;
+    /**
+     * Sequence of tasks that are chained together to form a pipeline
+     */
+    readonly tasks: Array<TaskJson>;
+    /**
+     * Set of information that are used as external knowledge in the pipeline
+     *
+     * @see https://github.com/webgptorg/promptbook/discussions/41
+     */
+    readonly knowledgeSources: Array<KnowledgeSourceJson | KnowledgeSourcePreparedJson>;
+    /**
+     * Set of information that are used as external knowledge in the pipeline
+     *
+     * @see https://github.com/webgptorg/promptbook/discussions/41
+     */
+    readonly knowledgePieces: Array<KnowledgePiecePreparedJson>;
+    /**
+     * List of prepared virtual personas that are used in the pipeline
+     *
+     * @see https://github.com/webgptorg/promptbook/discussions/22
+     */
+    readonly personas: Array<PersonaJson | PersonaPreparedJson>;
+    /**
+     * List of prepared virtual personas that are used in the pipeline
+     *
+     * @see https://github.com/webgptorg/promptbook/discussions/78
+     */
+    readonly preparations: Array<PreparationJson>;
+};
+/**
+ * TODO: [🛳] Default PERSONA for the pipeline `defaultPersonaName` (same as `defaultModelRequirements`)
+ * TODO: [🍙] Make some standard order of json properties
+ * TODO: [🧠] Maybe wrap all {parameterNames} in brackets for example { "resultingParameterName": "{foo}" }
+ * Note: [💼] There was a proposal for multiple types of promptbook objects 78816ff33e2705ee1a187aa2eb8affd976d4ea1a
+ *       But then immediately reverted back to the single type
+ *       With knowledge as part of the promptbook and collection just as a collection of promptbooks
+ */

package/esm/typings/src/pipeline/PipelineJson/PreparationJson.d.ts

@@ -0,0 +1,22 @@
+import type { PromptResultUsage } from '../../execution/PromptResultUsage';
+import type { number_id } from '../../types/typeAliases';
+import type { string_promptbook_version } from '../../version';
+export type PreparationJson = {
+    /**
+     * Incremental ID of the preparation
+     */
+    readonly id: number_id;
+    /**
+     * Version of the promptbook used for the preparation
+     */
+    readonly promptbookVersion: string_promptbook_version;
+    /**
+     * Usage of the prompt execution
+     */
+    readonly usage: PromptResultUsage;
+};
+/**
+ * TODO: [🍙] Make some standard order of json properties
+ * TODO: Maybe put here used `modelName`
+ * TODO: [🍥] When using `date` it changes all examples .book.json files each time so until some more elegant solution omit the time from prepared pipeline
+ */

package/esm/typings/src/pipeline/PipelineJson/PromptTaskJson.d.ts

@@ -0,0 +1,26 @@
+import type { ModelRequirements } from '../../types/ModelRequirements';
+import type { string_name } from '../../types/typeAliases';
+import type { CommonTaskJson } from './CommonTaskJson';
+/**
+ * Task for prompt to LLM
+ *
+ * Note: [🚉] This is fully serializable as JSON
+ */
+export type PromptTaskJson = CommonTaskJson & {
+    readonly taskType: 'PROMPT_TASK';
+    /**
+     * Name of the persona who will be responding to this prompt
+     */
+    readonly personaName?: string_name;
+    /**
+     * Requirements for the model
+     * - This is required only for taskType PROMPT_TASK
+     */
+    readonly modelRequirements?: Partial<ModelRequirements>;
+};
+/**
+ * TODO: [👙][🧠] Maybe add `knowledge`, `actions` and `instruments` to be available granularly for each task
+ *       @see https://github.com/webgptorg/promptbook/discussions/79
+ * TODO: [💕][🧠] Just selecting gpt3 or gpt4 level of model
+ * TODO: [🍙] Make some standard order of json properties
+ */

package/esm/typings/src/pipeline/PipelineJson/ScriptTaskJson.d.ts

@@ -0,0 +1,20 @@
+import type { ScriptLanguage } from '../../types/ScriptLanguage';
+import type { CommonTaskJson } from './CommonTaskJson';
+/**
+ * Task for script execution
+ *
+ * Note: [🚉] This is fully serializable as JSON
+ * @see https://github.com/webgptorg/promptbook/discussions/77
+ */
+export type ScriptTaskJson = CommonTaskJson & {
+    readonly taskType: 'SCRIPT_TASK';
+    /**
+     * Language of the script
+     * - This is required only for taskType SCRIPT
+     *
+     */
+    readonly contentLanguage?: ScriptLanguage;
+};
+/**
+ * TODO: [🍙] Make some standard order of json properties
+ */

package/esm/typings/src/pipeline/PipelineJson/SimpleTaskJson.d.ts

@@ -0,0 +1,13 @@
+import type { CommonTaskJson } from './CommonTaskJson';
+/**
+ * Task for simple concatenation of strings
+ *
+ * Note: [🚉] This is fully serializable as JSON
+ * @see https://github.com/webgptorg/promptbook/discussions/17
+ */
+export type SimpleTaskJson = CommonTaskJson & {
+    readonly taskType: 'SIMPLE_TASK';
+};
+/**
+ * TODO: [🍙] Make some standard order of json properties
+ */

package/esm/typings/src/pipeline/PipelineJson/TaskJson.d.ts

@@ -0,0 +1,11 @@
+import type { ___or___ } from '../../utils/organization/___or___';
+import type { DialogTaskJson } from './DialogTaskJson';
+import type { PromptTaskJson } from './PromptTaskJson';
+import type { ScriptTaskJson } from './ScriptTaskJson';
+import type { SimpleTaskJson } from './SimpleTaskJson';
+/**
+ * Describes one (prompt) task in the pipeline
+ *
+ * Note: [🚉] This is fully serializable as JSON
+ */
+export type TaskJson = PromptTaskJson | SimpleTaskJson | ScriptTaskJson | DialogTaskJson | ___or___ | ___or___;

package/esm/typings/src/pipeline/PipelineString.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Promptbook is the **core concept of this package**.
+ * It represents a series of tasks chained together to form a pipeline / one big task with input and result parameters.
+ *
+ * @see @@@ https://github.com/webgptorg/promptbook#promptbook
+ */
+export type PipelineString = string & {
+    readonly _type: 'Promptbook';
+};
+/**
+ * TODO: !! Better validation (validatePipelineString) or remove branded type and make it just string
+ */

package/esm/typings/src/postprocessing/utils/extractBlock.d.ts

@@ -0,0 +1,16 @@
+import type { string_markdown } from '../../types/typeAliases';
+/**
+ * Extracts code block from markdown.
+ *
+ * - When there are multiple or no code blocks the function throws a `ParseError`
+ *
+ * Note: There are multiple simmilar function:
+ * - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
+ * - `extractJsonBlock` extracts exactly one valid JSON code block
+ * - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
+ * - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block
+ *
+ * @public exported from `@promptbook/markdown-utils`
+ * @throws {ParseError} if there is not exactly one code block in the markdown
+ */
+export declare function extractBlock(markdown: string_markdown): string;

package/esm/typings/src/postprocessing/utils/extractJsonBlock.d.ts

@@ -0,0 +1,25 @@
+import type { string_json } from '../../types/typeAliases';
+import type { string_markdown } from '../../types/typeAliases';
+import type { really_unknown } from '../../utils/organization/really_unknown';
+/**
+ * Extracts  extracts exactly one valid JSON code block
+ *
+ * - When given string is a valid JSON as it is, it just returns it
+ * - When there is no JSON code block the function throws a `ParseError`
+ * - When there are multiple JSON code blocks the function throws a `ParseError`
+ *
+ * Note: It is not important if marked as ```json BUT if it is VALID JSON
+ * Note: There are multiple simmilar function:
+ * - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
+ * - `extractJsonBlock` extracts exactly one valid JSON code block
+ * - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
+ * - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block
+ *
+ * @public exported from `@promptbook/markdown-utils`
+ * @throws {ParseError} if there is no valid JSON block in the markdown
+ */
+export declare function extractJsonBlock(markdown: string_markdown): string_json<really_unknown>;
+/**
+ * TODO: Add some auto-healing logic + extract YAML, JSON5, TOML, etc.
+ * TODO: [🏢] Make this logic part of `JsonFormatDefinition` or `isValidJsonString`
+ */

package/esm/typings/src/postprocessing/utils/extractJsonBlock.test.d.ts

@@ -0,0 +1 @@
+export {};

package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts

@@ -0,0 +1,51 @@
+import type { IntermediateFilesStrategy } from '../types/IntermediateFilesStrategy';
+import type { string_dirname } from '../types/typeAliases';
+/**
+ * Options for preparation of the pipeline
+ */
+export type PrepareAndScrapeOptions = {
+    /**
+     * Path to the root folder of the pipeline
+     *
+     * Note: When the pipeline is not created from files, it is `null`
+     * Note: This folder must exist (=it is not created recursively)
+     *
+     * @default null or `process.cwd()` when created via `$provide...` function in node
+     */
+    readonly rootDirname?: string_dirname | null;
+    /**
+     * Path to the cache folder
+     *
+     * Note: When the folder does not exist, it is created recursively
+     *
+     * @default DEFAULT_SCRAPE_CACHE_DIRNAME
+     */
+    readonly cacheDirname?: string_dirname;
+    /**
+     * Strategy for caching the intermediate results for knowledge source
+     *
+     * @default DEFAULT_INTERMEDIATE_FILES_STRATEGY
+     */
+    readonly intermediateFilesStrategy?: IntermediateFilesStrategy;
+    /**
+     * Maximum number of tasks running in parallel
+     *
+     * @default DEFAULT_MAX_PARALLEL_COUNT
+     */
+    readonly maxParallelCount?: number;
+    /**
+     * If true, the missing software is automatically installed
+     *
+     * @default DEFAULT_IS_AUTO_INSTALLED
+     */
+    readonly isAutoInstalled?: boolean;
+    /**
+     * If true, the preparation logs additional information
+     *
+     * @default DEFAULT_IS_VERBOSE
+     */
+    readonly isVerbose?: boolean;
+};
+/**
+ * TODO: [🧠] Maybe split `PrepareAndScrapeOptions` and `ScrapeOptions` (`ScrapeOptions` should be extended from `PrepareAndScrapeOptions`)
+ */

package/esm/typings/src/prepare/isPipelinePrepared.d.ts

@@ -0,0 +1,16 @@
+import type { PipelineJson } from '../pipeline/PipelineJson/PipelineJson';
+/**
+ * Determine if the pipeline is fully prepared
+ *
+ * @public exported from `@promptbook/core`
+ */
+export declare function isPipelinePrepared(pipeline: PipelineJson): boolean;
+/**
+ * TODO: [🔃][main] !! If the pipeline was prepared with different version or different set of models, prepare it once again
+ * TODO: [🐠] Maybe base this on `makeValidator`
+ * TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared
+ * TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline`
+ *     - [🏍] ? Is context in each task
+ *     - [♨] Are examples prepared
+ *     - [♨] Are tasks prepared
+ */

package/esm/typings/src/prepare/isPipelinePrepared.test.d.ts

@@ -0,0 +1 @@
+export {};

package/esm/typings/src/prepare/preparePipeline.d.ts

@@ -0,0 +1,20 @@
+import type { ExecutionTools } from '../execution/ExecutionTools';
+import type { PipelineJson } from '../pipeline/PipelineJson/PipelineJson';
+import type { PrepareAndScrapeOptions } from './PrepareAndScrapeOptions';
+/**
+ * Prepare pipeline from string (markdown) format to JSON format
+ *
+ * Note: This function does not validate logic of the pipeline
+ * Note: This function acts as part of compilation process
+ * Note: When the pipeline is already prepared, it returns the same pipeline
+ * @public exported from `@promptbook/core`
+ */
+export declare function preparePipeline(pipeline: PipelineJson, tools: Pick<ExecutionTools, 'llm' | 'fs' | 'scrapers'>, options: PrepareAndScrapeOptions): Promise<PipelineJson>;
+/**
+ * TODO: Write tests for `preparePipeline`
+ * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
+ * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
+ * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
+ * TODO: [🧠][♏] Maybe if expecting JSON (In Anthropic Claude and other models without non-json) and its not specified in prompt content, append the instructions
+ *       @see https://docs.anthropic.com/en/docs/test-and-evaluate/strengthen-guardrails/increase-consistency#specify-the-desired-output-format
+ */

package/esm/typings/src/prepare/prepareTasks.d.ts

@@ -0,0 +1,32 @@
+import type { ExecutionTools } from '../execution/ExecutionTools';
+import type { PipelineJson } from '../pipeline/PipelineJson/PipelineJson';
+import type { TaskJson } from '../pipeline/PipelineJson/TaskJson';
+import type { PrepareAndScrapeOptions } from './PrepareAndScrapeOptions';
+type PrepareTaskInput = Pick<PipelineJson, 'tasks' | 'parameters'> & {
+    /**
+     * @@@
+     */
+    readonly knowledgePiecesCount: number;
+};
+type PreparedTasks = {
+    /**
+     * @@@ Sequence of tasks that are chained together to form a pipeline
+     */
+    readonly tasksPrepared: ReadonlyArray<TaskJson>;
+};
+/**
+ * @@@
+ *
+ * @public exported from `@promptbook/core`
+ */
+export declare function prepareTasks(pipeline: PrepareTaskInput, tools: Pick<ExecutionTools, 'llm' | 'fs' | 'scrapers'>, options: PrepareAndScrapeOptions): Promise<PreparedTasks>;
+export {};
+/**
+ * TODO: [🧠] Add context to each task (if missing)
+ * TODO: [🧠] What is better name `prepareTask` or `prepareTaskAndParameters`
+ * TODO: [♨][main] !!! Prepare index the examples and maybe tasks
+ * TODO: Write tests for `preparePipeline`
+ * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
+ * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
+ * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
+ */

package/esm/typings/src/prepare/unpreparePipeline.d.ts

@@ -0,0 +1,12 @@
+import type { PipelineJson } from '../pipeline/PipelineJson/PipelineJson';
+/**
+ * Unprepare just strips the preparation data of the pipeline
+ *
+ * @public exported from `@promptbook/core`
+ */
+export declare function unpreparePipeline(pipeline: PipelineJson): PipelineJson;
+/**
+ * TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline`
+ * TODO: Write tests for `preparePipeline`
+ * TODO: [🍙] Make some standard order of json properties
+ */

package/esm/typings/src/scrapers/_common/Converter.d.ts

@@ -0,0 +1,22 @@
+import type { ScraperAndConverterMetadata } from './register/ScraperAndConverterMetadata';
+import type { ScraperSourceHandler } from './Scraper';
+import type { ScraperIntermediateSource } from './ScraperIntermediateSource';
+/**
+ * @@@
+ *
+ */
+export type Converter = {
+    /**
+     * Metadata of the converter which includes title, mime types, etc.
+     */
+    readonly metadata: ScraperAndConverterMetadata;
+    /**
+     * Convert the the file and returns intermediate source or `null` if it can't convert it
+     *
+     * For example, convert a `.docx` to `.doc` file
+     * Or convert a `.pdf` to `.md` file
+     *
+     * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
+     */
+    $convert(source: ScraperSourceHandler): Promise<ScraperIntermediateSource>;
+};

package/esm/typings/src/scrapers/_common/Scraper.d.ts

@@ -0,0 +1,59 @@
+import type { Promisable } from 'type-fest';
+import type { KnowledgePiecePreparedJson } from '../../pipeline/PipelineJson/KnowledgePieceJson';
+import type { string_filename } from '../../types/typeAliases';
+import type { string_knowledge_source_link } from '../../types/typeAliases';
+import type { string_mime_type } from '../../types/typeAliases';
+import type { string_url } from '../../types/typeAliases';
+import type { ScraperAndConverterMetadata } from './register/ScraperAndConverterMetadata';
+/**
+ * @@@
+ *
+ */
+export type Scraper = {
+    /**
+     * Metadata of the scraper which includes title, mime types, etc.
+     */
+    readonly metadata: ScraperAndConverterMetadata;
+    /**
+     * Scrapes the markdown file and returns the knowledge pieces or `null` if it can't scrape it
+     */
+    scrape(source: ScraperSourceHandler): Promisable<ReadonlyArray<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
+};
+/**
+ * @@@
+ */
+export type ScraperSourceHandler = {
+    /**
+     * The source of the knowledge
+     */
+    readonly source: string_knowledge_source_link;
+    /**
+     * The path to the file, if it is a file
+     *
+     * Note: Typically one of the `filename` or `url` is set and the other is `null`
+     */
+    readonly filename: string_filename | null;
+    /**
+     * The URL, if it is online
+     *
+     * Note: Typically one of the `filename` or `url` is set and the other is `null`
+     */
+    readonly url: string_url | null;
+    /**
+     * Mime type of the source
+     */
+    readonly mimeType: string_mime_type;
+    /**
+     * Get the content as parsed JSON
+     */
+    asJson(): Promisable<unknown>;
+    /**
+     * Get the content as a utf-8 string
+     */
+    asText(): Promisable<string>;
+};
+/**
+ * TODO: [🥽] Add ` asBlob(): Promisable<Blob>;` or asFile
+ * TODO: [🐝] @@@ Annotate all
+ * TODO: [🔼] Export via types
+ */

package/esm/typings/src/scrapers/_common/ScraperIntermediateSource.d.ts

@@ -0,0 +1,11 @@
+import type { IDestroyable } from 'destroyable';
+import type { string_absolute_filename } from '../../types/typeAliases';
+/**
+ * @@@
+ */
+export type ScraperIntermediateSource = IDestroyable & {
+    /**
+     * @@@
+     */
+    readonly filename: string_absolute_filename;
+};

package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts

@@ -0,0 +1,19 @@
+import type { KnowledgePiecePreparedJson } from '../../pipeline/PipelineJson/KnowledgePieceJson';
+import type { KnowledgeSourceJson } from '../../pipeline/PipelineJson/KnowledgeSourceJson';
+import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
+import type { ExecutionTools } from '../../execution/ExecutionTools';
+/**
+ * Prepares the knowle
+ *
+ * @see https://github.com/webgptorg/promptbook/discussions/41
+ * @public exported from `@promptbook/core`
+ */
+export declare function prepareKnowledgePieces(knowledgeSources: ReadonlyArray<KnowledgeSourceJson>, tools: Pick<ExecutionTools, 'llm' | 'fs' | 'scrapers'>, options: PrepareAndScrapeOptions): Promise<ReadonlyArray<Omit<KnowledgePiecePreparedJson, 'preparationIds'>>>;
+/**
+ * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
+ *       Put `knowledgePieces` into `PrepareKnowledgeOptions`
+ * TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]]
+ * TODO: [🧠][❎] Do here propper M:N mapping
+ *       [x] One source can make multiple pieces
+ *       [ ] One piece can have multiple sources
+ */

package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.test.d.ts

@@ -0,0 +1 @@
+export {};