@promptbook/node 0.71.0-0 → 0.71.0-7

package/esm/typings/src/_packages/browser.index.d.ts

@@ -1,5 +1,5 @@
 import { PROMPTBOOK_VERSION } from '../version';
-import { SimplePromptInterfaceTools } from '../knowledge/dialogs/simple-prompt/SimplePromptInterfaceTools';
+import { SimplePromptInterfaceTools } from '../dialogs/simple-prompt/SimplePromptInterfaceTools';
 import { getLocalStorage } from '../storage/local-storage/getLocalStorage';
 import { getSessionStorage } from '../storage/local-storage/getSessionStorage';
 export { PROMPTBOOK_VERSION };

package/esm/typings/src/_packages/core.index.d.ts

@@ -13,6 +13,7 @@ import { MAX_FILENAME_LENGTH } from '../config';
 import { MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH } from '../config';
 import { MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL } from '../config';
 import { EXECUTIONS_CACHE_DIRNAME } from '../config';
+import { SCRAPE_CACHE_DIRNAME } from '../config';
 import { PIPELINE_COLLECTION_BASE_FILENAME } from '../config';
 import { RESERVED_PARAMETER_NAMES } from '../config';
 import { DEFAULT_REMOTE_URL } from '../config';
@@ -20,17 +21,21 @@ import { DEFAULT_REMOTE_URL_PATH } from '../config';
 import { DEFAULT_CSV_SETTINGS } from '../config';
 import { IS_VERBOSE } from '../config';
 import { pipelineJsonToString } from '../conversion/pipelineJsonToString';
-import type { PipelineStringToJsonOptions } from '../conversion/pipelineStringToJson';
 import { pipelineStringToJson } from '../conversion/pipelineStringToJson';
 import { pipelineStringToJsonSync } from '../conversion/pipelineStringToJsonSync';
 import { prettifyPipelineString } from '../conversion/prettify/prettifyPipelineString';
 import { stringifyPipelineJson } from '../conversion/utils/stringifyPipelineJson';
 import { validatePipeline } from '../conversion/validation/validatePipeline';
+import { CallbackInterfaceTools } from '../dialogs/callback/CallbackInterfaceTools';
+import type { CallbackInterfaceToolsOptions } from '../dialogs/callback/CallbackInterfaceToolsOptions';
+import { AbstractFormatError } from '../errors/AbstractFormatError';
 import { CollectionError } from '../errors/CollectionError';
 import { EnvironmentMismatchError } from '../errors/EnvironmentMismatchError';
 import { ExpectError } from '../errors/ExpectError';
 import { ERRORS } from '../errors/index';
+import { KnowledgeScrapeError } from '../errors/KnowledgeScrapeError';
 import { LimitReachedError } from '../errors/LimitReachedError';
+import { MissingToolsError } from '../errors/MissingToolsError';
 import { NotFoundError } from '../errors/NotFoundError';
 import { NotYetImplementedError } from '../errors/NotYetImplementedError';
 import { ParseError } from '../errors/ParseError';
@@ -47,12 +52,9 @@ import { isPassingExpectations } from '../execution/utils/checkExpectations';
 import { usageToHuman } from '../execution/utils/usageToHuman';
 import { usageToWorktime } from '../execution/utils/usageToWorktime';
 import { CsvFormatDefinition } from '../formats/csv/CsvFormatDefinition';
+import { CsvFormatError } from '../formats/csv/CsvFormatError';
 import { MANDATORY_CSV_SETTINGS } from '../formats/csv/CsvSettings';
 import { TextFormatDefinition } from '../formats/text/TextFormatDefinition';
-import { CallbackInterfaceTools } from '../knowledge/dialogs/callback/CallbackInterfaceTools';
-import type { CallbackInterfaceToolsOptions } from '../knowledge/dialogs/callback/CallbackInterfaceToolsOptions';
-import { prepareKnowledgePieces } from '../knowledge/prepare-knowledge/_common/prepareKnowledgePieces';
-import { prepareKnowledgeFromMarkdown } from '../knowledge/prepare-knowledge/markdown/prepareKnowledgeFromMarkdown';
 import { $llmToolsMetadataRegister } from '../llm-providers/_common/$llmToolsMetadataRegister';
 import { $llmToolsRegister } from '../llm-providers/_common/$llmToolsRegister';
 import { createLlmToolsFromConfiguration } from '../llm-providers/_common/createLlmToolsFromConfiguration';
@@ -68,6 +70,13 @@ import { isPipelinePrepared } from '../prepare/isPipelinePrepared';
 import { preparePipeline } from '../prepare/preparePipeline';
 import { prepareTemplates } from '../prepare/prepareTemplates';
 import { unpreparePipeline } from '../prepare/unpreparePipeline';
+import { prepareKnowledgePieces } from '../scrapers/_common/prepareKnowledgePieces';
+import { legacyDocumentScraper } from '../scrapers/document-legacy/legacyDocumentScraper';
+import { documentScraper } from '../scrapers/document/documentScraper';
+import { SCRAPERS } from '../scrapers/index';
+import { markdownScraper } from '../scrapers/markdown/markdownScraper';
+import { pdfScraper } from '../scrapers/pdf/pdfScraper';
+import { websiteScraper } from '../scrapers/website/websiteScraper';
 import { MemoryStorage } from '../storage/memory/MemoryStorage';
 import { PrefixStorage } from '../storage/utils/PrefixStorage';
 import { executionReportJsonToString } from '../types/execution-report/executionReportJsonToString';
@@ -90,6 +99,7 @@ export { MAX_FILENAME_LENGTH };
 export { MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH };
 export { MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL };
 export { EXECUTIONS_CACHE_DIRNAME };
+export { SCRAPE_CACHE_DIRNAME };
 export { PIPELINE_COLLECTION_BASE_FILENAME };
 export { RESERVED_PARAMETER_NAMES };
 export { DEFAULT_REMOTE_URL };
@@ -97,17 +107,21 @@ export { DEFAULT_REMOTE_URL_PATH };
 export { DEFAULT_CSV_SETTINGS };
 export { IS_VERBOSE };
 export { pipelineJsonToString };
-export type { PipelineStringToJsonOptions };
 export { pipelineStringToJson };
 export { pipelineStringToJsonSync };
 export { prettifyPipelineString };
 export { stringifyPipelineJson };
 export { validatePipeline };
+export { CallbackInterfaceTools };
+export type { CallbackInterfaceToolsOptions };
+export { AbstractFormatError };
 export { CollectionError };
 export { EnvironmentMismatchError };
 export { ExpectError };
 export { ERRORS };
+export { KnowledgeScrapeError };
 export { LimitReachedError };
+export { MissingToolsError };
 export { NotFoundError };
 export { NotYetImplementedError };
 export { ParseError };
@@ -124,12 +138,9 @@ export { isPassingExpectations };
 export { usageToHuman };
 export { usageToWorktime };
 export { CsvFormatDefinition };
+export { CsvFormatError };
 export { MANDATORY_CSV_SETTINGS };
 export { TextFormatDefinition };
-export { CallbackInterfaceTools };
-export type { CallbackInterfaceToolsOptions };
-export { prepareKnowledgePieces };
-export { prepareKnowledgeFromMarkdown };
 export { $llmToolsMetadataRegister };
 export { $llmToolsRegister };
 export { createLlmToolsFromConfiguration };
@@ -145,6 +156,13 @@ export { isPipelinePrepared };
 export { preparePipeline };
 export { prepareTemplates };
 export { unpreparePipeline };
+export { prepareKnowledgePieces };
+export { legacyDocumentScraper };
+export { documentScraper };
+export { SCRAPERS };
+export { markdownScraper };
+export { pdfScraper };
+export { websiteScraper };
 export { MemoryStorage };
 export { PrefixStorage };
 export { executionReportJsonToString };

package/esm/typings/src/_packages/node.index.d.ts

@@ -2,9 +2,13 @@ import { PROMPTBOOK_VERSION } from '../version';
 import { createCollectionFromDirectory } from '../collection/constructors/createCollectionFromDirectory';
 import { createLlmToolsFromConfigurationFromEnv } from '../llm-providers/_common/createLlmToolsFromConfigurationFromEnv';
 import { createLlmToolsFromEnv } from '../llm-providers/_common/createLlmToolsFromEnv';
-import { FilesStorage } from '../storage/files-storage/FilesStorage';
+import { FileCacheStorage } from '../storage/file-cache-storage/FileCacheStorage';
+import { $execCommand } from '../utils/execCommand/$execCommand';
+import { $execCommands } from '../utils/execCommand/$execCommands';
 export { PROMPTBOOK_VERSION };
 export { createCollectionFromDirectory };
 export { createLlmToolsFromConfigurationFromEnv };
 export { createLlmToolsFromEnv };
-export { FilesStorage };
+export { FileCacheStorage };
+export { $execCommand };
+export { $execCommands };

package/esm/typings/src/_packages/types.index.d.ts

@@ -10,9 +10,9 @@ import type { ExpectCommand } from '../commands/EXPECT/ExpectCommand';
 import type { ForeachJson } from '../commands/FOREACH/ForeachJson';
 import type { FormatCommand } from '../commands/FORMAT/FormatCommand';
 import type { TemplateType } from '../commands/TEMPLATE/TemplateTypes';
-import type { PipelineStringToJsonOptions } from '../conversion/pipelineStringToJson';
 import type { PrettifyOptions } from '../conversion/prettify/PrettifyOptions';
 import type { renderPipelineMermaidOptions } from '../conversion/prettify/renderPipelineMermaidOptions';
+import type { CallbackInterfaceToolsOptions } from '../dialogs/callback/CallbackInterfaceToolsOptions';
 import type { ErrorJson } from '../errors/utils/ErrorJson';
 import type { AvailableModel } from '../execution/AvailableModel';
 import type { CommonExecutionToolsOptions } from '../execution/CommonExecutionToolsOptions';
@@ -37,7 +37,6 @@ import type { UserInterfaceTools } from '../execution/UserInterfaceTools';
 import type { UserInterfaceToolsPromptDialogOptions } from '../execution/UserInterfaceTools';
 import type { FormatSubvalueDefinition } from '../formats/_common/FormatSubvalueDefinition';
 import type { CsvSettings } from '../formats/csv/CsvSettings';
-import type { CallbackInterfaceToolsOptions } from '../knowledge/dialogs/callback/CallbackInterfaceToolsOptions';
 import type { LlmToolsConfiguration } from '../llm-providers/_common/LlmToolsConfiguration';
 import type { LlmToolsMetadata } from '../llm-providers/_common/LlmToolsMetadata';
 import type { LlmToolsOptions } from '../llm-providers/_common/LlmToolsOptions';
@@ -65,11 +64,15 @@ import type { RemoteLlmExecutionToolsOptions } from '../llm-providers/remote/int
 import type { RemoteServerOptions } from '../llm-providers/remote/interfaces/RemoteServerOptions';
 import type { AnonymousRemoteServerOptions } from '../llm-providers/remote/interfaces/RemoteServerOptions';
 import type { CollectionRemoteServerOptions } from '../llm-providers/remote/interfaces/RemoteServerOptions';
-import type { PrepareOptions } from '../prepare/PrepareOptions';
+import type { PrepareAndScrapeOptions } from '../prepare/PrepareAndScrapeOptions';
+import type { Converter } from '../scrapers/_common/Converter';
+import type { Scraper } from '../scrapers/_common/Scraper';
+import type { ScraperSourceHandler } from '../scrapers/_common/Scraper';
+import type { ScraperIntermediateSource } from '../scrapers/_common/ScraperIntermediateSource';
 import type { JavascriptExecutionToolsOptions } from '../scripting/javascript/JavascriptExecutionToolsOptions';
 import type { PostprocessingFunction } from '../scripting/javascript/JavascriptExecutionToolsOptions';
 import type { PromptbookStorage } from '../storage/_common/PromptbookStorage';
-import type { FilesStorageOptions } from '../storage/files-storage/FilesStorageOptions';
+import type { FileCacheStorageOptions } from '../storage/file-cache-storage/FileCacheStorageOptions';
 import type { ExecutionPromptReportJson } from '../types/execution-report/ExecutionPromptReportJson';
 import type { ExecutionReportJson } from '../types/execution-report/ExecutionReportJson';
 import type { ExecutionReportString } from '../types/execution-report/ExecutionReportString';
@@ -166,13 +169,12 @@ import type { string_sha256 } from '../types/typeAliases';
 import type { string_semantic_version } from '../types/typeAliases';
 import type { string_version_dependency } from '../types/typeAliases';
 import type { string_file_extension } from '../types/typeAliases';
-import type { string_file_absolute_path } from '../types/typeAliases';
-import type { string_file_relative_path } from '../types/typeAliases';
-import type { string_file_path } from '../types/typeAliases';
-import type { string_folder_absolute_path } from '../types/typeAliases';
-import type { string_folder_relative_path } from '../types/typeAliases';
-import type { string_folder_path } from '../types/typeAliases';
+import type { string_absolute_filename } from '../types/typeAliases';
+import type { string_relative_filename } from '../types/typeAliases';
 import type { string_filename } from '../types/typeAliases';
+import type { string_absolute_dirname } from '../types/typeAliases';
+import type { string_relative_dirname } from '../types/typeAliases';
+import type { string_dirname } from '../types/typeAliases';
 import type { string_person_fullname } from '../types/typeAliases';
 import type { string_person_profile } from '../types/typeAliases';
 import type { string_license } from '../types/typeAliases';
@@ -215,6 +217,8 @@ import type { number_megabytes } from '../types/typeAliases';
 import type { number_gigabytes } from '../types/typeAliases';
 import type { number_terabytes } from '../types/typeAliases';
 import type { Registered } from '../utils/$Register';
+import type { IExecCommandOptions } from '../utils/execCommand/IExecCommandOptions';
+import type { IExecCommandOptionsAdvanced } from '../utils/execCommand/IExecCommandOptions';
 import type { FromtoItems } from '../utils/FromtoItems';
 import type { CodeBlock } from '../utils/markdown/extractAllBlocksFromMarkdown';
 import type { MarkdownSection } from '../utils/markdown/parseMarkdownSection';
@@ -241,9 +245,9 @@ export type { ExpectCommand };
 export type { ForeachJson };
 export type { FormatCommand };
 export type { TemplateType };
-export type { PipelineStringToJsonOptions };
 export type { PrettifyOptions };
 export type { renderPipelineMermaidOptions };
+export type { CallbackInterfaceToolsOptions };
 export type { ErrorJson };
 export type { AvailableModel };
 export type { CommonExecutionToolsOptions };
@@ -268,7 +272,6 @@ export type { UserInterfaceTools };
 export type { UserInterfaceToolsPromptDialogOptions };
 export type { FormatSubvalueDefinition };
 export type { CsvSettings };
-export type { CallbackInterfaceToolsOptions };
 export type { LlmToolsConfiguration };
 export type { LlmToolsMetadata };
 export type { LlmToolsOptions };
@@ -296,11 +299,15 @@ export type { RemoteLlmExecutionToolsOptions };
 export type { RemoteServerOptions };
 export type { AnonymousRemoteServerOptions };
 export type { CollectionRemoteServerOptions };
-export type { PrepareOptions };
+export type { PrepareAndScrapeOptions };
+export type { Converter };
+export type { Scraper };
+export type { ScraperSourceHandler };
+export type { ScraperIntermediateSource };
 export type { JavascriptExecutionToolsOptions };
 export type { PostprocessingFunction };
 export type { PromptbookStorage };
-export type { FilesStorageOptions };
+export type { FileCacheStorageOptions };
 export type { ExecutionPromptReportJson };
 export type { ExecutionReportJson };
 export type { ExecutionReportString };
@@ -397,13 +404,12 @@ export type { string_sha256 };
 export type { string_semantic_version };
 export type { string_version_dependency };
 export type { string_file_extension };
-export type { string_file_absolute_path };
-export type { string_file_relative_path };
-export type { string_file_path };
-export type { string_folder_absolute_path };
-export type { string_folder_relative_path };
-export type { string_folder_path };
+export type { string_absolute_filename };
+export type { string_relative_filename };
 export type { string_filename };
+export type { string_absolute_dirname };
+export type { string_relative_dirname };
+export type { string_dirname };
 export type { string_person_fullname };
 export type { string_person_profile };
 export type { string_license };
@@ -446,6 +452,8 @@ export type { number_megabytes };
 export type { number_gigabytes };
 export type { number_terabytes };
 export type { Registered };
+export type { IExecCommandOptions };
+export type { IExecCommandOptionsAdvanced };
 export type { FromtoItems };
 export type { CodeBlock };
 export type { MarkdownSection };

package/esm/typings/src/cli/cli-commands/about.d.ts

@@ -7,5 +7,5 @@ import type { Command as Program } from 'commander';
 export declare function initializeAboutCommand(program: Program): void;
 /**
  * TODO: [🗽] Unite branding and make single place for it
- * Note: [🟡] This code should never be published outside of `@promptbook/cli`
+ * Note: [🟡] Code in this file should never be published outside of `@promptbook/cli`
  */

package/esm/typings/src/cli/cli-commands/hello.d.ts

@@ -6,5 +6,5 @@ import type { Command as Program } from 'commander';
  */
 export declare function initializeHelloCommand(program: Program): void;
 /**
- * Note: [🟡] This code should never be published outside of `@promptbook/cli`
+ * Note: [🟡] Code in this file should never be published outside of `@promptbook/cli`
  */

package/esm/typings/src/cli/cli-commands/make.d.ts

@@ -9,6 +9,6 @@ export declare function initializeMakeCommand(program: Program): void;
  * TODO: [🥃][main] !!! Allow `ptbk make` without configuring any llm tools
  * TODO: Maybe remove this command - "about" command should be enough?
  * TODO: [0] DRY Javascript and typescript - Maybe make ONLY typescript and for javascript just remove types
- * Note: [🟡] This code should never be published outside of `@promptbook/cli`
+ * Note: [🟡] Code in this file should never be published outside of `@promptbook/cli`
  * TODO: [🖇] What about symlinks? Maybe flag --follow-symlinks
  */

package/esm/typings/src/cli/cli-commands/prettify.d.ts

@@ -6,6 +6,6 @@ import type { Command as Program } from 'commander';
  */
 export declare function initializePrettifyCommand(program: Program): void;
 /**
- * Note: [🟡] This code should never be published outside of `@promptbook/cli`
+ * Note: [🟡] Code in this file should never be published outside of `@promptbook/cli`
  * TODO: [🖇] What about symlinks? Maybe flag --follow-symlinks
  */

package/esm/typings/src/cli/main.d.ts

@@ -8,5 +8,5 @@ export declare const _CLI: {
     _initialize: typeof promptbookCli;
 };
 /**
- * Note: [🟡] This code should never be published outside of `@promptbook/cli`
+ * Note: [🟡] Code in this file should never be published outside of `@promptbook/cli`
  */

package/esm/typings/src/cli/promptbookCli.d.ts

@@ -8,5 +8,5 @@ export declare function promptbookCli(): Promise<void>;
  * TODO: [🥠] Do not export, its just for CLI script
  * TODO: [🕌] When more functionalities, rename
  * Note: 11:11
- * Note: [🟡] This code should never be published outside of `@promptbook/cli`
+ * Note: [🟡] Code in this file should never be published outside of `@promptbook/cli`
  */

package/esm/typings/src/collection/constructors/createCollectionFromDirectory.d.ts

@@ -1,10 +1,13 @@
-import type { PipelineStringToJsonOptions } from '../../conversion/pipelineStringToJson';
-import type { string_folder_path } from '../../types/typeAliases';
+import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
+import type { string_dirname } from '../../types/typeAliases';
 import type { PipelineCollection } from '../PipelineCollection';
 /**
  * Options for `createCollectionFromDirectory` function
+ *
+ * Note: `rootDirname` is not needed because it is the folder in which `.ptbk.md` file is located
+ *       This is not same as `path` which is the first argument of `createCollectionFromDirectory` - it can be a subfolder
  */
-type CreatePipelineCollectionFromDirectoryOptions = PipelineStringToJsonOptions & {
+type CreatePipelineCollectionFromDirectoryOptions = Omit<PrepareAndScrapeOptions, 'rootDirname'> & {
     /**
      * If true, the directory is searched recursively for pipelines
      *
@@ -41,9 +44,9 @@ type CreatePipelineCollectionFromDirectoryOptions = PipelineStringToJsonOptions
  * @returns PipelineCollection
  * @public exported from `@promptbook/node`
  */
-export declare function createCollectionFromDirectory(path: string_folder_path, options?: CreatePipelineCollectionFromDirectoryOptions): Promise<PipelineCollection>;
+export declare function createCollectionFromDirectory(path: string_dirname, options?: CreatePipelineCollectionFromDirectoryOptions): Promise<PipelineCollection>;
 export {};
 /**
- * Note: [🟢] This code should never be published outside of `@promptbook/node` and `@promptbook/cli` and `@promptbook/cli`
+ * Note: [🟢] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
  * TODO: [🖇] What about symlinks? Maybe option isSymlinksFollowed
  */

package/esm/typings/src/collection/constructors/createCollectionFromUrl.d.ts

@@ -25,5 +25,5 @@ type CreatePipelineCollectionFromUrlyOptions = {
 export declare function createCollectionFromUrl(url: string_url | URL, options: CreatePipelineCollectionFromUrlyOptions): Promise<PipelineCollection>;
 export {};
 /**
- * TODO:[main] !!!! [🧠] Library precompilation and do not mix markdown and json promptbooks
+ * TODO: [main] !!!! [🧠] Library precompilation and do not mix markdown and json promptbooks
  */

package/esm/typings/src/commands/EXPECT/expectCommandParser.d.ts

@@ -3,7 +3,7 @@ import type { ExpectCommand } from './ExpectCommand';
 /**
  * Parses the expect command
  *
- * @see ./EXPECT-README.md for more details
+ * @see `documentationUrl` for more details
  * @private within the commands folder
  */
 export declare const expectCommandParser: PipelineTemplateCommandParser<ExpectCommand>;

package/esm/typings/src/commands/FOREACH/ForeachJson.d.ts

@@ -19,5 +19,9 @@ export type ForeachJson = {
     /**
      * @@@
      */
-    readonly subparameterNames: Array<string_parameter_name>;
+    readonly inputSubparameterNames: Array<string_parameter_name>;
+    /**
+     * @@@
+     */
+    readonly outputSubparameterName: string_parameter_name;
 };

package/esm/typings/src/commands/FOREACH/foreachCommandParser.d.ts

@@ -5,7 +5,7 @@ import type { ForeachCommand } from './ForeachCommand';
  *
  * Note: @@@ This command is used as foreach for new commands - it should NOT be used in any `.ptbk.md` file
  *
- * @see ./FOREACH-README.md for more details <- TODO: @@@ Write theese README files OR remove this link + add annotation here (to all commands)
+ * @see `documentationUrl` for more details
  * @private within the commands folder
  */
 export declare const foreachCommandParser: PipelineTemplateCommandParser<ForeachCommand>;

package/esm/typings/src/commands/FORMAT/formatCommandParser.d.ts

@@ -3,7 +3,7 @@ import type { FormatCommand } from './FormatCommand';
 /**
  * Parses the format command
  *
- * @see ./FORMAT-README.md for more details
+ * @see `documentationUrl` for more details
  * @private within the commands folder
  */
 export declare const formatCommandParser: PipelineTemplateCommandParser<FormatCommand>;

package/esm/typings/src/commands/JOKER/jokerCommandParser.d.ts

@@ -3,7 +3,7 @@ import type { JokerCommand } from './JokerCommand';
 /**
  * Parses the joker command
  *
- * @see ./JOKER-README.md for more details
+ * @see `documentationUrl` for more details
  * @private within the commands folder
  */
 export declare const jokerCommandParser: PipelineTemplateCommandParser<JokerCommand>;

package/esm/typings/src/commands/KNOWLEDGE/knowledgeCommandParser.d.ts

@@ -3,7 +3,7 @@ import type { KnowledgeCommand } from './KnowledgeCommand';
 /**
  * Parses the knowledge command
  *
- * @see ./KNOWLEDGE-README.md for more details
+ * @see `documentationUrl` for more details
  * @private within the commands folder
  */
 export declare const knowledgeCommandParser: PipelineHeadCommandParser<KnowledgeCommand>;

package/esm/typings/src/commands/KNOWLEDGE/utils/sourceContentToName.d.ts

@@ -0,0 +1,11 @@
+import type { string_knowledge_source_content } from '../../../types/typeAliases';
+import type { string_name } from '../../../types/typeAliases';
+/**
+ * Creates unique name for the source
+ *
+ * @private within the repository
+ */
+export declare function sourceContentToName(sourceContent: string_knowledge_source_content): string_name;
+/**
+ * TODO: [🐱‍🐉][🧠] Make some smart crop NOT source-i-m-pavol-a-develop-... BUT source-i-m-pavol-a-developer-...
+ */

package/esm/typings/src/commands/KNOWLEDGE/utils/sourceContentToName.test.d.ts

@@ -0,0 +1,4 @@
+export {};
+/**
+ * TODO: [🐱‍🐉][🧠] Make some smart crop NOT source-i-m-pavol-a-develop-... BUT source-i-m-pavol-a-developer-...
+ */

package/esm/typings/src/commands/MODEL/modelCommandParser.d.ts

@@ -3,7 +3,7 @@ import type { ModelCommand } from './ModelCommand';
 /**
  * Parses the model command
  *
- * @see ./MODEL-README.md for more details
+ * @see `documentationUrl` for more details
  * @private within the commands folder
  */
 export declare const modelCommandParser: PipelineBothCommandParser<ModelCommand>;

package/esm/typings/src/commands/PARAMETER/parameterCommandParser.d.ts

@@ -3,7 +3,7 @@ import type { ParameterCommand } from './ParameterCommand';
 /**
  * Parses the parameter command
  *
- * @see ./PARAMETER-README.md for more details
+ * @see `documentationUrl` for more details
  * @private within the commands folder
  */
 export declare const parameterCommandParser: PipelineBothCommandParser<ParameterCommand>;

package/esm/typings/src/commands/PERSONA/personaCommandParser.d.ts

@@ -3,7 +3,7 @@ import type { PersonaCommand } from './PersonaCommand';
 /**
  * Parses the persona command
  *
- * @see ./PERSONA-README.md for more details
+ * @see `documentationUrl` for more details
  * @private within the commands folder
  */
 export declare const personaCommandParser: PipelineBothCommandParser<PersonaCommand>;

package/esm/typings/src/commands/POSTPROCESS/postprocessCommandParser.d.ts

@@ -3,7 +3,7 @@ import type { PostprocessCommand } from './PostprocessCommand';
 /**
  * Parses the postprocess command
  *
- * @see ./POSTPROCESS-README.md for more details
+ * @see `documentationUrl` for more details
  * @private within the commands folder
  */
 export declare const postprocessCommandParser: PipelineTemplateCommandParser<PostprocessCommand>;

package/esm/typings/src/commands/PROMPTBOOK_VERSION/promptbookVersionCommandParser.d.ts

@@ -3,7 +3,7 @@ import type { PromptbookVersionCommand } from './PromptbookVersionCommand';
 /**
  * Parses the PROMPTBOOK_VERSION command
  *
- * @see ./PROMPTBOOK_VERSION-README.md for more details
+ * @see `documentationUrl` for more details
  * @private within the commands folder
  */
 export declare const promptbookVersionCommandParser: PipelineHeadCommandParser<PromptbookVersionCommand>;

package/esm/typings/src/commands/TEMPLATE/templateCommandParser.d.ts

@@ -3,7 +3,7 @@ import type { TemplateCommand } from './TemplateCommand';
 /**
  * Parses the template command
  *
- * @see ./TEMPLATE-README.md for more details
+ * @see `documentationUrl` for more details
  * @private within the commands folder
  */
 export declare const templateCommandParser: PipelineTemplateCommandParser<TemplateCommand>;

package/esm/typings/src/commands/URL/urlCommandParser.d.ts

@@ -3,7 +3,7 @@ import type { UrlCommand } from './UrlCommand';
 /**
  * Parses the url command
  *
- * @see ./URL-README.md for more details
+ * @see `documentationUrl` for more details
  * @private within the commands folder
  */
 export declare const urlCommandParser: PipelineHeadCommandParser<UrlCommand>;

package/esm/typings/src/commands/X_ACTION/actionCommandParser.d.ts

@@ -3,7 +3,7 @@ import type { ActionCommand } from './ActionCommand';
 /**
  * Parses the action command
  *
- * @see ./ACTION-README.md for more details
+ * @see `documentationUrl` for more details
  * @private within the commands folder
  */
 export declare const actionCommandParser: PipelineHeadCommandParser<ActionCommand>;

package/esm/typings/src/commands/X_INSTRUMENT/instrumentCommandParser.d.ts

@@ -3,7 +3,7 @@ import type { InstrumentCommand } from './InstrumentCommand';
 /**
  * Parses the instrument command
  *
- * @see ./INSTRUMENT-README.md for more details
+ * @see `documentationUrl` for more details
  * @private within the commands folder
  */
 export declare const instrumentCommandParser: PipelineHeadCommandParser<InstrumentCommand>;

package/esm/typings/src/commands/_BOILERPLATE/boilerplateCommandParser.d.ts

@@ -5,7 +5,7 @@ import type { BoilerplateCommand } from './BoilerplateCommand';
  *
  * Note: @@@ This command is used as boilerplate for new commands - it should NOT be used in any `.ptbk.md` file
  *
- * @see ./BOILERPLATE-README.md for more details <- TODO: @@@ Write theese README files OR remove this link + add annotation here (to all commands)
+ * @see `documentationUrl` for more details
  * @private within the commands folder
  */
 export declare const boilerplateCommandParser: PipelineBothCommandParser<BoilerplateCommand>;

package/esm/typings/src/config.d.ts

@@ -84,9 +84,19 @@ export declare const MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL = 200;
 /**
  * Where to store the cache of executions for promptbook CLI
  *
+ * Note: When the folder does not exist, it is created recursively
+ *
  * @public exported from `@promptbook/core`
  */
 export declare const EXECUTIONS_CACHE_DIRNAME = "/.promptbook/executions-cache";
+/**
+ * Where to store the scrape cache
+ *
+ * Note: When the folder does not exist, it is created recursively
+ *
+ * @public exported from `@promptbook/core`
+ */
+export declare const SCRAPE_CACHE_DIRNAME = "/.promptbook/scrape-cache";
 /**
  * The name of the builded pipeline collection made by CLI `ptbk make` and for lookup in `createCollectionFromDirectory`
  *

package/esm/typings/src/conversion/pipelineStringToJson.d.ts

@@ -1,19 +1,6 @@
-import type { LlmExecutionTools } from '../execution/LlmExecutionTools';
+import type { PrepareAndScrapeOptions } from '../prepare/PrepareAndScrapeOptions';
 import type { PipelineJson } from '../types/PipelineJson/PipelineJson';
 import type { PipelineString } from '../types/PipelineString';
-/**
- * Options for `pipelineStringToJson`
- *
- * @public exported from `@promptbook/core`
- */
-export type PipelineStringToJsonOptions = {
-    /**
-     * Tools for processing required for preparation of knowledge *(not for actual execution)*
-     *
-     * Note: If you provide `null`, the knowledge will not be prepared
-     */
-    readonly llmTools: LlmExecutionTools | null;
-};
 /**
  * Compile pipeline from string (markdown) format to JSON format
  *
@@ -31,7 +18,7 @@ export type PipelineStringToJsonOptions = {
  * @throws {ParseError} if the promptbook string is not valid
  * @public exported from `@promptbook/core`
  */
-export declare function pipelineStringToJson(pipelineString: PipelineString, options?: PipelineStringToJsonOptions): Promise<PipelineJson>;
+export declare function pipelineStringToJson(pipelineString: PipelineString, options?: PrepareAndScrapeOptions): Promise<PipelineJson>;
 /**
  * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
  * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools

package/esm/typings/src/conversion/pipelineStringToJsonSync.d.ts

@@ -18,7 +18,7 @@ import type { PipelineString } from '../types/PipelineString';
  */
 export declare function pipelineStringToJsonSync(pipelineString: PipelineString): PipelineJson;
 /**
- * TODO:[main] !!!! Warn if used only sync version
+ * TODO: [main] !!!! Warn if used only sync version
  * TODO: [🚞] Report here line/column of error
  * TODO: Use spaceTrim more effectively
  * TODO: [🧠] Parameter flags - isInput, isOutput, isInternal

package/esm/typings/src/conversion/validation/_importPipeline.d.ts

@@ -25,5 +25,5 @@ export declare function importPipelineJson(path: `${string}.ptbk.json`): Pipelin
  */
 export declare function importPipelineJsonAsString(path: `${string}.ptbk.json`): string_json<PipelineJson>;
 /**
- * TODO: [🥯] This should be part of scripting + testing utilities set which are not builded into packages
+ * Note: [⚫] Code in this file should never be published in any package
  */

package/esm/typings/src/{knowledge/dialogs → dialogs}/callback/CallbackInterfaceTools.d.ts

@@ -1,5 +1,5 @@
-import type { UserInterfaceTools } from '../../../execution/UserInterfaceTools';
-import type { UserInterfaceToolsPromptDialogOptions } from '../../../execution/UserInterfaceTools';
+import type { UserInterfaceTools } from '../../execution/UserInterfaceTools';
+import type { UserInterfaceToolsPromptDialogOptions } from '../../execution/UserInterfaceTools';
 import type { CallbackInterfaceToolsOptions } from './CallbackInterfaceToolsOptions';
 /**
  * Delagates the user interaction to a async callback function