@promptbook/core 0.71.0-13 → 0.71.0-14

package/esm/typings/src/_packages/node.index.d.ts

@@ -3,6 +3,7 @@ import { createCollectionFromDirectory } from '../collection/constructors/create
 import { $provideExecutionToolsForNode } from '../execution/utils/$provideExecutionToolsForNode';
 import { $provideLlmToolsConfigurationFromEnv } from '../llm-providers/_common/register/$provideLlmToolsConfigurationFromEnv';
 import { $provideLlmToolsFromEnv } from '../llm-providers/_common/register/$provideLlmToolsFromEnv';
+import { $provideFilesystemForNode } from '../scrapers/_common/register/$provideFilesystemForNode';
 import { $provideScrapersForNode } from '../scrapers/_common/register/$provideScrapersForNode';
 import { FileCacheStorage } from '../storage/file-cache-storage/FileCacheStorage';
 import { $execCommand } from '../utils/execCommand/$execCommand';
@@ -12,6 +13,7 @@ export { createCollectionFromDirectory };
 export { $provideExecutionToolsForNode };
 export { $provideLlmToolsConfigurationFromEnv };
 export { $provideLlmToolsFromEnv };
+export { $provideFilesystemForNode };
 export { $provideScrapersForNode };
 export { FileCacheStorage };
 export { $execCommand };

package/esm/typings/src/_packages/types.index.d.ts

@@ -20,6 +20,7 @@ import type { CreatePipelineExecutorOptions } from '../execution/createPipelineE
 import type { CreatePipelineExecutorSettings } from '../execution/createPipelineExecutor/00-CreatePipelineExecutorSettings';
 import type { EmbeddingVector } from '../execution/EmbeddingVector';
 import type { ExecutionTools } from '../execution/ExecutionTools';
+import type { FilesystemTools } from '../execution/FilesystemTools';
 import type { LlmExecutionTools } from '../execution/LlmExecutionTools';
 import type { LlmExecutionToolsConstructor } from '../execution/LlmExecutionToolsConstructor';
 import type { PipelineExecutor } from '../execution/PipelineExecutor';
@@ -259,6 +260,7 @@ export type { CreatePipelineExecutorOptions };
 export type { CreatePipelineExecutorSettings };
 export type { EmbeddingVector };
 export type { ExecutionTools };
+export type { FilesystemTools };
 export type { LlmExecutionTools };
 export type { LlmExecutionToolsConstructor };
 export type { PipelineExecutor };

package/esm/typings/src/collection/constructors/createCollectionFromDirectory.d.ts

@@ -46,9 +46,9 @@ type CreatePipelineCollectionFromDirectoryOptions = Omit<PrepareAndScrapeOptions
  * @returns PipelineCollection
  * @public exported from `@promptbook/node`
  */
-export declare function createCollectionFromDirectory(path: string_dirname, tools?: Pick<ExecutionTools, 'llm' | 'scrapers'>, options?: CreatePipelineCollectionFromDirectoryOptions): Promise<PipelineCollection>;
+export declare function createCollectionFromDirectory(path: string_dirname, tools?: Pick<ExecutionTools, 'llm' | 'fs' | 'scrapers'>, options?: CreatePipelineCollectionFromDirectoryOptions): Promise<PipelineCollection>;
 export {};
 /**
- * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
  * TODO: [🖇] What about symlinks? Maybe option isSymlinksFollowed
+ * TODO: Maybe move from `@promptbook/node` to `@promptbook/core` as we removes direct dependency on `fs`
  */

package/esm/typings/src/conversion/pipelineStringToJson.d.ts

@@ -20,7 +20,7 @@ import type { PipelineString } from '../types/PipelineString';
  * @throws {ParseError} if the promptbook string is not valid
  * @public exported from `@promptbook/core`
  */
-export declare function pipelineStringToJson(pipelineString: PipelineString, tools?: Pick<ExecutionTools, 'llm' | 'scrapers'>, options?: PrepareAndScrapeOptions): Promise<PipelineJson>;
+export declare function pipelineStringToJson(pipelineString: PipelineString, tools?: Pick<ExecutionTools, 'llm' | 'fs' | 'scrapers'>, options?: PrepareAndScrapeOptions): Promise<PipelineJson>;
 /**
  * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
  * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools

package/esm/typings/src/execution/ExecutionTools.d.ts

@@ -1,5 +1,6 @@
 import type { Scraper } from '../scrapers/_common/Scraper';
 import type { Arrayable } from '../types/Arrayable';
+import type { FilesystemTools } from './FilesystemTools';
 import type { LlmExecutionTools } from './LlmExecutionTools';
 import type { ScriptExecutionTools } from './ScriptExecutionTools';
 import type { UserInterfaceTools } from './UserInterfaceTools';
@@ -12,11 +13,19 @@ export type ExecutionTools = {
     /**
      * Tools for executing prompts in large language models
      *
-     * Tip: Use `$provideLlmToolsFromEnv()` to use all available LLM providers you configured
+     * Tip: Use `$provideLlmToolsFromEnv` to use all available LLM providers you configured
      *
      * @default [] - If not provided, no LLM execution will be possible - it does not make sense in most cases
      */
     readonly llm?: Arrayable<LlmExecutionTools>;
+    /**
+     * Tools for filesystem operations
+     *
+     * Tip: Use `$provideFilesystemForNode` to use filesystem in Node.js
+     *
+     * @default undefined - If not provided, no filesystem operations will be possible
+     */
+    readonly fs?: FilesystemTools;
     /**
      * Scrapers for extracting knowledge from external sources
      *
@@ -30,7 +39,7 @@ export type ExecutionTools = {
      *
      * Note: You can pass multiple ScriptExecutionTools, they will be tried one by one until one of them supports the script
      *       If none of them supports the script, an error is thrown
-     * Tip: Use here `new JavascriptExecutionTools()`
+     * Tip: Use here `new JavascriptExecutionTools`
      *
      * @default [] - If not provided, no script execution will be possible
      */
@@ -43,6 +52,7 @@ export type ExecutionTools = {
     readonly userInterface?: UserInterfaceTools;
 };
 /**
+ * TODO: !!!!!! Move here also the executables and make $provideXxxxForNode
  * TODO: [🍂] Maybe make llm = $provideLlmToolsFromEnv() without problem with bundle contaminated by only `@promptbook/node` and `@promptbook/cli` stuff
  * TODO: [🍂] Same with scrapers
  */

package/esm/typings/src/execution/FilesystemTools.d.ts

@@ -0,0 +1,9 @@
+/// <reference types="node" />
+import type fs from 'fs/promises';
+/**
+ * Container for all the tools needed to manipulate with filesystem
+ */
+export type FilesystemTools = Pick<typeof fs, 'access' | 'constants' | 'readFile' | 'stat' | 'readdir'>;
+/**
+ * TODO: Implement destroyable pattern to free resources
+ */

package/esm/typings/src/execution/translation/automatic-translate/translateMessages.d.ts

@@ -8,4 +8,5 @@ export declare function translateMessages({ automaticTranslator, from, to, }: {
 } & TranslatorOptions): Promise<void>;
 /**
  * TODO: [😶] Unite floder listing
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
  */

package/esm/typings/src/prepare/preparePipeline.d.ts

@@ -9,7 +9,7 @@ import type { PrepareAndScrapeOptions } from './PrepareAndScrapeOptions';
  * Note: When the pipeline is already prepared, it returns the same pipeline
  * @public exported from `@promptbook/core`
  */
-export declare function preparePipeline(pipeline: PipelineJson, tools: Pick<ExecutionTools, 'llm' | 'scrapers'>, options: PrepareAndScrapeOptions): Promise<PipelineJson>;
+export declare function preparePipeline(pipeline: PipelineJson, tools: Pick<ExecutionTools, 'llm' | 'fs' | 'scrapers'>, options: PrepareAndScrapeOptions): Promise<PipelineJson>;
 /**
  * TODO: Write tests for `preparePipeline`
  * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch

package/esm/typings/src/prepare/prepareTemplates.d.ts

@@ -19,7 +19,7 @@ type PreparedTemplates = {
  *
  * @public exported from `@promptbook/core`
  */
-export declare function prepareTemplates(pipeline: PrepareTemplateInput, tools: Pick<ExecutionTools, 'llm' | 'scrapers'>, options: PrepareAndScrapeOptions): Promise<PreparedTemplates>;
+export declare function prepareTemplates(pipeline: PrepareTemplateInput, tools: Pick<ExecutionTools, 'llm' | 'fs' | 'scrapers'>, options: PrepareAndScrapeOptions): Promise<PreparedTemplates>;
 export {};
 /**
  * TODO: [🧠] Add context to each template (if missing)

package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts

@@ -8,7 +8,7 @@ import type { ExecutionTools } from '../../execution/ExecutionTools';
  * @see https://github.com/webgptorg/promptbook/discussions/41
  * @public exported from `@promptbook/core`
  */
-export declare function prepareKnowledgePieces(knowledgeSources: Array<KnowledgeSourceJson>, tools: Pick<ExecutionTools, 'llm' | 'scrapers'>, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'preparationIds'>>>;
+export declare function prepareKnowledgePieces(knowledgeSources: Array<KnowledgeSourceJson>, tools: Pick<ExecutionTools, 'llm' | 'fs' | 'scrapers'>, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'preparationIds'>>>;
 /**
  * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
  *       Put `knowledgePieces` into `PrepareKnowledgeOptions`

package/esm/typings/src/scrapers/_common/register/$provideFilesystemForNode.d.ts

@@ -0,0 +1,11 @@
+import type { FilesystemTools } from '../../../execution/FilesystemTools';
+import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
+/**
+ * @@@
+ *
+ * @public exported from `@promptbook/node`
+ */
+export declare function $provideFilesystemForNode(options?: Pick<PrepareAndScrapeOptions, 'isVerbose'>): FilesystemTools;
+/**
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ */

package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts

@@ -30,4 +30,5 @@ export {};
  * 1) Need to store more than serialized JSONs
  * 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: !!!!
  * TODO: [🐱‍🐉][🧠] Make some smart crop
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
  */

package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts

@@ -1,4 +1,5 @@
 import type { SetOptional } from 'type-fest';
+import type { ExecutionTools } from '../../../execution/ExecutionTools';
 import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
 import type { KnowledgeSourceJson } from '../../../types/PipelineJson/KnowledgeSourceJson';
 import type { ScraperSourceHandler } from '../Scraper';
@@ -7,7 +8,4 @@ import type { ScraperSourceHandler } from '../Scraper';
  *
  * @private for scraper utilities
  */
-export declare function makeKnowledgeSourceHandler(knowledgeSource: SetOptional<KnowledgeSourceJson, 'name'>, options?: Pick<PrepareAndScrapeOptions, 'rootDirname' | 'isVerbose'>): Promise<ScraperSourceHandler>;
-/**
- * TODO: !!!!!!! Maybe constrain to @promptbook/node bundle
- */
+export declare function makeKnowledgeSourceHandler(knowledgeSource: SetOptional<KnowledgeSourceJson, 'name'>, tools: Pick<ExecutionTools, 'fs'>, options?: Pick<PrepareAndScrapeOptions, 'rootDirname' | 'isVerbose'>): Promise<ScraperSourceHandler>;

package/esm/typings/src/scrapers/document/DocumentScraper.d.ts

@@ -23,7 +23,7 @@ export declare class DocumentScraper implements Converter, Scraper {
      * Markdown scraper is used internally
      */
     private readonly markdownScraper;
-    constructor(tools: Pick<ExecutionTools, 'llm'>, options: PrepareAndScrapeOptions);
+    constructor(tools: Pick<ExecutionTools, 'fs' | 'llm'>, options: PrepareAndScrapeOptions);
     /**
      * Convert the `.docx` or `.odt`  to `.md` file and returns intermediate source
      *

package/esm/typings/src/scrapers/document-legacy/LegacyDocumentScraper.d.ts

@@ -23,7 +23,7 @@ export declare class LegacyDocumentScraper implements Converter, Scraper {
      * Document scraper is used internally
      */
     private readonly documentScraper;
-    constructor(tools: Pick<ExecutionTools, 'llm'>, options: PrepareAndScrapeOptions);
+    constructor(tools: Pick<ExecutionTools, 'fs' | 'llm'>, options: PrepareAndScrapeOptions);
     /**
      * Convert the `.doc` or `.rtf`  to `.doc` file and returns intermediate source
      *

package/esm/typings/src/scrapers/website/WebsiteScraper.d.ts

@@ -24,7 +24,7 @@ export declare class WebsiteScraper implements Converter, Scraper {
      * Markdown scraper is used internally
      */
     private readonly markdownScraper;
-    constructor(tools: Pick<ExecutionTools, 'llm'>, options: PrepareAndScrapeOptions);
+    constructor(tools: Pick<ExecutionTools, 'fs' | 'llm'>, options: PrepareAndScrapeOptions);
     /**
      * Convert the website  to `.md` file and returns intermediate source
      *

package/esm/typings/src/storage/file-cache-storage/FileCacheStorage.d.ts

@@ -1,3 +1,4 @@
+import type { ExecutionTools } from '../../execution/ExecutionTools';
 import type { PromptbookStorage } from '../_common/PromptbookStorage';
 import type { FileCacheStorageOptions } from './FileCacheStorageOptions';
 /**
@@ -6,8 +7,9 @@ import type { FileCacheStorageOptions } from './FileCacheStorageOptions';
  * @public exported from `@promptbook/node`
  */
 export declare class FileCacheStorage<TItem> implements PromptbookStorage<TItem> {
+    private readonly tools;
     private readonly options;
-    constructor(options: FileCacheStorageOptions);
+    constructor(tools: Required<Pick<ExecutionTools, 'fs'>>, options: FileCacheStorageOptions);
     /**
      * @@@
      */

package/esm/typings/src/utils/files/{$isDirectoryExisting.d.ts → isDirectoryExisting.d.ts}

@@ -1,14 +1,13 @@
+import type { FilesystemTools } from '../../execution/FilesystemTools';
 import type { string_dirname } from '../../types/typeAliases';
 /**
  * Checks if the directory exists
  *
- * Note: `$` is used to indicate that this function is not a pure function - it looks at the filesystem
- *
  * @private within the repository
  */
-export declare function $isDirectoryExisting(directoryPath: string_dirname): Promise<boolean>;
+export declare function isDirectoryExisting(directoryPath: string_dirname, fs: FilesystemTools): Promise<boolean>;
 /**
- * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ * Note: Not [~🟢~] because it is not directly dependent on `fs
  * TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
  * TODO: [🧠][📂] "directory" vs "folder"
  * TODO: [🖇] What about symlinks?

package/esm/typings/src/utils/files/isFileExisting.d.ts

@@ -0,0 +1,13 @@
+import type { FilesystemTools } from '../../execution/FilesystemTools';
+import type { string_filename } from '../../types/typeAliases';
+/**
+ * Checks if the file exists
+ *
+ * @private within the repository
+ */
+export declare function isFileExisting(filename: string_filename, fs: FilesystemTools): Promise<boolean>;
+/**
+ * Note: Not [~🟢~] because it is not directly dependent on `fs
+ * TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
+ * TODO: [🖇] What about symlinks?
+ */

package/esm/typings/src/utils/files/{$listAllFiles.d.ts → listAllFiles.d.ts}

@@ -1,18 +1,17 @@
+import type { FilesystemTools } from '../../execution/FilesystemTools';
 import type { string_dirname } from '../../types/typeAliases';
 import type { string_filename } from '../../types/typeAliases';
 /**
  * Reads all files in the directory
  *
- * Note: `$` is used to indicate that this function is not a pure function - it looks at the filesystem
- *
  * @param path
  * @param isRecursive
  * @returns List of all files in the directory
  * @private internal function of `createCollectionFromDirectory`
  */
-export declare function $listAllFiles(path: string_dirname, isRecursive: boolean): Promise<Array<string_filename>>;
+export declare function listAllFiles(path: string_dirname, isRecursive: boolean, fs: FilesystemTools): Promise<Array<string_filename>>;
 /**
  * TODO: [😶] Unite floder listing
- * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ * Note: Not [~🟢~] because it is not directly dependent on `fs
  * TODO: [🖇] What about symlinks?
  */

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@promptbook/core",
-    "version": "0.71.0-13",
+    "version": "0.71.0-14",
     "description": "Supercharge your use of large language models",
     "private": false,
     "sideEffects": false,

package/umd/index.umd.js

@@ -16,7 +16,7 @@
     /**
      * The version of the Promptbook library
      */
-    var PROMPTBOOK_VERSION = '0.71.0-12';
+    var PROMPTBOOK_VERSION = '0.71.0-13';
     // TODO: [main] !!!! List here all the versions and annotate + put into script
 
     /*! *****************************************************************************
@@ -5263,39 +5263,45 @@
      */
 
     /**
-     * Detects if the code is running in a Node.js environment
+     * Convert file extension to mime type
      *
-     * Note: `$` is used to indicate that this function is not a pure function - it looks at the global object to determine the environment
+     * @private within the repository
+     */
+    function extensionToMimeType(value) {
+        return mimeTypes.lookup(value) || 'application/octet-stream';
+    }
+
+    /**
+     * Get the file extension from a file name
      *
-     * @public exported from `@promptbook/utils`
+     * @private within the repository
      */
-    var $isRunningInNode = new Function("\n    try {\n        return this === global;\n    } catch (e) {\n        return false;\n    }\n");
+    function getFileExtension(value) {
+        var match = value.match(/\.([0-9a-z]+)(?:[?#]|$)/i);
+        return match ? match[1].toLowerCase() : null;
+    }
 
     /**
      * Checks if the file exists
      *
-     * Note: `$` is used to indicate that this function is not a pure function - it looks at the filesystem
-     *
      * @private within the repository
      */
-    function $isFileExisting(filename) {
+    function isFileExisting(filename, fs) {
         return __awaiter(this, void 0, void 0, function () {
             var isReadAccessAllowed, isFile;
             return __generator(this, function (_a) {
                 switch (_a.label) {
-                    case 0:
-                        if (!$isRunningInNode()) {
-                            throw new EnvironmentMismatchError('Function `$isFileExisting` works only in Node environment.js');
-                        }
-                        return [4 /*yield*/, promises.access(filename, promises.constants.R_OK)
-                                .then(function () { return true; })
-                                .catch(function () { return false; })];
+                    case 0: return [4 /*yield*/, fs
+                            .access(filename, fs.constants.R_OK)
+                            .then(function () { return true; })
+                            .catch(function () { return false; })];
                     case 1:
                         isReadAccessAllowed = _a.sent();
                         if (!isReadAccessAllowed) {
                             return [2 /*return*/, false];
                         }
-                        return [4 /*yield*/, promises.stat(filename)
+                        return [4 /*yield*/, fs
+                                .stat(filename)
                                 .then(function (fileStat) { return fileStat.isFile(); })
                                 .catch(function () { return false; })];
                     case 2:
@@ -5306,30 +5312,11 @@
         });
     }
     /**
-     * Note: [🟢 !!!!!! After fix makeKnowledgeSourceHandler] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
+     * Note: Not [~🟢~] because it is not directly dependent on `fs
      * TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
      * TODO: [🖇] What about symlinks?
      */
 
-    /**
-     * Convert file extension to mime type
-     *
-     * @private within the repository
-     */
-    function extensionToMimeType(value) {
-        return mimeTypes.lookup(value) || 'application/octet-stream';
-    }
-
-    /**
-     * Get the file extension from a file name
-     *
-     * @private within the repository
-     */
-    function getFileExtension(value) {
-        var match = value.match(/\.([0-9a-z]+)(?:[?#]|$)/i);
-        return match ? match[1].toLowerCase() : null;
-    }
-
     /**
      * Tests if given string is valid URL.
      *
@@ -5361,7 +5348,7 @@
      *
      * @private for scraper utilities
      */
-    function makeKnowledgeSourceHandler(knowledgeSource, options) {
+    function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
         var _a;
         return __awaiter(this, void 0, void 0, function () {
             var sourceContent, name, _b, _c, rootDirname, _d, 
@@ -5430,8 +5417,9 @@
                             }];
                     case 2:
                         if (!(isValidFilePath(sourceContent) || /\.[a-z]{1,10}$/i.exec(sourceContent))) return [3 /*break*/, 4];
-                        if (!$isRunningInNode()) {
-                            throw new EnvironmentMismatchError('Importing knowledge source file works only in Node.js environment');
+                        if (tools.fs === undefined) {
+                            throw new EnvironmentMismatchError('Can not import file knowledge without filesystem tools');
+                            //          <- TODO: [🧠] What is the best error type here`
                         }
                         if (rootDirname === null) {
                             throw new EnvironmentMismatchError('Can not import file knowledge in non-file pipeline');
@@ -5440,7 +5428,7 @@
                         filename_1 = path.join(rootDirname, sourceContent).split('\\').join('/');
                         fileExtension = getFileExtension(filename_1);
                         mimeType_1 = extensionToMimeType(fileExtension || '');
-                        return [4 /*yield*/, $isFileExisting(filename_1)];
+                        return [4 /*yield*/, isFileExisting(filename_1, tools.fs)];
                     case 3:
                         if (!(_e.sent())) {
                             throw new NotFoundError(spaceTrim__default["default"](function (block) { return "\n                          Can not make source handler for file which does not exist:\n\n                          File:\n                          ".concat(block(filename_1), "\n                      "); }));
@@ -5456,7 +5444,7 @@
                                         var content;
                                         return __generator(this, function (_a) {
                                             switch (_a.label) {
-                                                case 0: return [4 /*yield*/, promises.readFile(filename_1)];
+                                                case 0: return [4 /*yield*/, tools.fs.readFile(filename_1)];
                                                 case 1:
                                                     content = _a.sent();
                                                     return [2 /*return*/, new Blob([
@@ -5510,9 +5498,6 @@
             });
         });
     }
-    /**
-     * TODO: !!!!!!! Maybe constrain to @promptbook/node bundle
-     */
 
     /**
      * Prepares the knowle
@@ -5536,7 +5521,7 @@
                                     switch (_d.label) {
                                         case 0:
                                             partialPieces = null;
-                                            return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, { rootDirname: rootDirname, isVerbose: isVerbose })];
+                                            return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname: rootDirname, isVerbose: isVerbose })];
                                         case 1:
                                             sourceHandler = _d.sent();
                                             _d.label = 2;