@promptbook/cli 0.71.0-13 → 0.71.0-14

package/README.md

@@ -61,6 +61,7 @@ Then just use it:
 ```typescript
 import { createPipelineExecutor, assertsExecutionSuccessful } from '@promptbook/core';
 import { $provideExecutionToolsForNode } from '@promptbook/node';
+import { $provideFilesystemForNode } from '@promptbook/node';
 import { getPipelineCollection } from './promptbook-collection'; // <- Importing from pre-built library
 import { JavascriptExecutionTools } from '@promptbook/execute-javascript';
 import { OpenAiExecutionTools } from '@promptbook/openai';

package/esm/index.es.js

@@ -2,7 +2,7 @@ import commander from 'commander';
 import spaceTrim$1, { spaceTrim } from 'spacetrim';
 import colors from 'colors';
 import { forTime } from 'waitasecond';
-import { stat, access, constants, readFile, readdir, writeFile, mkdir, unlink, rm, rmdir, rename } from 'fs/promises';
+import { readFile, stat, access, constants, readdir, writeFile, mkdir, unlink, rm, rmdir, rename } from 'fs/promises';
 import { join, basename, dirname } from 'path';
 import { format } from 'prettier';
 import parserHtml from 'prettier/parser-html';
@@ -26,7 +26,7 @@ import { Converter } from 'showdown';
 /**
  * The version of the Promptbook library
  */
-var PROMPTBOOK_VERSION = '0.71.0-12';
+var PROMPTBOOK_VERSION = '0.71.0-13';
 // TODO: [main] !!!! List here all the versions and annotate + put into script
 
 /*! *****************************************************************************
@@ -5094,30 +5094,45 @@ function sourceContentToName(sourceContent) {
  */
 
 /**
- * Checks if the file exists
+ * Convert file extension to mime type
  *
- * Note: `$` is used to indicate that this function is not a pure function - it looks at the filesystem
+ * @private within the repository
+ */
+function extensionToMimeType(value) {
+    return lookup(value) || 'application/octet-stream';
+}
+
+/**
+ * Get the file extension from a file name
  *
  * @private within the repository
  */
-function $isFileExisting(filename) {
+function getFileExtension(value) {
+    var match = value.match(/\.([0-9a-z]+)(?:[?#]|$)/i);
+    return match ? match[1].toLowerCase() : null;
+}
+
+/**
+ * Checks if the file exists
+ *
+ * @private within the repository
+ */
+function isFileExisting(filename, fs) {
     return __awaiter(this, void 0, void 0, function () {
         var isReadAccessAllowed, isFile;
         return __generator(this, function (_a) {
             switch (_a.label) {
-                case 0:
-                    if (!$isRunningInNode()) {
-                        throw new EnvironmentMismatchError('Function `$isFileExisting` works only in Node environment.js');
-                    }
-                    return [4 /*yield*/, access(filename, constants.R_OK)
-                            .then(function () { return true; })
-                            .catch(function () { return false; })];
+                case 0: return [4 /*yield*/, fs
+                        .access(filename, fs.constants.R_OK)
+                        .then(function () { return true; })
+                        .catch(function () { return false; })];
                 case 1:
                     isReadAccessAllowed = _a.sent();
                     if (!isReadAccessAllowed) {
                         return [2 /*return*/, false];
                     }
-                    return [4 /*yield*/, stat(filename)
+                    return [4 /*yield*/, fs
+                            .stat(filename)
                             .then(function (fileStat) { return fileStat.isFile(); })
                             .catch(function () { return false; })];
                 case 2:
@@ -5128,30 +5143,11 @@ function $isFileExisting(filename) {
     });
 }
 /**
- * Note: [🟢 !!!!!! After fix makeKnowledgeSourceHandler] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
+ * Note: Not [~🟢~] because it is not directly dependent on `fs
  * TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
  * TODO: [🖇] What about symlinks?
  */
 
-/**
- * Convert file extension to mime type
- *
- * @private within the repository
- */
-function extensionToMimeType(value) {
-    return lookup(value) || 'application/octet-stream';
-}
-
-/**
- * Get the file extension from a file name
- *
- * @private within the repository
- */
-function getFileExtension(value) {
-    var match = value.match(/\.([0-9a-z]+)(?:[?#]|$)/i);
-    return match ? match[1].toLowerCase() : null;
-}
-
 /**
  * Tests if given string is valid URL.
  *
@@ -5183,7 +5179,7 @@ function isValidFilePath(filename) {
  *
  * @private for scraper utilities
  */
-function makeKnowledgeSourceHandler(knowledgeSource, options) {
+function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
     var _a;
     return __awaiter(this, void 0, void 0, function () {
         var sourceContent, name, _b, _c, rootDirname, _d, 
@@ -5252,8 +5248,9 @@ function makeKnowledgeSourceHandler(knowledgeSource, options) {
                         }];
                 case 2:
                     if (!(isValidFilePath(sourceContent) || /\.[a-z]{1,10}$/i.exec(sourceContent))) return [3 /*break*/, 4];
-                    if (!$isRunningInNode()) {
-                        throw new EnvironmentMismatchError('Importing knowledge source file works only in Node.js environment');
+                    if (tools.fs === undefined) {
+                        throw new EnvironmentMismatchError('Can not import file knowledge without filesystem tools');
+                        //          <- TODO: [🧠] What is the best error type here`
                     }
                     if (rootDirname === null) {
                         throw new EnvironmentMismatchError('Can not import file knowledge in non-file pipeline');
@@ -5262,7 +5259,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, options) {
                     filename_1 = join(rootDirname, sourceContent).split('\\').join('/');
                     fileExtension = getFileExtension(filename_1);
                     mimeType_1 = extensionToMimeType(fileExtension || '');
-                    return [4 /*yield*/, $isFileExisting(filename_1)];
+                    return [4 /*yield*/, isFileExisting(filename_1, tools.fs)];
                 case 3:
                     if (!(_e.sent())) {
                         throw new NotFoundError(spaceTrim$1(function (block) { return "\n                          Can not make source handler for file which does not exist:\n\n                          File:\n                          ".concat(block(filename_1), "\n                      "); }));
@@ -5278,7 +5275,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, options) {
                                     var content;
                                     return __generator(this, function (_a) {
                                         switch (_a.label) {
-                                            case 0: return [4 /*yield*/, readFile(filename_1)];
+                                            case 0: return [4 /*yield*/, tools.fs.readFile(filename_1)];
                                             case 1:
                                                 content = _a.sent();
                                                 return [2 /*return*/, new Blob([
@@ -5332,9 +5329,6 @@ function makeKnowledgeSourceHandler(knowledgeSource, options) {
         });
     });
 }
-/**
- * TODO: !!!!!!! Maybe constrain to @promptbook/node bundle
- */
 
 /**
  * Prepares the knowle
@@ -5358,7 +5352,7 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
                                 switch (_d.label) {
                                     case 0:
                                         partialPieces = null;
-                                        return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, { rootDirname: rootDirname, isVerbose: isVerbose })];
+                                        return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname: rootDirname, isVerbose: isVerbose })];
                                     case 1:
                                         sourceHandler = _d.sent();
                                         _d.label = 2;
@@ -8590,6 +8584,29 @@ function $provideLlmToolsFromEnv(options) {
  * TODO: [🍂] Maybe make llm = $provideLlmToolsFromEnv() without problem with bundle contaminated by only `@promptbook/node` and `@promptbook/cli` stuff
  */
 
+/**
+ * @@@
+ *
+ * @public exported from `@promptbook/node`
+ */
+function $provideFilesystemForNode(options) {
+    if (!$isRunningInNode()) {
+        throw new EnvironmentMismatchError('Function `$provideFilesystemForNode` works only in Node.js environment');
+    }
+    var _a = (options || {}).isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
+    TODO_USE(isVerbose);
+    return {
+        stat: stat,
+        access: access,
+        constants: constants,
+        readFile: readFile,
+        readdir: readdir,
+    };
+}
+/**
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ */
+
 /**
  * !!!!!!
  *
@@ -9112,7 +9129,8 @@ function $provideExecutionToolsForNode(options) {
                     }
                     llm = $provideLlmToolsFromEnv(options);
                     _a = {
-                        llm: llm
+                        llm: llm,
+                        fs: $provideFilesystemForNode()
                     };
                     return [4 /*yield*/, $provideScrapersForNode({ llm: llm }, options)];
                 case 1:
@@ -9131,28 +9149,24 @@ function $provideExecutionToolsForNode(options) {
 /**
  * Checks if the directory exists
  *
- * Note: `$` is used to indicate that this function is not a pure function - it looks at the filesystem
- *
  * @private within the repository
  */
-function $isDirectoryExisting(directoryPath) {
+function isDirectoryExisting(directoryPath, fs) {
     return __awaiter(this, void 0, void 0, function () {
         var isReadAccessAllowed, isDirectory;
         return __generator(this, function (_a) {
             switch (_a.label) {
-                case 0:
-                    if (!$isRunningInNode()) {
-                        throw new EnvironmentMismatchError('Function `$isDirectoryExisting` works only in Node environment.js');
-                    }
-                    return [4 /*yield*/, access(directoryPath, constants.R_OK)
-                            .then(function () { return true; })
-                            .catch(function () { return false; })];
+                case 0: return [4 /*yield*/, fs
+                        .access(directoryPath, fs.constants.R_OK)
+                        .then(function () { return true; })
+                        .catch(function () { return false; })];
                 case 1:
                     isReadAccessAllowed = _a.sent();
                     if (!isReadAccessAllowed) {
                         return [2 /*return*/, false];
                     }
-                    return [4 /*yield*/, stat(directoryPath)
+                    return [4 /*yield*/, fs
+                            .stat(directoryPath)
                             .then(function (fileStat) { return fileStat.isDirectory(); })
                             .catch(function () { return false; })];
                 case 2:
@@ -9163,7 +9177,7 @@ function $isDirectoryExisting(directoryPath) {
     });
 }
 /**
- * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ * Note: Not [~🟢~] because it is not directly dependent on `fs
  * TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
  * TODO: [🧠][📂] "directory" vs "folder"
  * TODO: [🖇] What about symlinks?
@@ -9172,30 +9186,24 @@ function $isDirectoryExisting(directoryPath) {
 /**
  * Reads all files in the directory
  *
- * Note: `$` is used to indicate that this function is not a pure function - it looks at the filesystem
- *
  * @param path
  * @param isRecursive
  * @returns List of all files in the directory
  * @private internal function of `createCollectionFromDirectory`
  */
-function $listAllFiles(path, isRecursive) {
+function listAllFiles(path, isRecursive, fs) {
     return __awaiter(this, void 0, void 0, function () {
         var dirents, fileNames, _a, _b, dirent, subPath, _c, _d, _e, _f, e_1_1;
         var e_1, _g;
         return __generator(this, function (_h) {
             switch (_h.label) {
-                case 0:
-                    if (!$isRunningInNode()) {
-                        throw new EnvironmentMismatchError('Function `$listAllFiles` works only in Node environment.js');
-                    }
-                    return [4 /*yield*/, $isDirectoryExisting(path)];
+                case 0: return [4 /*yield*/, isDirectoryExisting(path, fs)];
                 case 1:
                     if (!(_h.sent())) {
                         throw new Error("Directory \"".concat(path, "\" does not exist or is not readable"));
                         //           <- TODO: Use some custom error class
                     }
-                    return [4 /*yield*/, readdir(path, {
+                    return [4 /*yield*/, fs.readdir(path, {
                             withFileTypes: true /* Note: This is not working: recursive: isRecursive */,
                         })];
                 case 2:
@@ -9219,7 +9227,7 @@ function $listAllFiles(path, isRecursive) {
                     _d = (_c = fileNames.push).apply;
                     _e = [fileNames];
                     _f = [[]];
-                    return [4 /*yield*/, $listAllFiles(subPath, isRecursive)];
+                    return [4 /*yield*/, listAllFiles(subPath, isRecursive, fs)];
                 case 5:
                     _d.apply(_c, _e.concat([__spreadArray.apply(void 0, _f.concat([__read.apply(void 0, [(_h.sent()).map(function (filename) { return filename; })]), false]))]));
                     _h.label = 6;
@@ -9244,7 +9252,7 @@ function $listAllFiles(path, isRecursive) {
 }
 /**
  * TODO: [😶] Unite floder listing
- * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ * Note: Not [~🟢~] because it is not directly dependent on `fs
  * TODO: [🖇] What about symlinks?
  */
 
@@ -9353,17 +9361,18 @@ function createCollectionFromDirectory(path, tools, options) {
         return __generator(this, function (_f) {
             switch (_f.label) {
                 case 0:
-                    if (!$isRunningInNode()) {
-                        throw new Error('Function `createCollectionFromDirectory` can only be run in Node.js environment because it reads the file system.');
-                    }
                     if (!(tools === undefined)) return [3 /*break*/, 2];
                     return [4 /*yield*/, $provideExecutionToolsForNode()];
                 case 1:
                     tools = _f.sent();
                     _f.label = 2;
                 case 2:
+                    if (tools === undefined || tools.fs === undefined) {
+                        throw new EnvironmentMismatchError('Can not create collection without filesystem tools');
+                        //          <- TODO: [🧠] What is the best error type here`
+                    }
                     makedLibraryFilePath = join(path, "".concat(PIPELINE_COLLECTION_BASE_FILENAME, ".json"));
-                    return [4 /*yield*/, $isFileExisting(makedLibraryFilePath)];
+                    return [4 /*yield*/, isFileExisting(makedLibraryFilePath, tools.fs)];
                 case 3:
                     if (!(_f.sent())) {
                         console.info(colors.yellow("Tip: Prebuild your pipeline collection (file with supposed prebuild ".concat(makedLibraryFilePath, " not found) with CLI util \"ptbk make\" to speed up the collection creation.")));
@@ -9383,7 +9392,7 @@ function createCollectionFromDirectory(path, tools, options) {
                                     if (isVerbose) {
                                         console.info(colors.cyan("Creating pipeline collection from path ".concat(path.split('\\').join('/'))));
                                     }
-                                    return [4 /*yield*/, $listAllFiles(path, isRecursive)];
+                                    return [4 /*yield*/, listAllFiles(path, isRecursive, tools.fs)];
                                 case 1:
                                     fileNames = _b.sent();
                                     // Note: First load all .ptbk.json and then .ptbk.md files
@@ -9531,8 +9540,8 @@ function createCollectionFromDirectory(path, tools, options) {
     });
 }
 /**
- * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
  * TODO: [🖇] What about symlinks? Maybe option isSymlinksFollowed
+ * TODO: Maybe move from `@promptbook/node` to `@promptbook/core` as we removes direct dependency on `fs`
  */
 
 /**
@@ -9675,7 +9684,8 @@ function nameToSubfolderPath(name) {
  * @public exported from `@promptbook/node`
  */
 var FileCacheStorage = /** @class */ (function () {
-    function FileCacheStorage(options) {
+    function FileCacheStorage(tools, options) {
+        this.tools = tools;
         this.options = options;
         if (!$isRunningInNode()) {
             throw new EnvironmentMismatchError("FileCacheStorage works only in Node.js environment");
@@ -9701,7 +9711,7 @@ var FileCacheStorage = /** @class */ (function () {
                 switch (_a.label) {
                     case 0:
                         filename = this.getFilenameForKey(key);
-                        return [4 /*yield*/, $isFileExisting(filename)];
+                        return [4 /*yield*/, isFileExisting(filename, this.tools.fs)];
                     case 1:
                         if (!(_a.sent())) {
                             return [2 /*return*/, null];
@@ -9956,7 +9966,7 @@ function $provideLlmToolsForCli(options) {
     return cacheLlmTools(countTotalUsage(
     //        <- Note: for example here we don`t want the [🌯]
     $provideLlmToolsFromEnv()), {
-        storage: new FileCacheStorage({ rootFolderPath: join(process.cwd(), EXECUTIONS_CACHE_DIRNAME) }),
+        storage: new FileCacheStorage({ fs: $provideFilesystemForNode() }, { rootFolderPath: join(process.cwd(), EXECUTIONS_CACHE_DIRNAME) }),
         isReloaded: isCacheReloaded,
     });
 }
@@ -12743,6 +12753,7 @@ function getScraperIntermediateSource(source, options) {
  * 1) Need to store more than serialized JSONs
  * 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: !!!!
  * TODO: [🐱‍🐉][🧠] Make some smart crop
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
  */
 
 /**
@@ -13000,6 +13011,10 @@ var DocumentScraper = /** @class */ (function () {
                         if (!$isRunningInNode()) {
                             throw new KnowledgeScrapeError('Scraping .docx files is only supported in Node environment');
                         }
+                        if (this.tools.fs === undefined) {
+                            throw new EnvironmentMismatchError('Can not scrape documents without filesystem tools');
+                            //          <- TODO: [🧠] What is the best error type here`
+                        }
                         if (externalProgramsPaths.pandocPath === undefined) {
                             throw new MissingToolsError('Pandoc is required for scraping .docx files');
                         }
@@ -13017,7 +13032,7 @@ var DocumentScraper = /** @class */ (function () {
                             })];
                     case 1:
                         cacheFilehandler = _g.sent();
-                        return [4 /*yield*/, $isFileExisting(cacheFilehandler.filename)];
+                        return [4 /*yield*/, isFileExisting(cacheFilehandler.filename, this.tools.fs)];
                     case 2:
                         if (!!(_g.sent())) return [3 /*break*/, 5];
                         command_1 = "\"".concat(externalProgramsPaths.pandocPath, "\" -f ").concat(extension, " -t markdown \"").concat(source.filename, "\" -o \"").concat(cacheFilehandler.filename, "\"");
@@ -13026,7 +13041,7 @@ var DocumentScraper = /** @class */ (function () {
                     case 3:
                         // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
                         _g.sent();
-                        return [4 /*yield*/, $isFileExisting(cacheFilehandler.filename)];
+                        return [4 /*yield*/, isFileExisting(cacheFilehandler.filename, this.tools.fs)];
                     case 4:
                         // Note: [0]
                         if (!(_g.sent())) {
@@ -13152,6 +13167,10 @@ var LegacyDocumentScraper = /** @class */ (function () {
                         if (!$isRunningInNode()) {
                             throw new KnowledgeScrapeError('Scraping .doc files is only supported in Node environment');
                         }
+                        if (this.tools.fs === undefined) {
+                            throw new EnvironmentMismatchError('Can not scrape (legacy) documents without filesystem tools');
+                            //          <- TODO: [🧠] What is the best error type here`
+                        }
                         if (externalProgramsPaths.libreOfficePath === undefined) {
                             throw new MissingToolsError('LibreOffice is required for scraping .doc and .rtf files');
                         }
@@ -13172,7 +13191,7 @@ var LegacyDocumentScraper = /** @class */ (function () {
                         if (isVerbose) {
                             console.info("documentScraper: Converting .".concat(extension, " -> .docx"));
                         }
-                        return [4 /*yield*/, $isFileExisting(cacheFilehandler.filename)];
+                        return [4 /*yield*/, isFileExisting(cacheFilehandler.filename, this.tools.fs)];
                     case 2:
                         if (!!(_g.sent())) return [3 /*break*/, 8];
                         documentSourceOutdirPathForLibreOffice_1 = join(dirname(cacheFilehandler.filename), 'libreoffice')
@@ -13197,7 +13216,7 @@ var LegacyDocumentScraper = /** @class */ (function () {
                         return [4 /*yield*/, rmdir(documentSourceOutdirPathForLibreOffice_1)];
                     case 6:
                         _g.sent();
-                        return [4 /*yield*/, $isFileExisting(cacheFilehandler.filename)];
+                        return [4 /*yield*/, isFileExisting(cacheFilehandler.filename, this.tools.fs)];
                     case 7:
                         if (!(_g.sent())) {
                             throw new UnexpectedError(spaceTrim$1(function (block) { return "\n                            File that was supposed to be created by LibreOffice does not exist for unknown reason\n\n                            Expected file:\n                            ".concat(block(cacheFilehandler.filename), "\n\n                            The temporary folder:\n                            ").concat(block(documentSourceOutdirPathForLibreOffice_1), "\n\n                            Command:\n                            > ").concat(block(command_1), "\n\n                        "); }));
@@ -13277,7 +13296,6 @@ var _LegacyDocumentScraperRegistration = $scrapersRegister.register(createLegacy
  * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
  */
 
-// <- TODO: !!!!!!! Are theese changed to import type { ... } from ... correctly
 /**
  * @@@
  *