@promptbook/openai 0.71.0-13 → 0.71.0-15

package/README.md

@@ -47,12 +47,16 @@ npm install @promptbook/openai
 
 ```typescript
 import { createPipelineExecutor, assertsExecutionSuccessful } from '@promptbook/core';
-import { createCollectionFromDirectory } from '@promptbook/node';
-import { $provideExecutionToolsForNode } from '@promptbook/node';
+import {
+    createCollectionFromDirectory,
+    $provideExecutionToolsForNode,
+    $provideFilesystemForNode,
+} from '@promptbook/node';
 import { JavascriptExecutionTools } from '@promptbook/execute-javascript';
 import { OpenAiExecutionTools } from '@promptbook/openai';
 
 // ▶ Prepare tools
+const fs = $provideFilesystemForNode();
 const llm = new OpenAiExecutionTools(
     //            <- TODO: [🧱] Implement in a functional (not new Class) way
     {
@@ -62,7 +66,8 @@ const llm = new OpenAiExecutionTools(
 );
 const tools = {
     llm,
-    scrapers: await $provideScrapersForNode({ llm }),
+    fs,
+    scrapers: await $provideScrapersForNode({ fs, llm }),
     script: [new JavascriptExecutionTools()],
 };
 
@@ -97,6 +102,7 @@ You can just use `$provideExecutionToolsForNode` function to create all required
 import { createPipelineExecutor, createCollectionFromDirectory, assertsExecutionSuccessful } from '@promptbook/core';
 import { JavascriptExecutionTools } from '@promptbook/execute-javascript';
 import { $provideExecutionToolsForNode } from '@promptbook/node';
+import { $provideFilesystemForNode } from '@promptbook/node';
 
 // ▶ Prepare tools
 const tools = await $provideExecutionToolsForNode();
@@ -130,14 +136,18 @@ You can use multiple LLM providers in one Promptbook execution. The best model w
 
 ```typescript
 import { createPipelineExecutor, assertsExecutionSuccessful } from '@promptbook/core';
-import { createCollectionFromDirectory } from '@promptbook/node';
-import { $provideExecutionToolsForNode } from '@promptbook/node';
+import {
+    createCollectionFromDirectory,
+    $provideExecutionToolsForNode,
+    $provideFilesystemForNode,
+} from '@promptbook/node';
 import { JavascriptExecutionTools } from '@promptbook/execute-javascript';
 import { OpenAiExecutionTools } from '@promptbook/openai';
 import { AnthropicClaudeExecutionTools } from '@promptbook/anthropic-claude';
 import { AzureOpenAiExecutionTools } from '@promptbook/azure-openai';
 
 // ▶ Prepare multiple tools
+const fs = $provideFilesystemForNode();
 const llm = [
     // Note: You can use multiple LLM providers in one Promptbook execution.
     //       The best model will be chosen automatically according to the prompt and the model's capabilities.
@@ -164,7 +174,8 @@ const llm = [
 ];
 const tools = {
     llm,
-    scrapers: await $provideScrapersForNode({ llm }),
+    fs,
+    scrapers: await $provideScrapersForNode({ fs, llm }),
     script: [new JavascriptExecutionTools()],
 };
 

package/esm/index.es.js

@@ -6,7 +6,7 @@ import spaceTrim$1, { spaceTrim } from 'spacetrim';
 /**
  * The version of the Promptbook library
  */
-var PROMPTBOOK_VERSION = '0.71.0-12';
+var PROMPTBOOK_VERSION = '0.71.0-14';
 // TODO: [main] !!!! List here all the versions and annotate + put into script
 
 /*! *****************************************************************************

package/esm/typings/src/_packages/node.index.d.ts

@@ -3,6 +3,7 @@ import { createCollectionFromDirectory } from '../collection/constructors/create
 import { $provideExecutionToolsForNode } from '../execution/utils/$provideExecutionToolsForNode';
 import { $provideLlmToolsConfigurationFromEnv } from '../llm-providers/_common/register/$provideLlmToolsConfigurationFromEnv';
 import { $provideLlmToolsFromEnv } from '../llm-providers/_common/register/$provideLlmToolsFromEnv';
+import { $provideFilesystemForNode } from '../scrapers/_common/register/$provideFilesystemForNode';
 import { $provideScrapersForNode } from '../scrapers/_common/register/$provideScrapersForNode';
 import { FileCacheStorage } from '../storage/file-cache-storage/FileCacheStorage';
 import { $execCommand } from '../utils/execCommand/$execCommand';
@@ -12,6 +13,7 @@ export { createCollectionFromDirectory };
 export { $provideExecutionToolsForNode };
 export { $provideLlmToolsConfigurationFromEnv };
 export { $provideLlmToolsFromEnv };
+export { $provideFilesystemForNode };
 export { $provideScrapersForNode };
 export { FileCacheStorage };
 export { $execCommand };

package/esm/typings/src/_packages/types.index.d.ts

@@ -20,6 +20,7 @@ import type { CreatePipelineExecutorOptions } from '../execution/createPipelineE
 import type { CreatePipelineExecutorSettings } from '../execution/createPipelineExecutor/00-CreatePipelineExecutorSettings';
 import type { EmbeddingVector } from '../execution/EmbeddingVector';
 import type { ExecutionTools } from '../execution/ExecutionTools';
+import type { FilesystemTools } from '../execution/FilesystemTools';
 import type { LlmExecutionTools } from '../execution/LlmExecutionTools';
 import type { LlmExecutionToolsConstructor } from '../execution/LlmExecutionToolsConstructor';
 import type { PipelineExecutor } from '../execution/PipelineExecutor';
@@ -259,6 +260,7 @@ export type { CreatePipelineExecutorOptions };
 export type { CreatePipelineExecutorSettings };
 export type { EmbeddingVector };
 export type { ExecutionTools };
+export type { FilesystemTools };
 export type { LlmExecutionTools };
 export type { LlmExecutionToolsConstructor };
 export type { PipelineExecutor };

package/esm/typings/src/collection/constructors/createCollectionFromDirectory.d.ts

@@ -46,9 +46,9 @@ type CreatePipelineCollectionFromDirectoryOptions = Omit<PrepareAndScrapeOptions
  * @returns PipelineCollection
  * @public exported from `@promptbook/node`
  */
-export declare function createCollectionFromDirectory(path: string_dirname, tools?: Pick<ExecutionTools, 'llm' | 'scrapers'>, options?: CreatePipelineCollectionFromDirectoryOptions): Promise<PipelineCollection>;
+export declare function createCollectionFromDirectory(path: string_dirname, tools?: Pick<ExecutionTools, 'llm' | 'fs' | 'scrapers'>, options?: CreatePipelineCollectionFromDirectoryOptions): Promise<PipelineCollection>;
 export {};
 /**
- * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
  * TODO: [🖇] What about symlinks? Maybe option isSymlinksFollowed
+ * TODO: Maybe move from `@promptbook/node` to `@promptbook/core` as we removes direct dependency on `fs`
  */

package/esm/typings/src/conversion/pipelineStringToJson.d.ts

@@ -20,7 +20,7 @@ import type { PipelineString } from '../types/PipelineString';
  * @throws {ParseError} if the promptbook string is not valid
  * @public exported from `@promptbook/core`
  */
-export declare function pipelineStringToJson(pipelineString: PipelineString, tools?: Pick<ExecutionTools, 'llm' | 'scrapers'>, options?: PrepareAndScrapeOptions): Promise<PipelineJson>;
+export declare function pipelineStringToJson(pipelineString: PipelineString, tools?: Pick<ExecutionTools, 'llm' | 'fs' | 'scrapers'>, options?: PrepareAndScrapeOptions): Promise<PipelineJson>;
 /**
  * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
  * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools

package/esm/typings/src/execution/ExecutionTools.d.ts

@@ -1,5 +1,6 @@
 import type { Scraper } from '../scrapers/_common/Scraper';
 import type { Arrayable } from '../types/Arrayable';
+import type { FilesystemTools } from './FilesystemTools';
 import type { LlmExecutionTools } from './LlmExecutionTools';
 import type { ScriptExecutionTools } from './ScriptExecutionTools';
 import type { UserInterfaceTools } from './UserInterfaceTools';
@@ -12,11 +13,19 @@ export type ExecutionTools = {
     /**
      * Tools for executing prompts in large language models
      *
-     * Tip: Use `$provideLlmToolsFromEnv()` to use all available LLM providers you configured
+     * Tip: Use `$provideLlmToolsFromEnv` to use all available LLM providers you configured
      *
      * @default [] - If not provided, no LLM execution will be possible - it does not make sense in most cases
      */
     readonly llm?: Arrayable<LlmExecutionTools>;
+    /**
+     * Tools for filesystem operations
+     *
+     * Tip: Use `$provideFilesystemForNode` to use filesystem in Node.js
+     *
+     * @default undefined - If not provided, no filesystem operations will be possible
+     */
+    readonly fs?: FilesystemTools;
     /**
      * Scrapers for extracting knowledge from external sources
      *
@@ -30,7 +39,7 @@ export type ExecutionTools = {
      *
      * Note: You can pass multiple ScriptExecutionTools, they will be tried one by one until one of them supports the script
      *       If none of them supports the script, an error is thrown
-     * Tip: Use here `new JavascriptExecutionTools()`
+     * Tip: Use here `new JavascriptExecutionTools`
      *
      * @default [] - If not provided, no script execution will be possible
      */
@@ -43,6 +52,5 @@ export type ExecutionTools = {
     readonly userInterface?: UserInterfaceTools;
 };
 /**
- * TODO: [🍂] Maybe make llm = $provideLlmToolsFromEnv() without problem with bundle contaminated by only `@promptbook/node` and `@promptbook/cli` stuff
- * TODO: [🍂] Same with scrapers
+ * TODO: !!!!!! Move here also the executables and make $provideXxxxForNode
  */

package/esm/typings/src/execution/FilesystemTools.d.ts

@@ -0,0 +1,9 @@
+/// <reference types="node" />
+import type fs from 'fs/promises';
+/**
+ * Container for all the tools needed to manipulate with filesystem
+ */
+export type FilesystemTools = Pick<typeof fs, 'access' | 'constants' | 'readFile' | 'stat' | 'readdir'>;
+/**
+ * TODO: Implement destroyable pattern to free resources
+ */

package/esm/typings/src/execution/translation/automatic-translate/translateMessages.d.ts

@@ -8,4 +8,5 @@ export declare function translateMessages({ automaticTranslator, from, to, }: {
 } & TranslatorOptions): Promise<void>;
 /**
  * TODO: [😶] Unite floder listing
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
  */

package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsFromEnv.d.ts

@@ -23,5 +23,4 @@ export declare function $provideLlmToolsFromEnv(options?: CreateLlmToolsFromConf
  * TODO: [🥃] Allow `ptbk make` without llm tools
  * TODO: This should be maybe not under `_common` but under `utils`
  * TODO: [®] DRY Register logic
- * TODO: [🍂] Maybe make llm = $provideLlmToolsFromEnv() without problem with bundle contaminated by only `@promptbook/node` and `@promptbook/cli` stuff
  */

package/esm/typings/src/prepare/preparePipeline.d.ts

@@ -9,7 +9,7 @@ import type { PrepareAndScrapeOptions } from './PrepareAndScrapeOptions';
  * Note: When the pipeline is already prepared, it returns the same pipeline
  * @public exported from `@promptbook/core`
  */
-export declare function preparePipeline(pipeline: PipelineJson, tools: Pick<ExecutionTools, 'llm' | 'scrapers'>, options: PrepareAndScrapeOptions): Promise<PipelineJson>;
+export declare function preparePipeline(pipeline: PipelineJson, tools: Pick<ExecutionTools, 'llm' | 'fs' | 'scrapers'>, options: PrepareAndScrapeOptions): Promise<PipelineJson>;
 /**
  * TODO: Write tests for `preparePipeline`
  * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch

package/esm/typings/src/prepare/prepareTemplates.d.ts

@@ -19,7 +19,7 @@ type PreparedTemplates = {
  *
  * @public exported from `@promptbook/core`
  */
-export declare function prepareTemplates(pipeline: PrepareTemplateInput, tools: Pick<ExecutionTools, 'llm' | 'scrapers'>, options: PrepareAndScrapeOptions): Promise<PreparedTemplates>;
+export declare function prepareTemplates(pipeline: PrepareTemplateInput, tools: Pick<ExecutionTools, 'llm' | 'fs' | 'scrapers'>, options: PrepareAndScrapeOptions): Promise<PreparedTemplates>;
 export {};
 /**
  * TODO: [🧠] Add context to each template (if missing)

package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts

@@ -8,7 +8,7 @@ import type { ExecutionTools } from '../../execution/ExecutionTools';
  * @see https://github.com/webgptorg/promptbook/discussions/41
  * @public exported from `@promptbook/core`
  */
-export declare function prepareKnowledgePieces(knowledgeSources: Array<KnowledgeSourceJson>, tools: Pick<ExecutionTools, 'llm' | 'scrapers'>, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'preparationIds'>>>;
+export declare function prepareKnowledgePieces(knowledgeSources: Array<KnowledgeSourceJson>, tools: Pick<ExecutionTools, 'llm' | 'fs' | 'scrapers'>, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'preparationIds'>>>;
 /**
  * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
  *       Put `knowledgePieces` into `PrepareKnowledgeOptions`

package/esm/typings/src/scrapers/_common/register/$provideFilesystemForNode.d.ts

@@ -0,0 +1,11 @@
+import type { FilesystemTools } from '../../../execution/FilesystemTools';
+import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
+/**
+ * @@@
+ *
+ * @public exported from `@promptbook/node`
+ */
+export declare function $provideFilesystemForNode(options?: Pick<PrepareAndScrapeOptions, 'isVerbose'>): FilesystemTools;
+/**
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ */

package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts

@@ -9,7 +9,7 @@ import type { Scraper } from '../Scraper';
  *
  * @public exported from `@promptbook/node`
  */
-export declare function $provideScrapersForNode(tools: Pick<ExecutionTools, 'llm'>, options?: PrepareAndScrapeOptions): Promise<Array<Scraper>>;
+export declare function $provideScrapersForNode(tools: Pick<ExecutionTools, 'fs' | 'llm'>, options?: PrepareAndScrapeOptions): Promise<Array<Scraper>>;
 /**
  * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
  */

package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts

@@ -30,4 +30,5 @@ export {};
  * 1) Need to store more than serialized JSONs
  * 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: !!!!
  * TODO: [🐱‍🐉][🧠] Make some smart crop
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
  */

package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts

@@ -1,4 +1,5 @@
 import type { SetOptional } from 'type-fest';
+import type { ExecutionTools } from '../../../execution/ExecutionTools';
 import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
 import type { KnowledgeSourceJson } from '../../../types/PipelineJson/KnowledgeSourceJson';
 import type { ScraperSourceHandler } from '../Scraper';
@@ -7,7 +8,4 @@ import type { ScraperSourceHandler } from '../Scraper';
  *
  * @private for scraper utilities
  */
-export declare function makeKnowledgeSourceHandler(knowledgeSource: SetOptional<KnowledgeSourceJson, 'name'>, options?: Pick<PrepareAndScrapeOptions, 'rootDirname' | 'isVerbose'>): Promise<ScraperSourceHandler>;
-/**
- * TODO: !!!!!!! Maybe constrain to @promptbook/node bundle
- */
+export declare function makeKnowledgeSourceHandler(knowledgeSource: SetOptional<KnowledgeSourceJson, 'name'>, tools: Pick<ExecutionTools, 'fs'>, options?: Pick<PrepareAndScrapeOptions, 'rootDirname' | 'isVerbose'>): Promise<ScraperSourceHandler>;

package/esm/typings/src/scrapers/document/DocumentScraper.d.ts

@@ -23,7 +23,7 @@ export declare class DocumentScraper implements Converter, Scraper {
      * Markdown scraper is used internally
      */
     private readonly markdownScraper;
-    constructor(tools: Pick<ExecutionTools, 'llm'>, options: PrepareAndScrapeOptions);
+    constructor(tools: Pick<ExecutionTools, 'fs' | 'llm'>, options: PrepareAndScrapeOptions);
     /**
      * Convert the `.docx` or `.odt`  to `.md` file and returns intermediate source
      *

package/esm/typings/src/scrapers/document-legacy/LegacyDocumentScraper.d.ts

@@ -23,7 +23,7 @@ export declare class LegacyDocumentScraper implements Converter, Scraper {
      * Document scraper is used internally
      */
     private readonly documentScraper;
-    constructor(tools: Pick<ExecutionTools, 'llm'>, options: PrepareAndScrapeOptions);
+    constructor(tools: Pick<ExecutionTools, 'fs' | 'llm'>, options: PrepareAndScrapeOptions);
     /**
      * Convert the `.doc` or `.rtf`  to `.doc` file and returns intermediate source
      *

package/esm/typings/src/scrapers/website/WebsiteScraper.d.ts

@@ -24,7 +24,7 @@ export declare class WebsiteScraper implements Converter, Scraper {
      * Markdown scraper is used internally
      */
     private readonly markdownScraper;
-    constructor(tools: Pick<ExecutionTools, 'llm'>, options: PrepareAndScrapeOptions);
+    constructor(tools: Pick<ExecutionTools, 'fs' | 'llm'>, options: PrepareAndScrapeOptions);
     /**
      * Convert the website  to `.md` file and returns intermediate source
      *

package/esm/typings/src/storage/file-cache-storage/FileCacheStorage.d.ts

@@ -1,3 +1,4 @@
+import type { ExecutionTools } from '../../execution/ExecutionTools';
 import type { PromptbookStorage } from '../_common/PromptbookStorage';
 import type { FileCacheStorageOptions } from './FileCacheStorageOptions';
 /**
@@ -6,8 +7,9 @@ import type { FileCacheStorageOptions } from './FileCacheStorageOptions';
  * @public exported from `@promptbook/node`
  */
 export declare class FileCacheStorage<TItem> implements PromptbookStorage<TItem> {
+    private readonly tools;
     private readonly options;
-    constructor(options: FileCacheStorageOptions);
+    constructor(tools: Required<Pick<ExecutionTools, 'fs'>>, options: FileCacheStorageOptions);
     /**
      * @@@
      */

package/esm/typings/src/utils/files/{$isDirectoryExisting.d.ts → isDirectoryExisting.d.ts}

@@ -1,14 +1,13 @@
+import type { FilesystemTools } from '../../execution/FilesystemTools';
 import type { string_dirname } from '../../types/typeAliases';
 /**
  * Checks if the directory exists
  *
- * Note: `$` is used to indicate that this function is not a pure function - it looks at the filesystem
- *
  * @private within the repository
  */
-export declare function $isDirectoryExisting(directoryPath: string_dirname): Promise<boolean>;
+export declare function isDirectoryExisting(directoryPath: string_dirname, fs: FilesystemTools): Promise<boolean>;
 /**
- * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ * Note: Not [~🟢~] because it is not directly dependent on `fs
  * TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
  * TODO: [🧠][📂] "directory" vs "folder"
  * TODO: [🖇] What about symlinks?

package/esm/typings/src/utils/files/isFileExisting.d.ts

@@ -0,0 +1,13 @@
+import type { FilesystemTools } from '../../execution/FilesystemTools';
+import type { string_filename } from '../../types/typeAliases';
+/**
+ * Checks if the file exists
+ *
+ * @private within the repository
+ */
+export declare function isFileExisting(filename: string_filename, fs: FilesystemTools): Promise<boolean>;
+/**
+ * Note: Not [~🟢~] because it is not directly dependent on `fs
+ * TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
+ * TODO: [🖇] What about symlinks?
+ */

package/esm/typings/src/utils/files/{$listAllFiles.d.ts → listAllFiles.d.ts}

@@ -1,18 +1,17 @@
+import type { FilesystemTools } from '../../execution/FilesystemTools';
 import type { string_dirname } from '../../types/typeAliases';
 import type { string_filename } from '../../types/typeAliases';
 /**
  * Reads all files in the directory
  *
- * Note: `$` is used to indicate that this function is not a pure function - it looks at the filesystem
- *
  * @param path
  * @param isRecursive
  * @returns List of all files in the directory
  * @private internal function of `createCollectionFromDirectory`
  */
-export declare function $listAllFiles(path: string_dirname, isRecursive: boolean): Promise<Array<string_filename>>;
+export declare function listAllFiles(path: string_dirname, isRecursive: boolean, fs: FilesystemTools): Promise<Array<string_filename>>;
 /**
  * TODO: [😶] Unite floder listing
- * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ * Note: Not [~🟢~] because it is not directly dependent on `fs
  * TODO: [🖇] What about symlinks?
  */

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@promptbook/openai",
-    "version": "0.71.0-13",
+    "version": "0.71.0-15",
     "description": "Supercharge your use of large language models",
     "private": false,
     "sideEffects": false,
@@ -51,7 +51,7 @@
     "module": "./esm/index.es.js",
     "typings": "./esm/typings/src/_packages/openai.index.d.ts",
     "peerDependencies": {
-        "@promptbook/core": "0.71.0-13"
+        "@promptbook/core": "0.71.0-15"
     },
     "dependencies": {
         "colors": "1.4.0",

package/umd/index.umd.js

@@ -14,7 +14,7 @@
     /**
      * The version of the Promptbook library
      */
-    var PROMPTBOOK_VERSION = '0.71.0-12';
+    var PROMPTBOOK_VERSION = '0.71.0-14';
     // TODO: [main] !!!! List here all the versions and annotate + put into script
 
     /*! *****************************************************************************

package/esm/typings/src/utils/files/$isFileExisting.d.ts

@@ -1,14 +0,0 @@
-import type { string_filename } from '../../types/typeAliases';
-/**
- * Checks if the file exists
- *
- * Note: `$` is used to indicate that this function is not a pure function - it looks at the filesystem
- *
- * @private within the repository
- */
-export declare function $isFileExisting(filename: string_filename): Promise<boolean>;
-/**
- * Note: [🟢 !!!!!! After fix makeKnowledgeSourceHandler] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
- * TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
- * TODO: [🖇] What about symlinks?
- */