@promptbook/core 0.92.0-20 → 0.92.0-22

package/esm/index.es.js

@@ -27,7 +27,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-const PROMPTBOOK_ENGINE_VERSION = '0.92.0-20';
+const PROMPTBOOK_ENGINE_VERSION = '0.92.0-22';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -462,6 +462,14 @@ const DEFAULT_IS_AUTO_INSTALLED = false;
  * @public exported from `@promptbook/core`
  */
 const DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME = `getPipelineCollection`;
+/**
+ * Default rate limits (requests per minute)
+ *
+ * Note: Adjust based on the provider tier you are have
+ *
+ * @public exported from `@promptbook/core`
+ */
+const DEFAULT_RPM = 60;
 /**
  * @@@
  *
@@ -2185,15 +2193,12 @@ function jsonParse(value) {
         }
         throw new Error(spaceTrim((block) => `
                     ${block(error.message)}
-            
+
                     The JSON text:
                     ${block(value)}
                 `));
     }
 }
-/**
- * TODO: !!!! Use in Promptbook.studio
- */
 
 /**
  * Recursively converts JSON strings to JSON objects
@@ -2817,7 +2822,8 @@ const CsvFormatParser = {
     subvalueParsers: [
         {
             subvalueName: 'ROW',
-            async mapValues(value, outputParameterName, settings, mapCallback) {
+            async mapValues(options) {
+                const { value, outputParameterName, settings, mapCallback, onProgress } = options;
                 const csv = csvParse(value, settings);
                 if (csv.errors.length !== 0) {
                     throw new CsvFormatError(spaceTrim((block) => `
@@ -2833,21 +2839,29 @@ const CsvFormatParser = {
                                 ${block(value)}
                             `));
                 }
-                const mappedData = await Promise.all(csv.data.map(async (row, index) => {
+                const mappedData = [];
+                for (let index = 0; index < csv.data.length; index++) {
+                    const row = csv.data[index];
                     if (row[outputParameterName]) {
                         throw new CsvFormatError(`Can not overwrite existing column "${outputParameterName}" in CSV row`);
                     }
-                    return {
+                    const mappedRow = {
                         ...row,
                         [outputParameterName]: await mapCallback(row, index),
                     };
-                }));
+                    mappedData.push(mappedRow);
+                    if (onProgress) {
+                        // Note: Report the CSV with all rows mapped so far
+                        await onProgress(unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS }));
+                    }
+                }
                 return unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
             },
         },
         {
             subvalueName: 'CELL',
-            async mapValues(value, outputParameterName, settings, mapCallback) {
+            async mapValues(options) {
+                const { value, settings, mapCallback, onProgress } = options;
                 const csv = csvParse(value, settings);
                 if (csv.errors.length !== 0) {
                     throw new CsvFormatError(spaceTrim((block) => `
@@ -2934,7 +2948,8 @@ const TextFormatParser = {
     subvalueParsers: [
         {
             subvalueName: 'LINE',
-            async mapValues(value, outputParameterName, settings, mapCallback) {
+            async mapValues(options) {
+                const { value, mapCallback, onProgress } = options;
                 const lines = value.split('\n');
                 const mappedLines = await Promise.all(lines.map((lineContent, lineNumber) => 
                 // TODO: [🧠] Maybe option to skip empty line
@@ -4266,7 +4281,7 @@ async function executeAttempts(options) {
  * @private internal utility of `createPipelineExecutor`
  */
 async function executeFormatSubvalues(options) {
-    const { task, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification } = options;
+    const { task, jokerParameterNames, parameters, priority, csvSettings, onProgress, pipelineIdentification } = options;
     if (task.foreach === undefined) {
         return /* not await */ executeAttempts(options);
     }
@@ -4320,21 +4335,32 @@ async function executeFormatSubvalues(options) {
         formatSettings = csvSettings;
         // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
     }
-    const resultString = await subvalueParser.mapValues(parameterValue, task.foreach.outputSubparameterName, formatSettings, async (subparameters, index) => {
-        let mappedParameters;
-        // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions
-        // TODO: When done [🐚] Report progress also for each subvalue here
-        try {
-            mappedParameters = mapAvailableToExpectedParameters({
-                expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
-                availableParameters: subparameters,
-            });
-        }
-        catch (error) {
-            if (!(error instanceof PipelineExecutionError)) {
-                throw error;
+    const resultString = await subvalueParser.mapValues({
+        value: parameterValue,
+        outputParameterName: task.foreach.outputSubparameterName,
+        settings: formatSettings,
+        onProgress(partialResultString) {
+            return onProgress(Object.freeze({
+                [task.resultingParameterName]: 
+                // <- Note: [👩‍👩‍👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
+                partialResultString,
+            }));
+        },
+        async mapCallback(subparameters, index) {
+            let mappedParameters;
+            // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions
+            // TODO: When done [🐚] Report progress also for each subvalue here
+            try {
+                mappedParameters = mapAvailableToExpectedParameters({
+                    expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
+                    availableParameters: subparameters,
+                });
             }
-            throw new PipelineExecutionError(spaceTrim((block) => `
+            catch (error) {
+                if (!(error instanceof PipelineExecutionError)) {
+                    throw error;
+                }
+                throw new PipelineExecutionError(spaceTrim((block) => `
                         ${error.message}
 
                         This is error in FOREACH command
@@ -4343,23 +4369,24 @@ async function executeFormatSubvalues(options) {
                         ${block(pipelineIdentification)}
                         Subparameter index: ${index}
                     `));
-        }
-        const allSubparameters = {
-            ...parameters,
-            ...mappedParameters,
-        };
-        // Note: [👨‍👨‍👧] Now we can freeze `subparameters` because we are sure that all and only used parameters are defined and are not going to be changed
-        Object.freeze(allSubparameters);
-        const subresultString = await executeAttempts({
-            ...options,
-            priority: priority + index,
-            parameters: allSubparameters,
-            pipelineIdentification: spaceTrim((block) => `
+            }
+            const allSubparameters = {
+                ...parameters,
+                ...mappedParameters,
+            };
+            // Note: [👨‍👨‍👧] Now we can freeze `subparameters` because we are sure that all and only used parameters are defined and are not going to be changed
+            Object.freeze(allSubparameters);
+            const subresultString = await executeAttempts({
+                ...options,
+                priority: priority + index,
+                parameters: allSubparameters,
+                pipelineIdentification: spaceTrim((block) => `
                         ${block(pipelineIdentification)}
                         Subparameter index: ${index}
                     `),
-        });
-        return subresultString;
+            });
+            return subresultString;
+        },
     });
     return resultString;
 }
@@ -4533,11 +4560,6 @@ async function getReservedParametersForTask(options) {
 async function executeTask(options) {
     const { currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, maxExecutionAttempts, maxParallelCount, csvSettings, isVerbose, rootDirname, cacheDirname, intermediateFilesStrategy, isAutoInstalled, isNotPreparedWarningSupressed, } = options;
     const priority = preparedPipeline.tasks.length - preparedPipeline.tasks.indexOf(currentTask);
-    await onProgress({
-        outputParameters: {
-            [currentTask.resultingParameterName]: '', // <- TODO: [🧠] What is the best value here?
-        },
-    });
     // Note: Check consistency of used and dependent parameters which was also done in `validatePipeline`, but it’s good to doublecheck
     const usedParameterNames = extractParameterNamesFromTask(currentTask);
     const dependentParameterNames = new Set(currentTask.dependentParameterNames);
@@ -4612,6 +4634,7 @@ async function executeTask(options) {
         preparedPipeline,
         tools,
         $executionReport,
+        onProgress,
         pipelineIdentification,
         maxExecutionAttempts,
         maxParallelCount,
@@ -7212,6 +7235,43 @@ const ChatbotFormfactorDefinition = {
     },
 };
 
+/**
+ * Completion is formfactor that emulates completion models
+ *
+ * @public exported from `@promptbook/core`
+ */
+const CompletionFormfactorDefinition = {
+    name: 'COMPLETION',
+    description: `@@@`,
+    documentationUrl: `https://github.com/webgptorg/promptbook/discussions/@@`,
+    //                                                                     <- TODO: https://github.com/webgptorg/promptbook/discussions/new?category=concepts
+    //                                                                              "🔠 Completion Formfactor"
+    pipelineInterface: {
+        inputParameters: [
+            {
+                name: 'inputText',
+                description: `Input text to be completed`,
+                isInput: true,
+                isOutput: false,
+            },
+            {
+                name: 'instructions',
+                description: `Additional instructions for the model, for example the required length, empty by default`,
+                isInput: true,
+                isOutput: false,
+            },
+        ],
+        outputParameters: [
+            {
+                name: 'followingText',
+                description: `Text that follows the input text`,
+                isInput: false,
+                isOutput: true,
+            },
+        ],
+    },
+};
+
 /**
  * Generator is form of app that @@@
  *
@@ -7396,6 +7456,8 @@ const FORMFACTOR_DEFINITIONS = [
     MatcherFormfactorDefinition,
     GeneratorFormfactorDefinition,
     ImageGeneratorFormfactorDefinition,
+    CompletionFormfactorDefinition,
+    // <- [🛬] When making new formfactor, copy the _boilerplate and link it here
 ];
 /**
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -10888,6 +10950,7 @@ const _OpenAiMetadataRegistration = $llmToolsMetadataRegister.register({
             className: 'OpenAiExecutionTools',
             options: {
                 apiKey: 'sk-',
+                maxRequestsPerMinute: DEFAULT_RPM,
             },
         };
     },
@@ -10960,20 +11023,28 @@ const _OpenAiAssistantMetadataRegistration = $llmToolsMetadataRegister.register(
 /**
  * Migrates the pipeline to the latest version
  *
+ * Note: Migration does not do heavy lifting like calling the LLMs, just lightweight changes of the structure
+ *
  * @public exported from `@promptbook/core`
  */
 function migratePipeline(deprecatedPipeline) {
     /* eslint-disable prefer-const */
     let { pipelineUrl, sourceFile, title, bookVersion, description, formfactorName, parameters, tasks, knowledgeSources, knowledgePieces, personas, preparations, sources, } = deprecatedPipeline;
+    let isChanged = false;
     personas = personas.map((persona) => {
         const migratedPersona = { ...persona }; /* <- TODO: [🌪] */
         if (migratedPersona.modelRequirements !== undefined) {
+            isChanged = true;
             migratedPersona.modelsRequirements = [migratedPersona.modelRequirements];
             delete migratedPersona.modelRequirements;
         }
         return migratedPersona;
     });
-    return {
+    if (!isChanged) {
+        // Note: If nothing to migrate, return the same pipeline
+        return deprecatedPipeline;
+    }
+    const migratedPipeline = {
         pipelineUrl,
         sourceFile,
         title,
@@ -10989,6 +11060,10 @@ function migratePipeline(deprecatedPipeline) {
         sources,
         // <- TODO: [🍙] Make some standard order of json properties
     };
+    console.info(`Book automatically migrated`, { deprecatedPipeline, migratedPipeline });
+    // console.info(`Book automatically migrated from ${} -> ${}`, {deprecatedPipeline,migratedPipeline})
+    // <- TODO: Report the versions of the migration, DO not migrate backwards, throw `CompatibilityError` when given newer version than current version of the engine and link the NPM + Docker packages
+    return migratedPipeline;
 }
 
 /**
@@ -11423,5 +11498,5 @@ class PrefixStorage {
     }
 }
 
-export { $llmToolsMetadataRegister, $llmToolsRegister, $scrapersMetadataRegister, $scrapersRegister, ADMIN_EMAIL, ADMIN_GITHUB_NAME, AbstractFormatError, AuthenticationError, BOOK_LANGUAGE_VERSION, BlackholeStorage, BoilerplateError, BoilerplateFormfactorDefinition, CLAIM, CLI_APP_ID, CallbackInterfaceTools, ChatbotFormfactorDefinition, CollectionError, CsvFormatError, CsvFormatParser, DEFAULT_BOOKS_DIRNAME, DEFAULT_BOOK_OUTPUT_PARAMETER_NAME, DEFAULT_BOOK_TITLE, DEFAULT_CSV_SETTINGS, DEFAULT_DOWNLOAD_CACHE_DIRNAME, DEFAULT_EXECUTION_CACHE_DIRNAME, DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME, DEFAULT_INTERMEDIATE_FILES_STRATEGY, DEFAULT_IS_AUTO_INSTALLED, DEFAULT_IS_VERBOSE, DEFAULT_MAX_EXECUTION_ATTEMPTS, DEFAULT_MAX_FILE_SIZE, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL, DEFAULT_MAX_PARALLEL_COUNT, DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, DEFAULT_PROMPT_TASK_TITLE, DEFAULT_REMOTE_SERVER_URL, DEFAULT_SCRAPE_CACHE_DIRNAME, DEFAULT_TASK_TITLE, EXPECTATION_UNITS, EnvironmentMismatchError, ExecutionReportStringOptionsDefaults, ExpectError, FORMFACTOR_DEFINITIONS, GENERIC_PIPELINE_INTERFACE, GeneratorFormfactorDefinition, GenericFormfactorDefinition, ImageGeneratorFormfactorDefinition, KnowledgeScrapeError, LimitReachedError, MANDATORY_CSV_SETTINGS, MAX_FILENAME_LENGTH, MODEL_ORDER, MODEL_TRUST_LEVEL, MODEL_VARIANTS, MatcherFormfactorDefinition, MemoryStorage, MissingToolsError, MultipleLlmExecutionTools, NAME, NonTaskSectionTypes, NotFoundError, NotYetImplementedError, ORDER_OF_PIPELINE_JSON, PLAYGROUND_APP_ID, PROMPTBOOK_ENGINE_VERSION, PROMPTBOOK_ERRORS, ParseError, PipelineExecutionError, PipelineLogicError, PipelineUrlError, PrefixStorage, PromptbookFetchError, REMOTE_SERVER_URLS, RESERVED_PARAMETER_NAMES, SET_IS_VERBOSE, SectionTypes, SheetsFormfactorDefinition, TaskTypes, TextFormatParser, TranslatorFormfactorDefinition, UNCERTAIN_USAGE, UNCERTAIN_ZERO_VALUE, UnexpectedError, WrappedError, ZERO_USAGE, ZERO_VALUE, _AnthropicClaudeMetadataRegistration, _AzureOpenAiMetadataRegistration, _BoilerplateScraperMetadataRegistration, _DeepseekMetadataRegistration, _DocumentScraperMetadataRegistration, _GoogleMetadataRegistration, _LegacyDocumentScraperMetadataRegistration, _MarkdownScraperMetadataRegistration, _MarkitdownScraperMetadataRegistration, _OpenAiAssistantMetadataRegistration, _OpenAiMetadataRegistration, _PdfScraperMetadataRegistration, _WebsiteScraperMetadataRegistration, addUsage, book, cacheLlmTools, collectionToJson, compilePipeline, computeCosineSimilarity, countUsage, createCollectionFromJson, createCollectionFromPromise, createCollectionFromUrl, createLlmToolsFromConfiguration, createPipelineExecutor, createSubcollection, embeddingVectorToString, executionReportJsonToString, extractParameterNamesFromTask, filterModels, getPipelineInterface, identificationToPromptbookToken, isPassingExpectations, isPipelineImplementingInterface, isPipelineInterfacesEqual, isPipelinePrepared, isValidPipelineString, joinLlmExecutionTools, limitTotalUsage, makeKnowledgeSourceHandler, migratePipeline, parsePipeline, pipelineJsonToString, prepareKnowledgePieces, preparePersona, preparePipeline, prepareTasks, prettifyPipelineString, promptbookFetch, promptbookTokenToIdentification, unpreparePipeline, usageToHuman, usageToWorktime, validatePipeline, validatePipelineString };
+export { $llmToolsMetadataRegister, $llmToolsRegister, $scrapersMetadataRegister, $scrapersRegister, ADMIN_EMAIL, ADMIN_GITHUB_NAME, AbstractFormatError, AuthenticationError, BOOK_LANGUAGE_VERSION, BlackholeStorage, BoilerplateError, BoilerplateFormfactorDefinition, CLAIM, CLI_APP_ID, CallbackInterfaceTools, ChatbotFormfactorDefinition, CollectionError, CompletionFormfactorDefinition, CsvFormatError, CsvFormatParser, DEFAULT_BOOKS_DIRNAME, DEFAULT_BOOK_OUTPUT_PARAMETER_NAME, DEFAULT_BOOK_TITLE, DEFAULT_CSV_SETTINGS, DEFAULT_DOWNLOAD_CACHE_DIRNAME, DEFAULT_EXECUTION_CACHE_DIRNAME, DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME, DEFAULT_INTERMEDIATE_FILES_STRATEGY, DEFAULT_IS_AUTO_INSTALLED, DEFAULT_IS_VERBOSE, DEFAULT_MAX_EXECUTION_ATTEMPTS, DEFAULT_MAX_FILE_SIZE, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL, DEFAULT_MAX_PARALLEL_COUNT, DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, DEFAULT_PROMPT_TASK_TITLE, DEFAULT_REMOTE_SERVER_URL, DEFAULT_RPM, DEFAULT_SCRAPE_CACHE_DIRNAME, DEFAULT_TASK_TITLE, EXPECTATION_UNITS, EnvironmentMismatchError, ExecutionReportStringOptionsDefaults, ExpectError, FORMFACTOR_DEFINITIONS, GENERIC_PIPELINE_INTERFACE, GeneratorFormfactorDefinition, GenericFormfactorDefinition, ImageGeneratorFormfactorDefinition, KnowledgeScrapeError, LimitReachedError, MANDATORY_CSV_SETTINGS, MAX_FILENAME_LENGTH, MODEL_ORDER, MODEL_TRUST_LEVEL, MODEL_VARIANTS, MatcherFormfactorDefinition, MemoryStorage, MissingToolsError, MultipleLlmExecutionTools, NAME, NonTaskSectionTypes, NotFoundError, NotYetImplementedError, ORDER_OF_PIPELINE_JSON, PLAYGROUND_APP_ID, PROMPTBOOK_ENGINE_VERSION, PROMPTBOOK_ERRORS, ParseError, PipelineExecutionError, PipelineLogicError, PipelineUrlError, PrefixStorage, PromptbookFetchError, REMOTE_SERVER_URLS, RESERVED_PARAMETER_NAMES, SET_IS_VERBOSE, SectionTypes, SheetsFormfactorDefinition, TaskTypes, TextFormatParser, TranslatorFormfactorDefinition, UNCERTAIN_USAGE, UNCERTAIN_ZERO_VALUE, UnexpectedError, WrappedError, ZERO_USAGE, ZERO_VALUE, _AnthropicClaudeMetadataRegistration, _AzureOpenAiMetadataRegistration, _BoilerplateScraperMetadataRegistration, _DeepseekMetadataRegistration, _DocumentScraperMetadataRegistration, _GoogleMetadataRegistration, _LegacyDocumentScraperMetadataRegistration, _MarkdownScraperMetadataRegistration, _MarkitdownScraperMetadataRegistration, _OpenAiAssistantMetadataRegistration, _OpenAiMetadataRegistration, _PdfScraperMetadataRegistration, _WebsiteScraperMetadataRegistration, addUsage, book, cacheLlmTools, collectionToJson, compilePipeline, computeCosineSimilarity, countUsage, createCollectionFromJson, createCollectionFromPromise, createCollectionFromUrl, createLlmToolsFromConfiguration, createPipelineExecutor, createSubcollection, embeddingVectorToString, executionReportJsonToString, extractParameterNamesFromTask, filterModels, getPipelineInterface, identificationToPromptbookToken, isPassingExpectations, isPipelineImplementingInterface, isPipelineInterfacesEqual, isPipelinePrepared, isValidPipelineString, joinLlmExecutionTools, limitTotalUsage, makeKnowledgeSourceHandler, migratePipeline, parsePipeline, pipelineJsonToString, prepareKnowledgePieces, preparePersona, preparePipeline, prepareTasks, prettifyPipelineString, promptbookFetch, promptbookTokenToIdentification, unpreparePipeline, usageToHuman, usageToWorktime, validatePipeline, validatePipelineString };
 //# sourceMappingURL=index.es.js.map