@promptbook/node 0.88.0 → 0.89.0-11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/esm/index.es.js +218 -116
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/core.index.d.ts +18 -6
- package/esm/typings/src/_packages/remote-client.index.d.ts +6 -8
- package/esm/typings/src/_packages/remote-server.index.d.ts +6 -6
- package/esm/typings/src/_packages/types.index.d.ts +18 -10
- package/esm/typings/src/cli/cli-commands/login.d.ts +15 -0
- package/esm/typings/src/cli/common/$addGlobalOptionsToCommand.d.ts +7 -0
- package/esm/typings/src/cli/common/$provideLlmToolsForCli.d.ts +15 -0
- package/esm/typings/src/config.d.ts +15 -8
- package/esm/typings/src/errors/0-index.d.ts +6 -0
- package/esm/typings/src/errors/AuthenticationError.d.ts +9 -0
- package/esm/typings/src/errors/PromptbookFetchError.d.ts +9 -0
- package/esm/typings/src/execution/PipelineExecutorResult.d.ts +2 -2
- package/esm/typings/src/execution/PromptResult.d.ts +2 -2
- package/esm/typings/src/execution/{PromptResultUsage.d.ts → Usage.d.ts} +5 -5
- package/esm/typings/src/execution/utils/addUsage.d.ts +2 -2
- package/esm/typings/src/execution/utils/computeUsageCounts.d.ts +3 -3
- package/esm/typings/src/execution/utils/usage-constants.d.ts +77 -60
- package/esm/typings/src/execution/utils/usageToHuman.d.ts +5 -5
- package/esm/typings/src/execution/utils/usageToWorktime.d.ts +5 -5
- package/esm/typings/src/llm-providers/_common/register/$provideEnvFilename.d.ts +12 -0
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsConfigurationFromEnv.d.ts +2 -8
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground.d.ts +2 -0
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForWizzardOrCli.d.ts +36 -1
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsFromEnv.d.ts +1 -0
- package/esm/typings/src/llm-providers/_common/utils/count-total-usage/LlmExecutionToolsWithTotalUsage.d.ts +9 -2
- package/esm/typings/src/llm-providers/_common/utils/count-total-usage/{countTotalUsage.d.ts → countUsage.d.ts} +1 -1
- package/esm/typings/src/llm-providers/_common/utils/count-total-usage/limitTotalUsage.d.ts +2 -2
- package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions.d.ts +1 -1
- package/esm/typings/src/llm-providers/anthropic-claude/computeAnthropicClaudeUsage.d.ts +2 -2
- package/esm/typings/src/llm-providers/anthropic-claude/register-configuration.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +0 -9
- package/esm/typings/src/llm-providers/openai/computeOpenAiUsage.d.ts +2 -2
- package/esm/typings/src/pipeline/PipelineJson/PreparationJson.d.ts +2 -2
- package/esm/typings/src/playground/playground.d.ts +5 -0
- package/esm/typings/src/remote-server/RemoteServer.d.ts +23 -0
- package/esm/typings/src/remote-server/socket-types/_subtypes/{PromptbookServer_Identification.d.ts → Identification.d.ts} +5 -4
- package/esm/typings/src/remote-server/socket-types/listModels/PromptbookServer_ListModels_Request.d.ts +2 -2
- package/esm/typings/src/remote-server/socket-types/prepare/PromptbookServer_PreparePipeline_Request.d.ts +2 -2
- package/esm/typings/src/remote-server/socket-types/prompt/PromptbookServer_Prompt_Request.d.ts +2 -2
- package/esm/typings/src/remote-server/startRemoteServer.d.ts +2 -2
- package/esm/typings/src/remote-server/types/RemoteClientOptions.d.ts +4 -12
- package/esm/typings/src/remote-server/types/RemoteServerOptions.d.ts +88 -6
- package/esm/typings/src/scrapers/_common/utils/{scraperFetch.d.ts → promptbookFetch.d.ts} +2 -2
- package/esm/typings/src/storage/env-storage/$EnvStorage.d.ts +37 -0
- package/esm/typings/src/types/typeAliases.d.ts +8 -2
- package/esm/typings/src/utils/organization/TODO_narrow.d.ts +6 -0
- package/package.json +2 -2
- package/umd/index.umd.js +218 -116
- package/umd/index.umd.js.map +1 -1
package/README.md
CHANGED
|
@@ -23,6 +23,10 @@
|
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
|
|
26
|
+
<blockquote style="color: #ff8811">
|
|
27
|
+
<b>⚠ Warning:</b> This is a pre-release version of the library. It is not yet ready for production use. Please look at <a href="https://www.npmjs.com/package/@promptbook/core?activeTab=versions">latest stable release</a>.
|
|
28
|
+
</blockquote>
|
|
29
|
+
|
|
26
30
|
## 📦 Package `@promptbook/node`
|
|
27
31
|
|
|
28
32
|
- Promptbooks are [divided into several](#-packages) packages, all are published from [single monorepo](https://github.com/webgptorg/promptbook).
|
|
@@ -200,7 +204,7 @@ Each part of the book defines one of 3 circles:
|
|
|
200
204
|
|
|
201
205
|
### **What:** Workflows, Tasks and Parameters
|
|
202
206
|
|
|
203
|
-
What work needs to be done. Each book defines a workflow, which is one or more tasks. Each workflow has a fixed input and output. For example, you have a book that generates an article from a topic. Once it generates an article about AI, once about marketing, once about cooking. The workflow (= your AI program) is the same, only the input and output change.
|
|
207
|
+
What work needs to be done. Each book defines a [workflow *(scenario or pipeline)*](https://github.com/webgptorg/promptbook/discussions/88), which is one or more tasks. Each workflow has a fixed input and output. For example, you have a book that generates an article from a topic. Once it generates an article about AI, once about marketing, once about cooking. The workflow (= your AI program) is the same, only the input and output change.
|
|
204
208
|
|
|
205
209
|
**Related commands:**
|
|
206
210
|
|
|
@@ -310,6 +314,8 @@ The following glossary is used to clarify certain concepts:
|
|
|
310
314
|
|
|
311
315
|
|
|
312
316
|
|
|
317
|
+
|
|
318
|
+
|
|
313
319
|
_Note: This section is not complete dictionary, more list of general AI / LLM terms that has connection with Promptbook_
|
|
314
320
|
|
|
315
321
|
|
package/esm/index.es.js
CHANGED
|
@@ -30,7 +30,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
|
|
|
30
30
|
* @generated
|
|
31
31
|
* @see https://github.com/webgptorg/promptbook
|
|
32
32
|
*/
|
|
33
|
-
const PROMPTBOOK_ENGINE_VERSION = '0.
|
|
33
|
+
const PROMPTBOOK_ENGINE_VERSION = '0.89.0-11';
|
|
34
34
|
/**
|
|
35
35
|
* TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
|
|
36
36
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -159,6 +159,7 @@ const DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹♂️]
|
|
|
159
159
|
*/
|
|
160
160
|
const DEFAULT_MAX_EXECUTION_ATTEMPTS = 10; // <- TODO: [🤹♂️]
|
|
161
161
|
// <- TODO: [🕝] Make also `BOOKS_DIRNAME_ALTERNATIVES`
|
|
162
|
+
// TODO: !!!!!! Just .promptbook dir, hardocode others
|
|
162
163
|
/**
|
|
163
164
|
* Where to store the temporary downloads
|
|
164
165
|
*
|
|
@@ -175,6 +176,15 @@ const DEFAULT_DOWNLOAD_CACHE_DIRNAME = './.promptbook/download-cache';
|
|
|
175
176
|
* @public exported from `@promptbook/core`
|
|
176
177
|
*/
|
|
177
178
|
const DEFAULT_SCRAPE_CACHE_DIRNAME = './.promptbook/scrape-cache';
|
|
179
|
+
/*
|
|
180
|
+
TODO: [🌃]
|
|
181
|
+
/**
|
|
182
|
+
* Id of application for the wizzard when using remote server
|
|
183
|
+
*
|
|
184
|
+
* @public exported from `@promptbook/core`
|
|
185
|
+
* /
|
|
186
|
+
ex-port const WIZZARD_APP_ID: string_app_id = 'wizzard';
|
|
187
|
+
*/
|
|
178
188
|
/**
|
|
179
189
|
* The name of the builded pipeline collection made by CLI `ptbk make` and for lookup in `createCollectionFromDirectory`
|
|
180
190
|
*
|
|
@@ -216,7 +226,7 @@ const IS_PIPELINE_LOGIC_VALIDATED = just(
|
|
|
216
226
|
true);
|
|
217
227
|
/**
|
|
218
228
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
219
|
-
* TODO: [🧠][🧜♂️] Maybe join
|
|
229
|
+
* TODO: [🧠][🧜♂️] Maybe join remoteServerUrl and path into single value
|
|
220
230
|
*/
|
|
221
231
|
|
|
222
232
|
/**
|
|
@@ -1684,6 +1694,19 @@ class CsvFormatError extends AbstractFormatError {
|
|
|
1684
1694
|
}
|
|
1685
1695
|
}
|
|
1686
1696
|
|
|
1697
|
+
/**
|
|
1698
|
+
* AuthenticationError is thrown from login function which is dependency of remote server
|
|
1699
|
+
*
|
|
1700
|
+
* @public exported from `@promptbook/core`
|
|
1701
|
+
*/
|
|
1702
|
+
class AuthenticationError extends Error {
|
|
1703
|
+
constructor(message) {
|
|
1704
|
+
super(message);
|
|
1705
|
+
this.name = 'AuthenticationError';
|
|
1706
|
+
Object.setPrototypeOf(this, AuthenticationError.prototype);
|
|
1707
|
+
}
|
|
1708
|
+
}
|
|
1709
|
+
|
|
1687
1710
|
/**
|
|
1688
1711
|
* This error indicates that the pipeline collection cannot be propperly loaded
|
|
1689
1712
|
*
|
|
@@ -1775,6 +1798,19 @@ class NotYetImplementedError extends Error {
|
|
|
1775
1798
|
}
|
|
1776
1799
|
}
|
|
1777
1800
|
|
|
1801
|
+
/**
|
|
1802
|
+
* Error thrown when a fetch request fails
|
|
1803
|
+
*
|
|
1804
|
+
* @public exported from `@promptbook/core`
|
|
1805
|
+
*/
|
|
1806
|
+
class PromptbookFetchError extends Error {
|
|
1807
|
+
constructor(message) {
|
|
1808
|
+
super(message);
|
|
1809
|
+
this.name = 'PromptbookFetchError';
|
|
1810
|
+
Object.setPrototypeOf(this, PromptbookFetchError.prototype);
|
|
1811
|
+
}
|
|
1812
|
+
}
|
|
1813
|
+
|
|
1778
1814
|
/**
|
|
1779
1815
|
* Index of all custom errors
|
|
1780
1816
|
*
|
|
@@ -1812,6 +1848,8 @@ const COMMON_JAVASCRIPT_ERRORS = {
|
|
|
1812
1848
|
TypeError,
|
|
1813
1849
|
URIError,
|
|
1814
1850
|
AggregateError,
|
|
1851
|
+
AuthenticationError,
|
|
1852
|
+
PromptbookFetchError,
|
|
1815
1853
|
/*
|
|
1816
1854
|
Note: Not widely supported
|
|
1817
1855
|
> InternalError,
|
|
@@ -1873,6 +1911,7 @@ function assertsTaskSuccessful(executionResult) {
|
|
|
1873
1911
|
const { isSuccessful, errors, warnings } = executionResult;
|
|
1874
1912
|
for (const warning of warnings) {
|
|
1875
1913
|
console.warn(warning.message);
|
|
1914
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
1876
1915
|
}
|
|
1877
1916
|
if (isSuccessful === true) {
|
|
1878
1917
|
return;
|
|
@@ -2010,6 +2049,10 @@ function serializeError(error) {
|
|
|
2010
2049
|
|
|
2011
2050
|
Cannot serialize error with name "${name}"
|
|
2012
2051
|
|
|
2052
|
+
Authors of Promptbook probably forgot to add this error into the list of errors:
|
|
2053
|
+
https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
|
|
2054
|
+
|
|
2055
|
+
|
|
2013
2056
|
${block(stack || message)}
|
|
2014
2057
|
|
|
2015
2058
|
`));
|
|
@@ -2097,30 +2140,42 @@ function valueToString(value) {
|
|
|
2097
2140
|
}
|
|
2098
2141
|
}
|
|
2099
2142
|
|
|
2143
|
+
/**
|
|
2144
|
+
* Represents the uncertain value
|
|
2145
|
+
*
|
|
2146
|
+
* @public exported from `@promptbook/core`
|
|
2147
|
+
*/
|
|
2148
|
+
const ZERO_VALUE = $deepFreeze({ value: 0 });
|
|
2149
|
+
/**
|
|
2150
|
+
* Represents the uncertain value
|
|
2151
|
+
*
|
|
2152
|
+
* @public exported from `@promptbook/core`
|
|
2153
|
+
*/
|
|
2154
|
+
const UNCERTAIN_ZERO_VALUE = $deepFreeze({ value: 0, isUncertain: true });
|
|
2100
2155
|
/**
|
|
2101
2156
|
* Represents the usage with no resources consumed
|
|
2102
2157
|
*
|
|
2103
2158
|
* @public exported from `@promptbook/core`
|
|
2104
2159
|
*/
|
|
2105
2160
|
const ZERO_USAGE = $deepFreeze({
|
|
2106
|
-
price:
|
|
2161
|
+
price: ZERO_VALUE,
|
|
2107
2162
|
input: {
|
|
2108
|
-
tokensCount:
|
|
2109
|
-
charactersCount:
|
|
2110
|
-
wordsCount:
|
|
2111
|
-
sentencesCount:
|
|
2112
|
-
linesCount:
|
|
2113
|
-
paragraphsCount:
|
|
2114
|
-
pagesCount:
|
|
2163
|
+
tokensCount: ZERO_VALUE,
|
|
2164
|
+
charactersCount: ZERO_VALUE,
|
|
2165
|
+
wordsCount: ZERO_VALUE,
|
|
2166
|
+
sentencesCount: ZERO_VALUE,
|
|
2167
|
+
linesCount: ZERO_VALUE,
|
|
2168
|
+
paragraphsCount: ZERO_VALUE,
|
|
2169
|
+
pagesCount: ZERO_VALUE,
|
|
2115
2170
|
},
|
|
2116
2171
|
output: {
|
|
2117
|
-
tokensCount:
|
|
2118
|
-
charactersCount:
|
|
2119
|
-
wordsCount:
|
|
2120
|
-
sentencesCount:
|
|
2121
|
-
linesCount:
|
|
2122
|
-
paragraphsCount:
|
|
2123
|
-
pagesCount:
|
|
2172
|
+
tokensCount: ZERO_VALUE,
|
|
2173
|
+
charactersCount: ZERO_VALUE,
|
|
2174
|
+
wordsCount: ZERO_VALUE,
|
|
2175
|
+
sentencesCount: ZERO_VALUE,
|
|
2176
|
+
linesCount: ZERO_VALUE,
|
|
2177
|
+
paragraphsCount: ZERO_VALUE,
|
|
2178
|
+
pagesCount: ZERO_VALUE,
|
|
2124
2179
|
},
|
|
2125
2180
|
});
|
|
2126
2181
|
/**
|
|
@@ -2129,24 +2184,24 @@ const ZERO_USAGE = $deepFreeze({
|
|
|
2129
2184
|
* @public exported from `@promptbook/core`
|
|
2130
2185
|
*/
|
|
2131
2186
|
$deepFreeze({
|
|
2132
|
-
price:
|
|
2187
|
+
price: UNCERTAIN_ZERO_VALUE,
|
|
2133
2188
|
input: {
|
|
2134
|
-
tokensCount:
|
|
2135
|
-
charactersCount:
|
|
2136
|
-
wordsCount:
|
|
2137
|
-
sentencesCount:
|
|
2138
|
-
linesCount:
|
|
2139
|
-
paragraphsCount:
|
|
2140
|
-
pagesCount:
|
|
2189
|
+
tokensCount: UNCERTAIN_ZERO_VALUE,
|
|
2190
|
+
charactersCount: UNCERTAIN_ZERO_VALUE,
|
|
2191
|
+
wordsCount: UNCERTAIN_ZERO_VALUE,
|
|
2192
|
+
sentencesCount: UNCERTAIN_ZERO_VALUE,
|
|
2193
|
+
linesCount: UNCERTAIN_ZERO_VALUE,
|
|
2194
|
+
paragraphsCount: UNCERTAIN_ZERO_VALUE,
|
|
2195
|
+
pagesCount: UNCERTAIN_ZERO_VALUE,
|
|
2141
2196
|
},
|
|
2142
2197
|
output: {
|
|
2143
|
-
tokensCount:
|
|
2144
|
-
charactersCount:
|
|
2145
|
-
wordsCount:
|
|
2146
|
-
sentencesCount:
|
|
2147
|
-
linesCount:
|
|
2148
|
-
paragraphsCount:
|
|
2149
|
-
pagesCount:
|
|
2198
|
+
tokensCount: UNCERTAIN_ZERO_VALUE,
|
|
2199
|
+
charactersCount: UNCERTAIN_ZERO_VALUE,
|
|
2200
|
+
wordsCount: UNCERTAIN_ZERO_VALUE,
|
|
2201
|
+
sentencesCount: UNCERTAIN_ZERO_VALUE,
|
|
2202
|
+
linesCount: UNCERTAIN_ZERO_VALUE,
|
|
2203
|
+
paragraphsCount: UNCERTAIN_ZERO_VALUE,
|
|
2204
|
+
pagesCount: UNCERTAIN_ZERO_VALUE,
|
|
2150
2205
|
},
|
|
2151
2206
|
});
|
|
2152
2207
|
/**
|
|
@@ -2317,7 +2372,7 @@ function extractParameterNamesFromTask(task) {
|
|
|
2317
2372
|
if (parameterNames.has(subparameterName)) {
|
|
2318
2373
|
parameterNames.delete(subparameterName);
|
|
2319
2374
|
parameterNames.add(foreach.parameterName);
|
|
2320
|
-
// <- TODO: [
|
|
2375
|
+
// <- TODO: [🏮] Warn/logic error when `subparameterName` not used
|
|
2321
2376
|
}
|
|
2322
2377
|
}
|
|
2323
2378
|
}
|
|
@@ -2811,6 +2866,7 @@ function joinLlmExecutionTools(...llmExecutionTools) {
|
|
|
2811
2866
|
`);
|
|
2812
2867
|
// TODO: [🟥] Detect browser / node and make it colorfull
|
|
2813
2868
|
console.warn(warningMessage);
|
|
2869
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
2814
2870
|
/*
|
|
2815
2871
|
return {
|
|
2816
2872
|
async listModels() {
|
|
@@ -4400,6 +4456,7 @@ function createPipelineExecutor(options) {
|
|
|
4400
4456
|
|
|
4401
4457
|
@see more at https://ptbk.io/prepare-pipeline
|
|
4402
4458
|
`));
|
|
4459
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
4403
4460
|
}
|
|
4404
4461
|
let runCount = 0;
|
|
4405
4462
|
const pipelineExecutorWithCallback = async (inputParameters, onProgress) => {
|
|
@@ -4476,8 +4533,9 @@ async function forEachAsync(array, options, callbackfunction) {
|
|
|
4476
4533
|
* @returns LLM tools with same functionality with added total cost counting
|
|
4477
4534
|
* @public exported from `@promptbook/core`
|
|
4478
4535
|
*/
|
|
4479
|
-
function
|
|
4536
|
+
function countUsage(llmTools) {
|
|
4480
4537
|
let totalUsage = ZERO_USAGE;
|
|
4538
|
+
const spending = new Subject();
|
|
4481
4539
|
const proxyTools = {
|
|
4482
4540
|
get title() {
|
|
4483
4541
|
// TODO: [🧠] Maybe put here some suffix
|
|
@@ -4487,12 +4545,15 @@ function countTotalUsage(llmTools) {
|
|
|
4487
4545
|
// TODO: [🧠] Maybe put here some suffix
|
|
4488
4546
|
return llmTools.description;
|
|
4489
4547
|
},
|
|
4490
|
-
|
|
4548
|
+
checkConfiguration() {
|
|
4491
4549
|
return /* not await */ llmTools.checkConfiguration();
|
|
4492
4550
|
},
|
|
4493
4551
|
listModels() {
|
|
4494
4552
|
return /* not await */ llmTools.listModels();
|
|
4495
4553
|
},
|
|
4554
|
+
spending() {
|
|
4555
|
+
return spending.asObservable();
|
|
4556
|
+
},
|
|
4496
4557
|
getTotalUsage() {
|
|
4497
4558
|
// <- Note: [🥫] Not using getter `get totalUsage` but `getTotalUsage` to allow this object to be proxied
|
|
4498
4559
|
return totalUsage;
|
|
@@ -4503,6 +4564,7 @@ function countTotalUsage(llmTools) {
|
|
|
4503
4564
|
// console.info('[🚕] callChatModel through countTotalUsage');
|
|
4504
4565
|
const promptResult = await llmTools.callChatModel(prompt);
|
|
4505
4566
|
totalUsage = addUsage(totalUsage, promptResult.usage);
|
|
4567
|
+
spending.next(promptResult.usage);
|
|
4506
4568
|
return promptResult;
|
|
4507
4569
|
};
|
|
4508
4570
|
}
|
|
@@ -4511,6 +4573,7 @@ function countTotalUsage(llmTools) {
|
|
|
4511
4573
|
// console.info('[🚕] callCompletionModel through countTotalUsage');
|
|
4512
4574
|
const promptResult = await llmTools.callCompletionModel(prompt);
|
|
4513
4575
|
totalUsage = addUsage(totalUsage, promptResult.usage);
|
|
4576
|
+
spending.next(promptResult.usage);
|
|
4514
4577
|
return promptResult;
|
|
4515
4578
|
};
|
|
4516
4579
|
}
|
|
@@ -4519,6 +4582,7 @@ function countTotalUsage(llmTools) {
|
|
|
4519
4582
|
// console.info('[🚕] callEmbeddingModel through countTotalUsage');
|
|
4520
4583
|
const promptResult = await llmTools.callEmbeddingModel(prompt);
|
|
4521
4584
|
totalUsage = addUsage(totalUsage, promptResult.usage);
|
|
4585
|
+
spending.next(promptResult.usage);
|
|
4522
4586
|
return promptResult;
|
|
4523
4587
|
};
|
|
4524
4588
|
}
|
|
@@ -4997,17 +5061,24 @@ function titleToName(value) {
|
|
|
4997
5061
|
/**
|
|
4998
5062
|
* The built-in `fetch' function with a lightweight error handling wrapper as default fetch function used in Promptbook scrapers
|
|
4999
5063
|
*
|
|
5000
|
-
* @
|
|
5064
|
+
* @public exported from `@promptbook/core`
|
|
5001
5065
|
*/
|
|
5002
|
-
const
|
|
5066
|
+
const promptbookFetch = async (urlOrRequest, init) => {
|
|
5003
5067
|
try {
|
|
5004
|
-
return await fetch(
|
|
5068
|
+
return await fetch(urlOrRequest, init);
|
|
5005
5069
|
}
|
|
5006
5070
|
catch (error) {
|
|
5007
5071
|
if (!(error instanceof Error)) {
|
|
5008
5072
|
throw error;
|
|
5009
5073
|
}
|
|
5010
|
-
|
|
5074
|
+
let url;
|
|
5075
|
+
if (typeof urlOrRequest === 'string') {
|
|
5076
|
+
url = urlOrRequest;
|
|
5077
|
+
}
|
|
5078
|
+
else if (urlOrRequest instanceof Request) {
|
|
5079
|
+
url = urlOrRequest.url;
|
|
5080
|
+
}
|
|
5081
|
+
throw new PromptbookFetchError(spaceTrim((block) => `
|
|
5011
5082
|
Can not fetch "${url}"
|
|
5012
5083
|
|
|
5013
5084
|
Fetch error:
|
|
@@ -5028,7 +5099,7 @@ const scraperFetch = async (url, init) => {
|
|
|
5028
5099
|
async function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
5029
5100
|
// console.log('!! makeKnowledgeSourceHandler', knowledgeSource);
|
|
5030
5101
|
var _a;
|
|
5031
|
-
const { fetch =
|
|
5102
|
+
const { fetch = promptbookFetch } = tools;
|
|
5032
5103
|
const { knowledgeSourceContent } = knowledgeSource;
|
|
5033
5104
|
let { name } = knowledgeSource;
|
|
5034
5105
|
const { rootDirname = null,
|
|
@@ -5169,63 +5240,73 @@ async function prepareKnowledgePieces(knowledgeSources, tools, options) {
|
|
|
5169
5240
|
const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT, rootDirname, isVerbose = DEFAULT_IS_VERBOSE } = options;
|
|
5170
5241
|
const knowledgePreparedUnflatten = new Array(knowledgeSources.length);
|
|
5171
5242
|
await forEachAsync(knowledgeSources, { maxParallelCount }, async (knowledgeSource, index) => {
|
|
5172
|
-
|
|
5173
|
-
|
|
5174
|
-
|
|
5175
|
-
|
|
5176
|
-
|
|
5177
|
-
|
|
5178
|
-
|
|
5179
|
-
|
|
5180
|
-
|
|
5181
|
-
|
|
5182
|
-
|
|
5183
|
-
|
|
5184
|
-
|
|
5185
|
-
|
|
5186
|
-
|
|
5187
|
-
|
|
5188
|
-
|
|
5243
|
+
try {
|
|
5244
|
+
let partialPieces = null;
|
|
5245
|
+
const sourceHandler = await makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname, isVerbose });
|
|
5246
|
+
const scrapers = arrayableToArray(tools.scrapers);
|
|
5247
|
+
for (const scraper of scrapers) {
|
|
5248
|
+
if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
|
|
5249
|
+
// <- TODO: [🦔] Implement mime-type wildcards
|
|
5250
|
+
) {
|
|
5251
|
+
continue;
|
|
5252
|
+
}
|
|
5253
|
+
const partialPiecesUnchecked = await scraper.scrape(sourceHandler);
|
|
5254
|
+
if (partialPiecesUnchecked !== null) {
|
|
5255
|
+
partialPieces = [...partialPiecesUnchecked];
|
|
5256
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
|
|
5257
|
+
break;
|
|
5258
|
+
}
|
|
5259
|
+
console.warn(spaceTrim((block) => `
|
|
5260
|
+
Cannot scrape knowledge from source despite the scraper \`${scraper.metadata.className}\` supports the mime type "${sourceHandler.mimeType}".
|
|
5189
5261
|
|
|
5190
|
-
|
|
5191
|
-
|
|
5192
|
-
|
|
5193
|
-
|
|
5194
|
-
|
|
5262
|
+
The source:
|
|
5263
|
+
${block(knowledgeSource.knowledgeSourceContent
|
|
5264
|
+
.split('\n')
|
|
5265
|
+
.map((line) => `> ${line}`)
|
|
5266
|
+
.join('\n'))}
|
|
5195
5267
|
|
|
5196
|
-
|
|
5268
|
+
${block($registeredScrapersMessage(scrapers))}
|
|
5197
5269
|
|
|
5198
5270
|
|
|
5199
|
-
|
|
5200
|
-
|
|
5201
|
-
|
|
5202
|
-
|
|
5203
|
-
|
|
5271
|
+
`));
|
|
5272
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
5273
|
+
}
|
|
5274
|
+
if (partialPieces === null) {
|
|
5275
|
+
throw new KnowledgeScrapeError(spaceTrim((block) => `
|
|
5276
|
+
Cannot scrape knowledge
|
|
5204
5277
|
|
|
5205
|
-
|
|
5206
|
-
|
|
5207
|
-
|
|
5208
|
-
|
|
5209
|
-
|
|
5278
|
+
The source:
|
|
5279
|
+
> ${block(knowledgeSource.knowledgeSourceContent
|
|
5280
|
+
.split('\n')
|
|
5281
|
+
.map((line) => `> ${line}`)
|
|
5282
|
+
.join('\n'))}
|
|
5210
5283
|
|
|
5211
|
-
|
|
5284
|
+
No scraper found for the mime type "${sourceHandler.mimeType}"
|
|
5212
5285
|
|
|
5213
|
-
|
|
5286
|
+
${block($registeredScrapersMessage(scrapers))}
|
|
5214
5287
|
|
|
5215
5288
|
|
|
5216
|
-
|
|
5289
|
+
`));
|
|
5290
|
+
}
|
|
5291
|
+
const pieces = partialPieces.map((partialPiece) => ({
|
|
5292
|
+
...partialPiece,
|
|
5293
|
+
sources: [
|
|
5294
|
+
{
|
|
5295
|
+
name: knowledgeSource.name,
|
|
5296
|
+
// line, column <- TODO: [☀]
|
|
5297
|
+
// <- TODO: [❎]
|
|
5298
|
+
},
|
|
5299
|
+
],
|
|
5300
|
+
}));
|
|
5301
|
+
knowledgePreparedUnflatten[index] = pieces;
|
|
5302
|
+
}
|
|
5303
|
+
catch (error) {
|
|
5304
|
+
if (!(error instanceof Error)) {
|
|
5305
|
+
throw error;
|
|
5306
|
+
}
|
|
5307
|
+
console.warn(error);
|
|
5308
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
5217
5309
|
}
|
|
5218
|
-
const pieces = partialPieces.map((partialPiece) => ({
|
|
5219
|
-
...partialPiece,
|
|
5220
|
-
sources: [
|
|
5221
|
-
{
|
|
5222
|
-
name: knowledgeSource.name,
|
|
5223
|
-
// line, column <- TODO: [☀]
|
|
5224
|
-
// <- TODO: [❎]
|
|
5225
|
-
},
|
|
5226
|
-
],
|
|
5227
|
-
}));
|
|
5228
|
-
knowledgePreparedUnflatten[index] = pieces;
|
|
5229
5310
|
});
|
|
5230
5311
|
const knowledgePrepared = knowledgePreparedUnflatten.flat();
|
|
5231
5312
|
return knowledgePrepared;
|
|
@@ -5331,7 +5412,7 @@ async function preparePipeline(pipeline, tools, options) {
|
|
|
5331
5412
|
// TODO: [🚐] Make arrayable LLMs -> single LLM DRY
|
|
5332
5413
|
const _llms = arrayableToArray(tools.llm);
|
|
5333
5414
|
const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
|
|
5334
|
-
const llmToolsWithUsage =
|
|
5415
|
+
const llmToolsWithUsage = countUsage(llmTools);
|
|
5335
5416
|
// <- TODO: [🌯]
|
|
5336
5417
|
/*
|
|
5337
5418
|
TODO: [🧠][🪑][🔃] Should this be done or not
|
|
@@ -7050,7 +7131,8 @@ const modelCommandParser = {
|
|
|
7050
7131
|
if ($pipelineJson.defaultModelRequirements[command.key] !== undefined) {
|
|
7051
7132
|
if ($pipelineJson.defaultModelRequirements[command.key] === command.value) {
|
|
7052
7133
|
console.warn(`Multiple commands \`MODEL ${command.key} ${command.value}\` in the pipeline head`);
|
|
7053
|
-
// <- TODO: [
|
|
7134
|
+
// <- TODO: [🏮] Some better way how to get warnings from pipeline parsing / logic
|
|
7135
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
7054
7136
|
}
|
|
7055
7137
|
else {
|
|
7056
7138
|
throw new ParseError(spaceTrim(`
|
|
@@ -7082,6 +7164,7 @@ const modelCommandParser = {
|
|
|
7082
7164
|
modelVariant: 'VARIANT',
|
|
7083
7165
|
maxTokens: '???',
|
|
7084
7166
|
}[command.key]} ${command.value}\` in the task "${$taskJson.title || $taskJson.name}"`);
|
|
7167
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
7085
7168
|
}
|
|
7086
7169
|
else {
|
|
7087
7170
|
throw new ParseError(spaceTrim(`
|
|
@@ -7361,15 +7444,15 @@ function $applyToTaskJson(command, $taskJson, $pipelineJson) {
|
|
|
7361
7444
|
}
|
|
7362
7445
|
console.warn(spaceTrim(`
|
|
7363
7446
|
|
|
7364
|
-
|
|
7447
|
+
Persona "${personaName}" is defined multiple times with different description:
|
|
7365
7448
|
|
|
7366
|
-
|
|
7367
|
-
|
|
7449
|
+
First definition:
|
|
7450
|
+
${persona.description}
|
|
7368
7451
|
|
|
7369
|
-
|
|
7370
|
-
|
|
7452
|
+
Second definition:
|
|
7453
|
+
${personaDescription}
|
|
7371
7454
|
|
|
7372
|
-
|
|
7455
|
+
`));
|
|
7373
7456
|
persona.description += spaceTrim('\n\n' + personaDescription);
|
|
7374
7457
|
}
|
|
7375
7458
|
|
|
@@ -9005,6 +9088,7 @@ function $execCommand(options) {
|
|
|
9005
9088
|
}
|
|
9006
9089
|
else {
|
|
9007
9090
|
console.warn(`Command "${humanReadableCommand}" exceeded time limit of ${timeout}ms but continues running`);
|
|
9091
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
9008
9092
|
resolve('Command exceeded time limit');
|
|
9009
9093
|
}
|
|
9010
9094
|
});
|
|
@@ -9030,6 +9114,7 @@ function $execCommand(options) {
|
|
|
9030
9114
|
output.push(stderr.toString());
|
|
9031
9115
|
if (isVerbose && stderr.toString().trim()) {
|
|
9032
9116
|
console.warn(stderr.toString());
|
|
9117
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
9033
9118
|
}
|
|
9034
9119
|
});
|
|
9035
9120
|
const finishWithCode = (code) => {
|
|
@@ -9041,6 +9126,7 @@ function $execCommand(options) {
|
|
|
9041
9126
|
else {
|
|
9042
9127
|
if (isVerbose) {
|
|
9043
9128
|
console.warn(`Command "${humanReadableCommand}" exited with code ${code}`);
|
|
9129
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
9044
9130
|
}
|
|
9045
9131
|
resolve(spaceTrim$1(output.join('\n')));
|
|
9046
9132
|
}
|
|
@@ -9062,6 +9148,7 @@ function $execCommand(options) {
|
|
|
9062
9148
|
else {
|
|
9063
9149
|
if (isVerbose) {
|
|
9064
9150
|
console.warn(error);
|
|
9151
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
9065
9152
|
}
|
|
9066
9153
|
resolve(spaceTrim$1(output.join('\n')));
|
|
9067
9154
|
}
|
|
@@ -9492,21 +9579,15 @@ function $registeredLlmToolsMessage() {
|
|
|
9492
9579
|
*/
|
|
9493
9580
|
|
|
9494
9581
|
/**
|
|
9495
|
-
*
|
|
9496
|
-
*
|
|
9497
|
-
* @@@ .env
|
|
9582
|
+
* Provides the path to the `.env` file
|
|
9498
9583
|
*
|
|
9499
|
-
*
|
|
9500
|
-
* - `process.env.OPENAI_API_KEY`
|
|
9501
|
-
* - `process.env.ANTHROPIC_CLAUDE_API_KEY`
|
|
9502
|
-
* - ...
|
|
9584
|
+
* Note: `$` is used to indicate that this function is not a pure function - it uses filesystem to access `.env` file
|
|
9503
9585
|
*
|
|
9504
|
-
* @
|
|
9505
|
-
* @public exported from `@promptbook/node`
|
|
9586
|
+
* @private within the repository - for CLI utils
|
|
9506
9587
|
*/
|
|
9507
|
-
async function $
|
|
9588
|
+
async function $provideEnvFilename() {
|
|
9508
9589
|
if (!$isRunningInNode()) {
|
|
9509
|
-
throw new EnvironmentMismatchError('Function `$
|
|
9590
|
+
throw new EnvironmentMismatchError('Function `$provideEnvFilename` works only in Node.js environment');
|
|
9510
9591
|
}
|
|
9511
9592
|
const envFilePatterns = [
|
|
9512
9593
|
'.env',
|
|
@@ -9526,8 +9607,7 @@ async function $provideLlmToolsConfigurationFromEnv() {
|
|
|
9526
9607
|
const envFilename = join(rootDirname, pattern);
|
|
9527
9608
|
if (await isFileExisting(envFilename, $provideFilesystemForNode())) {
|
|
9528
9609
|
$setUsedEnvFilename(envFilename);
|
|
9529
|
-
|
|
9530
|
-
break up_to_root;
|
|
9610
|
+
return envFilename;
|
|
9531
9611
|
}
|
|
9532
9612
|
}
|
|
9533
9613
|
if (isRootPath(rootDirname)) {
|
|
@@ -9536,6 +9616,34 @@ async function $provideLlmToolsConfigurationFromEnv() {
|
|
|
9536
9616
|
// Note: If the directory does not exist, try the parent directory
|
|
9537
9617
|
rootDirname = join(rootDirname, '..');
|
|
9538
9618
|
}
|
|
9619
|
+
return null;
|
|
9620
|
+
}
|
|
9621
|
+
/**
|
|
9622
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
9623
|
+
*/
|
|
9624
|
+
|
|
9625
|
+
/**
|
|
9626
|
+
* @@@
|
|
9627
|
+
*
|
|
9628
|
+
* @@@ .env
|
|
9629
|
+
* Note: `$` is used to indicate that this function is not a pure function - it uses filesystem to access `.env` file
|
|
9630
|
+
*
|
|
9631
|
+
* It looks for environment variables:
|
|
9632
|
+
* - `process.env.OPENAI_API_KEY`
|
|
9633
|
+
* - `process.env.ANTHROPIC_CLAUDE_API_KEY`
|
|
9634
|
+
* - ...
|
|
9635
|
+
*
|
|
9636
|
+
* @returns @@@
|
|
9637
|
+
* @public exported from `@promptbook/node`
|
|
9638
|
+
*/
|
|
9639
|
+
async function $provideLlmToolsConfigurationFromEnv() {
|
|
9640
|
+
if (!$isRunningInNode()) {
|
|
9641
|
+
throw new EnvironmentMismatchError('Function `$provideLlmToolsFromEnv` works only in Node.js environment');
|
|
9642
|
+
}
|
|
9643
|
+
const envFilepath = await $provideEnvFilename();
|
|
9644
|
+
if (envFilepath !== null) {
|
|
9645
|
+
dotenv.config({ path: envFilepath });
|
|
9646
|
+
}
|
|
9539
9647
|
const llmToolsConfiguration = $llmToolsMetadataRegister
|
|
9540
9648
|
.list()
|
|
9541
9649
|
.map((metadata) => metadata.createConfigurationFromEnv(process.env))
|
|
@@ -9543,15 +9651,8 @@ async function $provideLlmToolsConfigurationFromEnv() {
|
|
|
9543
9651
|
return llmToolsConfiguration;
|
|
9544
9652
|
}
|
|
9545
9653
|
/**
|
|
9546
|
-
* TODO: [🧠][🪁] Maybe do allow to do auto-install if package not registered and not found
|
|
9547
|
-
* TODO: Add Azure OpenAI
|
|
9548
|
-
* TODO: [🧠][🍛]
|
|
9549
|
-
* TODO: [🧠] Is there some meaningfull way how to test this util
|
|
9550
9654
|
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
9551
|
-
|
|
9552
|
-
* TODO: This should be maybe not under `_common` but under `utils`
|
|
9553
|
-
* TODO: [🧠][⚛] Maybe pass env as argument
|
|
9554
|
-
* TODO: [®] DRY Register logic */
|
|
9655
|
+
*/
|
|
9555
9656
|
|
|
9556
9657
|
/**
|
|
9557
9658
|
* @@@
|
|
@@ -9610,6 +9711,7 @@ function createLlmToolsFromConfiguration(configuration, options = {}) {
|
|
|
9610
9711
|
* Note: This function is not cached, every call creates new instance of `MultipleLlmExecutionTools`
|
|
9611
9712
|
*
|
|
9612
9713
|
* @@@ .env
|
|
9714
|
+
* Note: `$` is used to indicate that this function is not a pure function - it uses filesystem to access `.env` file
|
|
9613
9715
|
*
|
|
9614
9716
|
* It looks for environment variables:
|
|
9615
9717
|
* - `process.env.OPENAI_API_KEY`
|