@promptbook/core 0.89.0-1 → 0.89.0-11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/esm/index.es.js +186 -104
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/core.index.d.ts +16 -4
- package/esm/typings/src/_packages/remote-client.index.d.ts +6 -8
- package/esm/typings/src/_packages/remote-server.index.d.ts +6 -6
- package/esm/typings/src/_packages/types.index.d.ts +18 -20
- package/esm/typings/src/cli/cli-commands/login.d.ts +15 -0
- package/esm/typings/src/cli/common/$addGlobalOptionsToCommand.d.ts +7 -0
- package/esm/typings/src/cli/common/$provideLlmToolsForCli.d.ts +15 -0
- package/esm/typings/src/config.d.ts +15 -8
- package/esm/typings/src/errors/0-index.d.ts +6 -0
- package/esm/typings/src/errors/AuthenticationError.d.ts +9 -0
- package/esm/typings/src/errors/PromptbookFetchError.d.ts +9 -0
- package/esm/typings/src/execution/PipelineExecutorResult.d.ts +2 -2
- package/esm/typings/src/execution/PromptResult.d.ts +2 -2
- package/esm/typings/src/execution/{PromptResultUsage.d.ts → Usage.d.ts} +5 -5
- package/esm/typings/src/execution/utils/addUsage.d.ts +2 -2
- package/esm/typings/src/execution/utils/computeUsageCounts.d.ts +3 -3
- package/esm/typings/src/execution/utils/usage-constants.d.ts +77 -60
- package/esm/typings/src/execution/utils/usageToHuman.d.ts +5 -5
- package/esm/typings/src/execution/utils/usageToWorktime.d.ts +5 -5
- package/esm/typings/src/llm-providers/_common/register/$provideEnvFilename.d.ts +12 -0
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsConfigurationFromEnv.d.ts +2 -8
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground.d.ts +2 -0
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForWizzardOrCli.d.ts +36 -1
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsFromEnv.d.ts +1 -0
- package/esm/typings/src/llm-providers/_common/utils/count-total-usage/LlmExecutionToolsWithTotalUsage.d.ts +3 -3
- package/esm/typings/src/llm-providers/_common/utils/count-total-usage/limitTotalUsage.d.ts +2 -2
- package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions.d.ts +1 -1
- package/esm/typings/src/llm-providers/anthropic-claude/computeAnthropicClaudeUsage.d.ts +2 -2
- package/esm/typings/src/llm-providers/anthropic-claude/register-configuration.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +0 -9
- package/esm/typings/src/llm-providers/openai/computeOpenAiUsage.d.ts +2 -2
- package/esm/typings/src/pipeline/PipelineJson/PreparationJson.d.ts +2 -2
- package/esm/typings/src/remote-server/RemoteServer.d.ts +23 -0
- package/esm/typings/src/remote-server/socket-types/_subtypes/{PromptbookServer_Identification.d.ts → Identification.d.ts} +3 -3
- package/esm/typings/src/remote-server/socket-types/listModels/PromptbookServer_ListModels_Request.d.ts +2 -2
- package/esm/typings/src/remote-server/socket-types/prepare/PromptbookServer_PreparePipeline_Request.d.ts +2 -2
- package/esm/typings/src/remote-server/socket-types/prompt/PromptbookServer_Prompt_Request.d.ts +2 -2
- package/esm/typings/src/remote-server/startRemoteServer.d.ts +2 -2
- package/esm/typings/src/remote-server/types/RemoteClientOptions.d.ts +4 -12
- package/esm/typings/src/remote-server/types/RemoteServerOptions.d.ts +73 -3
- package/esm/typings/src/scrapers/_common/utils/{scraperFetch.d.ts → promptbookFetch.d.ts} +2 -2
- package/esm/typings/src/storage/env-storage/$EnvStorage.d.ts +37 -0
- package/esm/typings/src/types/typeAliases.d.ts +6 -0
- package/esm/typings/src/utils/organization/TODO_narrow.d.ts +6 -0
- package/package.json +1 -1
- package/umd/index.umd.js +193 -105
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/playground/BrjappConnector.d.ts +0 -64
- package/esm/typings/src/playground/brjapp-api-schema.d.ts +0 -12879
package/README.md
CHANGED
|
@@ -204,7 +204,7 @@ Each part of the book defines one of 3 circles:
|
|
|
204
204
|
|
|
205
205
|
### **What:** Workflows, Tasks and Parameters
|
|
206
206
|
|
|
207
|
-
What work needs to be done. Each book defines a workflow, which is one or more tasks. Each workflow has a fixed input and output. For example, you have a book that generates an article from a topic. Once it generates an article about AI, once about marketing, once about cooking. The workflow (= your AI program) is the same, only the input and output change.
|
|
207
|
+
What work needs to be done. Each book defines a [workflow *(scenario or pipeline)*](https://github.com/webgptorg/promptbook/discussions/88), which is one or more tasks. Each workflow has a fixed input and output. For example, you have a book that generates an article from a topic. Once it generates an article about AI, once about marketing, once about cooking. The workflow (= your AI program) is the same, only the input and output change.
|
|
208
208
|
|
|
209
209
|
**Related commands:**
|
|
210
210
|
|
|
@@ -314,6 +314,8 @@ The following glossary is used to clarify certain concepts:
|
|
|
314
314
|
|
|
315
315
|
|
|
316
316
|
|
|
317
|
+
|
|
318
|
+
|
|
317
319
|
_Note: This section is not complete dictionary, more list of general AI / LLM terms that has connection with Promptbook_
|
|
318
320
|
|
|
319
321
|
|
package/esm/index.es.js
CHANGED
|
@@ -27,7 +27,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
|
|
|
27
27
|
* @generated
|
|
28
28
|
* @see https://github.com/webgptorg/promptbook
|
|
29
29
|
*/
|
|
30
|
-
const PROMPTBOOK_ENGINE_VERSION = '0.89.0-
|
|
30
|
+
const PROMPTBOOK_ENGINE_VERSION = '0.89.0-11';
|
|
31
31
|
/**
|
|
32
32
|
* TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
|
|
33
33
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -572,6 +572,7 @@ const DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL = 200;
|
|
|
572
572
|
*/
|
|
573
573
|
const DEFAULT_BOOKS_DIRNAME = './books';
|
|
574
574
|
// <- TODO: [🕝] Make also `BOOKS_DIRNAME_ALTERNATIVES`
|
|
575
|
+
// TODO: !!!!!! Just .promptbook dir, hardocode others
|
|
575
576
|
/**
|
|
576
577
|
* Where to store the temporary downloads
|
|
577
578
|
*
|
|
@@ -596,6 +597,27 @@ const DEFAULT_EXECUTION_CACHE_DIRNAME = './.promptbook/execution-cache';
|
|
|
596
597
|
* @public exported from `@promptbook/core`
|
|
597
598
|
*/
|
|
598
599
|
const DEFAULT_SCRAPE_CACHE_DIRNAME = './.promptbook/scrape-cache';
|
|
600
|
+
/**
|
|
601
|
+
* Id of application for the CLI when using remote server
|
|
602
|
+
*
|
|
603
|
+
* @public exported from `@promptbook/core`
|
|
604
|
+
*/
|
|
605
|
+
const CLI_APP_ID = 'cli';
|
|
606
|
+
/**
|
|
607
|
+
* Id of application for the playground
|
|
608
|
+
*
|
|
609
|
+
* @public exported from `@promptbook/core`
|
|
610
|
+
*/
|
|
611
|
+
const PLAYGROUND_APP_ID = 'playground';
|
|
612
|
+
/*
|
|
613
|
+
TODO: [🌃]
|
|
614
|
+
/**
|
|
615
|
+
* Id of application for the wizzard when using remote server
|
|
616
|
+
*
|
|
617
|
+
* @public exported from `@promptbook/core`
|
|
618
|
+
* /
|
|
619
|
+
ex-port const WIZZARD_APP_ID: string_app_id = 'wizzard';
|
|
620
|
+
*/
|
|
599
621
|
/**
|
|
600
622
|
* The name of the builded pipeline collection made by CLI `ptbk make` and for lookup in `createCollectionFromDirectory`
|
|
601
623
|
*
|
|
@@ -616,13 +638,7 @@ const MOMENT_ARG_THRESHOLDS = {
|
|
|
616
638
|
*
|
|
617
639
|
* @public exported from `@promptbook/core`
|
|
618
640
|
*/
|
|
619
|
-
const
|
|
620
|
-
/**
|
|
621
|
-
* @@@
|
|
622
|
-
*
|
|
623
|
-
* @public exported from `@promptbook/core`
|
|
624
|
-
*/
|
|
625
|
-
const DEFAULT_REMOTE_URL_PATH = '/promptbook/socket.io';
|
|
641
|
+
const DEFAULT_REMOTE_SERVER_URL = 'https://api.pavolhejny.com/promptbook';
|
|
626
642
|
// <- TODO: [🧜♂️]
|
|
627
643
|
/**
|
|
628
644
|
* @@@
|
|
@@ -674,7 +690,7 @@ const IS_PIPELINE_LOGIC_VALIDATED = just(
|
|
|
674
690
|
true);
|
|
675
691
|
/**
|
|
676
692
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
677
|
-
* TODO: [🧠][🧜♂️] Maybe join
|
|
693
|
+
* TODO: [🧠][🧜♂️] Maybe join remoteServerUrl and path into single value
|
|
678
694
|
*/
|
|
679
695
|
|
|
680
696
|
/**
|
|
@@ -1893,6 +1909,19 @@ class CsvFormatError extends AbstractFormatError {
|
|
|
1893
1909
|
}
|
|
1894
1910
|
}
|
|
1895
1911
|
|
|
1912
|
+
/**
|
|
1913
|
+
* AuthenticationError is thrown from login function which is dependency of remote server
|
|
1914
|
+
*
|
|
1915
|
+
* @public exported from `@promptbook/core`
|
|
1916
|
+
*/
|
|
1917
|
+
class AuthenticationError extends Error {
|
|
1918
|
+
constructor(message) {
|
|
1919
|
+
super(message);
|
|
1920
|
+
this.name = 'AuthenticationError';
|
|
1921
|
+
Object.setPrototypeOf(this, AuthenticationError.prototype);
|
|
1922
|
+
}
|
|
1923
|
+
}
|
|
1924
|
+
|
|
1896
1925
|
/**
|
|
1897
1926
|
* This error indicates that the pipeline collection cannot be propperly loaded
|
|
1898
1927
|
*
|
|
@@ -1984,6 +2013,19 @@ class NotYetImplementedError extends Error {
|
|
|
1984
2013
|
}
|
|
1985
2014
|
}
|
|
1986
2015
|
|
|
2016
|
+
/**
|
|
2017
|
+
* Error thrown when a fetch request fails
|
|
2018
|
+
*
|
|
2019
|
+
* @public exported from `@promptbook/core`
|
|
2020
|
+
*/
|
|
2021
|
+
class PromptbookFetchError extends Error {
|
|
2022
|
+
constructor(message) {
|
|
2023
|
+
super(message);
|
|
2024
|
+
this.name = 'PromptbookFetchError';
|
|
2025
|
+
Object.setPrototypeOf(this, PromptbookFetchError.prototype);
|
|
2026
|
+
}
|
|
2027
|
+
}
|
|
2028
|
+
|
|
1987
2029
|
/**
|
|
1988
2030
|
* Index of all custom errors
|
|
1989
2031
|
*
|
|
@@ -2021,6 +2063,8 @@ const COMMON_JAVASCRIPT_ERRORS = {
|
|
|
2021
2063
|
TypeError,
|
|
2022
2064
|
URIError,
|
|
2023
2065
|
AggregateError,
|
|
2066
|
+
AuthenticationError,
|
|
2067
|
+
PromptbookFetchError,
|
|
2024
2068
|
/*
|
|
2025
2069
|
Note: Not widely supported
|
|
2026
2070
|
> InternalError,
|
|
@@ -2082,6 +2126,7 @@ function assertsTaskSuccessful(executionResult) {
|
|
|
2082
2126
|
const { isSuccessful, errors, warnings } = executionResult;
|
|
2083
2127
|
for (const warning of warnings) {
|
|
2084
2128
|
console.warn(warning.message);
|
|
2129
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
2085
2130
|
}
|
|
2086
2131
|
if (isSuccessful === true) {
|
|
2087
2132
|
return;
|
|
@@ -2219,6 +2264,10 @@ function serializeError(error) {
|
|
|
2219
2264
|
|
|
2220
2265
|
Cannot serialize error with name "${name}"
|
|
2221
2266
|
|
|
2267
|
+
Authors of Promptbook probably forgot to add this error into the list of errors:
|
|
2268
|
+
https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
|
|
2269
|
+
|
|
2270
|
+
|
|
2222
2271
|
${block(stack || message)}
|
|
2223
2272
|
|
|
2224
2273
|
`));
|
|
@@ -2306,30 +2355,42 @@ function valueToString(value) {
|
|
|
2306
2355
|
}
|
|
2307
2356
|
}
|
|
2308
2357
|
|
|
2358
|
+
/**
|
|
2359
|
+
* Represents the uncertain value
|
|
2360
|
+
*
|
|
2361
|
+
* @public exported from `@promptbook/core`
|
|
2362
|
+
*/
|
|
2363
|
+
const ZERO_VALUE = $deepFreeze({ value: 0 });
|
|
2364
|
+
/**
|
|
2365
|
+
* Represents the uncertain value
|
|
2366
|
+
*
|
|
2367
|
+
* @public exported from `@promptbook/core`
|
|
2368
|
+
*/
|
|
2369
|
+
const UNCERTAIN_ZERO_VALUE = $deepFreeze({ value: 0, isUncertain: true });
|
|
2309
2370
|
/**
|
|
2310
2371
|
* Represents the usage with no resources consumed
|
|
2311
2372
|
*
|
|
2312
2373
|
* @public exported from `@promptbook/core`
|
|
2313
2374
|
*/
|
|
2314
2375
|
const ZERO_USAGE = $deepFreeze({
|
|
2315
|
-
price:
|
|
2376
|
+
price: ZERO_VALUE,
|
|
2316
2377
|
input: {
|
|
2317
|
-
tokensCount:
|
|
2318
|
-
charactersCount:
|
|
2319
|
-
wordsCount:
|
|
2320
|
-
sentencesCount:
|
|
2321
|
-
linesCount:
|
|
2322
|
-
paragraphsCount:
|
|
2323
|
-
pagesCount:
|
|
2378
|
+
tokensCount: ZERO_VALUE,
|
|
2379
|
+
charactersCount: ZERO_VALUE,
|
|
2380
|
+
wordsCount: ZERO_VALUE,
|
|
2381
|
+
sentencesCount: ZERO_VALUE,
|
|
2382
|
+
linesCount: ZERO_VALUE,
|
|
2383
|
+
paragraphsCount: ZERO_VALUE,
|
|
2384
|
+
pagesCount: ZERO_VALUE,
|
|
2324
2385
|
},
|
|
2325
2386
|
output: {
|
|
2326
|
-
tokensCount:
|
|
2327
|
-
charactersCount:
|
|
2328
|
-
wordsCount:
|
|
2329
|
-
sentencesCount:
|
|
2330
|
-
linesCount:
|
|
2331
|
-
paragraphsCount:
|
|
2332
|
-
pagesCount:
|
|
2387
|
+
tokensCount: ZERO_VALUE,
|
|
2388
|
+
charactersCount: ZERO_VALUE,
|
|
2389
|
+
wordsCount: ZERO_VALUE,
|
|
2390
|
+
sentencesCount: ZERO_VALUE,
|
|
2391
|
+
linesCount: ZERO_VALUE,
|
|
2392
|
+
paragraphsCount: ZERO_VALUE,
|
|
2393
|
+
pagesCount: ZERO_VALUE,
|
|
2333
2394
|
},
|
|
2334
2395
|
});
|
|
2335
2396
|
/**
|
|
@@ -2338,24 +2399,24 @@ const ZERO_USAGE = $deepFreeze({
|
|
|
2338
2399
|
* @public exported from `@promptbook/core`
|
|
2339
2400
|
*/
|
|
2340
2401
|
const UNCERTAIN_USAGE = $deepFreeze({
|
|
2341
|
-
price:
|
|
2402
|
+
price: UNCERTAIN_ZERO_VALUE,
|
|
2342
2403
|
input: {
|
|
2343
|
-
tokensCount:
|
|
2344
|
-
charactersCount:
|
|
2345
|
-
wordsCount:
|
|
2346
|
-
sentencesCount:
|
|
2347
|
-
linesCount:
|
|
2348
|
-
paragraphsCount:
|
|
2349
|
-
pagesCount:
|
|
2404
|
+
tokensCount: UNCERTAIN_ZERO_VALUE,
|
|
2405
|
+
charactersCount: UNCERTAIN_ZERO_VALUE,
|
|
2406
|
+
wordsCount: UNCERTAIN_ZERO_VALUE,
|
|
2407
|
+
sentencesCount: UNCERTAIN_ZERO_VALUE,
|
|
2408
|
+
linesCount: UNCERTAIN_ZERO_VALUE,
|
|
2409
|
+
paragraphsCount: UNCERTAIN_ZERO_VALUE,
|
|
2410
|
+
pagesCount: UNCERTAIN_ZERO_VALUE,
|
|
2350
2411
|
},
|
|
2351
2412
|
output: {
|
|
2352
|
-
tokensCount:
|
|
2353
|
-
charactersCount:
|
|
2354
|
-
wordsCount:
|
|
2355
|
-
sentencesCount:
|
|
2356
|
-
linesCount:
|
|
2357
|
-
paragraphsCount:
|
|
2358
|
-
pagesCount:
|
|
2413
|
+
tokensCount: UNCERTAIN_ZERO_VALUE,
|
|
2414
|
+
charactersCount: UNCERTAIN_ZERO_VALUE,
|
|
2415
|
+
wordsCount: UNCERTAIN_ZERO_VALUE,
|
|
2416
|
+
sentencesCount: UNCERTAIN_ZERO_VALUE,
|
|
2417
|
+
linesCount: UNCERTAIN_ZERO_VALUE,
|
|
2418
|
+
paragraphsCount: UNCERTAIN_ZERO_VALUE,
|
|
2419
|
+
pagesCount: UNCERTAIN_ZERO_VALUE,
|
|
2359
2420
|
},
|
|
2360
2421
|
});
|
|
2361
2422
|
/**
|
|
@@ -2526,7 +2587,7 @@ function extractParameterNamesFromTask(task) {
|
|
|
2526
2587
|
if (parameterNames.has(subparameterName)) {
|
|
2527
2588
|
parameterNames.delete(subparameterName);
|
|
2528
2589
|
parameterNames.add(foreach.parameterName);
|
|
2529
|
-
// <- TODO: [
|
|
2590
|
+
// <- TODO: [🏮] Warn/logic error when `subparameterName` not used
|
|
2530
2591
|
}
|
|
2531
2592
|
}
|
|
2532
2593
|
}
|
|
@@ -3020,6 +3081,7 @@ function joinLlmExecutionTools(...llmExecutionTools) {
|
|
|
3020
3081
|
`);
|
|
3021
3082
|
// TODO: [🟥] Detect browser / node and make it colorfull
|
|
3022
3083
|
console.warn(warningMessage);
|
|
3084
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
3023
3085
|
/*
|
|
3024
3086
|
return {
|
|
3025
3087
|
async listModels() {
|
|
@@ -4631,6 +4693,7 @@ function createPipelineExecutor(options) {
|
|
|
4631
4693
|
|
|
4632
4694
|
@see more at https://ptbk.io/prepare-pipeline
|
|
4633
4695
|
`));
|
|
4696
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
4634
4697
|
}
|
|
4635
4698
|
let runCount = 0;
|
|
4636
4699
|
const pipelineExecutorWithCallback = async (inputParameters, onProgress) => {
|
|
@@ -4927,7 +4990,7 @@ class $Register {
|
|
|
4927
4990
|
const existingRegistration = this.storage[existingRegistrationIndex];
|
|
4928
4991
|
if (!existingRegistration) {
|
|
4929
4992
|
if (DEFAULT_IS_VERBOSE) {
|
|
4930
|
-
console.
|
|
4993
|
+
console.info(`[📦] Registering \`${packageName}.${className}\` to \`${this.registerName}\``);
|
|
4931
4994
|
}
|
|
4932
4995
|
this.storage.push(registered);
|
|
4933
4996
|
}
|
|
@@ -5241,17 +5304,24 @@ function titleToName(value) {
|
|
|
5241
5304
|
/**
|
|
5242
5305
|
* The built-in `fetch' function with a lightweight error handling wrapper as default fetch function used in Promptbook scrapers
|
|
5243
5306
|
*
|
|
5244
|
-
* @
|
|
5307
|
+
* @public exported from `@promptbook/core`
|
|
5245
5308
|
*/
|
|
5246
|
-
const
|
|
5309
|
+
const promptbookFetch = async (urlOrRequest, init) => {
|
|
5247
5310
|
try {
|
|
5248
|
-
return await fetch(
|
|
5311
|
+
return await fetch(urlOrRequest, init);
|
|
5249
5312
|
}
|
|
5250
5313
|
catch (error) {
|
|
5251
5314
|
if (!(error instanceof Error)) {
|
|
5252
5315
|
throw error;
|
|
5253
5316
|
}
|
|
5254
|
-
|
|
5317
|
+
let url;
|
|
5318
|
+
if (typeof urlOrRequest === 'string') {
|
|
5319
|
+
url = urlOrRequest;
|
|
5320
|
+
}
|
|
5321
|
+
else if (urlOrRequest instanceof Request) {
|
|
5322
|
+
url = urlOrRequest.url;
|
|
5323
|
+
}
|
|
5324
|
+
throw new PromptbookFetchError(spaceTrim((block) => `
|
|
5255
5325
|
Can not fetch "${url}"
|
|
5256
5326
|
|
|
5257
5327
|
Fetch error:
|
|
@@ -5272,7 +5342,7 @@ const scraperFetch = async (url, init) => {
|
|
|
5272
5342
|
async function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
5273
5343
|
// console.log('!! makeKnowledgeSourceHandler', knowledgeSource);
|
|
5274
5344
|
var _a;
|
|
5275
|
-
const { fetch =
|
|
5345
|
+
const { fetch = promptbookFetch } = tools;
|
|
5276
5346
|
const { knowledgeSourceContent } = knowledgeSource;
|
|
5277
5347
|
let { name } = knowledgeSource;
|
|
5278
5348
|
const { rootDirname = null,
|
|
@@ -5413,63 +5483,73 @@ async function prepareKnowledgePieces(knowledgeSources, tools, options) {
|
|
|
5413
5483
|
const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT, rootDirname, isVerbose = DEFAULT_IS_VERBOSE } = options;
|
|
5414
5484
|
const knowledgePreparedUnflatten = new Array(knowledgeSources.length);
|
|
5415
5485
|
await forEachAsync(knowledgeSources, { maxParallelCount }, async (knowledgeSource, index) => {
|
|
5416
|
-
|
|
5417
|
-
|
|
5418
|
-
|
|
5419
|
-
|
|
5420
|
-
|
|
5421
|
-
|
|
5422
|
-
|
|
5423
|
-
|
|
5424
|
-
|
|
5425
|
-
|
|
5426
|
-
|
|
5427
|
-
|
|
5428
|
-
|
|
5429
|
-
|
|
5430
|
-
|
|
5431
|
-
|
|
5432
|
-
|
|
5486
|
+
try {
|
|
5487
|
+
let partialPieces = null;
|
|
5488
|
+
const sourceHandler = await makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname, isVerbose });
|
|
5489
|
+
const scrapers = arrayableToArray(tools.scrapers);
|
|
5490
|
+
for (const scraper of scrapers) {
|
|
5491
|
+
if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
|
|
5492
|
+
// <- TODO: [🦔] Implement mime-type wildcards
|
|
5493
|
+
) {
|
|
5494
|
+
continue;
|
|
5495
|
+
}
|
|
5496
|
+
const partialPiecesUnchecked = await scraper.scrape(sourceHandler);
|
|
5497
|
+
if (partialPiecesUnchecked !== null) {
|
|
5498
|
+
partialPieces = [...partialPiecesUnchecked];
|
|
5499
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
|
|
5500
|
+
break;
|
|
5501
|
+
}
|
|
5502
|
+
console.warn(spaceTrim((block) => `
|
|
5503
|
+
Cannot scrape knowledge from source despite the scraper \`${scraper.metadata.className}\` supports the mime type "${sourceHandler.mimeType}".
|
|
5433
5504
|
|
|
5434
|
-
|
|
5435
|
-
|
|
5436
|
-
|
|
5437
|
-
|
|
5438
|
-
|
|
5505
|
+
The source:
|
|
5506
|
+
${block(knowledgeSource.knowledgeSourceContent
|
|
5507
|
+
.split('\n')
|
|
5508
|
+
.map((line) => `> ${line}`)
|
|
5509
|
+
.join('\n'))}
|
|
5439
5510
|
|
|
5440
|
-
|
|
5511
|
+
${block($registeredScrapersMessage(scrapers))}
|
|
5441
5512
|
|
|
5442
5513
|
|
|
5443
|
-
|
|
5444
|
-
|
|
5445
|
-
|
|
5446
|
-
|
|
5447
|
-
|
|
5514
|
+
`));
|
|
5515
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
5516
|
+
}
|
|
5517
|
+
if (partialPieces === null) {
|
|
5518
|
+
throw new KnowledgeScrapeError(spaceTrim((block) => `
|
|
5519
|
+
Cannot scrape knowledge
|
|
5448
5520
|
|
|
5449
|
-
|
|
5450
|
-
|
|
5451
|
-
|
|
5452
|
-
|
|
5453
|
-
|
|
5521
|
+
The source:
|
|
5522
|
+
> ${block(knowledgeSource.knowledgeSourceContent
|
|
5523
|
+
.split('\n')
|
|
5524
|
+
.map((line) => `> ${line}`)
|
|
5525
|
+
.join('\n'))}
|
|
5454
5526
|
|
|
5455
|
-
|
|
5527
|
+
No scraper found for the mime type "${sourceHandler.mimeType}"
|
|
5456
5528
|
|
|
5457
|
-
|
|
5529
|
+
${block($registeredScrapersMessage(scrapers))}
|
|
5458
5530
|
|
|
5459
5531
|
|
|
5460
|
-
|
|
5532
|
+
`));
|
|
5533
|
+
}
|
|
5534
|
+
const pieces = partialPieces.map((partialPiece) => ({
|
|
5535
|
+
...partialPiece,
|
|
5536
|
+
sources: [
|
|
5537
|
+
{
|
|
5538
|
+
name: knowledgeSource.name,
|
|
5539
|
+
// line, column <- TODO: [☀]
|
|
5540
|
+
// <- TODO: [❎]
|
|
5541
|
+
},
|
|
5542
|
+
],
|
|
5543
|
+
}));
|
|
5544
|
+
knowledgePreparedUnflatten[index] = pieces;
|
|
5545
|
+
}
|
|
5546
|
+
catch (error) {
|
|
5547
|
+
if (!(error instanceof Error)) {
|
|
5548
|
+
throw error;
|
|
5549
|
+
}
|
|
5550
|
+
console.warn(error);
|
|
5551
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
5461
5552
|
}
|
|
5462
|
-
const pieces = partialPieces.map((partialPiece) => ({
|
|
5463
|
-
...partialPiece,
|
|
5464
|
-
sources: [
|
|
5465
|
-
{
|
|
5466
|
-
name: knowledgeSource.name,
|
|
5467
|
-
// line, column <- TODO: [☀]
|
|
5468
|
-
// <- TODO: [❎]
|
|
5469
|
-
},
|
|
5470
|
-
],
|
|
5471
|
-
}));
|
|
5472
|
-
knowledgePreparedUnflatten[index] = pieces;
|
|
5473
5553
|
});
|
|
5474
5554
|
const knowledgePrepared = knowledgePreparedUnflatten.flat();
|
|
5475
5555
|
return knowledgePrepared;
|
|
@@ -7294,7 +7374,8 @@ const modelCommandParser = {
|
|
|
7294
7374
|
if ($pipelineJson.defaultModelRequirements[command.key] !== undefined) {
|
|
7295
7375
|
if ($pipelineJson.defaultModelRequirements[command.key] === command.value) {
|
|
7296
7376
|
console.warn(`Multiple commands \`MODEL ${command.key} ${command.value}\` in the pipeline head`);
|
|
7297
|
-
// <- TODO: [
|
|
7377
|
+
// <- TODO: [🏮] Some better way how to get warnings from pipeline parsing / logic
|
|
7378
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
7298
7379
|
}
|
|
7299
7380
|
else {
|
|
7300
7381
|
throw new ParseError(spaceTrim(`
|
|
@@ -7326,6 +7407,7 @@ const modelCommandParser = {
|
|
|
7326
7407
|
modelVariant: 'VARIANT',
|
|
7327
7408
|
maxTokens: '???',
|
|
7328
7409
|
}[command.key]} ${command.value}\` in the task "${$taskJson.title || $taskJson.name}"`);
|
|
7410
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
7329
7411
|
}
|
|
7330
7412
|
else {
|
|
7331
7413
|
throw new ParseError(spaceTrim(`
|
|
@@ -7605,15 +7687,15 @@ function $applyToTaskJson(command, $taskJson, $pipelineJson) {
|
|
|
7605
7687
|
}
|
|
7606
7688
|
console.warn(spaceTrim(`
|
|
7607
7689
|
|
|
7608
|
-
|
|
7690
|
+
Persona "${personaName}" is defined multiple times with different description:
|
|
7609
7691
|
|
|
7610
|
-
|
|
7611
|
-
|
|
7692
|
+
First definition:
|
|
7693
|
+
${persona.description}
|
|
7612
7694
|
|
|
7613
|
-
|
|
7614
|
-
|
|
7695
|
+
Second definition:
|
|
7696
|
+
${personaDescription}
|
|
7615
7697
|
|
|
7616
|
-
|
|
7698
|
+
`));
|
|
7617
7699
|
persona.description += spaceTrim('\n\n' + personaDescription);
|
|
7618
7700
|
}
|
|
7619
7701
|
|
|
@@ -9175,7 +9257,8 @@ function addAutoGeneratedSection(content, options) {
|
|
|
9175
9257
|
`));
|
|
9176
9258
|
}
|
|
9177
9259
|
console.warn(`No place where to put the section <!--${sectionName}-->, using the end of the file`);
|
|
9178
|
-
// <- TODO: [
|
|
9260
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
9261
|
+
// <- TODO: [🏮] Some better way how to get warnings from pipeline parsing / logic
|
|
9179
9262
|
return spaceTrim$1((block) => `
|
|
9180
9263
|
${block(content)}
|
|
9181
9264
|
|
|
@@ -10139,8 +10222,7 @@ const _AnthropicClaudeMetadataRegistration = $llmToolsMetadataRegister.register(
|
|
|
10139
10222
|
options: {
|
|
10140
10223
|
apiKey: 'sk-ant-api03-',
|
|
10141
10224
|
isProxied: true,
|
|
10142
|
-
|
|
10143
|
-
path: DEFAULT_REMOTE_URL_PATH,
|
|
10225
|
+
remoteServerUrl: DEFAULT_REMOTE_SERVER_URL,
|
|
10144
10226
|
},
|
|
10145
10227
|
};
|
|
10146
10228
|
},
|
|
@@ -10835,5 +10917,5 @@ class PrefixStorage {
|
|
|
10835
10917
|
}
|
|
10836
10918
|
}
|
|
10837
10919
|
|
|
10838
|
-
export { $llmToolsMetadataRegister, $llmToolsRegister, $scrapersMetadataRegister, $scrapersRegister, ADMIN_EMAIL, ADMIN_GITHUB_NAME, AbstractFormatError, BOOK_LANGUAGE_VERSION, BlackholeStorage, BoilerplateError, BoilerplateFormfactorDefinition, CLAIM, CallbackInterfaceTools, ChatbotFormfactorDefinition, CollectionError, CsvFormatDefinition, CsvFormatError, DEFAULT_BOOKS_DIRNAME, DEFAULT_BOOK_OUTPUT_PARAMETER_NAME, DEFAULT_BOOK_TITLE, DEFAULT_CSV_SETTINGS, DEFAULT_DOWNLOAD_CACHE_DIRNAME, DEFAULT_EXECUTION_CACHE_DIRNAME, DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME, DEFAULT_INTERMEDIATE_FILES_STRATEGY, DEFAULT_IS_AUTO_INSTALLED, DEFAULT_IS_VERBOSE, DEFAULT_MAX_EXECUTION_ATTEMPTS, DEFAULT_MAX_FILE_SIZE, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL, DEFAULT_MAX_PARALLEL_COUNT, DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, DEFAULT_PROMPT_TASK_TITLE,
|
|
10920
|
+
export { $llmToolsMetadataRegister, $llmToolsRegister, $scrapersMetadataRegister, $scrapersRegister, ADMIN_EMAIL, ADMIN_GITHUB_NAME, AbstractFormatError, AuthenticationError, BOOK_LANGUAGE_VERSION, BlackholeStorage, BoilerplateError, BoilerplateFormfactorDefinition, CLAIM, CLI_APP_ID, CallbackInterfaceTools, ChatbotFormfactorDefinition, CollectionError, CsvFormatDefinition, CsvFormatError, DEFAULT_BOOKS_DIRNAME, DEFAULT_BOOK_OUTPUT_PARAMETER_NAME, DEFAULT_BOOK_TITLE, DEFAULT_CSV_SETTINGS, DEFAULT_DOWNLOAD_CACHE_DIRNAME, DEFAULT_EXECUTION_CACHE_DIRNAME, DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME, DEFAULT_INTERMEDIATE_FILES_STRATEGY, DEFAULT_IS_AUTO_INSTALLED, DEFAULT_IS_VERBOSE, DEFAULT_MAX_EXECUTION_ATTEMPTS, DEFAULT_MAX_FILE_SIZE, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL, DEFAULT_MAX_PARALLEL_COUNT, DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, DEFAULT_PROMPT_TASK_TITLE, DEFAULT_REMOTE_SERVER_URL, DEFAULT_SCRAPE_CACHE_DIRNAME, DEFAULT_TASK_TITLE, EXPECTATION_UNITS, EnvironmentMismatchError, ExecutionReportStringOptionsDefaults, ExpectError, FORMFACTOR_DEFINITIONS, GENERIC_PIPELINE_INTERFACE, GeneratorFormfactorDefinition, GenericFormfactorDefinition, ImageGeneratorFormfactorDefinition, KnowledgeScrapeError, LOGO_DARK_SRC, LOGO_LIGHT_SRC, LimitReachedError, MANDATORY_CSV_SETTINGS, MAX_FILENAME_LENGTH, MODEL_VARIANTS, MatcherFormfactorDefinition, MemoryStorage, MissingToolsError, MultipleLlmExecutionTools, NAME, NonTaskSectionTypes, NotFoundError, NotYetImplementedError, ORDER_OF_PIPELINE_JSON, PLAYGROUND_APP_ID, PROMPTBOOK_ENGINE_VERSION, PROMPTBOOK_ERRORS, ParseError, PipelineExecutionError, PipelineLogicError, PipelineUrlError, PrefixStorage, PromptbookFetchError, RESERVED_PARAMETER_NAMES, SET_IS_VERBOSE, SectionTypes, SheetsFormfactorDefinition, TaskTypes, TextFormatDefinition, TranslatorFormfactorDefinition, UNCERTAIN_USAGE, UNCERTAIN_ZERO_VALUE, UnexpectedError, ZERO_USAGE, ZERO_VALUE, _AnthropicClaudeMetadataRegistration, _AzureOpenAiMetadataRegistration, _BoilerplateScraperMetadataRegistration, _DeepseekMetadataRegistration, _DocumentScraperMetadataRegistration, _GoogleMetadataRegistration, _LegacyDocumentScraperMetadataRegistration, _MarkdownScraperMetadataRegistration, _MarkitdownScraperMetadataRegistration, _OpenAiAssistantMetadataRegistration, _OpenAiMetadataRegistration, _PdfScraperMetadataRegistration, _WebsiteScraperMetadataRegistration, addUsage, book, cacheLlmTools, collectionToJson, compilePipeline, countUsage, createCollectionFromJson, createCollectionFromPromise, createCollectionFromUrl, createLlmToolsFromConfiguration, createPipelineExecutor, createSubcollection, embeddingVectorToString, executionReportJsonToString, extractParameterNamesFromTask, getPipelineInterface, isPassingExpectations, isPipelineImplementingInterface, isPipelineInterfacesEqual, isPipelinePrepared, isValidPipelineString, joinLlmExecutionTools, limitTotalUsage, makeKnowledgeSourceHandler, parsePipeline, pipelineJsonToString, prepareKnowledgePieces, preparePersona, preparePipeline, prepareTasks, prettifyPipelineString, promptbookFetch, unpreparePipeline, usageToHuman, usageToWorktime, validatePipeline, validatePipelineString };
|
|
10839
10921
|
//# sourceMappingURL=index.es.js.map
|