@promptbook/documents 0.92.0-11 → 0.92.0-13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/index.es.js +222 -199
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/browser.index.d.ts +2 -0
- package/esm/typings/src/_packages/core.index.d.ts +6 -4
- package/esm/typings/src/_packages/types.index.d.ts +2 -2
- package/esm/typings/src/execution/PipelineExecutorResult.d.ts +3 -1
- package/esm/typings/src/execution/createPipelineExecutor/computeCosineSimilarity.d.ts +13 -0
- package/esm/typings/src/execution/utils/checkExpectations.d.ts +1 -1
- package/esm/typings/src/formats/_common/{FormatDefinition.d.ts → FormatParser.d.ts} +3 -3
- package/esm/typings/src/formats/_common/{FormatSubvalueDefinition.d.ts → FormatSubvalueParser.d.ts} +1 -1
- package/esm/typings/src/formats/csv/CsvFormatParser.d.ts +17 -0
- package/esm/typings/src/formats/index.d.ts +2 -2
- package/esm/typings/src/formats/json/{JsonFormatDefinition.d.ts → JsonFormatParser.d.ts} +6 -6
- package/esm/typings/src/formats/text/{TextFormatDefinition.d.ts → TextFormatParser.d.ts} +7 -7
- package/esm/typings/src/formats/xml/XmlFormatParser.d.ts +19 -0
- package/esm/typings/src/postprocessing/utils/extractJsonBlock.d.ts +1 -1
- package/esm/typings/src/storage/local-storage/getIndexedDbStorage.d.ts +10 -0
- package/esm/typings/src/storage/local-storage/utils/makePromptbookStorageFromIndexedDb.d.ts +7 -0
- package/esm/typings/src/utils/expectation-counters/index.d.ts +1 -1
- package/package.json +2 -2
- package/umd/index.umd.js +222 -199
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/formats/csv/CsvFormatDefinition.d.ts +0 -17
- package/esm/typings/src/formats/xml/XmlFormatDefinition.d.ts +0 -19
package/esm/index.es.js
CHANGED
|
@@ -28,7 +28,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
|
|
|
28
28
|
* @generated
|
|
29
29
|
* @see https://github.com/webgptorg/promptbook
|
|
30
30
|
*/
|
|
31
|
-
const PROMPTBOOK_ENGINE_VERSION = '0.92.0-
|
|
31
|
+
const PROMPTBOOK_ENGINE_VERSION = '0.92.0-13';
|
|
32
32
|
/**
|
|
33
33
|
* TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
|
|
34
34
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -2298,75 +2298,6 @@ function isPipelinePrepared(pipeline) {
|
|
|
2298
2298
|
* - [♨] Are tasks prepared
|
|
2299
2299
|
*/
|
|
2300
2300
|
|
|
2301
|
-
/**
|
|
2302
|
-
* Converts a JavaScript Object Notation (JSON) string into an object.
|
|
2303
|
-
*
|
|
2304
|
-
* Note: This is wrapper around `JSON.parse()` with better error and type handling
|
|
2305
|
-
*
|
|
2306
|
-
* @public exported from `@promptbook/utils`
|
|
2307
|
-
*/
|
|
2308
|
-
function jsonParse(value) {
|
|
2309
|
-
if (value === undefined) {
|
|
2310
|
-
throw new Error(`Can not parse JSON from undefined value.`);
|
|
2311
|
-
}
|
|
2312
|
-
else if (typeof value !== 'string') {
|
|
2313
|
-
console.error('Can not parse JSON from non-string value.', { text: value });
|
|
2314
|
-
throw new Error(spaceTrim$1(`
|
|
2315
|
-
Can not parse JSON from non-string value.
|
|
2316
|
-
|
|
2317
|
-
The value type: ${typeof value}
|
|
2318
|
-
See more in console.
|
|
2319
|
-
`));
|
|
2320
|
-
}
|
|
2321
|
-
try {
|
|
2322
|
-
return JSON.parse(value);
|
|
2323
|
-
}
|
|
2324
|
-
catch (error) {
|
|
2325
|
-
if (!(error instanceof Error)) {
|
|
2326
|
-
throw error;
|
|
2327
|
-
}
|
|
2328
|
-
throw new Error(spaceTrim$1((block) => `
|
|
2329
|
-
${block(error.message)}
|
|
2330
|
-
|
|
2331
|
-
The JSON text:
|
|
2332
|
-
${block(value)}
|
|
2333
|
-
`));
|
|
2334
|
-
}
|
|
2335
|
-
}
|
|
2336
|
-
/**
|
|
2337
|
-
* TODO: !!!! Use in Promptbook.studio
|
|
2338
|
-
*/
|
|
2339
|
-
|
|
2340
|
-
/**
|
|
2341
|
-
* Recursively converts JSON strings to JSON objects
|
|
2342
|
-
|
|
2343
|
-
* @public exported from `@promptbook/utils`
|
|
2344
|
-
*/
|
|
2345
|
-
function jsonStringsToJsons(object) {
|
|
2346
|
-
if (object === null) {
|
|
2347
|
-
return object;
|
|
2348
|
-
}
|
|
2349
|
-
if (Array.isArray(object)) {
|
|
2350
|
-
return object.map(jsonStringsToJsons);
|
|
2351
|
-
}
|
|
2352
|
-
if (typeof object !== 'object') {
|
|
2353
|
-
return object;
|
|
2354
|
-
}
|
|
2355
|
-
const newObject = { ...object };
|
|
2356
|
-
for (const [key, value] of Object.entries(object)) {
|
|
2357
|
-
if (typeof value === 'string' && isValidJsonString(value)) {
|
|
2358
|
-
newObject[key] = jsonParse(value);
|
|
2359
|
-
}
|
|
2360
|
-
else {
|
|
2361
|
-
newObject[key] = jsonStringsToJsons(value);
|
|
2362
|
-
}
|
|
2363
|
-
}
|
|
2364
|
-
return newObject;
|
|
2365
|
-
}
|
|
2366
|
-
/**
|
|
2367
|
-
* TODO: Type the return type correctly
|
|
2368
|
-
*/
|
|
2369
|
-
|
|
2370
2301
|
/**
|
|
2371
2302
|
* This error indicates problems parsing the format value
|
|
2372
2303
|
*
|
|
@@ -2550,6 +2481,104 @@ const ALL_ERRORS = {
|
|
|
2550
2481
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
2551
2482
|
*/
|
|
2552
2483
|
|
|
2484
|
+
/**
|
|
2485
|
+
* Serializes an error into a [🚉] JSON-serializable object
|
|
2486
|
+
*
|
|
2487
|
+
* @public exported from `@promptbook/utils`
|
|
2488
|
+
*/
|
|
2489
|
+
function serializeError(error) {
|
|
2490
|
+
const { name, message, stack } = error;
|
|
2491
|
+
const { id } = error;
|
|
2492
|
+
if (!Object.keys(ALL_ERRORS).includes(name)) {
|
|
2493
|
+
console.error(spaceTrim$1((block) => `
|
|
2494
|
+
|
|
2495
|
+
Cannot serialize error with name "${name}"
|
|
2496
|
+
|
|
2497
|
+
Authors of Promptbook probably forgot to add this error into the list of errors:
|
|
2498
|
+
https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
|
|
2499
|
+
|
|
2500
|
+
|
|
2501
|
+
${block(stack || message)}
|
|
2502
|
+
|
|
2503
|
+
`));
|
|
2504
|
+
}
|
|
2505
|
+
return {
|
|
2506
|
+
name: name,
|
|
2507
|
+
message,
|
|
2508
|
+
stack,
|
|
2509
|
+
id, // Include id in the serialized object
|
|
2510
|
+
};
|
|
2511
|
+
}
|
|
2512
|
+
|
|
2513
|
+
/**
|
|
2514
|
+
* Converts a JavaScript Object Notation (JSON) string into an object.
|
|
2515
|
+
*
|
|
2516
|
+
* Note: This is wrapper around `JSON.parse()` with better error and type handling
|
|
2517
|
+
*
|
|
2518
|
+
* @public exported from `@promptbook/utils`
|
|
2519
|
+
*/
|
|
2520
|
+
function jsonParse(value) {
|
|
2521
|
+
if (value === undefined) {
|
|
2522
|
+
throw new Error(`Can not parse JSON from undefined value.`);
|
|
2523
|
+
}
|
|
2524
|
+
else if (typeof value !== 'string') {
|
|
2525
|
+
console.error('Can not parse JSON from non-string value.', { text: value });
|
|
2526
|
+
throw new Error(spaceTrim$1(`
|
|
2527
|
+
Can not parse JSON from non-string value.
|
|
2528
|
+
|
|
2529
|
+
The value type: ${typeof value}
|
|
2530
|
+
See more in console.
|
|
2531
|
+
`));
|
|
2532
|
+
}
|
|
2533
|
+
try {
|
|
2534
|
+
return JSON.parse(value);
|
|
2535
|
+
}
|
|
2536
|
+
catch (error) {
|
|
2537
|
+
if (!(error instanceof Error)) {
|
|
2538
|
+
throw error;
|
|
2539
|
+
}
|
|
2540
|
+
throw new Error(spaceTrim$1((block) => `
|
|
2541
|
+
${block(error.message)}
|
|
2542
|
+
|
|
2543
|
+
The JSON text:
|
|
2544
|
+
${block(value)}
|
|
2545
|
+
`));
|
|
2546
|
+
}
|
|
2547
|
+
}
|
|
2548
|
+
/**
|
|
2549
|
+
* TODO: !!!! Use in Promptbook.studio
|
|
2550
|
+
*/
|
|
2551
|
+
|
|
2552
|
+
/**
|
|
2553
|
+
* Recursively converts JSON strings to JSON objects
|
|
2554
|
+
|
|
2555
|
+
* @public exported from `@promptbook/utils`
|
|
2556
|
+
*/
|
|
2557
|
+
function jsonStringsToJsons(object) {
|
|
2558
|
+
if (object === null) {
|
|
2559
|
+
return object;
|
|
2560
|
+
}
|
|
2561
|
+
if (Array.isArray(object)) {
|
|
2562
|
+
return object.map(jsonStringsToJsons);
|
|
2563
|
+
}
|
|
2564
|
+
if (typeof object !== 'object') {
|
|
2565
|
+
return object;
|
|
2566
|
+
}
|
|
2567
|
+
const newObject = { ...object };
|
|
2568
|
+
for (const [key, value] of Object.entries(object)) {
|
|
2569
|
+
if (typeof value === 'string' && isValidJsonString(value)) {
|
|
2570
|
+
newObject[key] = jsonParse(value);
|
|
2571
|
+
}
|
|
2572
|
+
else {
|
|
2573
|
+
newObject[key] = jsonStringsToJsons(value);
|
|
2574
|
+
}
|
|
2575
|
+
}
|
|
2576
|
+
return newObject;
|
|
2577
|
+
}
|
|
2578
|
+
/**
|
|
2579
|
+
* TODO: Type the return type correctly
|
|
2580
|
+
*/
|
|
2581
|
+
|
|
2553
2582
|
/**
|
|
2554
2583
|
* Deserializes the error object
|
|
2555
2584
|
*
|
|
@@ -2715,64 +2744,6 @@ function createTask(options) {
|
|
|
2715
2744
|
* TODO: [🐚] Split into more files and make `PrepareTask` & `RemoteTask` + split the function
|
|
2716
2745
|
*/
|
|
2717
2746
|
|
|
2718
|
-
/**
|
|
2719
|
-
* Serializes an error into a [🚉] JSON-serializable object
|
|
2720
|
-
*
|
|
2721
|
-
* @public exported from `@promptbook/utils`
|
|
2722
|
-
*/
|
|
2723
|
-
function serializeError(error) {
|
|
2724
|
-
const { name, message, stack } = error;
|
|
2725
|
-
const { id } = error;
|
|
2726
|
-
if (!Object.keys(ALL_ERRORS).includes(name)) {
|
|
2727
|
-
console.error(spaceTrim$1((block) => `
|
|
2728
|
-
|
|
2729
|
-
Cannot serialize error with name "${name}"
|
|
2730
|
-
|
|
2731
|
-
Authors of Promptbook probably forgot to add this error into the list of errors:
|
|
2732
|
-
https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
${block(stack || message)}
|
|
2736
|
-
|
|
2737
|
-
`));
|
|
2738
|
-
}
|
|
2739
|
-
return {
|
|
2740
|
-
name: name,
|
|
2741
|
-
message,
|
|
2742
|
-
stack,
|
|
2743
|
-
id, // Include id in the serialized object
|
|
2744
|
-
};
|
|
2745
|
-
}
|
|
2746
|
-
|
|
2747
|
-
/**
|
|
2748
|
-
* Async version of Array.forEach
|
|
2749
|
-
*
|
|
2750
|
-
* @param array - Array to iterate over
|
|
2751
|
-
* @param options - Options for the function
|
|
2752
|
-
* @param callbackfunction - Function to call for each item
|
|
2753
|
-
* @public exported from `@promptbook/utils`
|
|
2754
|
-
* @deprecated [🪂] Use queues instead
|
|
2755
|
-
*/
|
|
2756
|
-
async function forEachAsync(array, options, callbackfunction) {
|
|
2757
|
-
const { maxParallelCount = Infinity } = options;
|
|
2758
|
-
let index = 0;
|
|
2759
|
-
let runningTasks = [];
|
|
2760
|
-
const tasks = [];
|
|
2761
|
-
for (const item of array) {
|
|
2762
|
-
const currentIndex = index++;
|
|
2763
|
-
const task = callbackfunction(item, currentIndex, array);
|
|
2764
|
-
tasks.push(task);
|
|
2765
|
-
runningTasks.push(task);
|
|
2766
|
-
/* not await */ Promise.resolve(task).then(() => {
|
|
2767
|
-
runningTasks = runningTasks.filter((t) => t !== task);
|
|
2768
|
-
});
|
|
2769
|
-
if (maxParallelCount < runningTasks.length) {
|
|
2770
|
-
await Promise.race(runningTasks);
|
|
2771
|
-
}
|
|
2772
|
-
}
|
|
2773
|
-
await Promise.all(tasks);
|
|
2774
|
-
}
|
|
2775
|
-
|
|
2776
2747
|
/**
|
|
2777
2748
|
* Represents the uncertain value
|
|
2778
2749
|
*
|
|
@@ -2816,7 +2787,7 @@ const ZERO_USAGE = $deepFreeze({
|
|
|
2816
2787
|
*
|
|
2817
2788
|
* @public exported from `@promptbook/core`
|
|
2818
2789
|
*/
|
|
2819
|
-
$deepFreeze({
|
|
2790
|
+
const UNCERTAIN_USAGE = $deepFreeze({
|
|
2820
2791
|
price: UNCERTAIN_ZERO_VALUE,
|
|
2821
2792
|
input: {
|
|
2822
2793
|
tokensCount: UNCERTAIN_ZERO_VALUE,
|
|
@@ -2841,6 +2812,35 @@ $deepFreeze({
|
|
|
2841
2812
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
2842
2813
|
*/
|
|
2843
2814
|
|
|
2815
|
+
/**
|
|
2816
|
+
* Async version of Array.forEach
|
|
2817
|
+
*
|
|
2818
|
+
* @param array - Array to iterate over
|
|
2819
|
+
* @param options - Options for the function
|
|
2820
|
+
* @param callbackfunction - Function to call for each item
|
|
2821
|
+
* @public exported from `@promptbook/utils`
|
|
2822
|
+
* @deprecated [🪂] Use queues instead
|
|
2823
|
+
*/
|
|
2824
|
+
async function forEachAsync(array, options, callbackfunction) {
|
|
2825
|
+
const { maxParallelCount = Infinity } = options;
|
|
2826
|
+
let index = 0;
|
|
2827
|
+
let runningTasks = [];
|
|
2828
|
+
const tasks = [];
|
|
2829
|
+
for (const item of array) {
|
|
2830
|
+
const currentIndex = index++;
|
|
2831
|
+
const task = callbackfunction(item, currentIndex, array);
|
|
2832
|
+
tasks.push(task);
|
|
2833
|
+
runningTasks.push(task);
|
|
2834
|
+
/* not await */ Promise.resolve(task).then(() => {
|
|
2835
|
+
runningTasks = runningTasks.filter((t) => t !== task);
|
|
2836
|
+
});
|
|
2837
|
+
if (maxParallelCount < runningTasks.length) {
|
|
2838
|
+
await Promise.race(runningTasks);
|
|
2839
|
+
}
|
|
2840
|
+
}
|
|
2841
|
+
await Promise.all(tasks);
|
|
2842
|
+
}
|
|
2843
|
+
|
|
2844
2844
|
/**
|
|
2845
2845
|
* Function `addUsage` will add multiple usages into one
|
|
2846
2846
|
*
|
|
@@ -4238,6 +4238,24 @@ const MANDATORY_CSV_SETTINGS = Object.freeze({
|
|
|
4238
4238
|
// encoding: 'utf-8',
|
|
4239
4239
|
});
|
|
4240
4240
|
|
|
4241
|
+
/**
|
|
4242
|
+
* Converts a CSV string into an object
|
|
4243
|
+
*
|
|
4244
|
+
* Note: This is wrapper around `papaparse.parse()` with better autohealing
|
|
4245
|
+
*
|
|
4246
|
+
* @private - for now until `@promptbook/csv` is released
|
|
4247
|
+
*/
|
|
4248
|
+
function csvParse(value /* <- TODO: string_csv */, settings, schema /* <- TODO: Make CSV Schemas */) {
|
|
4249
|
+
settings = { ...settings, ...MANDATORY_CSV_SETTINGS };
|
|
4250
|
+
// Note: Autoheal invalid '\n' characters
|
|
4251
|
+
if (settings.newline && !settings.newline.includes('\r') && value.includes('\r')) {
|
|
4252
|
+
console.warn('CSV string contains carriage return characters, but in the CSV settings the `newline` setting does not include them. Autohealing the CSV string.');
|
|
4253
|
+
value = value.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
|
|
4254
|
+
}
|
|
4255
|
+
const csv = parse(value, settings);
|
|
4256
|
+
return csv;
|
|
4257
|
+
}
|
|
4258
|
+
|
|
4241
4259
|
/**
|
|
4242
4260
|
* Function to check if a string is valid CSV
|
|
4243
4261
|
*
|
|
@@ -4260,31 +4278,13 @@ function isValidCsvString(value) {
|
|
|
4260
4278
|
}
|
|
4261
4279
|
}
|
|
4262
4280
|
|
|
4263
|
-
/**
|
|
4264
|
-
* Converts a CSV string into an object
|
|
4265
|
-
*
|
|
4266
|
-
* Note: This is wrapper around `papaparse.parse()` with better autohealing
|
|
4267
|
-
*
|
|
4268
|
-
* @private - for now until `@promptbook/csv` is released
|
|
4269
|
-
*/
|
|
4270
|
-
function csvParse(value /* <- TODO: string_csv */, settings, schema /* <- TODO: Make CSV Schemas */) {
|
|
4271
|
-
settings = { ...settings, ...MANDATORY_CSV_SETTINGS };
|
|
4272
|
-
// Note: Autoheal invalid '\n' characters
|
|
4273
|
-
if (settings.newline && !settings.newline.includes('\r') && value.includes('\r')) {
|
|
4274
|
-
console.warn('CSV string contains carriage return characters, but in the CSV settings the `newline` setting does not include them. Autohealing the CSV string.');
|
|
4275
|
-
value = value.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
|
|
4276
|
-
}
|
|
4277
|
-
const csv = parse(value, settings);
|
|
4278
|
-
return csv;
|
|
4279
|
-
}
|
|
4280
|
-
|
|
4281
4281
|
/**
|
|
4282
4282
|
* Definition for CSV spreadsheet
|
|
4283
4283
|
*
|
|
4284
4284
|
* @public exported from `@promptbook/core`
|
|
4285
4285
|
* <- TODO: [🏢] Export from package `@promptbook/csv`
|
|
4286
4286
|
*/
|
|
4287
|
-
const
|
|
4287
|
+
const CsvFormatParser = {
|
|
4288
4288
|
formatName: 'CSV',
|
|
4289
4289
|
aliases: ['SPREADSHEET', 'TABLE'],
|
|
4290
4290
|
isValid(value, settings, schema) {
|
|
@@ -4296,7 +4296,7 @@ const CsvFormatDefinition = {
|
|
|
4296
4296
|
heal(value, settings, schema) {
|
|
4297
4297
|
throw new Error('Not implemented');
|
|
4298
4298
|
},
|
|
4299
|
-
|
|
4299
|
+
subvalueParsers: [
|
|
4300
4300
|
{
|
|
4301
4301
|
subvalueName: 'ROW',
|
|
4302
4302
|
async mapValues(value, outputParameterName, settings, mapCallback) {
|
|
@@ -4357,10 +4357,10 @@ const CsvFormatDefinition = {
|
|
|
4357
4357
|
],
|
|
4358
4358
|
};
|
|
4359
4359
|
/**
|
|
4360
|
-
* TODO: [🍓] In `
|
|
4361
|
-
* TODO: [🍓] In `
|
|
4362
|
-
* TODO: [🍓] In `
|
|
4363
|
-
* TODO: [🍓] In `
|
|
4360
|
+
* TODO: [🍓] In `CsvFormatParser` implement simple `isValid`
|
|
4361
|
+
* TODO: [🍓] In `CsvFormatParser` implement partial `canBeValid`
|
|
4362
|
+
* TODO: [🍓] In `CsvFormatParser` implement `heal
|
|
4363
|
+
* TODO: [🍓] In `CsvFormatParser` implement `subvalueParsers`
|
|
4364
4364
|
* TODO: [🏢] Allow to expect something inside CSV objects and other formats
|
|
4365
4365
|
*/
|
|
4366
4366
|
|
|
@@ -4369,7 +4369,7 @@ const CsvFormatDefinition = {
|
|
|
4369
4369
|
*
|
|
4370
4370
|
* @private still in development [🏢]
|
|
4371
4371
|
*/
|
|
4372
|
-
const
|
|
4372
|
+
const JsonFormatParser = {
|
|
4373
4373
|
formatName: 'JSON',
|
|
4374
4374
|
mimeType: 'application/json',
|
|
4375
4375
|
isValid(value, settings, schema) {
|
|
@@ -4381,28 +4381,28 @@ const JsonFormatDefinition = {
|
|
|
4381
4381
|
heal(value, settings, schema) {
|
|
4382
4382
|
throw new Error('Not implemented');
|
|
4383
4383
|
},
|
|
4384
|
-
|
|
4384
|
+
subvalueParsers: [],
|
|
4385
4385
|
};
|
|
4386
4386
|
/**
|
|
4387
4387
|
* TODO: [🧠] Maybe propper instance of object
|
|
4388
4388
|
* TODO: [0] Make string_serialized_json
|
|
4389
4389
|
* TODO: [1] Make type for JSON Settings and Schema
|
|
4390
4390
|
* TODO: [🧠] What to use for validating JSONs - JSON Schema, ZoD, typescript types/interfaces,...?
|
|
4391
|
-
* TODO: [🍓] In `
|
|
4392
|
-
* TODO: [🍓] In `
|
|
4393
|
-
* TODO: [🍓] In `
|
|
4394
|
-
* TODO: [🍓] In `
|
|
4391
|
+
* TODO: [🍓] In `JsonFormatParser` implement simple `isValid`
|
|
4392
|
+
* TODO: [🍓] In `JsonFormatParser` implement partial `canBeValid`
|
|
4393
|
+
* TODO: [🍓] In `JsonFormatParser` implement `heal
|
|
4394
|
+
* TODO: [🍓] In `JsonFormatParser` implement `subvalueParsers`
|
|
4395
4395
|
* TODO: [🏢] Allow to expect something inside JSON objects and other formats
|
|
4396
4396
|
*/
|
|
4397
4397
|
|
|
4398
4398
|
/**
|
|
4399
4399
|
* Definition for any text - this will be always valid
|
|
4400
4400
|
*
|
|
4401
|
-
* Note: This is not useful for validation, but for splitting and mapping with `
|
|
4401
|
+
* Note: This is not useful for validation, but for splitting and mapping with `subvalueParsers`
|
|
4402
4402
|
*
|
|
4403
4403
|
* @public exported from `@promptbook/core`
|
|
4404
4404
|
*/
|
|
4405
|
-
const
|
|
4405
|
+
const TextFormatParser = {
|
|
4406
4406
|
formatName: 'TEXT',
|
|
4407
4407
|
isValid(value) {
|
|
4408
4408
|
return typeof value === 'string';
|
|
@@ -4411,9 +4411,9 @@ const TextFormatDefinition = {
|
|
|
4411
4411
|
return typeof partialValue === 'string';
|
|
4412
4412
|
},
|
|
4413
4413
|
heal() {
|
|
4414
|
-
throw new UnexpectedError('It does not make sense to call `
|
|
4414
|
+
throw new UnexpectedError('It does not make sense to call `TextFormatParser.heal`');
|
|
4415
4415
|
},
|
|
4416
|
-
|
|
4416
|
+
subvalueParsers: [
|
|
4417
4417
|
{
|
|
4418
4418
|
subvalueName: 'LINE',
|
|
4419
4419
|
async mapValues(value, outputParameterName, settings, mapCallback) {
|
|
@@ -4433,10 +4433,10 @@ const TextFormatDefinition = {
|
|
|
4433
4433
|
/**
|
|
4434
4434
|
* TODO: [1] Make type for XML Text and Schema
|
|
4435
4435
|
* TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages available as subvalues
|
|
4436
|
-
* TODO: [🍓] In `
|
|
4437
|
-
* TODO: [🍓] In `
|
|
4438
|
-
* TODO: [🍓] In `
|
|
4439
|
-
* TODO: [🍓] In `
|
|
4436
|
+
* TODO: [🍓] In `TextFormatParser` implement simple `isValid`
|
|
4437
|
+
* TODO: [🍓] In `TextFormatParser` implement partial `canBeValid`
|
|
4438
|
+
* TODO: [🍓] In `TextFormatParser` implement `heal
|
|
4439
|
+
* TODO: [🍓] In `TextFormatParser` implement `subvalueParsers`
|
|
4440
4440
|
* TODO: [🏢] Allow to expect something inside each item of list and other formats
|
|
4441
4441
|
*/
|
|
4442
4442
|
|
|
@@ -4469,7 +4469,7 @@ function isValidXmlString(value) {
|
|
|
4469
4469
|
*
|
|
4470
4470
|
* @private still in development [🏢]
|
|
4471
4471
|
*/
|
|
4472
|
-
const
|
|
4472
|
+
const XmlFormatParser = {
|
|
4473
4473
|
formatName: 'XML',
|
|
4474
4474
|
mimeType: 'application/xml',
|
|
4475
4475
|
isValid(value, settings, schema) {
|
|
@@ -4481,17 +4481,17 @@ const XmlFormatDefinition = {
|
|
|
4481
4481
|
heal(value, settings, schema) {
|
|
4482
4482
|
throw new Error('Not implemented');
|
|
4483
4483
|
},
|
|
4484
|
-
|
|
4484
|
+
subvalueParsers: [],
|
|
4485
4485
|
};
|
|
4486
4486
|
/**
|
|
4487
4487
|
* TODO: [🧠] Maybe propper instance of object
|
|
4488
4488
|
* TODO: [0] Make string_serialized_xml
|
|
4489
4489
|
* TODO: [1] Make type for XML Settings and Schema
|
|
4490
4490
|
* TODO: [🧠] What to use for validating XMLs - XSD,...
|
|
4491
|
-
* TODO: [🍓] In `
|
|
4492
|
-
* TODO: [🍓] In `
|
|
4493
|
-
* TODO: [🍓] In `
|
|
4494
|
-
* TODO: [🍓] In `
|
|
4491
|
+
* TODO: [🍓] In `XmlFormatParser` implement simple `isValid`
|
|
4492
|
+
* TODO: [🍓] In `XmlFormatParser` implement partial `canBeValid`
|
|
4493
|
+
* TODO: [🍓] In `XmlFormatParser` implement `heal
|
|
4494
|
+
* TODO: [🍓] In `XmlFormatParser` implement `subvalueParsers`
|
|
4495
4495
|
* TODO: [🏢] Allow to expect something inside XML and other formats
|
|
4496
4496
|
*/
|
|
4497
4497
|
|
|
@@ -4500,12 +4500,7 @@ const XmlFormatDefinition = {
|
|
|
4500
4500
|
*
|
|
4501
4501
|
* @private internal index of `...` <- TODO [🏢]
|
|
4502
4502
|
*/
|
|
4503
|
-
const FORMAT_DEFINITIONS = [
|
|
4504
|
-
JsonFormatDefinition,
|
|
4505
|
-
XmlFormatDefinition,
|
|
4506
|
-
TextFormatDefinition,
|
|
4507
|
-
CsvFormatDefinition,
|
|
4508
|
-
];
|
|
4503
|
+
const FORMAT_DEFINITIONS = [JsonFormatParser, XmlFormatParser, TextFormatParser, CsvFormatParser];
|
|
4509
4504
|
/**
|
|
4510
4505
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
4511
4506
|
*/
|
|
@@ -4675,7 +4670,7 @@ function extractJsonBlock(markdown) {
|
|
|
4675
4670
|
}
|
|
4676
4671
|
/**
|
|
4677
4672
|
* TODO: Add some auto-healing logic + extract YAML, JSON5, TOML, etc.
|
|
4678
|
-
* TODO: [🏢] Make this logic part of `
|
|
4673
|
+
* TODO: [🏢] Make this logic part of `JsonFormatParser` or `isValidJsonString`
|
|
4679
4674
|
*/
|
|
4680
4675
|
|
|
4681
4676
|
/**
|
|
@@ -4877,7 +4872,7 @@ const CountUtils = {
|
|
|
4877
4872
|
PAGES: countPages,
|
|
4878
4873
|
};
|
|
4879
4874
|
/**
|
|
4880
|
-
* TODO: [🧠][🤠] This should be probbably as part of `
|
|
4875
|
+
* TODO: [🧠][🤠] This should be probbably as part of `TextFormatParser`
|
|
4881
4876
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
4882
4877
|
*/
|
|
4883
4878
|
|
|
@@ -4905,7 +4900,7 @@ function checkExpectations(expectations, value) {
|
|
|
4905
4900
|
}
|
|
4906
4901
|
/**
|
|
4907
4902
|
* TODO: [💝] Unite object for expecting amount and format
|
|
4908
|
-
* TODO: [🧠][🤠] This should be part of `
|
|
4903
|
+
* TODO: [🧠][🤠] This should be part of `TextFormatParser`
|
|
4909
4904
|
* Note: [💝] and [🤠] are interconnected together
|
|
4910
4905
|
*/
|
|
4911
4906
|
|
|
@@ -5133,7 +5128,7 @@ async function executeAttempts(options) {
|
|
|
5133
5128
|
if (task.format) {
|
|
5134
5129
|
if (task.format === 'JSON') {
|
|
5135
5130
|
if (!isValidJsonString($ongoingTaskResult.$resultString || '')) {
|
|
5136
|
-
// TODO: [🏢] Do more universally via `
|
|
5131
|
+
// TODO: [🏢] Do more universally via `FormatParser`
|
|
5137
5132
|
try {
|
|
5138
5133
|
$ongoingTaskResult.$resultString = extractJsonBlock($ongoingTaskResult.$resultString || '');
|
|
5139
5134
|
}
|
|
@@ -5271,16 +5266,16 @@ async function executeFormatSubvalues(options) {
|
|
|
5271
5266
|
${block(pipelineIdentification)}
|
|
5272
5267
|
`));
|
|
5273
5268
|
}
|
|
5274
|
-
const
|
|
5275
|
-
if (
|
|
5269
|
+
const subvalueParser = formatDefinition.subvalueParsers.find((subvalueParser) => [subvalueParser.subvalueName, ...(subvalueParser.aliases || [])].includes(task.foreach.subformatName));
|
|
5270
|
+
if (subvalueParser === undefined) {
|
|
5276
5271
|
throw new UnexpectedError(
|
|
5277
5272
|
// <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError)
|
|
5278
5273
|
spaceTrim$1((block) => `
|
|
5279
5274
|
Unsupported subformat name "${task.foreach.subformatName}" for format "${task.foreach.formatName}"
|
|
5280
5275
|
|
|
5281
5276
|
Available subformat names for format "${formatDefinition.formatName}":
|
|
5282
|
-
${block(formatDefinition.
|
|
5283
|
-
.map((
|
|
5277
|
+
${block(formatDefinition.subvalueParsers
|
|
5278
|
+
.map((subvalueParser) => subvalueParser.subvalueName)
|
|
5284
5279
|
.map((subvalueName) => `- ${subvalueName}`)
|
|
5285
5280
|
.join('\n'))}
|
|
5286
5281
|
|
|
@@ -5294,7 +5289,7 @@ async function executeFormatSubvalues(options) {
|
|
|
5294
5289
|
formatSettings = csvSettings;
|
|
5295
5290
|
// <- TODO: [🤹♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
|
|
5296
5291
|
}
|
|
5297
|
-
const resultString = await
|
|
5292
|
+
const resultString = await subvalueParser.mapValues(parameterValue, task.foreach.outputSubparameterName, formatSettings, async (subparameters, index) => {
|
|
5298
5293
|
let mappedParameters;
|
|
5299
5294
|
// TODO: [🤹♂️][🪂] Limit to N concurrent executions
|
|
5300
5295
|
// TODO: When done [🐚] Report progress also for each subvalue here
|
|
@@ -5356,6 +5351,27 @@ async function getExamplesForTask(task) {
|
|
|
5356
5351
|
return RESERVED_PARAMETER_MISSING_VALUE /* <- TODO: [♨] Implement */;
|
|
5357
5352
|
}
|
|
5358
5353
|
|
|
5354
|
+
/**
|
|
5355
|
+
* Computes the cosine similarity between two embedding vectors
|
|
5356
|
+
*
|
|
5357
|
+
* Note: This is helping function for RAG (retrieval-augmented generation)
|
|
5358
|
+
*
|
|
5359
|
+
* @param embeddingVector1
|
|
5360
|
+
* @param embeddingVector2
|
|
5361
|
+
* @returns Cosine similarity between the two vectors
|
|
5362
|
+
*
|
|
5363
|
+
* @public exported from `@promptbook/core`
|
|
5364
|
+
*/
|
|
5365
|
+
function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
|
|
5366
|
+
if (embeddingVector1.length !== embeddingVector2.length) {
|
|
5367
|
+
throw new TypeError('Embedding vectors must have the same length');
|
|
5368
|
+
}
|
|
5369
|
+
const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
|
|
5370
|
+
const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
|
|
5371
|
+
const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
|
|
5372
|
+
return 1 - dotProduct / (magnitude1 * magnitude2);
|
|
5373
|
+
}
|
|
5374
|
+
|
|
5359
5375
|
/**
|
|
5360
5376
|
* @@@
|
|
5361
5377
|
*
|
|
@@ -5382,7 +5398,7 @@ async function getKnowledgeForTask(options) {
|
|
|
5382
5398
|
},
|
|
5383
5399
|
content: task.content,
|
|
5384
5400
|
parameters: {
|
|
5385
|
-
/*
|
|
5401
|
+
/* !!!! */
|
|
5386
5402
|
},
|
|
5387
5403
|
};
|
|
5388
5404
|
const taskEmbeddingResult = await llmTools.callEmbeddingModel(taskEmbeddingPrompt);
|
|
@@ -5417,16 +5433,6 @@ async function getKnowledgeForTask(options) {
|
|
|
5417
5433
|
return knowledgePiecesLimited.map(({ content }) => `- ${content}`).join('\n');
|
|
5418
5434
|
// <- TODO: [🧠] Some smart aggregation of knowledge pieces, single-line vs multi-line vs mixed
|
|
5419
5435
|
}
|
|
5420
|
-
// TODO: !!!!!! Annotate + to new file
|
|
5421
|
-
function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
|
|
5422
|
-
if (embeddingVector1.length !== embeddingVector2.length) {
|
|
5423
|
-
throw new TypeError('Embedding vectors must have the same length');
|
|
5424
|
-
}
|
|
5425
|
-
const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
|
|
5426
|
-
const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
|
|
5427
|
-
const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
|
|
5428
|
-
return 1 - dotProduct / (magnitude1 * magnitude2);
|
|
5429
|
-
}
|
|
5430
5436
|
/**
|
|
5431
5437
|
* TODO: !!!! Verify if this is working
|
|
5432
5438
|
* TODO: [♨] Implement Better - use keyword search
|
|
@@ -5635,6 +5641,7 @@ async function executePipeline(options) {
|
|
|
5635
5641
|
* Note: This is a flag to prevent `onProgress` call after the pipeline execution is finished
|
|
5636
5642
|
*/
|
|
5637
5643
|
let isReturned = false;
|
|
5644
|
+
console.log(`!!! preparedPipeline`, preparedPipeline);
|
|
5638
5645
|
// Note: Check that all input input parameters are defined
|
|
5639
5646
|
for (const parameter of preparedPipeline.parameters.filter(({ isInput }) => isInput)) {
|
|
5640
5647
|
if (inputParameters[parameter.name] === undefined) {
|
|
@@ -5929,6 +5936,22 @@ function createPipelineExecutor(options) {
|
|
|
5929
5936
|
cacheDirname,
|
|
5930
5937
|
intermediateFilesStrategy,
|
|
5931
5938
|
isAutoInstalled,
|
|
5939
|
+
}).catch((error) => {
|
|
5940
|
+
assertsError(error);
|
|
5941
|
+
return exportJson({
|
|
5942
|
+
name: 'pipelineExecutorResult',
|
|
5943
|
+
message: `Unuccessful PipelineExecutorResult, last catch`,
|
|
5944
|
+
order: [],
|
|
5945
|
+
value: {
|
|
5946
|
+
isSuccessful: false,
|
|
5947
|
+
errors: [serializeError(error)],
|
|
5948
|
+
warnings: [],
|
|
5949
|
+
usage: UNCERTAIN_USAGE,
|
|
5950
|
+
executionReport: null,
|
|
5951
|
+
outputParameters: {},
|
|
5952
|
+
preparedPipeline,
|
|
5953
|
+
},
|
|
5954
|
+
});
|
|
5932
5955
|
});
|
|
5933
5956
|
};
|
|
5934
5957
|
const pipelineExecutor = (inputParameters) => createTask({
|