@promptbook/pdf 0.92.0-11 → 0.92.0-13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/index.es.js +222 -199
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/browser.index.d.ts +2 -0
- package/esm/typings/src/_packages/core.index.d.ts +6 -4
- package/esm/typings/src/_packages/types.index.d.ts +2 -2
- package/esm/typings/src/execution/PipelineExecutorResult.d.ts +3 -1
- package/esm/typings/src/execution/createPipelineExecutor/computeCosineSimilarity.d.ts +13 -0
- package/esm/typings/src/execution/utils/checkExpectations.d.ts +1 -1
- package/esm/typings/src/formats/_common/{FormatDefinition.d.ts → FormatParser.d.ts} +3 -3
- package/esm/typings/src/formats/_common/{FormatSubvalueDefinition.d.ts → FormatSubvalueParser.d.ts} +1 -1
- package/esm/typings/src/formats/csv/CsvFormatParser.d.ts +17 -0
- package/esm/typings/src/formats/index.d.ts +2 -2
- package/esm/typings/src/formats/json/{JsonFormatDefinition.d.ts → JsonFormatParser.d.ts} +6 -6
- package/esm/typings/src/formats/text/{TextFormatDefinition.d.ts → TextFormatParser.d.ts} +7 -7
- package/esm/typings/src/formats/xml/XmlFormatParser.d.ts +19 -0
- package/esm/typings/src/postprocessing/utils/extractJsonBlock.d.ts +1 -1
- package/esm/typings/src/storage/local-storage/getIndexedDbStorage.d.ts +10 -0
- package/esm/typings/src/storage/local-storage/utils/makePromptbookStorageFromIndexedDb.d.ts +7 -0
- package/esm/typings/src/utils/expectation-counters/index.d.ts +1 -1
- package/package.json +2 -2
- package/umd/index.umd.js +222 -199
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/formats/csv/CsvFormatDefinition.d.ts +0 -17
- package/esm/typings/src/formats/xml/XmlFormatDefinition.d.ts +0 -19
package/esm/index.es.js
CHANGED
|
@@ -26,7 +26,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
|
|
|
26
26
|
* @generated
|
|
27
27
|
* @see https://github.com/webgptorg/promptbook
|
|
28
28
|
*/
|
|
29
|
-
const PROMPTBOOK_ENGINE_VERSION = '0.92.0-
|
|
29
|
+
const PROMPTBOOK_ENGINE_VERSION = '0.92.0-13';
|
|
30
30
|
/**
|
|
31
31
|
* TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
|
|
32
32
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -2137,75 +2137,6 @@ function isPipelinePrepared(pipeline) {
|
|
|
2137
2137
|
* - [♨] Are tasks prepared
|
|
2138
2138
|
*/
|
|
2139
2139
|
|
|
2140
|
-
/**
|
|
2141
|
-
* Converts a JavaScript Object Notation (JSON) string into an object.
|
|
2142
|
-
*
|
|
2143
|
-
* Note: This is wrapper around `JSON.parse()` with better error and type handling
|
|
2144
|
-
*
|
|
2145
|
-
* @public exported from `@promptbook/utils`
|
|
2146
|
-
*/
|
|
2147
|
-
function jsonParse(value) {
|
|
2148
|
-
if (value === undefined) {
|
|
2149
|
-
throw new Error(`Can not parse JSON from undefined value.`);
|
|
2150
|
-
}
|
|
2151
|
-
else if (typeof value !== 'string') {
|
|
2152
|
-
console.error('Can not parse JSON from non-string value.', { text: value });
|
|
2153
|
-
throw new Error(spaceTrim(`
|
|
2154
|
-
Can not parse JSON from non-string value.
|
|
2155
|
-
|
|
2156
|
-
The value type: ${typeof value}
|
|
2157
|
-
See more in console.
|
|
2158
|
-
`));
|
|
2159
|
-
}
|
|
2160
|
-
try {
|
|
2161
|
-
return JSON.parse(value);
|
|
2162
|
-
}
|
|
2163
|
-
catch (error) {
|
|
2164
|
-
if (!(error instanceof Error)) {
|
|
2165
|
-
throw error;
|
|
2166
|
-
}
|
|
2167
|
-
throw new Error(spaceTrim((block) => `
|
|
2168
|
-
${block(error.message)}
|
|
2169
|
-
|
|
2170
|
-
The JSON text:
|
|
2171
|
-
${block(value)}
|
|
2172
|
-
`));
|
|
2173
|
-
}
|
|
2174
|
-
}
|
|
2175
|
-
/**
|
|
2176
|
-
* TODO: !!!! Use in Promptbook.studio
|
|
2177
|
-
*/
|
|
2178
|
-
|
|
2179
|
-
/**
|
|
2180
|
-
* Recursively converts JSON strings to JSON objects
|
|
2181
|
-
|
|
2182
|
-
* @public exported from `@promptbook/utils`
|
|
2183
|
-
*/
|
|
2184
|
-
function jsonStringsToJsons(object) {
|
|
2185
|
-
if (object === null) {
|
|
2186
|
-
return object;
|
|
2187
|
-
}
|
|
2188
|
-
if (Array.isArray(object)) {
|
|
2189
|
-
return object.map(jsonStringsToJsons);
|
|
2190
|
-
}
|
|
2191
|
-
if (typeof object !== 'object') {
|
|
2192
|
-
return object;
|
|
2193
|
-
}
|
|
2194
|
-
const newObject = { ...object };
|
|
2195
|
-
for (const [key, value] of Object.entries(object)) {
|
|
2196
|
-
if (typeof value === 'string' && isValidJsonString(value)) {
|
|
2197
|
-
newObject[key] = jsonParse(value);
|
|
2198
|
-
}
|
|
2199
|
-
else {
|
|
2200
|
-
newObject[key] = jsonStringsToJsons(value);
|
|
2201
|
-
}
|
|
2202
|
-
}
|
|
2203
|
-
return newObject;
|
|
2204
|
-
}
|
|
2205
|
-
/**
|
|
2206
|
-
* TODO: Type the return type correctly
|
|
2207
|
-
*/
|
|
2208
|
-
|
|
2209
2140
|
/**
|
|
2210
2141
|
* This error indicates problems parsing the format value
|
|
2211
2142
|
*
|
|
@@ -2389,6 +2320,104 @@ const ALL_ERRORS = {
|
|
|
2389
2320
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
2390
2321
|
*/
|
|
2391
2322
|
|
|
2323
|
+
/**
|
|
2324
|
+
* Serializes an error into a [🚉] JSON-serializable object
|
|
2325
|
+
*
|
|
2326
|
+
* @public exported from `@promptbook/utils`
|
|
2327
|
+
*/
|
|
2328
|
+
function serializeError(error) {
|
|
2329
|
+
const { name, message, stack } = error;
|
|
2330
|
+
const { id } = error;
|
|
2331
|
+
if (!Object.keys(ALL_ERRORS).includes(name)) {
|
|
2332
|
+
console.error(spaceTrim((block) => `
|
|
2333
|
+
|
|
2334
|
+
Cannot serialize error with name "${name}"
|
|
2335
|
+
|
|
2336
|
+
Authors of Promptbook probably forgot to add this error into the list of errors:
|
|
2337
|
+
https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
|
|
2338
|
+
|
|
2339
|
+
|
|
2340
|
+
${block(stack || message)}
|
|
2341
|
+
|
|
2342
|
+
`));
|
|
2343
|
+
}
|
|
2344
|
+
return {
|
|
2345
|
+
name: name,
|
|
2346
|
+
message,
|
|
2347
|
+
stack,
|
|
2348
|
+
id, // Include id in the serialized object
|
|
2349
|
+
};
|
|
2350
|
+
}
|
|
2351
|
+
|
|
2352
|
+
/**
|
|
2353
|
+
* Converts a JavaScript Object Notation (JSON) string into an object.
|
|
2354
|
+
*
|
|
2355
|
+
* Note: This is wrapper around `JSON.parse()` with better error and type handling
|
|
2356
|
+
*
|
|
2357
|
+
* @public exported from `@promptbook/utils`
|
|
2358
|
+
*/
|
|
2359
|
+
function jsonParse(value) {
|
|
2360
|
+
if (value === undefined) {
|
|
2361
|
+
throw new Error(`Can not parse JSON from undefined value.`);
|
|
2362
|
+
}
|
|
2363
|
+
else if (typeof value !== 'string') {
|
|
2364
|
+
console.error('Can not parse JSON from non-string value.', { text: value });
|
|
2365
|
+
throw new Error(spaceTrim(`
|
|
2366
|
+
Can not parse JSON from non-string value.
|
|
2367
|
+
|
|
2368
|
+
The value type: ${typeof value}
|
|
2369
|
+
See more in console.
|
|
2370
|
+
`));
|
|
2371
|
+
}
|
|
2372
|
+
try {
|
|
2373
|
+
return JSON.parse(value);
|
|
2374
|
+
}
|
|
2375
|
+
catch (error) {
|
|
2376
|
+
if (!(error instanceof Error)) {
|
|
2377
|
+
throw error;
|
|
2378
|
+
}
|
|
2379
|
+
throw new Error(spaceTrim((block) => `
|
|
2380
|
+
${block(error.message)}
|
|
2381
|
+
|
|
2382
|
+
The JSON text:
|
|
2383
|
+
${block(value)}
|
|
2384
|
+
`));
|
|
2385
|
+
}
|
|
2386
|
+
}
|
|
2387
|
+
/**
|
|
2388
|
+
* TODO: !!!! Use in Promptbook.studio
|
|
2389
|
+
*/
|
|
2390
|
+
|
|
2391
|
+
/**
|
|
2392
|
+
* Recursively converts JSON strings to JSON objects
|
|
2393
|
+
|
|
2394
|
+
* @public exported from `@promptbook/utils`
|
|
2395
|
+
*/
|
|
2396
|
+
function jsonStringsToJsons(object) {
|
|
2397
|
+
if (object === null) {
|
|
2398
|
+
return object;
|
|
2399
|
+
}
|
|
2400
|
+
if (Array.isArray(object)) {
|
|
2401
|
+
return object.map(jsonStringsToJsons);
|
|
2402
|
+
}
|
|
2403
|
+
if (typeof object !== 'object') {
|
|
2404
|
+
return object;
|
|
2405
|
+
}
|
|
2406
|
+
const newObject = { ...object };
|
|
2407
|
+
for (const [key, value] of Object.entries(object)) {
|
|
2408
|
+
if (typeof value === 'string' && isValidJsonString(value)) {
|
|
2409
|
+
newObject[key] = jsonParse(value);
|
|
2410
|
+
}
|
|
2411
|
+
else {
|
|
2412
|
+
newObject[key] = jsonStringsToJsons(value);
|
|
2413
|
+
}
|
|
2414
|
+
}
|
|
2415
|
+
return newObject;
|
|
2416
|
+
}
|
|
2417
|
+
/**
|
|
2418
|
+
* TODO: Type the return type correctly
|
|
2419
|
+
*/
|
|
2420
|
+
|
|
2392
2421
|
/**
|
|
2393
2422
|
* Deserializes the error object
|
|
2394
2423
|
*
|
|
@@ -2554,64 +2583,6 @@ function createTask(options) {
|
|
|
2554
2583
|
* TODO: [🐚] Split into more files and make `PrepareTask` & `RemoteTask` + split the function
|
|
2555
2584
|
*/
|
|
2556
2585
|
|
|
2557
|
-
/**
|
|
2558
|
-
* Serializes an error into a [🚉] JSON-serializable object
|
|
2559
|
-
*
|
|
2560
|
-
* @public exported from `@promptbook/utils`
|
|
2561
|
-
*/
|
|
2562
|
-
function serializeError(error) {
|
|
2563
|
-
const { name, message, stack } = error;
|
|
2564
|
-
const { id } = error;
|
|
2565
|
-
if (!Object.keys(ALL_ERRORS).includes(name)) {
|
|
2566
|
-
console.error(spaceTrim((block) => `
|
|
2567
|
-
|
|
2568
|
-
Cannot serialize error with name "${name}"
|
|
2569
|
-
|
|
2570
|
-
Authors of Promptbook probably forgot to add this error into the list of errors:
|
|
2571
|
-
https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
${block(stack || message)}
|
|
2575
|
-
|
|
2576
|
-
`));
|
|
2577
|
-
}
|
|
2578
|
-
return {
|
|
2579
|
-
name: name,
|
|
2580
|
-
message,
|
|
2581
|
-
stack,
|
|
2582
|
-
id, // Include id in the serialized object
|
|
2583
|
-
};
|
|
2584
|
-
}
|
|
2585
|
-
|
|
2586
|
-
/**
|
|
2587
|
-
* Async version of Array.forEach
|
|
2588
|
-
*
|
|
2589
|
-
* @param array - Array to iterate over
|
|
2590
|
-
* @param options - Options for the function
|
|
2591
|
-
* @param callbackfunction - Function to call for each item
|
|
2592
|
-
* @public exported from `@promptbook/utils`
|
|
2593
|
-
* @deprecated [🪂] Use queues instead
|
|
2594
|
-
*/
|
|
2595
|
-
async function forEachAsync(array, options, callbackfunction) {
|
|
2596
|
-
const { maxParallelCount = Infinity } = options;
|
|
2597
|
-
let index = 0;
|
|
2598
|
-
let runningTasks = [];
|
|
2599
|
-
const tasks = [];
|
|
2600
|
-
for (const item of array) {
|
|
2601
|
-
const currentIndex = index++;
|
|
2602
|
-
const task = callbackfunction(item, currentIndex, array);
|
|
2603
|
-
tasks.push(task);
|
|
2604
|
-
runningTasks.push(task);
|
|
2605
|
-
/* not await */ Promise.resolve(task).then(() => {
|
|
2606
|
-
runningTasks = runningTasks.filter((t) => t !== task);
|
|
2607
|
-
});
|
|
2608
|
-
if (maxParallelCount < runningTasks.length) {
|
|
2609
|
-
await Promise.race(runningTasks);
|
|
2610
|
-
}
|
|
2611
|
-
}
|
|
2612
|
-
await Promise.all(tasks);
|
|
2613
|
-
}
|
|
2614
|
-
|
|
2615
2586
|
/**
|
|
2616
2587
|
* Represents the uncertain value
|
|
2617
2588
|
*
|
|
@@ -2655,7 +2626,7 @@ const ZERO_USAGE = $deepFreeze({
|
|
|
2655
2626
|
*
|
|
2656
2627
|
* @public exported from `@promptbook/core`
|
|
2657
2628
|
*/
|
|
2658
|
-
$deepFreeze({
|
|
2629
|
+
const UNCERTAIN_USAGE = $deepFreeze({
|
|
2659
2630
|
price: UNCERTAIN_ZERO_VALUE,
|
|
2660
2631
|
input: {
|
|
2661
2632
|
tokensCount: UNCERTAIN_ZERO_VALUE,
|
|
@@ -2680,6 +2651,35 @@ $deepFreeze({
|
|
|
2680
2651
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
2681
2652
|
*/
|
|
2682
2653
|
|
|
2654
|
+
/**
|
|
2655
|
+
* Async version of Array.forEach
|
|
2656
|
+
*
|
|
2657
|
+
* @param array - Array to iterate over
|
|
2658
|
+
* @param options - Options for the function
|
|
2659
|
+
* @param callbackfunction - Function to call for each item
|
|
2660
|
+
* @public exported from `@promptbook/utils`
|
|
2661
|
+
* @deprecated [🪂] Use queues instead
|
|
2662
|
+
*/
|
|
2663
|
+
async function forEachAsync(array, options, callbackfunction) {
|
|
2664
|
+
const { maxParallelCount = Infinity } = options;
|
|
2665
|
+
let index = 0;
|
|
2666
|
+
let runningTasks = [];
|
|
2667
|
+
const tasks = [];
|
|
2668
|
+
for (const item of array) {
|
|
2669
|
+
const currentIndex = index++;
|
|
2670
|
+
const task = callbackfunction(item, currentIndex, array);
|
|
2671
|
+
tasks.push(task);
|
|
2672
|
+
runningTasks.push(task);
|
|
2673
|
+
/* not await */ Promise.resolve(task).then(() => {
|
|
2674
|
+
runningTasks = runningTasks.filter((t) => t !== task);
|
|
2675
|
+
});
|
|
2676
|
+
if (maxParallelCount < runningTasks.length) {
|
|
2677
|
+
await Promise.race(runningTasks);
|
|
2678
|
+
}
|
|
2679
|
+
}
|
|
2680
|
+
await Promise.all(tasks);
|
|
2681
|
+
}
|
|
2682
|
+
|
|
2683
2683
|
/**
|
|
2684
2684
|
* Function `addUsage` will add multiple usages into one
|
|
2685
2685
|
*
|
|
@@ -4087,6 +4087,24 @@ const MANDATORY_CSV_SETTINGS = Object.freeze({
|
|
|
4087
4087
|
// encoding: 'utf-8',
|
|
4088
4088
|
});
|
|
4089
4089
|
|
|
4090
|
+
/**
|
|
4091
|
+
* Converts a CSV string into an object
|
|
4092
|
+
*
|
|
4093
|
+
* Note: This is wrapper around `papaparse.parse()` with better autohealing
|
|
4094
|
+
*
|
|
4095
|
+
* @private - for now until `@promptbook/csv` is released
|
|
4096
|
+
*/
|
|
4097
|
+
function csvParse(value /* <- TODO: string_csv */, settings, schema /* <- TODO: Make CSV Schemas */) {
|
|
4098
|
+
settings = { ...settings, ...MANDATORY_CSV_SETTINGS };
|
|
4099
|
+
// Note: Autoheal invalid '\n' characters
|
|
4100
|
+
if (settings.newline && !settings.newline.includes('\r') && value.includes('\r')) {
|
|
4101
|
+
console.warn('CSV string contains carriage return characters, but in the CSV settings the `newline` setting does not include them. Autohealing the CSV string.');
|
|
4102
|
+
value = value.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
|
|
4103
|
+
}
|
|
4104
|
+
const csv = parse(value, settings);
|
|
4105
|
+
return csv;
|
|
4106
|
+
}
|
|
4107
|
+
|
|
4090
4108
|
/**
|
|
4091
4109
|
* Function to check if a string is valid CSV
|
|
4092
4110
|
*
|
|
@@ -4109,31 +4127,13 @@ function isValidCsvString(value) {
|
|
|
4109
4127
|
}
|
|
4110
4128
|
}
|
|
4111
4129
|
|
|
4112
|
-
/**
|
|
4113
|
-
* Converts a CSV string into an object
|
|
4114
|
-
*
|
|
4115
|
-
* Note: This is wrapper around `papaparse.parse()` with better autohealing
|
|
4116
|
-
*
|
|
4117
|
-
* @private - for now until `@promptbook/csv` is released
|
|
4118
|
-
*/
|
|
4119
|
-
function csvParse(value /* <- TODO: string_csv */, settings, schema /* <- TODO: Make CSV Schemas */) {
|
|
4120
|
-
settings = { ...settings, ...MANDATORY_CSV_SETTINGS };
|
|
4121
|
-
// Note: Autoheal invalid '\n' characters
|
|
4122
|
-
if (settings.newline && !settings.newline.includes('\r') && value.includes('\r')) {
|
|
4123
|
-
console.warn('CSV string contains carriage return characters, but in the CSV settings the `newline` setting does not include them. Autohealing the CSV string.');
|
|
4124
|
-
value = value.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
|
|
4125
|
-
}
|
|
4126
|
-
const csv = parse(value, settings);
|
|
4127
|
-
return csv;
|
|
4128
|
-
}
|
|
4129
|
-
|
|
4130
4130
|
/**
|
|
4131
4131
|
* Definition for CSV spreadsheet
|
|
4132
4132
|
*
|
|
4133
4133
|
* @public exported from `@promptbook/core`
|
|
4134
4134
|
* <- TODO: [🏢] Export from package `@promptbook/csv`
|
|
4135
4135
|
*/
|
|
4136
|
-
const
|
|
4136
|
+
const CsvFormatParser = {
|
|
4137
4137
|
formatName: 'CSV',
|
|
4138
4138
|
aliases: ['SPREADSHEET', 'TABLE'],
|
|
4139
4139
|
isValid(value, settings, schema) {
|
|
@@ -4145,7 +4145,7 @@ const CsvFormatDefinition = {
|
|
|
4145
4145
|
heal(value, settings, schema) {
|
|
4146
4146
|
throw new Error('Not implemented');
|
|
4147
4147
|
},
|
|
4148
|
-
|
|
4148
|
+
subvalueParsers: [
|
|
4149
4149
|
{
|
|
4150
4150
|
subvalueName: 'ROW',
|
|
4151
4151
|
async mapValues(value, outputParameterName, settings, mapCallback) {
|
|
@@ -4206,10 +4206,10 @@ const CsvFormatDefinition = {
|
|
|
4206
4206
|
],
|
|
4207
4207
|
};
|
|
4208
4208
|
/**
|
|
4209
|
-
* TODO: [🍓] In `
|
|
4210
|
-
* TODO: [🍓] In `
|
|
4211
|
-
* TODO: [🍓] In `
|
|
4212
|
-
* TODO: [🍓] In `
|
|
4209
|
+
* TODO: [🍓] In `CsvFormatParser` implement simple `isValid`
|
|
4210
|
+
* TODO: [🍓] In `CsvFormatParser` implement partial `canBeValid`
|
|
4211
|
+
* TODO: [🍓] In `CsvFormatParser` implement `heal
|
|
4212
|
+
* TODO: [🍓] In `CsvFormatParser` implement `subvalueParsers`
|
|
4213
4213
|
* TODO: [🏢] Allow to expect something inside CSV objects and other formats
|
|
4214
4214
|
*/
|
|
4215
4215
|
|
|
@@ -4218,7 +4218,7 @@ const CsvFormatDefinition = {
|
|
|
4218
4218
|
*
|
|
4219
4219
|
* @private still in development [🏢]
|
|
4220
4220
|
*/
|
|
4221
|
-
const
|
|
4221
|
+
const JsonFormatParser = {
|
|
4222
4222
|
formatName: 'JSON',
|
|
4223
4223
|
mimeType: 'application/json',
|
|
4224
4224
|
isValid(value, settings, schema) {
|
|
@@ -4230,28 +4230,28 @@ const JsonFormatDefinition = {
|
|
|
4230
4230
|
heal(value, settings, schema) {
|
|
4231
4231
|
throw new Error('Not implemented');
|
|
4232
4232
|
},
|
|
4233
|
-
|
|
4233
|
+
subvalueParsers: [],
|
|
4234
4234
|
};
|
|
4235
4235
|
/**
|
|
4236
4236
|
* TODO: [🧠] Maybe propper instance of object
|
|
4237
4237
|
* TODO: [0] Make string_serialized_json
|
|
4238
4238
|
* TODO: [1] Make type for JSON Settings and Schema
|
|
4239
4239
|
* TODO: [🧠] What to use for validating JSONs - JSON Schema, ZoD, typescript types/interfaces,...?
|
|
4240
|
-
* TODO: [🍓] In `
|
|
4241
|
-
* TODO: [🍓] In `
|
|
4242
|
-
* TODO: [🍓] In `
|
|
4243
|
-
* TODO: [🍓] In `
|
|
4240
|
+
* TODO: [🍓] In `JsonFormatParser` implement simple `isValid`
|
|
4241
|
+
* TODO: [🍓] In `JsonFormatParser` implement partial `canBeValid`
|
|
4242
|
+
* TODO: [🍓] In `JsonFormatParser` implement `heal
|
|
4243
|
+
* TODO: [🍓] In `JsonFormatParser` implement `subvalueParsers`
|
|
4244
4244
|
* TODO: [🏢] Allow to expect something inside JSON objects and other formats
|
|
4245
4245
|
*/
|
|
4246
4246
|
|
|
4247
4247
|
/**
|
|
4248
4248
|
* Definition for any text - this will be always valid
|
|
4249
4249
|
*
|
|
4250
|
-
* Note: This is not useful for validation, but for splitting and mapping with `
|
|
4250
|
+
* Note: This is not useful for validation, but for splitting and mapping with `subvalueParsers`
|
|
4251
4251
|
*
|
|
4252
4252
|
* @public exported from `@promptbook/core`
|
|
4253
4253
|
*/
|
|
4254
|
-
const
|
|
4254
|
+
const TextFormatParser = {
|
|
4255
4255
|
formatName: 'TEXT',
|
|
4256
4256
|
isValid(value) {
|
|
4257
4257
|
return typeof value === 'string';
|
|
@@ -4260,9 +4260,9 @@ const TextFormatDefinition = {
|
|
|
4260
4260
|
return typeof partialValue === 'string';
|
|
4261
4261
|
},
|
|
4262
4262
|
heal() {
|
|
4263
|
-
throw new UnexpectedError('It does not make sense to call `
|
|
4263
|
+
throw new UnexpectedError('It does not make sense to call `TextFormatParser.heal`');
|
|
4264
4264
|
},
|
|
4265
|
-
|
|
4265
|
+
subvalueParsers: [
|
|
4266
4266
|
{
|
|
4267
4267
|
subvalueName: 'LINE',
|
|
4268
4268
|
async mapValues(value, outputParameterName, settings, mapCallback) {
|
|
@@ -4282,10 +4282,10 @@ const TextFormatDefinition = {
|
|
|
4282
4282
|
/**
|
|
4283
4283
|
* TODO: [1] Make type for XML Text and Schema
|
|
4284
4284
|
* TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages available as subvalues
|
|
4285
|
-
* TODO: [🍓] In `
|
|
4286
|
-
* TODO: [🍓] In `
|
|
4287
|
-
* TODO: [🍓] In `
|
|
4288
|
-
* TODO: [🍓] In `
|
|
4285
|
+
* TODO: [🍓] In `TextFormatParser` implement simple `isValid`
|
|
4286
|
+
* TODO: [🍓] In `TextFormatParser` implement partial `canBeValid`
|
|
4287
|
+
* TODO: [🍓] In `TextFormatParser` implement `heal
|
|
4288
|
+
* TODO: [🍓] In `TextFormatParser` implement `subvalueParsers`
|
|
4289
4289
|
* TODO: [🏢] Allow to expect something inside each item of list and other formats
|
|
4290
4290
|
*/
|
|
4291
4291
|
|
|
@@ -4318,7 +4318,7 @@ function isValidXmlString(value) {
|
|
|
4318
4318
|
*
|
|
4319
4319
|
* @private still in development [🏢]
|
|
4320
4320
|
*/
|
|
4321
|
-
const
|
|
4321
|
+
const XmlFormatParser = {
|
|
4322
4322
|
formatName: 'XML',
|
|
4323
4323
|
mimeType: 'application/xml',
|
|
4324
4324
|
isValid(value, settings, schema) {
|
|
@@ -4330,17 +4330,17 @@ const XmlFormatDefinition = {
|
|
|
4330
4330
|
heal(value, settings, schema) {
|
|
4331
4331
|
throw new Error('Not implemented');
|
|
4332
4332
|
},
|
|
4333
|
-
|
|
4333
|
+
subvalueParsers: [],
|
|
4334
4334
|
};
|
|
4335
4335
|
/**
|
|
4336
4336
|
* TODO: [🧠] Maybe propper instance of object
|
|
4337
4337
|
* TODO: [0] Make string_serialized_xml
|
|
4338
4338
|
* TODO: [1] Make type for XML Settings and Schema
|
|
4339
4339
|
* TODO: [🧠] What to use for validating XMLs - XSD,...
|
|
4340
|
-
* TODO: [🍓] In `
|
|
4341
|
-
* TODO: [🍓] In `
|
|
4342
|
-
* TODO: [🍓] In `
|
|
4343
|
-
* TODO: [🍓] In `
|
|
4340
|
+
* TODO: [🍓] In `XmlFormatParser` implement simple `isValid`
|
|
4341
|
+
* TODO: [🍓] In `XmlFormatParser` implement partial `canBeValid`
|
|
4342
|
+
* TODO: [🍓] In `XmlFormatParser` implement `heal
|
|
4343
|
+
* TODO: [🍓] In `XmlFormatParser` implement `subvalueParsers`
|
|
4344
4344
|
* TODO: [🏢] Allow to expect something inside XML and other formats
|
|
4345
4345
|
*/
|
|
4346
4346
|
|
|
@@ -4349,12 +4349,7 @@ const XmlFormatDefinition = {
|
|
|
4349
4349
|
*
|
|
4350
4350
|
* @private internal index of `...` <- TODO [🏢]
|
|
4351
4351
|
*/
|
|
4352
|
-
const FORMAT_DEFINITIONS = [
|
|
4353
|
-
JsonFormatDefinition,
|
|
4354
|
-
XmlFormatDefinition,
|
|
4355
|
-
TextFormatDefinition,
|
|
4356
|
-
CsvFormatDefinition,
|
|
4357
|
-
];
|
|
4352
|
+
const FORMAT_DEFINITIONS = [JsonFormatParser, XmlFormatParser, TextFormatParser, CsvFormatParser];
|
|
4358
4353
|
/**
|
|
4359
4354
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
4360
4355
|
*/
|
|
@@ -4524,7 +4519,7 @@ function extractJsonBlock(markdown) {
|
|
|
4524
4519
|
}
|
|
4525
4520
|
/**
|
|
4526
4521
|
* TODO: Add some auto-healing logic + extract YAML, JSON5, TOML, etc.
|
|
4527
|
-
* TODO: [🏢] Make this logic part of `
|
|
4522
|
+
* TODO: [🏢] Make this logic part of `JsonFormatParser` or `isValidJsonString`
|
|
4528
4523
|
*/
|
|
4529
4524
|
|
|
4530
4525
|
/**
|
|
@@ -4726,7 +4721,7 @@ const CountUtils = {
|
|
|
4726
4721
|
PAGES: countPages,
|
|
4727
4722
|
};
|
|
4728
4723
|
/**
|
|
4729
|
-
* TODO: [🧠][🤠] This should be probbably as part of `
|
|
4724
|
+
* TODO: [🧠][🤠] This should be probbably as part of `TextFormatParser`
|
|
4730
4725
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
4731
4726
|
*/
|
|
4732
4727
|
|
|
@@ -4754,7 +4749,7 @@ function checkExpectations(expectations, value) {
|
|
|
4754
4749
|
}
|
|
4755
4750
|
/**
|
|
4756
4751
|
* TODO: [💝] Unite object for expecting amount and format
|
|
4757
|
-
* TODO: [🧠][🤠] This should be part of `
|
|
4752
|
+
* TODO: [🧠][🤠] This should be part of `TextFormatParser`
|
|
4758
4753
|
* Note: [💝] and [🤠] are interconnected together
|
|
4759
4754
|
*/
|
|
4760
4755
|
|
|
@@ -4982,7 +4977,7 @@ async function executeAttempts(options) {
|
|
|
4982
4977
|
if (task.format) {
|
|
4983
4978
|
if (task.format === 'JSON') {
|
|
4984
4979
|
if (!isValidJsonString($ongoingTaskResult.$resultString || '')) {
|
|
4985
|
-
// TODO: [🏢] Do more universally via `
|
|
4980
|
+
// TODO: [🏢] Do more universally via `FormatParser`
|
|
4986
4981
|
try {
|
|
4987
4982
|
$ongoingTaskResult.$resultString = extractJsonBlock($ongoingTaskResult.$resultString || '');
|
|
4988
4983
|
}
|
|
@@ -5120,16 +5115,16 @@ async function executeFormatSubvalues(options) {
|
|
|
5120
5115
|
${block(pipelineIdentification)}
|
|
5121
5116
|
`));
|
|
5122
5117
|
}
|
|
5123
|
-
const
|
|
5124
|
-
if (
|
|
5118
|
+
const subvalueParser = formatDefinition.subvalueParsers.find((subvalueParser) => [subvalueParser.subvalueName, ...(subvalueParser.aliases || [])].includes(task.foreach.subformatName));
|
|
5119
|
+
if (subvalueParser === undefined) {
|
|
5125
5120
|
throw new UnexpectedError(
|
|
5126
5121
|
// <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError)
|
|
5127
5122
|
spaceTrim((block) => `
|
|
5128
5123
|
Unsupported subformat name "${task.foreach.subformatName}" for format "${task.foreach.formatName}"
|
|
5129
5124
|
|
|
5130
5125
|
Available subformat names for format "${formatDefinition.formatName}":
|
|
5131
|
-
${block(formatDefinition.
|
|
5132
|
-
.map((
|
|
5126
|
+
${block(formatDefinition.subvalueParsers
|
|
5127
|
+
.map((subvalueParser) => subvalueParser.subvalueName)
|
|
5133
5128
|
.map((subvalueName) => `- ${subvalueName}`)
|
|
5134
5129
|
.join('\n'))}
|
|
5135
5130
|
|
|
@@ -5143,7 +5138,7 @@ async function executeFormatSubvalues(options) {
|
|
|
5143
5138
|
formatSettings = csvSettings;
|
|
5144
5139
|
// <- TODO: [🤹♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
|
|
5145
5140
|
}
|
|
5146
|
-
const resultString = await
|
|
5141
|
+
const resultString = await subvalueParser.mapValues(parameterValue, task.foreach.outputSubparameterName, formatSettings, async (subparameters, index) => {
|
|
5147
5142
|
let mappedParameters;
|
|
5148
5143
|
// TODO: [🤹♂️][🪂] Limit to N concurrent executions
|
|
5149
5144
|
// TODO: When done [🐚] Report progress also for each subvalue here
|
|
@@ -5205,6 +5200,27 @@ async function getExamplesForTask(task) {
|
|
|
5205
5200
|
return RESERVED_PARAMETER_MISSING_VALUE /* <- TODO: [♨] Implement */;
|
|
5206
5201
|
}
|
|
5207
5202
|
|
|
5203
|
+
/**
|
|
5204
|
+
* Computes the cosine similarity between two embedding vectors
|
|
5205
|
+
*
|
|
5206
|
+
* Note: This is helping function for RAG (retrieval-augmented generation)
|
|
5207
|
+
*
|
|
5208
|
+
* @param embeddingVector1
|
|
5209
|
+
* @param embeddingVector2
|
|
5210
|
+
* @returns Cosine similarity between the two vectors
|
|
5211
|
+
*
|
|
5212
|
+
* @public exported from `@promptbook/core`
|
|
5213
|
+
*/
|
|
5214
|
+
function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
|
|
5215
|
+
if (embeddingVector1.length !== embeddingVector2.length) {
|
|
5216
|
+
throw new TypeError('Embedding vectors must have the same length');
|
|
5217
|
+
}
|
|
5218
|
+
const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
|
|
5219
|
+
const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
|
|
5220
|
+
const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
|
|
5221
|
+
return 1 - dotProduct / (magnitude1 * magnitude2);
|
|
5222
|
+
}
|
|
5223
|
+
|
|
5208
5224
|
/**
|
|
5209
5225
|
* @@@
|
|
5210
5226
|
*
|
|
@@ -5231,7 +5247,7 @@ async function getKnowledgeForTask(options) {
|
|
|
5231
5247
|
},
|
|
5232
5248
|
content: task.content,
|
|
5233
5249
|
parameters: {
|
|
5234
|
-
/*
|
|
5250
|
+
/* !!!! */
|
|
5235
5251
|
},
|
|
5236
5252
|
};
|
|
5237
5253
|
const taskEmbeddingResult = await llmTools.callEmbeddingModel(taskEmbeddingPrompt);
|
|
@@ -5266,16 +5282,6 @@ async function getKnowledgeForTask(options) {
|
|
|
5266
5282
|
return knowledgePiecesLimited.map(({ content }) => `- ${content}`).join('\n');
|
|
5267
5283
|
// <- TODO: [🧠] Some smart aggregation of knowledge pieces, single-line vs multi-line vs mixed
|
|
5268
5284
|
}
|
|
5269
|
-
// TODO: !!!!!! Annotate + to new file
|
|
5270
|
-
function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
|
|
5271
|
-
if (embeddingVector1.length !== embeddingVector2.length) {
|
|
5272
|
-
throw new TypeError('Embedding vectors must have the same length');
|
|
5273
|
-
}
|
|
5274
|
-
const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
|
|
5275
|
-
const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
|
|
5276
|
-
const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
|
|
5277
|
-
return 1 - dotProduct / (magnitude1 * magnitude2);
|
|
5278
|
-
}
|
|
5279
5285
|
/**
|
|
5280
5286
|
* TODO: !!!! Verify if this is working
|
|
5281
5287
|
* TODO: [♨] Implement Better - use keyword search
|
|
@@ -5484,6 +5490,7 @@ async function executePipeline(options) {
|
|
|
5484
5490
|
* Note: This is a flag to prevent `onProgress` call after the pipeline execution is finished
|
|
5485
5491
|
*/
|
|
5486
5492
|
let isReturned = false;
|
|
5493
|
+
console.log(`!!! preparedPipeline`, preparedPipeline);
|
|
5487
5494
|
// Note: Check that all input input parameters are defined
|
|
5488
5495
|
for (const parameter of preparedPipeline.parameters.filter(({ isInput }) => isInput)) {
|
|
5489
5496
|
if (inputParameters[parameter.name] === undefined) {
|
|
@@ -5778,6 +5785,22 @@ function createPipelineExecutor(options) {
|
|
|
5778
5785
|
cacheDirname,
|
|
5779
5786
|
intermediateFilesStrategy,
|
|
5780
5787
|
isAutoInstalled,
|
|
5788
|
+
}).catch((error) => {
|
|
5789
|
+
assertsError(error);
|
|
5790
|
+
return exportJson({
|
|
5791
|
+
name: 'pipelineExecutorResult',
|
|
5792
|
+
message: `Unuccessful PipelineExecutorResult, last catch`,
|
|
5793
|
+
order: [],
|
|
5794
|
+
value: {
|
|
5795
|
+
isSuccessful: false,
|
|
5796
|
+
errors: [serializeError(error)],
|
|
5797
|
+
warnings: [],
|
|
5798
|
+
usage: UNCERTAIN_USAGE,
|
|
5799
|
+
executionReport: null,
|
|
5800
|
+
outputParameters: {},
|
|
5801
|
+
preparedPipeline,
|
|
5802
|
+
},
|
|
5803
|
+
});
|
|
5781
5804
|
});
|
|
5782
5805
|
};
|
|
5783
5806
|
const pipelineExecutor = (inputParameters) => createTask({
|