@promptbook/node 0.92.0-10 → 0.92.0-12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/index.es.js +255 -233
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/browser.index.d.ts +2 -0
- package/esm/typings/src/_packages/core.index.d.ts +6 -4
- package/esm/typings/src/_packages/types.index.d.ts +2 -2
- package/esm/typings/src/execution/PipelineExecutorResult.d.ts +3 -1
- package/esm/typings/src/execution/createPipelineExecutor/computeCosineSimilarity.d.ts +13 -0
- package/esm/typings/src/execution/utils/checkExpectations.d.ts +1 -1
- package/esm/typings/src/formats/_common/{FormatDefinition.d.ts → FormatParser.d.ts} +3 -3
- package/esm/typings/src/formats/_common/{FormatSubvalueDefinition.d.ts → FormatSubvalueParser.d.ts} +1 -1
- package/esm/typings/src/formats/csv/CsvFormatParser.d.ts +17 -0
- package/esm/typings/src/formats/index.d.ts +2 -2
- package/esm/typings/src/formats/json/{JsonFormatDefinition.d.ts → JsonFormatParser.d.ts} +6 -6
- package/esm/typings/src/formats/text/{TextFormatDefinition.d.ts → TextFormatParser.d.ts} +7 -7
- package/esm/typings/src/formats/xml/XmlFormatParser.d.ts +19 -0
- package/esm/typings/src/postprocessing/utils/extractJsonBlock.d.ts +1 -1
- package/esm/typings/src/storage/local-storage/getIndexedDbStorage.d.ts +10 -0
- package/esm/typings/src/storage/local-storage/utils/makePromptbookStorageFromIndexedDb.d.ts +7 -0
- package/esm/typings/src/utils/expectation-counters/index.d.ts +1 -1
- package/package.json +2 -2
- package/umd/index.umd.js +258 -236
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/formats/csv/CsvFormatDefinition.d.ts +0 -17
- package/esm/typings/src/formats/xml/XmlFormatDefinition.d.ts +0 -19
package/umd/index.umd.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
(function (global, factory) {
|
|
2
|
-
typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('colors'), require('fs/promises'), require('path'), require('spacetrim'), require('jszip'), require('prettier'), require('prettier/parser-html'), require('
|
|
3
|
-
typeof define === 'function' && define.amd ? define(['exports', 'colors', 'fs/promises', 'path', 'spacetrim', 'jszip', 'prettier', 'prettier/parser-html', '
|
|
4
|
-
(global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-node"] = {}, global.colors, global.promises, global.path, global.spaceTrim, global.JSZip, global.prettier, global.parserHtml, global.
|
|
5
|
-
})(this, (function (exports, colors, promises, path, spaceTrim, JSZip, prettier, parserHtml,
|
|
2
|
+
typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('colors'), require('fs/promises'), require('path'), require('spacetrim'), require('jszip'), require('prettier'), require('prettier/parser-html'), require('crypto'), require('rxjs'), require('waitasecond'), require('papaparse'), require('crypto-js/enc-hex'), require('crypto-js/sha256'), require('crypto-js'), require('mime-types'), require('child_process'), require('dotenv')) :
|
|
3
|
+
typeof define === 'function' && define.amd ? define(['exports', 'colors', 'fs/promises', 'path', 'spacetrim', 'jszip', 'prettier', 'prettier/parser-html', 'crypto', 'rxjs', 'waitasecond', 'papaparse', 'crypto-js/enc-hex', 'crypto-js/sha256', 'crypto-js', 'mime-types', 'child_process', 'dotenv'], factory) :
|
|
4
|
+
(global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-node"] = {}, global.colors, global.promises, global.path, global.spaceTrim, global.JSZip, global.prettier, global.parserHtml, global.crypto, global.rxjs, global.waitasecond, global.papaparse, global.hexEncoder, global.sha256, global.cryptoJs, global.mimeTypes, global.child_process, global.dotenv));
|
|
5
|
+
})(this, (function (exports, colors, promises, path, spaceTrim, JSZip, prettier, parserHtml, crypto, rxjs, waitasecond, papaparse, hexEncoder, sha256, cryptoJs, mimeTypes, child_process, dotenv) { 'use strict';
|
|
6
6
|
|
|
7
7
|
function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
|
|
8
8
|
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
* @generated
|
|
47
47
|
* @see https://github.com/webgptorg/promptbook
|
|
48
48
|
*/
|
|
49
|
-
const PROMPTBOOK_ENGINE_VERSION = '0.92.0-
|
|
49
|
+
const PROMPTBOOK_ENGINE_VERSION = '0.92.0-12';
|
|
50
50
|
/**
|
|
51
51
|
* TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
|
|
52
52
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -1704,70 +1704,6 @@
|
|
|
1704
1704
|
* - [♨] Are tasks prepared
|
|
1705
1705
|
*/
|
|
1706
1706
|
|
|
1707
|
-
/**
|
|
1708
|
-
* Generates random token
|
|
1709
|
-
*
|
|
1710
|
-
* Note: This function is cryptographically secure (it uses crypto.randomBytes internally)
|
|
1711
|
-
*
|
|
1712
|
-
* @private internal helper function
|
|
1713
|
-
* @returns secure random token
|
|
1714
|
-
*/
|
|
1715
|
-
function $randomToken(randomness) {
|
|
1716
|
-
return crypto.randomBytes(randomness).toString('hex');
|
|
1717
|
-
}
|
|
1718
|
-
/**
|
|
1719
|
-
* TODO: Maybe use nanoid instead https://github.com/ai/nanoid
|
|
1720
|
-
*/
|
|
1721
|
-
|
|
1722
|
-
/**
|
|
1723
|
-
* Recursively converts JSON strings to JSON objects
|
|
1724
|
-
|
|
1725
|
-
* @public exported from `@promptbook/utils`
|
|
1726
|
-
*/
|
|
1727
|
-
function jsonStringsToJsons(object) {
|
|
1728
|
-
if (object === null) {
|
|
1729
|
-
return object;
|
|
1730
|
-
}
|
|
1731
|
-
if (Array.isArray(object)) {
|
|
1732
|
-
return object.map(jsonStringsToJsons);
|
|
1733
|
-
}
|
|
1734
|
-
if (typeof object !== 'object') {
|
|
1735
|
-
return object;
|
|
1736
|
-
}
|
|
1737
|
-
const newObject = { ...object };
|
|
1738
|
-
for (const [key, value] of Object.entries(object)) {
|
|
1739
|
-
if (typeof value === 'string' && isValidJsonString(value)) {
|
|
1740
|
-
newObject[key] = jsonParse(value);
|
|
1741
|
-
}
|
|
1742
|
-
else {
|
|
1743
|
-
newObject[key] = jsonStringsToJsons(value);
|
|
1744
|
-
}
|
|
1745
|
-
}
|
|
1746
|
-
return newObject;
|
|
1747
|
-
}
|
|
1748
|
-
/**
|
|
1749
|
-
* TODO: Type the return type correctly
|
|
1750
|
-
*/
|
|
1751
|
-
|
|
1752
|
-
/**
|
|
1753
|
-
* This error indicates errors during the execution of the pipeline
|
|
1754
|
-
*
|
|
1755
|
-
* @public exported from `@promptbook/core`
|
|
1756
|
-
*/
|
|
1757
|
-
class PipelineExecutionError extends Error {
|
|
1758
|
-
constructor(message) {
|
|
1759
|
-
// Added id parameter
|
|
1760
|
-
super(message);
|
|
1761
|
-
this.name = 'PipelineExecutionError';
|
|
1762
|
-
// TODO: [🐙] DRY - Maybe $randomId
|
|
1763
|
-
this.id = `error-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid simmilar char conflicts */)}`;
|
|
1764
|
-
Object.setPrototypeOf(this, PipelineExecutionError.prototype);
|
|
1765
|
-
}
|
|
1766
|
-
}
|
|
1767
|
-
/**
|
|
1768
|
-
* TODO: [🧠][🌂] Add id to all errors
|
|
1769
|
-
*/
|
|
1770
|
-
|
|
1771
1707
|
/**
|
|
1772
1708
|
* This error indicates problems parsing the format value
|
|
1773
1709
|
*
|
|
@@ -1902,6 +1838,40 @@
|
|
|
1902
1838
|
}
|
|
1903
1839
|
}
|
|
1904
1840
|
|
|
1841
|
+
/**
|
|
1842
|
+
* Generates random token
|
|
1843
|
+
*
|
|
1844
|
+
* Note: This function is cryptographically secure (it uses crypto.randomBytes internally)
|
|
1845
|
+
*
|
|
1846
|
+
* @private internal helper function
|
|
1847
|
+
* @returns secure random token
|
|
1848
|
+
*/
|
|
1849
|
+
function $randomToken(randomness) {
|
|
1850
|
+
return crypto.randomBytes(randomness).toString('hex');
|
|
1851
|
+
}
|
|
1852
|
+
/**
|
|
1853
|
+
* TODO: Maybe use nanoid instead https://github.com/ai/nanoid
|
|
1854
|
+
*/
|
|
1855
|
+
|
|
1856
|
+
/**
|
|
1857
|
+
* This error indicates errors during the execution of the pipeline
|
|
1858
|
+
*
|
|
1859
|
+
* @public exported from `@promptbook/core`
|
|
1860
|
+
*/
|
|
1861
|
+
class PipelineExecutionError extends Error {
|
|
1862
|
+
constructor(message) {
|
|
1863
|
+
// Added id parameter
|
|
1864
|
+
super(message);
|
|
1865
|
+
this.name = 'PipelineExecutionError';
|
|
1866
|
+
// TODO: [🐙] DRY - Maybe $randomId
|
|
1867
|
+
this.id = `error-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid simmilar char conflicts */)}`;
|
|
1868
|
+
Object.setPrototypeOf(this, PipelineExecutionError.prototype);
|
|
1869
|
+
}
|
|
1870
|
+
}
|
|
1871
|
+
/**
|
|
1872
|
+
* TODO: [🧠][🌂] Add id to all errors
|
|
1873
|
+
*/
|
|
1874
|
+
|
|
1905
1875
|
/**
|
|
1906
1876
|
* Error thrown when a fetch request fails
|
|
1907
1877
|
*
|
|
@@ -1977,6 +1947,65 @@
|
|
|
1977
1947
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
1978
1948
|
*/
|
|
1979
1949
|
|
|
1950
|
+
/**
|
|
1951
|
+
* Serializes an error into a [🚉] JSON-serializable object
|
|
1952
|
+
*
|
|
1953
|
+
* @public exported from `@promptbook/utils`
|
|
1954
|
+
*/
|
|
1955
|
+
function serializeError(error) {
|
|
1956
|
+
const { name, message, stack } = error;
|
|
1957
|
+
const { id } = error;
|
|
1958
|
+
if (!Object.keys(ALL_ERRORS).includes(name)) {
|
|
1959
|
+
console.error(spaceTrim__default["default"]((block) => `
|
|
1960
|
+
|
|
1961
|
+
Cannot serialize error with name "${name}"
|
|
1962
|
+
|
|
1963
|
+
Authors of Promptbook probably forgot to add this error into the list of errors:
|
|
1964
|
+
https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
|
|
1965
|
+
|
|
1966
|
+
|
|
1967
|
+
${block(stack || message)}
|
|
1968
|
+
|
|
1969
|
+
`));
|
|
1970
|
+
}
|
|
1971
|
+
return {
|
|
1972
|
+
name: name,
|
|
1973
|
+
message,
|
|
1974
|
+
stack,
|
|
1975
|
+
id, // Include id in the serialized object
|
|
1976
|
+
};
|
|
1977
|
+
}
|
|
1978
|
+
|
|
1979
|
+
/**
|
|
1980
|
+
* Recursively converts JSON strings to JSON objects
|
|
1981
|
+
|
|
1982
|
+
* @public exported from `@promptbook/utils`
|
|
1983
|
+
*/
|
|
1984
|
+
function jsonStringsToJsons(object) {
|
|
1985
|
+
if (object === null) {
|
|
1986
|
+
return object;
|
|
1987
|
+
}
|
|
1988
|
+
if (Array.isArray(object)) {
|
|
1989
|
+
return object.map(jsonStringsToJsons);
|
|
1990
|
+
}
|
|
1991
|
+
if (typeof object !== 'object') {
|
|
1992
|
+
return object;
|
|
1993
|
+
}
|
|
1994
|
+
const newObject = { ...object };
|
|
1995
|
+
for (const [key, value] of Object.entries(object)) {
|
|
1996
|
+
if (typeof value === 'string' && isValidJsonString(value)) {
|
|
1997
|
+
newObject[key] = jsonParse(value);
|
|
1998
|
+
}
|
|
1999
|
+
else {
|
|
2000
|
+
newObject[key] = jsonStringsToJsons(value);
|
|
2001
|
+
}
|
|
2002
|
+
}
|
|
2003
|
+
return newObject;
|
|
2004
|
+
}
|
|
2005
|
+
/**
|
|
2006
|
+
* TODO: Type the return type correctly
|
|
2007
|
+
*/
|
|
2008
|
+
|
|
1980
2009
|
/**
|
|
1981
2010
|
* Deserializes the error object
|
|
1982
2011
|
*
|
|
@@ -2143,33 +2172,72 @@
|
|
|
2143
2172
|
*/
|
|
2144
2173
|
|
|
2145
2174
|
/**
|
|
2146
|
-
*
|
|
2175
|
+
* Represents the uncertain value
|
|
2147
2176
|
*
|
|
2148
|
-
* @public exported from `@promptbook/
|
|
2177
|
+
* @public exported from `@promptbook/core`
|
|
2178
|
+
*/
|
|
2179
|
+
const ZERO_VALUE = $deepFreeze({ value: 0 });
|
|
2180
|
+
/**
|
|
2181
|
+
* Represents the uncertain value
|
|
2182
|
+
*
|
|
2183
|
+
* @public exported from `@promptbook/core`
|
|
2184
|
+
*/
|
|
2185
|
+
const UNCERTAIN_ZERO_VALUE = $deepFreeze({ value: 0, isUncertain: true });
|
|
2186
|
+
/**
|
|
2187
|
+
* Represents the usage with no resources consumed
|
|
2188
|
+
*
|
|
2189
|
+
* @public exported from `@promptbook/core`
|
|
2190
|
+
*/
|
|
2191
|
+
const ZERO_USAGE = $deepFreeze({
|
|
2192
|
+
price: ZERO_VALUE,
|
|
2193
|
+
input: {
|
|
2194
|
+
tokensCount: ZERO_VALUE,
|
|
2195
|
+
charactersCount: ZERO_VALUE,
|
|
2196
|
+
wordsCount: ZERO_VALUE,
|
|
2197
|
+
sentencesCount: ZERO_VALUE,
|
|
2198
|
+
linesCount: ZERO_VALUE,
|
|
2199
|
+
paragraphsCount: ZERO_VALUE,
|
|
2200
|
+
pagesCount: ZERO_VALUE,
|
|
2201
|
+
},
|
|
2202
|
+
output: {
|
|
2203
|
+
tokensCount: ZERO_VALUE,
|
|
2204
|
+
charactersCount: ZERO_VALUE,
|
|
2205
|
+
wordsCount: ZERO_VALUE,
|
|
2206
|
+
sentencesCount: ZERO_VALUE,
|
|
2207
|
+
linesCount: ZERO_VALUE,
|
|
2208
|
+
paragraphsCount: ZERO_VALUE,
|
|
2209
|
+
pagesCount: ZERO_VALUE,
|
|
2210
|
+
},
|
|
2211
|
+
});
|
|
2212
|
+
/**
|
|
2213
|
+
* Represents the usage with unknown resources consumed
|
|
2214
|
+
*
|
|
2215
|
+
* @public exported from `@promptbook/core`
|
|
2216
|
+
*/
|
|
2217
|
+
const UNCERTAIN_USAGE = $deepFreeze({
|
|
2218
|
+
price: UNCERTAIN_ZERO_VALUE,
|
|
2219
|
+
input: {
|
|
2220
|
+
tokensCount: UNCERTAIN_ZERO_VALUE,
|
|
2221
|
+
charactersCount: UNCERTAIN_ZERO_VALUE,
|
|
2222
|
+
wordsCount: UNCERTAIN_ZERO_VALUE,
|
|
2223
|
+
sentencesCount: UNCERTAIN_ZERO_VALUE,
|
|
2224
|
+
linesCount: UNCERTAIN_ZERO_VALUE,
|
|
2225
|
+
paragraphsCount: UNCERTAIN_ZERO_VALUE,
|
|
2226
|
+
pagesCount: UNCERTAIN_ZERO_VALUE,
|
|
2227
|
+
},
|
|
2228
|
+
output: {
|
|
2229
|
+
tokensCount: UNCERTAIN_ZERO_VALUE,
|
|
2230
|
+
charactersCount: UNCERTAIN_ZERO_VALUE,
|
|
2231
|
+
wordsCount: UNCERTAIN_ZERO_VALUE,
|
|
2232
|
+
sentencesCount: UNCERTAIN_ZERO_VALUE,
|
|
2233
|
+
linesCount: UNCERTAIN_ZERO_VALUE,
|
|
2234
|
+
paragraphsCount: UNCERTAIN_ZERO_VALUE,
|
|
2235
|
+
pagesCount: UNCERTAIN_ZERO_VALUE,
|
|
2236
|
+
},
|
|
2237
|
+
});
|
|
2238
|
+
/**
|
|
2239
|
+
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
2149
2240
|
*/
|
|
2150
|
-
function serializeError(error) {
|
|
2151
|
-
const { name, message, stack } = error;
|
|
2152
|
-
const { id } = error;
|
|
2153
|
-
if (!Object.keys(ALL_ERRORS).includes(name)) {
|
|
2154
|
-
console.error(spaceTrim__default["default"]((block) => `
|
|
2155
|
-
|
|
2156
|
-
Cannot serialize error with name "${name}"
|
|
2157
|
-
|
|
2158
|
-
Authors of Promptbook probably forgot to add this error into the list of errors:
|
|
2159
|
-
https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
${block(stack || message)}
|
|
2163
|
-
|
|
2164
|
-
`));
|
|
2165
|
-
}
|
|
2166
|
-
return {
|
|
2167
|
-
name: name,
|
|
2168
|
-
message,
|
|
2169
|
-
stack,
|
|
2170
|
-
id, // Include id in the serialized object
|
|
2171
|
-
};
|
|
2172
|
-
}
|
|
2173
2241
|
|
|
2174
2242
|
/**
|
|
2175
2243
|
* Format either small or big number
|
|
@@ -2252,74 +2320,6 @@
|
|
|
2252
2320
|
}
|
|
2253
2321
|
}
|
|
2254
2322
|
|
|
2255
|
-
/**
|
|
2256
|
-
* Represents the uncertain value
|
|
2257
|
-
*
|
|
2258
|
-
* @public exported from `@promptbook/core`
|
|
2259
|
-
*/
|
|
2260
|
-
const ZERO_VALUE = $deepFreeze({ value: 0 });
|
|
2261
|
-
/**
|
|
2262
|
-
* Represents the uncertain value
|
|
2263
|
-
*
|
|
2264
|
-
* @public exported from `@promptbook/core`
|
|
2265
|
-
*/
|
|
2266
|
-
const UNCERTAIN_ZERO_VALUE = $deepFreeze({ value: 0, isUncertain: true });
|
|
2267
|
-
/**
|
|
2268
|
-
* Represents the usage with no resources consumed
|
|
2269
|
-
*
|
|
2270
|
-
* @public exported from `@promptbook/core`
|
|
2271
|
-
*/
|
|
2272
|
-
const ZERO_USAGE = $deepFreeze({
|
|
2273
|
-
price: ZERO_VALUE,
|
|
2274
|
-
input: {
|
|
2275
|
-
tokensCount: ZERO_VALUE,
|
|
2276
|
-
charactersCount: ZERO_VALUE,
|
|
2277
|
-
wordsCount: ZERO_VALUE,
|
|
2278
|
-
sentencesCount: ZERO_VALUE,
|
|
2279
|
-
linesCount: ZERO_VALUE,
|
|
2280
|
-
paragraphsCount: ZERO_VALUE,
|
|
2281
|
-
pagesCount: ZERO_VALUE,
|
|
2282
|
-
},
|
|
2283
|
-
output: {
|
|
2284
|
-
tokensCount: ZERO_VALUE,
|
|
2285
|
-
charactersCount: ZERO_VALUE,
|
|
2286
|
-
wordsCount: ZERO_VALUE,
|
|
2287
|
-
sentencesCount: ZERO_VALUE,
|
|
2288
|
-
linesCount: ZERO_VALUE,
|
|
2289
|
-
paragraphsCount: ZERO_VALUE,
|
|
2290
|
-
pagesCount: ZERO_VALUE,
|
|
2291
|
-
},
|
|
2292
|
-
});
|
|
2293
|
-
/**
|
|
2294
|
-
* Represents the usage with unknown resources consumed
|
|
2295
|
-
*
|
|
2296
|
-
* @public exported from `@promptbook/core`
|
|
2297
|
-
*/
|
|
2298
|
-
$deepFreeze({
|
|
2299
|
-
price: UNCERTAIN_ZERO_VALUE,
|
|
2300
|
-
input: {
|
|
2301
|
-
tokensCount: UNCERTAIN_ZERO_VALUE,
|
|
2302
|
-
charactersCount: UNCERTAIN_ZERO_VALUE,
|
|
2303
|
-
wordsCount: UNCERTAIN_ZERO_VALUE,
|
|
2304
|
-
sentencesCount: UNCERTAIN_ZERO_VALUE,
|
|
2305
|
-
linesCount: UNCERTAIN_ZERO_VALUE,
|
|
2306
|
-
paragraphsCount: UNCERTAIN_ZERO_VALUE,
|
|
2307
|
-
pagesCount: UNCERTAIN_ZERO_VALUE,
|
|
2308
|
-
},
|
|
2309
|
-
output: {
|
|
2310
|
-
tokensCount: UNCERTAIN_ZERO_VALUE,
|
|
2311
|
-
charactersCount: UNCERTAIN_ZERO_VALUE,
|
|
2312
|
-
wordsCount: UNCERTAIN_ZERO_VALUE,
|
|
2313
|
-
sentencesCount: UNCERTAIN_ZERO_VALUE,
|
|
2314
|
-
linesCount: UNCERTAIN_ZERO_VALUE,
|
|
2315
|
-
paragraphsCount: UNCERTAIN_ZERO_VALUE,
|
|
2316
|
-
pagesCount: UNCERTAIN_ZERO_VALUE,
|
|
2317
|
-
},
|
|
2318
|
-
});
|
|
2319
|
-
/**
|
|
2320
|
-
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
2321
|
-
*/
|
|
2322
|
-
|
|
2323
2323
|
/**
|
|
2324
2324
|
* Function `addUsage` will add multiple usages into one
|
|
2325
2325
|
*
|
|
@@ -2537,6 +2537,24 @@
|
|
|
2537
2537
|
// encoding: 'utf-8',
|
|
2538
2538
|
});
|
|
2539
2539
|
|
|
2540
|
+
/**
|
|
2541
|
+
* Converts a CSV string into an object
|
|
2542
|
+
*
|
|
2543
|
+
* Note: This is wrapper around `papaparse.parse()` with better autohealing
|
|
2544
|
+
*
|
|
2545
|
+
* @private - for now until `@promptbook/csv` is released
|
|
2546
|
+
*/
|
|
2547
|
+
function csvParse(value /* <- TODO: string_csv */, settings, schema /* <- TODO: Make CSV Schemas */) {
|
|
2548
|
+
settings = { ...settings, ...MANDATORY_CSV_SETTINGS };
|
|
2549
|
+
// Note: Autoheal invalid '\n' characters
|
|
2550
|
+
if (settings.newline && !settings.newline.includes('\r') && value.includes('\r')) {
|
|
2551
|
+
console.warn('CSV string contains carriage return characters, but in the CSV settings the `newline` setting does not include them. Autohealing the CSV string.');
|
|
2552
|
+
value = value.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
|
|
2553
|
+
}
|
|
2554
|
+
const csv = papaparse.parse(value, settings);
|
|
2555
|
+
return csv;
|
|
2556
|
+
}
|
|
2557
|
+
|
|
2540
2558
|
/**
|
|
2541
2559
|
* Function to check if a string is valid CSV
|
|
2542
2560
|
*
|
|
@@ -2559,31 +2577,13 @@
|
|
|
2559
2577
|
}
|
|
2560
2578
|
}
|
|
2561
2579
|
|
|
2562
|
-
/**
|
|
2563
|
-
* Converts a CSV string into an object
|
|
2564
|
-
*
|
|
2565
|
-
* Note: This is wrapper around `papaparse.parse()` with better autohealing
|
|
2566
|
-
*
|
|
2567
|
-
* @private - for now until `@promptbook/csv` is released
|
|
2568
|
-
*/
|
|
2569
|
-
function csvParse(value /* <- TODO: string_csv */, settings, schema /* <- TODO: Make CSV Schemas */) {
|
|
2570
|
-
settings = { ...settings, ...MANDATORY_CSV_SETTINGS };
|
|
2571
|
-
// Note: Autoheal invalid '\n' characters
|
|
2572
|
-
if (settings.newline && !settings.newline.includes('\r') && value.includes('\r')) {
|
|
2573
|
-
console.warn('CSV string contains carriage return characters, but in the CSV settings the `newline` setting does not include them. Autohealing the CSV string.');
|
|
2574
|
-
value = value.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
|
|
2575
|
-
}
|
|
2576
|
-
const csv = papaparse.parse(value, settings);
|
|
2577
|
-
return csv;
|
|
2578
|
-
}
|
|
2579
|
-
|
|
2580
2580
|
/**
|
|
2581
2581
|
* Definition for CSV spreadsheet
|
|
2582
2582
|
*
|
|
2583
2583
|
* @public exported from `@promptbook/core`
|
|
2584
2584
|
* <- TODO: [🏢] Export from package `@promptbook/csv`
|
|
2585
2585
|
*/
|
|
2586
|
-
const
|
|
2586
|
+
const CsvFormatParser = {
|
|
2587
2587
|
formatName: 'CSV',
|
|
2588
2588
|
aliases: ['SPREADSHEET', 'TABLE'],
|
|
2589
2589
|
isValid(value, settings, schema) {
|
|
@@ -2595,7 +2595,7 @@
|
|
|
2595
2595
|
heal(value, settings, schema) {
|
|
2596
2596
|
throw new Error('Not implemented');
|
|
2597
2597
|
},
|
|
2598
|
-
|
|
2598
|
+
subvalueParsers: [
|
|
2599
2599
|
{
|
|
2600
2600
|
subvalueName: 'ROW',
|
|
2601
2601
|
async mapValues(value, outputParameterName, settings, mapCallback) {
|
|
@@ -2656,10 +2656,10 @@
|
|
|
2656
2656
|
],
|
|
2657
2657
|
};
|
|
2658
2658
|
/**
|
|
2659
|
-
* TODO: [🍓] In `
|
|
2660
|
-
* TODO: [🍓] In `
|
|
2661
|
-
* TODO: [🍓] In `
|
|
2662
|
-
* TODO: [🍓] In `
|
|
2659
|
+
* TODO: [🍓] In `CsvFormatParser` implement simple `isValid`
|
|
2660
|
+
* TODO: [🍓] In `CsvFormatParser` implement partial `canBeValid`
|
|
2661
|
+
* TODO: [🍓] In `CsvFormatParser` implement `heal
|
|
2662
|
+
* TODO: [🍓] In `CsvFormatParser` implement `subvalueParsers`
|
|
2663
2663
|
* TODO: [🏢] Allow to expect something inside CSV objects and other formats
|
|
2664
2664
|
*/
|
|
2665
2665
|
|
|
@@ -2668,7 +2668,7 @@
|
|
|
2668
2668
|
*
|
|
2669
2669
|
* @private still in development [🏢]
|
|
2670
2670
|
*/
|
|
2671
|
-
const
|
|
2671
|
+
const JsonFormatParser = {
|
|
2672
2672
|
formatName: 'JSON',
|
|
2673
2673
|
mimeType: 'application/json',
|
|
2674
2674
|
isValid(value, settings, schema) {
|
|
@@ -2680,28 +2680,28 @@
|
|
|
2680
2680
|
heal(value, settings, schema) {
|
|
2681
2681
|
throw new Error('Not implemented');
|
|
2682
2682
|
},
|
|
2683
|
-
|
|
2683
|
+
subvalueParsers: [],
|
|
2684
2684
|
};
|
|
2685
2685
|
/**
|
|
2686
2686
|
* TODO: [🧠] Maybe propper instance of object
|
|
2687
2687
|
* TODO: [0] Make string_serialized_json
|
|
2688
2688
|
* TODO: [1] Make type for JSON Settings and Schema
|
|
2689
2689
|
* TODO: [🧠] What to use for validating JSONs - JSON Schema, ZoD, typescript types/interfaces,...?
|
|
2690
|
-
* TODO: [🍓] In `
|
|
2691
|
-
* TODO: [🍓] In `
|
|
2692
|
-
* TODO: [🍓] In `
|
|
2693
|
-
* TODO: [🍓] In `
|
|
2690
|
+
* TODO: [🍓] In `JsonFormatParser` implement simple `isValid`
|
|
2691
|
+
* TODO: [🍓] In `JsonFormatParser` implement partial `canBeValid`
|
|
2692
|
+
* TODO: [🍓] In `JsonFormatParser` implement `heal
|
|
2693
|
+
* TODO: [🍓] In `JsonFormatParser` implement `subvalueParsers`
|
|
2694
2694
|
* TODO: [🏢] Allow to expect something inside JSON objects and other formats
|
|
2695
2695
|
*/
|
|
2696
2696
|
|
|
2697
2697
|
/**
|
|
2698
2698
|
* Definition for any text - this will be always valid
|
|
2699
2699
|
*
|
|
2700
|
-
* Note: This is not useful for validation, but for splitting and mapping with `
|
|
2700
|
+
* Note: This is not useful for validation, but for splitting and mapping with `subvalueParsers`
|
|
2701
2701
|
*
|
|
2702
2702
|
* @public exported from `@promptbook/core`
|
|
2703
2703
|
*/
|
|
2704
|
-
const
|
|
2704
|
+
const TextFormatParser = {
|
|
2705
2705
|
formatName: 'TEXT',
|
|
2706
2706
|
isValid(value) {
|
|
2707
2707
|
return typeof value === 'string';
|
|
@@ -2710,9 +2710,9 @@
|
|
|
2710
2710
|
return typeof partialValue === 'string';
|
|
2711
2711
|
},
|
|
2712
2712
|
heal() {
|
|
2713
|
-
throw new UnexpectedError('It does not make sense to call `
|
|
2713
|
+
throw new UnexpectedError('It does not make sense to call `TextFormatParser.heal`');
|
|
2714
2714
|
},
|
|
2715
|
-
|
|
2715
|
+
subvalueParsers: [
|
|
2716
2716
|
{
|
|
2717
2717
|
subvalueName: 'LINE',
|
|
2718
2718
|
async mapValues(value, outputParameterName, settings, mapCallback) {
|
|
@@ -2732,10 +2732,10 @@
|
|
|
2732
2732
|
/**
|
|
2733
2733
|
* TODO: [1] Make type for XML Text and Schema
|
|
2734
2734
|
* TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages available as subvalues
|
|
2735
|
-
* TODO: [🍓] In `
|
|
2736
|
-
* TODO: [🍓] In `
|
|
2737
|
-
* TODO: [🍓] In `
|
|
2738
|
-
* TODO: [🍓] In `
|
|
2735
|
+
* TODO: [🍓] In `TextFormatParser` implement simple `isValid`
|
|
2736
|
+
* TODO: [🍓] In `TextFormatParser` implement partial `canBeValid`
|
|
2737
|
+
* TODO: [🍓] In `TextFormatParser` implement `heal
|
|
2738
|
+
* TODO: [🍓] In `TextFormatParser` implement `subvalueParsers`
|
|
2739
2739
|
* TODO: [🏢] Allow to expect something inside each item of list and other formats
|
|
2740
2740
|
*/
|
|
2741
2741
|
|
|
@@ -2768,7 +2768,7 @@
|
|
|
2768
2768
|
*
|
|
2769
2769
|
* @private still in development [🏢]
|
|
2770
2770
|
*/
|
|
2771
|
-
const
|
|
2771
|
+
const XmlFormatParser = {
|
|
2772
2772
|
formatName: 'XML',
|
|
2773
2773
|
mimeType: 'application/xml',
|
|
2774
2774
|
isValid(value, settings, schema) {
|
|
@@ -2780,17 +2780,17 @@
|
|
|
2780
2780
|
heal(value, settings, schema) {
|
|
2781
2781
|
throw new Error('Not implemented');
|
|
2782
2782
|
},
|
|
2783
|
-
|
|
2783
|
+
subvalueParsers: [],
|
|
2784
2784
|
};
|
|
2785
2785
|
/**
|
|
2786
2786
|
* TODO: [🧠] Maybe propper instance of object
|
|
2787
2787
|
* TODO: [0] Make string_serialized_xml
|
|
2788
2788
|
* TODO: [1] Make type for XML Settings and Schema
|
|
2789
2789
|
* TODO: [🧠] What to use for validating XMLs - XSD,...
|
|
2790
|
-
* TODO: [🍓] In `
|
|
2791
|
-
* TODO: [🍓] In `
|
|
2792
|
-
* TODO: [🍓] In `
|
|
2793
|
-
* TODO: [🍓] In `
|
|
2790
|
+
* TODO: [🍓] In `XmlFormatParser` implement simple `isValid`
|
|
2791
|
+
* TODO: [🍓] In `XmlFormatParser` implement partial `canBeValid`
|
|
2792
|
+
* TODO: [🍓] In `XmlFormatParser` implement `heal
|
|
2793
|
+
* TODO: [🍓] In `XmlFormatParser` implement `subvalueParsers`
|
|
2794
2794
|
* TODO: [🏢] Allow to expect something inside XML and other formats
|
|
2795
2795
|
*/
|
|
2796
2796
|
|
|
@@ -2799,12 +2799,7 @@
|
|
|
2799
2799
|
*
|
|
2800
2800
|
* @private internal index of `...` <- TODO [🏢]
|
|
2801
2801
|
*/
|
|
2802
|
-
const FORMAT_DEFINITIONS = [
|
|
2803
|
-
JsonFormatDefinition,
|
|
2804
|
-
XmlFormatDefinition,
|
|
2805
|
-
TextFormatDefinition,
|
|
2806
|
-
CsvFormatDefinition,
|
|
2807
|
-
];
|
|
2802
|
+
const FORMAT_DEFINITIONS = [JsonFormatParser, XmlFormatParser, TextFormatParser, CsvFormatParser];
|
|
2808
2803
|
/**
|
|
2809
2804
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
2810
2805
|
*/
|
|
@@ -3166,7 +3161,7 @@
|
|
|
3166
3161
|
}
|
|
3167
3162
|
/**
|
|
3168
3163
|
* TODO: Add some auto-healing logic + extract YAML, JSON5, TOML, etc.
|
|
3169
|
-
* TODO: [🏢] Make this logic part of `
|
|
3164
|
+
* TODO: [🏢] Make this logic part of `JsonFormatParser` or `isValidJsonString`
|
|
3170
3165
|
*/
|
|
3171
3166
|
|
|
3172
3167
|
/**
|
|
@@ -3665,7 +3660,7 @@
|
|
|
3665
3660
|
PAGES: countPages,
|
|
3666
3661
|
};
|
|
3667
3662
|
/**
|
|
3668
|
-
* TODO: [🧠][🤠] This should be probbably as part of `
|
|
3663
|
+
* TODO: [🧠][🤠] This should be probbably as part of `TextFormatParser`
|
|
3669
3664
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
3670
3665
|
*/
|
|
3671
3666
|
|
|
@@ -3693,7 +3688,7 @@
|
|
|
3693
3688
|
}
|
|
3694
3689
|
/**
|
|
3695
3690
|
* TODO: [💝] Unite object for expecting amount and format
|
|
3696
|
-
* TODO: [🧠][🤠] This should be part of `
|
|
3691
|
+
* TODO: [🧠][🤠] This should be part of `TextFormatParser`
|
|
3697
3692
|
* Note: [💝] and [🤠] are interconnected together
|
|
3698
3693
|
*/
|
|
3699
3694
|
|
|
@@ -3921,7 +3916,7 @@
|
|
|
3921
3916
|
if (task.format) {
|
|
3922
3917
|
if (task.format === 'JSON') {
|
|
3923
3918
|
if (!isValidJsonString($ongoingTaskResult.$resultString || '')) {
|
|
3924
|
-
// TODO: [🏢] Do more universally via `
|
|
3919
|
+
// TODO: [🏢] Do more universally via `FormatParser`
|
|
3925
3920
|
try {
|
|
3926
3921
|
$ongoingTaskResult.$resultString = extractJsonBlock($ongoingTaskResult.$resultString || '');
|
|
3927
3922
|
}
|
|
@@ -4059,16 +4054,16 @@
|
|
|
4059
4054
|
${block(pipelineIdentification)}
|
|
4060
4055
|
`));
|
|
4061
4056
|
}
|
|
4062
|
-
const
|
|
4063
|
-
if (
|
|
4057
|
+
const subvalueParser = formatDefinition.subvalueParsers.find((subvalueParser) => [subvalueParser.subvalueName, ...(subvalueParser.aliases || [])].includes(task.foreach.subformatName));
|
|
4058
|
+
if (subvalueParser === undefined) {
|
|
4064
4059
|
throw new UnexpectedError(
|
|
4065
4060
|
// <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError)
|
|
4066
4061
|
spaceTrim__default["default"]((block) => `
|
|
4067
4062
|
Unsupported subformat name "${task.foreach.subformatName}" for format "${task.foreach.formatName}"
|
|
4068
4063
|
|
|
4069
4064
|
Available subformat names for format "${formatDefinition.formatName}":
|
|
4070
|
-
${block(formatDefinition.
|
|
4071
|
-
.map((
|
|
4065
|
+
${block(formatDefinition.subvalueParsers
|
|
4066
|
+
.map((subvalueParser) => subvalueParser.subvalueName)
|
|
4072
4067
|
.map((subvalueName) => `- ${subvalueName}`)
|
|
4073
4068
|
.join('\n'))}
|
|
4074
4069
|
|
|
@@ -4082,7 +4077,7 @@
|
|
|
4082
4077
|
formatSettings = csvSettings;
|
|
4083
4078
|
// <- TODO: [🤹♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
|
|
4084
4079
|
}
|
|
4085
|
-
const resultString = await
|
|
4080
|
+
const resultString = await subvalueParser.mapValues(parameterValue, task.foreach.outputSubparameterName, formatSettings, async (subparameters, index) => {
|
|
4086
4081
|
let mappedParameters;
|
|
4087
4082
|
// TODO: [🤹♂️][🪂] Limit to N concurrent executions
|
|
4088
4083
|
// TODO: When done [🐚] Report progress also for each subvalue here
|
|
@@ -4144,6 +4139,27 @@
|
|
|
4144
4139
|
return RESERVED_PARAMETER_MISSING_VALUE /* <- TODO: [♨] Implement */;
|
|
4145
4140
|
}
|
|
4146
4141
|
|
|
4142
|
+
/**
|
|
4143
|
+
* Computes the cosine similarity between two embedding vectors
|
|
4144
|
+
*
|
|
4145
|
+
* Note: This is helping function for RAG (retrieval-augmented generation)
|
|
4146
|
+
*
|
|
4147
|
+
* @param embeddingVector1
|
|
4148
|
+
* @param embeddingVector2
|
|
4149
|
+
* @returns Cosine similarity between the two vectors
|
|
4150
|
+
*
|
|
4151
|
+
* @public exported from `@promptbook/core`
|
|
4152
|
+
*/
|
|
4153
|
+
function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
|
|
4154
|
+
if (embeddingVector1.length !== embeddingVector2.length) {
|
|
4155
|
+
throw new TypeError('Embedding vectors must have the same length');
|
|
4156
|
+
}
|
|
4157
|
+
const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
|
|
4158
|
+
const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
|
|
4159
|
+
const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
|
|
4160
|
+
return 1 - dotProduct / (magnitude1 * magnitude2);
|
|
4161
|
+
}
|
|
4162
|
+
|
|
4147
4163
|
/**
|
|
4148
4164
|
* @@@
|
|
4149
4165
|
*
|
|
@@ -4170,7 +4186,7 @@
|
|
|
4170
4186
|
},
|
|
4171
4187
|
content: task.content,
|
|
4172
4188
|
parameters: {
|
|
4173
|
-
/*
|
|
4189
|
+
/* !!!! */
|
|
4174
4190
|
},
|
|
4175
4191
|
};
|
|
4176
4192
|
const taskEmbeddingResult = await llmTools.callEmbeddingModel(taskEmbeddingPrompt);
|
|
@@ -4205,16 +4221,6 @@
|
|
|
4205
4221
|
return knowledgePiecesLimited.map(({ content }) => `- ${content}`).join('\n');
|
|
4206
4222
|
// <- TODO: [🧠] Some smart aggregation of knowledge pieces, single-line vs multi-line vs mixed
|
|
4207
4223
|
}
|
|
4208
|
-
// TODO: !!!!!! Annotate + to new file
|
|
4209
|
-
function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
|
|
4210
|
-
if (embeddingVector1.length !== embeddingVector2.length) {
|
|
4211
|
-
throw new TypeError('Embedding vectors must have the same length');
|
|
4212
|
-
}
|
|
4213
|
-
const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
|
|
4214
|
-
const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
|
|
4215
|
-
const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
|
|
4216
|
-
return 1 - dotProduct / (magnitude1 * magnitude2);
|
|
4217
|
-
}
|
|
4218
4224
|
/**
|
|
4219
4225
|
* TODO: !!!! Verify if this is working
|
|
4220
4226
|
* TODO: [♨] Implement Better - use keyword search
|
|
@@ -4717,6 +4723,22 @@
|
|
|
4717
4723
|
cacheDirname,
|
|
4718
4724
|
intermediateFilesStrategy,
|
|
4719
4725
|
isAutoInstalled,
|
|
4726
|
+
}).catch((error) => {
|
|
4727
|
+
assertsError(error);
|
|
4728
|
+
return exportJson({
|
|
4729
|
+
name: 'pipelineExecutorResult',
|
|
4730
|
+
message: `Unuccessful PipelineExecutorResult, last catch`,
|
|
4731
|
+
order: [],
|
|
4732
|
+
value: {
|
|
4733
|
+
isSuccessful: false,
|
|
4734
|
+
errors: [serializeError(error)],
|
|
4735
|
+
warnings: [],
|
|
4736
|
+
usage: UNCERTAIN_USAGE,
|
|
4737
|
+
executionReport: null,
|
|
4738
|
+
outputParameters: {},
|
|
4739
|
+
preparedPipeline,
|
|
4740
|
+
},
|
|
4741
|
+
});
|
|
4720
4742
|
});
|
|
4721
4743
|
};
|
|
4722
4744
|
const pipelineExecutor = (inputParameters) => createTask({
|
|
@@ -6701,14 +6723,14 @@
|
|
|
6701
6723
|
`));
|
|
6702
6724
|
// <- TODO: [🏢] List all supported format names
|
|
6703
6725
|
}
|
|
6704
|
-
const
|
|
6705
|
-
if (
|
|
6726
|
+
const subvalueParser = formatDefinition.subvalueParsers.find((subvalueParser) => [subvalueParser.subvalueName, ...(subvalueParser.aliases || [])].includes(subformatName));
|
|
6727
|
+
if (subvalueParser === undefined) {
|
|
6706
6728
|
throw new ParseError(spaceTrim__default["default"]((block) => `
|
|
6707
6729
|
Unsupported subformat name "${subformatName}" for format "${formatName}"
|
|
6708
6730
|
|
|
6709
6731
|
Available subformat names for format "${formatDefinition.formatName}":
|
|
6710
|
-
${block(formatDefinition.
|
|
6711
|
-
.map((
|
|
6732
|
+
${block(formatDefinition.subvalueParsers
|
|
6733
|
+
.map((subvalueParser) => subvalueParser.subvalueName)
|
|
6712
6734
|
.map((subvalueName) => `- ${subvalueName}`)
|
|
6713
6735
|
.join('\n'))}
|
|
6714
6736
|
`));
|