bare-script 2.1.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/lib/data.js +537 -0
- package/lib/library.js +97 -0
- package/lib/model.js +3 -0
- package/lib/parser.js +11 -5
- package/lib/runtime.js +5 -2
- package/lib/runtimeAsync.js +10 -4
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -33,7 +33,7 @@ import {parseScript} from 'bare-script/lib/parser.js';
|
|
|
33
33
|
// Parse the script
|
|
34
34
|
const script = parseScript(`\
|
|
35
35
|
# Double a number
|
|
36
|
-
function double(n)
|
|
36
|
+
function double(n):
|
|
37
37
|
return n * 2
|
|
38
38
|
endfunction
|
|
39
39
|
|
|
@@ -58,9 +58,9 @@ This outputs:
|
|
|
58
58
|
includes a set of built-in functions for mathematical operations, object manipulation, array
|
|
59
59
|
manipulation, regular expressions, HTTP fetch and more. The following example demonstrates the use
|
|
60
60
|
of the
|
|
61
|
-
[systemFetch](https://craigahobbs.github.io/bare-script/library/#var.
|
|
62
|
-
[objectGet](https://craigahobbs.github.io/bare-script/library/#var.
|
|
63
|
-
[arrayLength](https://craigahobbs.github.io/bare-script/library/#var.
|
|
61
|
+
[systemFetch](https://craigahobbs.github.io/bare-script/library/#var.vGroup='System'&systemfetch),
|
|
62
|
+
[objectGet](https://craigahobbs.github.io/bare-script/library/#var.vGroup='Object'&objectget), and
|
|
63
|
+
[arrayLength](https://craigahobbs.github.io/bare-script/library/#var.vGroup='Array'&arraylength)
|
|
64
64
|
functions.
|
|
65
65
|
|
|
66
66
|
~~~ javascript
|
|
@@ -134,7 +134,7 @@ bare script.bare
|
|
|
134
134
|
~~~
|
|
135
135
|
|
|
136
136
|
**Note:** In the BareScript CLI, import statements and the
|
|
137
|
-
[systemFetch](https://craigahobbs.github.io/bare-script/library/#var.
|
|
137
|
+
[systemFetch](https://craigahobbs.github.io/bare-script/library/#var.vGroup='System'&systemfetch)
|
|
138
138
|
function read non-URL paths from the local file system.
|
|
139
139
|
|
|
140
140
|
|
package/lib/data.js
ADDED
|
@@ -0,0 +1,537 @@
|
|
|
1
|
+
// Licensed under the MIT License
|
|
2
|
+
// https://github.com/craigahobbs/bare-script/blob/main/LICENSE
|
|
3
|
+
|
|
4
|
+
/** @module lib/data */
|
|
5
|
+
|
|
6
|
+
import {evaluateExpression} from './runtime.js';
|
|
7
|
+
import {jsonStringifySortKeys} from 'schema-markdown/lib/encode.js';
|
|
8
|
+
import {parseExpression} from './parser.js';
|
|
9
|
+
import {parseSchemaMarkdown} from 'schema-markdown/lib/parser.js';
|
|
10
|
+
import {validateType} from 'schema-markdown/lib/schema.js';
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Parse and validate CSV text to a data array
|
|
15
|
+
*
|
|
16
|
+
* @param {string} text - The CSV text
|
|
17
|
+
* @returns {Object[]} The data array
|
|
18
|
+
*/
|
|
19
|
+
export function parseCSV(text) {
|
|
20
|
+
// Line-split the text
|
|
21
|
+
const lines = [];
|
|
22
|
+
if (typeof text === 'string') {
|
|
23
|
+
lines.push(...text.split(rCSVLineSplit));
|
|
24
|
+
} else {
|
|
25
|
+
for (const textPart of text) {
|
|
26
|
+
lines.push(...textPart.split(rCSVLineSplit));
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// Split lines into rows
|
|
31
|
+
const rows = lines.filter((line) => !line.match(rCSVBlankLine)).map((line) => {
|
|
32
|
+
const row = [];
|
|
33
|
+
let linePart = line;
|
|
34
|
+
while (linePart !== '') {
|
|
35
|
+
// Quoted field?
|
|
36
|
+
const mQuoted = linePart.match(rCSVQuotedField) ?? linePart.match(rCSVQuotedFieldEnd);
|
|
37
|
+
if (mQuoted !== null) {
|
|
38
|
+
row.push(mQuoted[1].replaceAll(rCSVQuoteEscape, '"'));
|
|
39
|
+
linePart = linePart.slice(mQuoted[0].length);
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// Non-quoted field
|
|
44
|
+
const ixComma = linePart.indexOf(',');
|
|
45
|
+
row.push(ixComma !== -1 ? linePart.slice(0, ixComma) : linePart);
|
|
46
|
+
linePart = (ixComma !== -1 ? linePart.slice(ixComma + 1) : '');
|
|
47
|
+
}
|
|
48
|
+
return row;
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
// Assemble the data rows
|
|
52
|
+
const result = [];
|
|
53
|
+
if (rows.length >= 2) {
|
|
54
|
+
const [fields] = rows;
|
|
55
|
+
for (let ixLine = 1; ixLine < rows.length; ixLine += 1) {
|
|
56
|
+
const row = rows[ixLine];
|
|
57
|
+
result.push(Object.fromEntries(fields.map(
|
|
58
|
+
(field, ixField) => [field, ixField < row.length ? row[ixField] : 'null']
|
|
59
|
+
)));
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
return result;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const rCSVLineSplit = /\r?\n/;
|
|
67
|
+
const rCSVBlankLine = /^\s*$/;
|
|
68
|
+
const rCSVQuotedField = /^"((?:""|[^"])*)",/;
|
|
69
|
+
const rCSVQuotedFieldEnd = /^"((?:""|[^"])*)"\s*$/;
|
|
70
|
+
const rCSVQuoteEscape = /""/g;
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Determine data field types and parse/validate field values
|
|
75
|
+
*
|
|
76
|
+
* @param {Object[]} data - The data array. Row objects are updated with parsed/validated values.
|
|
77
|
+
* @param {boolean} [csv=false] - If true, parse number and null strings
|
|
78
|
+
* @returns {Object} The map of field name to field type ("datetime", "number", "string")
|
|
79
|
+
* @throws Throws an error if data is invalid
|
|
80
|
+
*/
|
|
81
|
+
export function validateData(data, csv = false) {
|
|
82
|
+
// Determine field types
|
|
83
|
+
const types = {};
|
|
84
|
+
for (const row of data) {
|
|
85
|
+
for (const [field, value] of Object.entries(row)) {
|
|
86
|
+
if (!(field in types)) {
|
|
87
|
+
if (typeof value === 'number') {
|
|
88
|
+
types[field] = 'number';
|
|
89
|
+
} else if (value instanceof Date) {
|
|
90
|
+
types[field] = 'datetime';
|
|
91
|
+
} else if (typeof value === 'string' && (!csv || value !== 'null')) {
|
|
92
|
+
if (parseDatetime(value) !== null) {
|
|
93
|
+
types[field] = 'datetime';
|
|
94
|
+
} else if (csv && parseNumber(value) !== null) {
|
|
95
|
+
types[field] = 'number';
|
|
96
|
+
} else {
|
|
97
|
+
types[field] = 'string';
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// Validate field values
|
|
105
|
+
const throwFieldError = (field, fieldType, fieldValue) => {
|
|
106
|
+
throw new Error(`Invalid "${field}" field value ${JSON.stringify(fieldValue)}, expected type ${fieldType}`);
|
|
107
|
+
};
|
|
108
|
+
for (const row of data) {
|
|
109
|
+
for (const [field, value] of Object.entries(row)) {
|
|
110
|
+
const fieldType = types[field];
|
|
111
|
+
|
|
112
|
+
// Null string?
|
|
113
|
+
if (csv && value === 'null') {
|
|
114
|
+
row[field] = null;
|
|
115
|
+
|
|
116
|
+
// Number field
|
|
117
|
+
} else if (fieldType === 'number') {
|
|
118
|
+
if (csv && typeof value === 'string') {
|
|
119
|
+
const numberValue = parseNumber(value);
|
|
120
|
+
if (numberValue === null) {
|
|
121
|
+
throwFieldError(field, fieldType, value);
|
|
122
|
+
}
|
|
123
|
+
row[field] = numberValue;
|
|
124
|
+
} else if (value !== null && typeof value !== 'number') {
|
|
125
|
+
throwFieldError(field, fieldType, value);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Datetime field
|
|
129
|
+
} else if (fieldType === 'datetime') {
|
|
130
|
+
if (typeof value === 'string') {
|
|
131
|
+
const datetimeValue = parseDatetime(value);
|
|
132
|
+
if (datetimeValue === null) {
|
|
133
|
+
throwFieldError(field, fieldType, value);
|
|
134
|
+
}
|
|
135
|
+
row[field] = datetimeValue;
|
|
136
|
+
} else if (value !== null && !(value instanceof Date)) {
|
|
137
|
+
throwFieldError(field, fieldType, value);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// String field
|
|
141
|
+
} else {
|
|
142
|
+
if (value !== null && typeof value !== 'string') {
|
|
143
|
+
throwFieldError(field, fieldType, value);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
return types;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
function parseNumber(text) {
|
|
154
|
+
const value = Number.parseFloat(text);
|
|
155
|
+
if (Number.isNaN(value) || !Number.isFinite(value)) {
|
|
156
|
+
return null;
|
|
157
|
+
}
|
|
158
|
+
return value;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
export function parseDatetime(text) {
|
|
163
|
+
if (rDate.test(text)) {
|
|
164
|
+
const localDate = new Date(text);
|
|
165
|
+
return new Date(localDate.getUTCFullYear(), localDate.getUTCMonth(), localDate.getUTCDate());
|
|
166
|
+
} else if (rDatetime.test(text)) {
|
|
167
|
+
return new Date(text);
|
|
168
|
+
}
|
|
169
|
+
return null;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
const rDate = /^\d{4}-\d{2}-\d{2}$/;
|
|
173
|
+
const rDatetime = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?(?:Z|[+-]\d{2}:\d{2})$/;
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Join two data arrays
|
|
178
|
+
*
|
|
179
|
+
* @param {Object} leftData - The left data array
|
|
180
|
+
* @param {Object} rightData - The left data array
|
|
181
|
+
* @param {string} joinExpr - The join [expression]{@link https://craigahobbs.github.io/bare-script/language/#expressions}
|
|
182
|
+
* @param {?string} [rightExpr = null] - The right join [expression]{@link https://craigahobbs.github.io/bare-script/language/#expressions}
|
|
183
|
+
* @param {boolean} [isLeftJoin = false] - If true, perform a left join (always include left row)
|
|
184
|
+
* @param {?Object} [variables = null] - Additional variables for expression evaluation
|
|
185
|
+
* @param {?Object} [options = null] - The [script execution options]{@link module:lib/runtime~ExecuteScriptOptions}
|
|
186
|
+
* @returns {Object[]} The joined data array
|
|
187
|
+
*/
|
|
188
|
+
export function joinData(leftData, rightData, joinExpr, rightExpr = null, isLeftJoin = false, variables = null, options = null) {
|
|
189
|
+
// Compute the map of row field name to joined row field name
|
|
190
|
+
const leftNames = {};
|
|
191
|
+
for (const row of leftData) {
|
|
192
|
+
for (const fieldName of Object.keys(row)) {
|
|
193
|
+
if (!(fieldName in leftNames)) {
|
|
194
|
+
leftNames[fieldName] = fieldName;
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
const rightNames = {};
|
|
199
|
+
for (const row of rightData) {
|
|
200
|
+
for (const fieldName of Object.keys(row)) {
|
|
201
|
+
if (!(fieldName in rightNames)) {
|
|
202
|
+
if (!(fieldName in leftNames)) {
|
|
203
|
+
rightNames[fieldName] = fieldName;
|
|
204
|
+
} else {
|
|
205
|
+
let uniqueName = fieldName;
|
|
206
|
+
let ixUnique = 2;
|
|
207
|
+
do {
|
|
208
|
+
uniqueName = `${fieldName}${ixUnique}`;
|
|
209
|
+
ixUnique += 1;
|
|
210
|
+
} while (uniqueName in leftNames || uniqueName in rightNames);
|
|
211
|
+
rightNames[fieldName] = uniqueName;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// Create the evaluation options object
|
|
218
|
+
let evalOptions = options;
|
|
219
|
+
if (variables !== null) {
|
|
220
|
+
evalOptions = (options !== null ? {...options} : {});
|
|
221
|
+
if ('globals' in evalOptions) {
|
|
222
|
+
evalOptions.globals = {...evalOptions.globals, ...variables};
|
|
223
|
+
} else {
|
|
224
|
+
evalOptions.globals = variables;
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// Parse the left and right expressions
|
|
229
|
+
const leftExpression = parseExpression(joinExpr);
|
|
230
|
+
const rightExpression = (rightExpr !== null ? parseExpression(rightExpr) : leftExpression);
|
|
231
|
+
|
|
232
|
+
// Bucket the right rows by the right expression value
|
|
233
|
+
const rightCategoryRows = {};
|
|
234
|
+
for (const rightRow of rightData) {
|
|
235
|
+
const categoryKey = jsonStringifySortKeys(evaluateExpression(rightExpression, evalOptions, rightRow));
|
|
236
|
+
if (!(categoryKey in rightCategoryRows)) {
|
|
237
|
+
rightCategoryRows[categoryKey] = [];
|
|
238
|
+
}
|
|
239
|
+
rightCategoryRows[categoryKey].push(rightRow);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
// Join the left with the right
|
|
243
|
+
const data = [];
|
|
244
|
+
for (const leftRow of leftData) {
|
|
245
|
+
const categoryKey = jsonStringifySortKeys(evaluateExpression(leftExpression, evalOptions, leftRow));
|
|
246
|
+
if (categoryKey in rightCategoryRows) {
|
|
247
|
+
for (const rightRow of rightCategoryRows[categoryKey]) {
|
|
248
|
+
const joinRow = {...leftRow};
|
|
249
|
+
for (const [rightName, rightValue] of Object.entries(rightRow)) {
|
|
250
|
+
joinRow[rightNames[rightName]] = rightValue;
|
|
251
|
+
}
|
|
252
|
+
data.push(joinRow);
|
|
253
|
+
}
|
|
254
|
+
} else if (!isLeftJoin) {
|
|
255
|
+
data.push({...leftRow});
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
return data;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
/**
|
|
264
|
+
* Add a calculated field to each row of a data array
|
|
265
|
+
*
|
|
266
|
+
* @param {Object[]} data - The data array. Row objects are updated with the calculated field values.
|
|
267
|
+
* @param {string} fieldName - The calculated field name
|
|
268
|
+
* @param {string} expr - The calculated field expression
|
|
269
|
+
* @param {?Object} [variables = null] - Additional variables for expression evaluation
|
|
270
|
+
* @param {?Object} [options = null] - The [script execution options]{@link module:lib/runtime~ExecuteScriptOptions}
|
|
271
|
+
* @returns {Object[]} The updated data array
|
|
272
|
+
*/
|
|
273
|
+
export function addCalculatedField(data, fieldName, expr, variables = null, options = null) {
|
|
274
|
+
// Parse the calculation expression
|
|
275
|
+
const calcExpr = parseExpression(expr);
|
|
276
|
+
|
|
277
|
+
// Create the evaluation options object
|
|
278
|
+
let evalOptions = options;
|
|
279
|
+
if (variables !== null) {
|
|
280
|
+
evalOptions = (options !== null ? {...options} : {});
|
|
281
|
+
if ('globals' in evalOptions) {
|
|
282
|
+
evalOptions.globals = {...evalOptions.globals, ...variables};
|
|
283
|
+
} else {
|
|
284
|
+
evalOptions.globals = variables;
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
// Compute the calculated field for each row
|
|
289
|
+
for (const row of data) {
|
|
290
|
+
row[fieldName] = evaluateExpression(calcExpr, evalOptions, row);
|
|
291
|
+
}
|
|
292
|
+
return data;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* Filter data rows
|
|
298
|
+
*
|
|
299
|
+
* @param {Object[]} data - The data array
|
|
300
|
+
* @param {string} expr - The boolean filter [expression]{@link https://craigahobbs.github.io/bare-script/language/#expressions}
|
|
301
|
+
* @param {?Object} [variables = null] - Additional variables for expression evaluation
|
|
302
|
+
* @param {?Object} [options = null] - The [script execution options]{@link module:lib/runtime~ExecuteScriptOptions}
|
|
303
|
+
* @returns {Object[]} The filtered data array
|
|
304
|
+
*/
|
|
305
|
+
export function filterData(data, expr, variables = null, options = null) {
|
|
306
|
+
const result = [];
|
|
307
|
+
|
|
308
|
+
// Parse the filter expression
|
|
309
|
+
const filterExpr = parseExpression(expr);
|
|
310
|
+
|
|
311
|
+
// Create the evaluation options object
|
|
312
|
+
let evalOptions = options;
|
|
313
|
+
if (variables !== null) {
|
|
314
|
+
evalOptions = (options !== null ? {...options} : {});
|
|
315
|
+
if ('globals' in evalOptions) {
|
|
316
|
+
evalOptions.globals = {...evalOptions.globals, ...variables};
|
|
317
|
+
} else {
|
|
318
|
+
evalOptions.globals = variables;
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
// Filter the data
|
|
323
|
+
for (const row of data) {
|
|
324
|
+
if (evaluateExpression(filterExpr, evalOptions, row)) {
|
|
325
|
+
result.push(row);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
return result;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
// The aggregation model
|
|
334
|
+
export const aggregationTypes = parseSchemaMarkdown(`\
|
|
335
|
+
group "Aggregation"
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
# A data aggregation specification
|
|
339
|
+
struct Aggregation
|
|
340
|
+
|
|
341
|
+
# The aggregation category fields
|
|
342
|
+
optional string[len > 0] categories
|
|
343
|
+
|
|
344
|
+
# The aggregation measures
|
|
345
|
+
AggregationMeasure[len > 0] measures
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
# An aggregation measure specification
|
|
349
|
+
struct AggregationMeasure
|
|
350
|
+
|
|
351
|
+
# The aggregation measure field
|
|
352
|
+
string field
|
|
353
|
+
|
|
354
|
+
# The aggregation function
|
|
355
|
+
AggregationFunction function
|
|
356
|
+
|
|
357
|
+
# The aggregated-measure field name
|
|
358
|
+
optional string name
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
# An aggregation function
|
|
362
|
+
enum AggregationFunction
|
|
363
|
+
|
|
364
|
+
# The average of the measure's values
|
|
365
|
+
average
|
|
366
|
+
|
|
367
|
+
# The count of the measure's values
|
|
368
|
+
count
|
|
369
|
+
|
|
370
|
+
# The greatest of the measure's values
|
|
371
|
+
max
|
|
372
|
+
|
|
373
|
+
# The least of the measure's values
|
|
374
|
+
min
|
|
375
|
+
|
|
376
|
+
# The sum of the measure's values
|
|
377
|
+
sum
|
|
378
|
+
`);
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
/**
|
|
382
|
+
* Validate an aggregation model
|
|
383
|
+
*
|
|
384
|
+
* @param {Object} aggregation - The
|
|
385
|
+
* [aggregation model]{@link https://craigahobbs.github.io/bare-script/library/model.html#var.vName='Aggregation'}
|
|
386
|
+
* @returns {Object} The validated
|
|
387
|
+
* [aggregation model]{@link https://craigahobbs.github.io/bare-script/library/model.html#var.vName='Aggregation'}
|
|
388
|
+
* @throws [ValidationError]{@link https://craigahobbs.github.io/schema-markdown-js/module-lib_schema.ValidationError.html}
|
|
389
|
+
*/
|
|
390
|
+
export function validateAggregation(aggregation) {
|
|
391
|
+
return validateType(aggregationTypes, 'Aggregation', aggregation);
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
/**
|
|
396
|
+
* Aggregate data rows
|
|
397
|
+
*
|
|
398
|
+
* @param {Object[]} data - The data array
|
|
399
|
+
* @param {Object} aggregation - The
|
|
400
|
+
* [aggregation model]{@link https://craigahobbs.github.io/bare-script/library/model.html#var.vName='Aggregation'}
|
|
401
|
+
* @returns {Object[]} The aggregated data array
|
|
402
|
+
*/
|
|
403
|
+
export function aggregateData(data, aggregation) {
|
|
404
|
+
const categories = aggregation.categories ?? null;
|
|
405
|
+
|
|
406
|
+
// Create the aggregate rows
|
|
407
|
+
const categoryRows = {};
|
|
408
|
+
for (const row of data) {
|
|
409
|
+
// Compute the category values
|
|
410
|
+
const categoryValues = (categories !== null ? categories.map((categoryField) => row[categoryField]) : null);
|
|
411
|
+
|
|
412
|
+
// Get or create the aggregate row
|
|
413
|
+
let aggregateRow;
|
|
414
|
+
const rowKey = (categoryValues !== null ? jsonStringifySortKeys(categoryValues) : '');
|
|
415
|
+
if (rowKey in categoryRows) {
|
|
416
|
+
aggregateRow = categoryRows[rowKey];
|
|
417
|
+
} else {
|
|
418
|
+
aggregateRow = {};
|
|
419
|
+
categoryRows[rowKey] = aggregateRow;
|
|
420
|
+
if (categories !== null) {
|
|
421
|
+
for (let ixCategoryField = 0; ixCategoryField < categories.length; ixCategoryField++) {
|
|
422
|
+
aggregateRow[categories[ixCategoryField]] = categoryValues[ixCategoryField];
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
// Add to the aggregate measure values
|
|
428
|
+
for (const measure of aggregation.measures) {
|
|
429
|
+
const field = measure.name ?? measure.field;
|
|
430
|
+
const value = row[measure.field] ?? null;
|
|
431
|
+
if (!(field in aggregateRow)) {
|
|
432
|
+
aggregateRow[field] = [];
|
|
433
|
+
}
|
|
434
|
+
aggregateRow[field].push(value);
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
// Compute the measure values aggregate function value
|
|
439
|
+
const aggregateRows = Object.values(categoryRows);
|
|
440
|
+
for (const aggregateRow of aggregateRows) {
|
|
441
|
+
for (const measure of aggregation.measures) {
|
|
442
|
+
const field = measure.name ?? measure.field;
|
|
443
|
+
const func = measure.function;
|
|
444
|
+
const measureValues = aggregateRow[field];
|
|
445
|
+
if (func === 'count') {
|
|
446
|
+
aggregateRow[field] = measureValues.length;
|
|
447
|
+
} else if (func === 'max') {
|
|
448
|
+
aggregateRow[field] = measureValues.reduce((max, val) => (val > max ? val : max));
|
|
449
|
+
} else if (func === 'min') {
|
|
450
|
+
aggregateRow[field] = measureValues.reduce((min, val) => (val < min ? val : min));
|
|
451
|
+
} else if (func === 'sum') {
|
|
452
|
+
aggregateRow[field] = measureValues.reduce((sum, val) => sum + val, 0);
|
|
453
|
+
} else {
|
|
454
|
+
aggregateRow[field] = measureValues.reduce((sum, val) => sum + val, 0) / measureValues.length;
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
return aggregateRows;
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
/**
|
|
464
|
+
* Sort data rows
|
|
465
|
+
*
|
|
466
|
+
* @param {Object[]} data - The data array
|
|
467
|
+
* @param {Object[]} sorts - The sort field-name/descending-sort tuples
|
|
468
|
+
* @returns {Object[]} The sorted data array
|
|
469
|
+
*/
|
|
470
|
+
export function sortData(data, sorts) {
|
|
471
|
+
return data.sort((row1, row2) => sorts.reduce((result, sort) => {
|
|
472
|
+
if (result !== 0) {
|
|
473
|
+
return result;
|
|
474
|
+
}
|
|
475
|
+
const [field, desc = false] = sort;
|
|
476
|
+
const value1 = row1[field] ?? null;
|
|
477
|
+
const value2 = row2[field] ?? null;
|
|
478
|
+
const compare = compareValues(value1, value2);
|
|
479
|
+
return desc ? -compare : compare;
|
|
480
|
+
}, 0));
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
/**
|
|
485
|
+
* Top data rows
|
|
486
|
+
*
|
|
487
|
+
* @param {Object[]} data - The data array
|
|
488
|
+
* @param {number} count - The number of rows to keep
|
|
489
|
+
* @param {?string[]} [categoryFields = null] - The category fields
|
|
490
|
+
* @returns {Object[]} The top data array
|
|
491
|
+
*/
|
|
492
|
+
export function topData(data, count, categoryFields = null) {
|
|
493
|
+
// Bucket rows by category
|
|
494
|
+
const categoryRows = {};
|
|
495
|
+
const categoryOrder = [];
|
|
496
|
+
for (const row of data) {
|
|
497
|
+
const categoryKey = categoryFields === null ? ''
|
|
498
|
+
: jsonStringifySortKeys(categoryFields.map((field) => (field in row ? row[field] : null)));
|
|
499
|
+
if (!(categoryKey in categoryRows)) {
|
|
500
|
+
categoryRows[categoryKey] = [];
|
|
501
|
+
categoryOrder.push(categoryKey);
|
|
502
|
+
}
|
|
503
|
+
categoryRows[categoryKey].push(row);
|
|
504
|
+
}
|
|
505
|
+
// Take only the top rows
|
|
506
|
+
const dataTop = [];
|
|
507
|
+
const topCount = count;
|
|
508
|
+
for (const categoryKey of categoryOrder) {
|
|
509
|
+
const categoryKeyRows = categoryRows[categoryKey];
|
|
510
|
+
const categoryKeyLength = categoryKeyRows.length;
|
|
511
|
+
for (let ixRow = 0; ixRow < topCount && ixRow < categoryKeyLength; ixRow++) {
|
|
512
|
+
dataTop.push(categoryKeyRows[ixRow]);
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
return dataTop;
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
/**
|
|
520
|
+
* Compare two data values
|
|
521
|
+
*
|
|
522
|
+
* @param {*} value1 - The first value
|
|
523
|
+
* @param {*} value2 - The second value
|
|
524
|
+
* @returns {number} -1 if the first value is less, 1 if the first value is greater, and 0 if they are equal
|
|
525
|
+
*/
|
|
526
|
+
export function compareValues(value1, value2) {
|
|
527
|
+
if (value1 === null) {
|
|
528
|
+
return value2 === null ? 0 : -1;
|
|
529
|
+
} else if (value2 === null) {
|
|
530
|
+
return 1;
|
|
531
|
+
} else if (value1 instanceof Date) {
|
|
532
|
+
const time1 = value1.getTime();
|
|
533
|
+
const time2 = value2.getTime();
|
|
534
|
+
return time1 < time2 ? -1 : (time1 === time2 ? 0 : 1);
|
|
535
|
+
}
|
|
536
|
+
return value1 < value2 ? -1 : (value1 === value2 ? 0 : 1);
|
|
537
|
+
}
|
package/lib/library.js
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
// Licensed under the MIT License
|
|
2
2
|
// https://github.com/craigahobbs/bare-script/blob/main/LICENSE
|
|
3
3
|
|
|
4
|
+
import {
|
|
5
|
+
addCalculatedField, aggregateData, filterData, joinData, parseCSV, parseDatetime, sortData, topData, validateAggregation, validateData
|
|
6
|
+
} from './data.js';
|
|
4
7
|
import {validateType, validateTypeModel} from 'schema-markdown/lib/schema.js';
|
|
5
8
|
import {jsonStringifySortKeys} from 'schema-markdown/lib/encode.js';
|
|
6
9
|
import {parseSchemaMarkdown} from 'schema-markdown/lib/parser.js';
|
|
@@ -149,6 +152,93 @@ export const scriptFunctions = {
|
|
|
149
152
|
),
|
|
150
153
|
|
|
151
154
|
|
|
155
|
+
//
|
|
156
|
+
// Data functions
|
|
157
|
+
//
|
|
158
|
+
|
|
159
|
+
// $function: dataAggregate
|
|
160
|
+
// $group: Data
|
|
161
|
+
// $doc: Aggregate a data array
|
|
162
|
+
// $arg data: The data array
|
|
163
|
+
// $arg aggregation: The [aggregation model](model.html#var.vName='Aggregation')
|
|
164
|
+
// $return: The aggregated data array
|
|
165
|
+
'dataAggregate': ([data, aggregation]) => aggregateData(data, validateAggregation(aggregation)),
|
|
166
|
+
|
|
167
|
+
// $function: dataCalculatedField
|
|
168
|
+
// $group: Data
|
|
169
|
+
// $doc: Add a calculated field to a data array
|
|
170
|
+
// $arg data: The data array
|
|
171
|
+
// $arg fieldName: The calculated field name
|
|
172
|
+
// $arg expr: The calculated field expression
|
|
173
|
+
// $arg variables: Optional (default is null). A variables object the expression evaluation.
|
|
174
|
+
// $return: The updated data array
|
|
175
|
+
'dataCalculatedField': ([data, fieldName, expr, variables = null], options) => (
|
|
176
|
+
addCalculatedField(data, fieldName, expr, variables, options)
|
|
177
|
+
),
|
|
178
|
+
|
|
179
|
+
// $function: dataFilter
|
|
180
|
+
// $group: Data
|
|
181
|
+
// $doc: Filter a data array
|
|
182
|
+
// $arg data: The data array
|
|
183
|
+
// $arg expr: The filter expression
|
|
184
|
+
// $arg variables: Optional (default is null). A variables object the expression evaluation.
|
|
185
|
+
// $return: The filtered data array
|
|
186
|
+
'dataFilter': ([data, expr, variables = null], options) => filterData(data, expr, variables, options),
|
|
187
|
+
|
|
188
|
+
// $function: dataJoin
|
|
189
|
+
// $group: Data
|
|
190
|
+
// $doc: Join two data arrays
|
|
191
|
+
// $arg leftData: The left data array
|
|
192
|
+
// $arg rightData: The right data array
|
|
193
|
+
// $arg joinExpr: The [join expression](https://craigahobbs.github.io/bare-script/language/#expressions)
|
|
194
|
+
// $arg rightExpr: Optional (default is null).
|
|
195
|
+
// $arg rightExpr: The right [join expression](https://craigahobbs.github.io/bare-script/language/#expressions)
|
|
196
|
+
// $arg isLeftJoin: Optional (default is false). If true, perform a left join (always include left row).
|
|
197
|
+
// $arg variables: Optional (default is null). A variables object for join expression evaluation.
|
|
198
|
+
// $return: The joined data array
|
|
199
|
+
'dataJoin': ([leftData, rightData, joinExpr, rightExpr = null, isLeftJoin = false, variables = null], options) => (
|
|
200
|
+
joinData(leftData, rightData, joinExpr, rightExpr, isLeftJoin, variables, options)
|
|
201
|
+
),
|
|
202
|
+
|
|
203
|
+
// $function: dataParseCSV
|
|
204
|
+
// $group: Data
|
|
205
|
+
// $doc: Parse CSV text to a data array
|
|
206
|
+
// $arg text: The CSV text
|
|
207
|
+
// $return: The data array
|
|
208
|
+
'dataParseCSV': (text) => {
|
|
209
|
+
const data = parseCSV(text);
|
|
210
|
+
validateData(data, true);
|
|
211
|
+
return data;
|
|
212
|
+
},
|
|
213
|
+
|
|
214
|
+
// $function: dataSort
|
|
215
|
+
// $group: Data
|
|
216
|
+
// $doc: Sort a data array
|
|
217
|
+
// $arg data: The data array
|
|
218
|
+
// $arg sorts: The sort field-name/descending-sort tuples
|
|
219
|
+
// $return: The sorted data array
|
|
220
|
+
'dataSort': ([data, sorts]) => sortData(data, sorts),
|
|
221
|
+
|
|
222
|
+
// $function: dataTop
|
|
223
|
+
// $group: Data
|
|
224
|
+
// $doc: Keep the top rows for each category
|
|
225
|
+
// $arg data: The data array
|
|
226
|
+
// $arg count: The number of rows to keep
|
|
227
|
+
// $arg categoryFields: Optional (default is null). The category fields.
|
|
228
|
+
// $return: The top data array
|
|
229
|
+
'dataTop': ([data, count, categoryFields = null]) => topData(data, count, categoryFields),
|
|
230
|
+
|
|
231
|
+
// $function: dataValidate
|
|
232
|
+
// $group: Data
|
|
233
|
+
// $doc: Validate a data array
|
|
234
|
+
// $arg data: The data array
|
|
235
|
+
// $return: The validated data array
|
|
236
|
+
'dataValidate': ([data]) => {
|
|
237
|
+
validateData(data);
|
|
238
|
+
return data;
|
|
239
|
+
},
|
|
240
|
+
|
|
241
|
+
|
|
152
242
|
//
|
|
153
243
|
// Datetime functions
|
|
154
244
|
//
|
|
@@ -184,6 +274,13 @@ export const scriptFunctions = {
|
|
|
184
274
|
return result;
|
|
185
275
|
},
|
|
186
276
|
|
|
277
|
+
// $function: datetimeISOParse
|
|
278
|
+
// $group: Datetime
|
|
279
|
+
// $doc: Parse an ISO date/time string
|
|
280
|
+
// $arg str: The ISO date/time string
|
|
281
|
+
// $return: The datetime, or null if parsing fails
|
|
282
|
+
'datetimeISOParse': ([str]) => parseDatetime(str),
|
|
283
|
+
|
|
187
284
|
// $function: datetimeMinute
|
|
188
285
|
// $group: Datetime
|
|
189
286
|
// $doc: Get the number of minutes of a datetime
|
package/lib/model.js
CHANGED
|
@@ -79,6 +79,9 @@ struct FunctionStatement
|
|
|
79
79
|
# The function's argument names
|
|
80
80
|
optional string[len > 0] args
|
|
81
81
|
|
|
82
|
+
# If true, the function's last argument is the array of all remaining arguments
|
|
83
|
+
optional bool lastArgArray
|
|
84
|
+
|
|
82
85
|
# The function's statements
|
|
83
86
|
ScriptStatement[] statements
|
|
84
87
|
|
package/lib/parser.js
CHANGED
|
@@ -9,8 +9,10 @@ const rScriptLineSplit = /\r?\n/;
|
|
|
9
9
|
const rScriptContinuation = /\\\s*$/;
|
|
10
10
|
const rScriptComment = /^\s*(?:#.*)?$/;
|
|
11
11
|
const rScriptAssignment = /^\s*(?<name>[A-Za-z_]\w*)\s*=\s*(?<expr>.+)$/;
|
|
12
|
-
const rScriptFunctionBegin =
|
|
13
|
-
|
|
12
|
+
const rScriptFunctionBegin = new RegExp(
|
|
13
|
+
'^(?<async>\\s*async)?\\s*function\\s+(?<name>[A-Za-z_]\\w*)\\s*\\(' +
|
|
14
|
+
'\\s*(?<args>[A-Za-z_]\\w*(?:\\s*,\\s*[A-Za-z_]\\w*)*)?(?<lastArgArray>\\s*\\.\\.\\.)?\\s*\\)\\s*:\\s*$'
|
|
15
|
+
);
|
|
14
16
|
const rScriptFunctionArgSplit = /\s*,\s*/;
|
|
15
17
|
const rScriptFunctionEnd = /^\s*endfunction\s*$/;
|
|
16
18
|
const rScriptLabel = /^\s*(?<name>[A-Za-z_]\w*)\s*:\s*$/;
|
|
@@ -119,14 +121,18 @@ export function parseScript(scriptText, startLineNumber = 1) {
|
|
|
119
121
|
functionDef = {
|
|
120
122
|
'function': {
|
|
121
123
|
'name': matchFunctionBegin.groups.name,
|
|
122
|
-
'args': typeof matchFunctionBegin.groups.args !== 'undefined'
|
|
123
|
-
? matchFunctionBegin.groups.args.split(rScriptFunctionArgSplit) : [],
|
|
124
124
|
'statements': []
|
|
125
125
|
}
|
|
126
126
|
};
|
|
127
|
-
if (matchFunctionBegin.groups.
|
|
127
|
+
if (typeof matchFunctionBegin.groups.args !== 'undefined') {
|
|
128
|
+
functionDef.function.args = matchFunctionBegin.groups.args.split(rScriptFunctionArgSplit);
|
|
129
|
+
}
|
|
130
|
+
if (typeof matchFunctionBegin.groups.async !== 'undefined') {
|
|
128
131
|
functionDef.function.async = true;
|
|
129
132
|
}
|
|
133
|
+
if (typeof matchFunctionBegin.groups.lastArgArray !== 'undefined') {
|
|
134
|
+
functionDef.function.lastArgArray = true;
|
|
135
|
+
}
|
|
130
136
|
statements.push(functionDef);
|
|
131
137
|
continue;
|
|
132
138
|
}
|
package/lib/runtime.js
CHANGED
|
@@ -134,8 +134,11 @@ export function executeScriptHelper(statements, options, locals) {
|
|
|
134
134
|
const funcLocals = {};
|
|
135
135
|
if ('args' in statement.function) {
|
|
136
136
|
const argsLength = args.length;
|
|
137
|
-
|
|
138
|
-
|
|
137
|
+
const funcArgsLength = statement.function.args.length;
|
|
138
|
+
const ixArgLast = (statement.function.lastArgArray ?? null) && (funcArgsLength - 1);
|
|
139
|
+
for (let ixArg = 0; ixArg < funcArgsLength; ixArg++) {
|
|
140
|
+
const argName = statement.function.args[ixArg];
|
|
141
|
+
funcLocals[argName] = (ixArg < argsLength ? (ixArg === ixArgLast ? args.slice(ixArg) : args[ixArg]) : null);
|
|
139
142
|
}
|
|
140
143
|
}
|
|
141
144
|
return executeScriptHelper(statement.function.statements, fnOptions, funcLocals);
|
package/lib/runtimeAsync.js
CHANGED
|
@@ -104,8 +104,11 @@ async function executeScriptHelperAsync(statements, options, locals) {
|
|
|
104
104
|
const funcLocals = {};
|
|
105
105
|
if ('args' in statement.function) {
|
|
106
106
|
const argsLength = args.length;
|
|
107
|
-
|
|
108
|
-
|
|
107
|
+
const funcArgsLength = statement.function.args.length;
|
|
108
|
+
const ixArgLast = (statement.function.lastArgArray ?? null) && (funcArgsLength - 1);
|
|
109
|
+
for (let ixArg = 0; ixArg < funcArgsLength; ixArg++) {
|
|
110
|
+
const argName = statement.function.args[ixArg];
|
|
111
|
+
funcLocals[argName] = (ixArg < argsLength ? (ixArg === ixArgLast ? args.slice(ixArg) : args[ixArg]) : null);
|
|
109
112
|
}
|
|
110
113
|
}
|
|
111
114
|
return executeScriptHelperAsync(statement.function.statements, fnOptions, funcLocals);
|
|
@@ -115,8 +118,11 @@ async function executeScriptHelperAsync(statements, options, locals) {
|
|
|
115
118
|
const funcLocals = {};
|
|
116
119
|
if ('args' in statement.function) {
|
|
117
120
|
const argsLength = args.length;
|
|
118
|
-
|
|
119
|
-
|
|
121
|
+
const funcArgsLength = statement.function.args.length;
|
|
122
|
+
const ixArgLast = (statement.function.lastArgArray ?? null) && (funcArgsLength - 1);
|
|
123
|
+
for (let ixArg = 0; ixArg < funcArgsLength; ixArg++) {
|
|
124
|
+
const argName = statement.function.args[ixArg];
|
|
125
|
+
funcLocals[argName] = (ixArg < argsLength ? (ixArg === ixArgLast ? args.slice(ixArg) : args[ixArg]) : null);
|
|
120
126
|
}
|
|
121
127
|
}
|
|
122
128
|
return executeScriptHelper(statement.function.statements, fnOptions, funcLocals);
|