dbgate-datalib 6.5.4 → 6.5.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. package/lib/PerspectiveDataLoader.js +6 -0
  2. package/lib/chartDefinitions.d.ts +9 -1
  3. package/lib/chartDefinitions.js +3 -1
  4. package/lib/chartProcessor.d.ts +8 -0
  5. package/lib/chartProcessor.js +155 -61
  6. package/lib/chartScoring.js +10 -0
  7. package/lib/chartTools.d.ts +7 -1
  8. package/lib/chartTools.js +125 -23
  9. package/lib/tests/chartProcessor.test.js +23 -11
  10. package/package.json +5 -5
package/lib/PerspectiveDataLoader.js CHANGED
@@ -99,6 +99,7 @@ class PerspectiveDataLoader {
99
99
  conid: props.databaseConfig.conid,
100
100
  database: props.databaseConfig.database,
101
101
  select,
102
+ auditLogSessionGroup: 'perspective',
102
103
  });
103
104
  if (response.errorMessage)
104
105
  return response;
@@ -140,6 +141,7 @@ class PerspectiveDataLoader {
140
141
  pureName,
141
142
  aggregate,
142
143
  },
144
+ auditLogSessionGroup: 'perspective',
143
145
  });
144
146
  if (response.errorMessage)
145
147
  return response;
@@ -197,6 +199,7 @@ class PerspectiveDataLoader {
197
199
  conid: props.databaseConfig.conid,
198
200
  database: props.databaseConfig.database,
199
201
  select,
202
+ auditLogSessionGroup: 'perspective',
200
203
  });
201
204
  if (response.errorMessage)
202
205
  return response;
@@ -235,6 +238,7 @@ class PerspectiveDataLoader {
235
238
  conid: props.databaseConfig.conid,
236
239
  database: props.databaseConfig.database,
237
240
  options,
241
+ auditLogSessionGroup: 'perspective',
238
242
  });
239
243
  if (response.errorMessage)
240
244
  return response;
@@ -273,6 +277,7 @@ class PerspectiveDataLoader {
273
277
  conid: props.databaseConfig.conid,
274
278
  database: props.databaseConfig.database,
275
279
  select,
280
+ auditLogSessionGroup: 'perspective',
276
281
  });
277
282
  if (response.errorMessage)
278
283
  return response;
@@ -287,6 +292,7 @@ class PerspectiveDataLoader {
287
292
  conid: props.databaseConfig.conid,
288
293
  database: props.databaseConfig.database,
289
294
  options,
295
+ auditLogSessionGroup: 'perspective',
290
296
  });
291
297
  return response;
292
298
  });
package/lib/chartDefinitions.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- export type ChartTypeEnum = 'bar' | 'line' | 'pie' | 'polarArea';
1
+ export type ChartTypeEnum = 'bar' | 'line' | 'timeline' | 'pie' | 'polarArea';
2
2
  export type ChartXTransformFunction = 'identity' | 'date:minute' | 'date:hour' | 'date:day' | 'date:month' | 'date:year';
3
3
  export type ChartYAggregateFunction = 'sum' | 'first' | 'last' | 'min' | 'max' | 'count' | 'avg';
4
4
  export type ChartDataLabelFormatter = 'number' | 'size:bytes' | 'size:kb' | 'size:mb' | 'size:gb';
@@ -10,6 +10,7 @@ export declare const ChartConstDefaults: {
10
10
  };
11
11
  export declare const ChartLimits: {
12
12
  AUTODETECT_CHART_LIMIT: number;
13
+ AUTODETECT_CHART_TOTAL_LIMIT: number;
13
14
  AUTODETECT_MEASURES_LIMIT: number;
14
15
  APPLY_LIMIT_AFTER_ROWS: number;
15
16
  MAX_DISTINCT_VALUES: number;
@@ -17,6 +18,7 @@ export declare const ChartLimits: {
17
18
  PIE_RATIO_LIMIT: number;
18
19
  PIE_COUNT_LIMIT: number;
19
20
  CHART_FILL_LIMIT: number;
21
+ CHART_GROUP_LIMIT: number;
20
22
  };
21
23
  export interface ChartXFieldDefinition {
22
24
  field: string;
@@ -40,6 +42,8 @@ export interface ChartDefinition {
40
42
  trimXCountLimit?: number;
41
43
  xdef: ChartXFieldDefinition;
42
44
  ydefs: ChartYFieldDefinition[];
45
+ groupingField?: string;
46
+ groupTransformFunction?: ChartXTransformFunction;
43
47
  useDataLabels?: boolean;
44
48
  dataLabelFormatter?: ChartDataLabelFormatter;
45
49
  }
@@ -54,6 +58,7 @@ export interface ChartDateParsed {
54
58
  }
55
59
  export interface ChartAvailableColumn {
56
60
  field: string;
61
+ dataType: 'none' | 'string' | 'number' | 'date' | 'mixed';
57
62
  }
58
63
  export interface ProcessedChart {
59
64
  minX?: string;
@@ -63,6 +68,7 @@ export interface ProcessedChart {
63
68
  [key: string]: any;
64
69
  };
65
70
  bucketKeysOrdered: string[];
71
+ bucketKeysSet: Set<string>;
66
72
  bucketKeyDateParsed: {
67
73
  [key: string]: ChartDateParsed;
68
74
  };
@@ -74,6 +80,8 @@ export interface ProcessedChart {
74
80
  validYRows: {
75
81
  [key: string]: number;
76
82
  };
83
+ groups: string[];
84
+ groupSet: Set<string>;
77
85
  topDistinctValues: {
78
86
  [key: string]: Set<any>;
79
87
  };
package/lib/chartDefinitions.js CHANGED
@@ -9,11 +9,13 @@ exports.ChartConstDefaults = {
9
9
  };
10
10
  exports.ChartLimits = {
11
11
  AUTODETECT_CHART_LIMIT: 10,
12
+ AUTODETECT_CHART_TOTAL_LIMIT: 32,
12
13
  AUTODETECT_MEASURES_LIMIT: 10,
13
14
  APPLY_LIMIT_AFTER_ROWS: 100,
14
15
  MAX_DISTINCT_VALUES: 10,
15
16
  VALID_VALUE_RATIO_LIMIT: 0.5,
16
17
  PIE_RATIO_LIMIT: 0.05,
17
18
  PIE_COUNT_LIMIT: 10,
18
- CHART_FILL_LIMIT: 10000, // limit for filled charts (time intervals), to avoid too many points
19
+ CHART_FILL_LIMIT: 10000,
20
+ CHART_GROUP_LIMIT: 32, // limit for number of groups in a chart
19
21
  };
package/lib/chartProcessor.d.ts CHANGED
@@ -11,9 +11,17 @@ export declare class ChartProcessor {
11
11
  rowsAdded: number;
12
12
  errorMessage?: string;
13
13
  constructor(givenDefinitions?: ChartDefinition[]);
14
+ runAutoDetectCharts(dateColumns: {
15
+ [key: string]: ChartDateParsed;
16
+ }, numericColumnsForAutodetect: {
17
+ [key: string]: number;
18
+ }, stringColumns: {
19
+ [key: string]: string;
20
+ }): void;
14
21
  addRow(row: any): void;
15
22
  applyLimitsOnCharts(): void;
16
23
  addRows(...rows: any[]): void;
24
+ splitChartsByYDefs(): void;
17
25
  finalize(): void;
18
26
  groupPieOtherBuckets(chart: ProcessedChart): void;
19
27
  applyRawData(chart: ProcessedChart, row: any, dateParsed: ChartDateParsed, numericColumns: {
package/lib/chartProcessor.js CHANGED
@@ -7,6 +7,9 @@ exports.ChartProcessor = void 0;
7
7
  const chartDefinitions_1 = require("./chartDefinitions");
8
8
  const sortBy_1 = __importDefault(require("lodash/sortBy"));
9
9
  const sum_1 = __importDefault(require("lodash/sum"));
10
+ const zipObject_1 = __importDefault(require("lodash/zipObject"));
11
+ const mapValues_1 = __importDefault(require("lodash/mapValues"));
12
+ const pick_1 = __importDefault(require("lodash/pick"));
10
13
  const chartTools_1 = require("./chartTools");
11
14
  const chartScoring_1 = require("./chartScoring");
12
15
  class ChartProcessor {
@@ -31,6 +34,9 @@ class ChartProcessor {
31
34
  availableColumns: [],
32
35
  validYRows: {},
33
36
  topDistinctValues: {},
37
+ groups: [],
38
+ groupSet: new Set(),
39
+ bucketKeysSet: new Set(),
34
40
  });
35
41
  }
36
42
  this.autoDetectCharts = this.givenDefinitions.length == 0;
@@ -57,6 +63,74 @@ class ChartProcessor {
57
63
  // this.chartsBySignature[signature] = chart;
58
64
  // return chart;
59
65
  // }
66
+ runAutoDetectCharts(dateColumns, numericColumnsForAutodetect, stringColumns) {
67
+ const processColumnType = (columns, transformTest, chartType, transformFunction) => {
68
+ for (const xcol in columns) {
69
+ for (const groupingField of [undefined, ...Object.keys(stringColumns)]) {
70
+ if (xcol == groupingField) {
71
+ continue;
72
+ }
73
+ let usedChart = this.chartsProcessing.find(chart => !chart.isGivenDefinition &&
74
+ chart.definition.xdef.field === xcol &&
75
+ transformTest(chart.definition.xdef.transformFunction) &&
76
+ chart.definition.groupingField == groupingField);
77
+ if (!usedChart &&
78
+ (this.rowsAdded < chartDefinitions_1.ChartLimits.APPLY_LIMIT_AFTER_ROWS ||
79
+ this.chartsProcessing.length < chartDefinitions_1.ChartLimits.AUTODETECT_CHART_LIMIT)) {
80
+ usedChart = {
81
+ definition: {
82
+ chartType,
83
+ xdef: {
84
+ field: xcol,
85
+ transformFunction,
86
+ },
87
+ ydefs: [
88
+ {
89
+ field: '__count',
90
+ aggregateFunction: 'count',
91
+ },
92
+ ],
93
+ groupingField,
94
+ },
95
+ rowsAdded: 0,
96
+ bucketKeysOrdered: [],
97
+ buckets: {},
98
+ groups: [],
99
+ bucketKeyDateParsed: {},
100
+ isGivenDefinition: false,
101
+ invalidXRows: 0,
102
+ invalidYRows: {},
103
+ availableColumns: [],
104
+ validYRows: {},
105
+ topDistinctValues: {},
106
+ groupSet: new Set(),
107
+ bucketKeysSet: new Set(),
108
+ };
109
+ this.chartsProcessing.push(usedChart);
110
+ }
111
+ if (!usedChart) {
112
+ continue; // chart not created - probably too many charts already
113
+ }
114
+ for (const [key, value] of Object.entries(numericColumnsForAutodetect)) {
115
+ // if (value == null) continue;
116
+ // if (key == datecol) continue; // skip date column itself
117
+ const existingYDef = usedChart.definition.ydefs.find(y => y.field === key);
118
+ if (!existingYDef &&
119
+ (this.rowsAdded < chartDefinitions_1.ChartLimits.APPLY_LIMIT_AFTER_ROWS ||
120
+ usedChart.definition.ydefs.length < chartDefinitions_1.ChartLimits.AUTODETECT_MEASURES_LIMIT)) {
121
+ const newYDef = {
122
+ field: key,
123
+ aggregateFunction: 'sum',
124
+ };
125
+ usedChart.definition.ydefs.push(newYDef);
126
+ }
127
+ }
128
+ }
129
+ }
130
+ };
131
+ processColumnType(dateColumns, transform => transform === null || transform === void 0 ? void 0 : transform.startsWith('date:'), 'timeline', 'date:day');
132
+ processColumnType(stringColumns, transform => transform == 'identity', 'bar', 'identity');
133
+ }
60
134
  addRow(row) {
61
135
  const dateColumns = {};
62
136
  const numericColumns = {};
@@ -64,14 +138,25 @@ class ChartProcessor {
64
138
  const stringColumns = {};
65
139
  for (const [key, value] of Object.entries(row)) {
66
140
  const number = typeof value == 'string' ? Number(value) : typeof value == 'number' ? value : NaN;
67
- this.availableColumnsDict[key] = {
68
- field: key,
69
- };
141
+ let availableColumn = this.availableColumnsDict[key];
142
+ if (!availableColumn) {
143
+ availableColumn = {
144
+ field: key,
145
+ dataType: 'none',
146
+ };
147
+ this.availableColumnsDict[key] = availableColumn;
148
+ }
70
149
  const keyLower = key.toLowerCase();
71
150
  const keyIsId = keyLower.endsWith('_id') || keyLower == 'id' || key.endsWith('Id');
72
151
  const parsedDate = (0, chartTools_1.tryParseChartDate)(value);
73
152
  if (parsedDate) {
74
153
  dateColumns[key] = parsedDate;
154
+ if (availableColumn.dataType == 'none') {
155
+ availableColumn.dataType = 'date';
156
+ }
157
+ if (availableColumn.dataType != 'date') {
158
+ availableColumn.dataType = 'mixed';
159
+ }
75
160
  continue;
76
161
  }
77
162
  if (!isNaN(number) && isFinite(number)) {
@@ -79,71 +164,34 @@ class ChartProcessor {
79
164
  if (!keyIsId) {
80
165
  numericColumnsForAutodetect[key] = number; // for auto-detecting charts
81
166
  }
167
+ if (availableColumn.dataType == 'none') {
168
+ availableColumn.dataType = 'number';
169
+ }
170
+ if (availableColumn.dataType != 'number') {
171
+ availableColumn.dataType = 'mixed';
172
+ }
82
173
  continue;
83
174
  }
84
175
  if (typeof value === 'string' && isNaN(number) && value.length < 100) {
85
176
  stringColumns[key] = value;
177
+ if (availableColumn.dataType == 'none') {
178
+ availableColumn.dataType = 'string';
179
+ }
180
+ if (availableColumn.dataType != 'string') {
181
+ availableColumn.dataType = 'mixed';
182
+ }
86
183
  }
87
184
  }
88
185
  // const sortedNumericColumnns = Object.keys(numericColumns).sort();
89
186
  if (this.autoDetectCharts) {
90
- // create charts from data, if there are no given definitions
91
- for (const datecol in dateColumns) {
92
- let usedChart = this.chartsProcessing.find(chart => {
93
- var _a;
94
- return !chart.isGivenDefinition &&
95
- chart.definition.xdef.field === datecol &&
96
- ((_a = chart.definition.xdef.transformFunction) === null || _a === void 0 ? void 0 : _a.startsWith('date:'));
97
- });
98
- if (!usedChart &&
99
- (this.rowsAdded < chartDefinitions_1.ChartLimits.APPLY_LIMIT_AFTER_ROWS ||
100
- this.chartsProcessing.length < chartDefinitions_1.ChartLimits.AUTODETECT_CHART_LIMIT)) {
101
- usedChart = {
102
- definition: {
103
- chartType: 'line',
104
- xdef: {
105
- field: datecol,
106
- transformFunction: 'date:day',
107
- },
108
- ydefs: [],
109
- },
110
- rowsAdded: 0,
111
- bucketKeysOrdered: [],
112
- buckets: {},
113
- bucketKeyDateParsed: {},
114
- isGivenDefinition: false,
115
- invalidXRows: 0,
116
- invalidYRows: {},
117
- availableColumns: [],
118
- validYRows: {},
119
- topDistinctValues: {},
120
- };
121
- this.chartsProcessing.push(usedChart);
122
- }
123
- for (const [key, value] of Object.entries(row)) {
124
- if (value == null)
125
- continue;
126
- if (key == datecol)
127
- continue; // skip date column itself
128
- let existingYDef = usedChart.definition.ydefs.find(y => y.field === key);
129
- if (!existingYDef &&
130
- (this.rowsAdded < chartDefinitions_1.ChartLimits.APPLY_LIMIT_AFTER_ROWS ||
131
- usedChart.definition.ydefs.length < chartDefinitions_1.ChartLimits.AUTODETECT_MEASURES_LIMIT)) {
132
- existingYDef = {
133
- field: key,
134
- aggregateFunction: 'sum',
135
- };
136
- usedChart.definition.ydefs.push(existingYDef);
137
- }
138
- }
139
- }
187
+ this.runAutoDetectCharts(dateColumns, numericColumnsForAutodetect, stringColumns);
140
188
  }
141
189
  // apply on all charts with this date column
142
190
  for (const chart of this.chartsProcessing) {
143
191
  if (chart.errorMessage) {
144
192
  continue; // skip charts with errors
145
193
  }
146
- this.applyRawData(chart, row, dateColumns[chart.definition.xdef.field], chart.isGivenDefinition ? numericColumns : numericColumnsForAutodetect, stringColumns);
194
+ this.applyRawData(chart, row, dateColumns[chart.definition.xdef.field], numericColumns, stringColumns);
147
195
  if (Object.keys(chart.buckets).length > chartDefinitions_1.ChartLimits.CHART_FILL_LIMIT) {
148
196
  chart.errorMessage = `Chart has too many buckets, limit is ${chartDefinitions_1.ChartLimits.CHART_FILL_LIMIT}.`;
149
197
  }
@@ -152,6 +200,9 @@ class ChartProcessor {
152
200
  if (this.chartsProcessing[i].errorMessage) {
153
201
  continue; // skip charts with errors
154
202
  }
203
+ if (this.chartsProcessing[i].definition.chartType != 'timeline') {
204
+ continue; // skip non-timeline charts
205
+ }
155
206
  this.chartsProcessing[i] = (0, chartTools_1.autoAggregateCompactTimelineChart)(this.chartsProcessing[i]);
156
207
  }
157
208
  this.rowsAdded += 1;
@@ -177,8 +228,35 @@ class ChartProcessor {
177
228
  this.addRow(row);
178
229
  }
179
230
  }
231
+ splitChartsByYDefs() {
232
+ const newCharts = [];
233
+ for (const chart of this.chartsProcessing) {
234
+ if (chart.isGivenDefinition) {
235
+ newCharts.push(chart);
236
+ continue;
237
+ }
238
+ const yRanges = chart.definition.ydefs.map(ydef => (0, chartTools_1.getChartYRange)(chart, ydef).max);
239
+ const yRangeByField = (0, zipObject_1.default)(chart.definition.ydefs.map(ydef => ydef.field), yRanges);
240
+ let ydefsToAssign = chart.definition.ydefs.map(ydef => ydef.field);
241
+ while (ydefsToAssign.length > 0) {
242
+ const first = ydefsToAssign.shift();
243
+ const additionals = [];
244
+ for (const candidate of ydefsToAssign) {
245
+ if ((0, chartTools_1.chartsHaveSimilarRange)(yRangeByField[first], yRangeByField[candidate])) {
246
+ additionals.push(candidate);
247
+ }
248
+ }
249
+ const ydefsCurrent = [first, ...additionals];
250
+ const partialChart = Object.assign(Object.assign({}, chart), { definition: Object.assign(Object.assign({}, chart.definition), { ydefs: ydefsCurrent.map(y => chart.definition.ydefs.find(yd => yd.field === y)) }), buckets: (0, mapValues_1.default)(chart.buckets, bucket => (0, pick_1.default)(bucket, ydefsCurrent)) });
251
+ newCharts.push(partialChart);
252
+ ydefsToAssign = ydefsToAssign.filter(y => !additionals.includes(y));
253
+ }
254
+ }
255
+ this.chartsProcessing = newCharts;
256
+ }
180
257
  finalize() {
181
258
  var _a;
259
+ this.splitChartsByYDefs();
182
260
  this.applyLimitsOnCharts();
183
261
  this.availableColumns = Object.values(this.availableColumnsDict);
184
262
  for (const chart of this.chartsProcessing) {
@@ -193,7 +271,7 @@ class ChartProcessor {
193
271
  const sortOrder = (_a = chart.definition.xdef.sortOrder) !== null && _a !== void 0 ? _a : 'ascKeys';
194
272
  if (sortOrder != 'natural') {
195
273
  if (sortOrder == 'ascKeys' || sortOrder == 'descKeys') {
196
- if (chart.definition.chartType == 'line' && chart.definition.xdef.transformFunction.startsWith('date:')) {
274
+ if (chart.definition.chartType == 'timeline' && chart.definition.xdef.transformFunction.startsWith('date:')) {
197
275
  addedChart = (0, chartTools_1.autoAggregateCompactTimelineChart)(addedChart);
198
276
  (0, chartTools_1.fillChartTimelineBuckets)(addedChart);
199
277
  }
@@ -201,13 +279,13 @@ class ChartProcessor {
201
279
  this.charts.push(addedChart);
202
280
  continue;
203
281
  }
204
- addedChart.bucketKeysOrdered = (0, sortBy_1.default)(Object.keys(addedChart.buckets));
282
+ addedChart.bucketKeysOrdered = (0, sortBy_1.default)([...addedChart.bucketKeysSet]);
205
283
  if (sortOrder == 'descKeys') {
206
284
  addedChart.bucketKeysOrdered.reverse();
207
285
  }
208
286
  }
209
287
  if (sortOrder == 'ascValues' || sortOrder == 'descValues') {
210
- addedChart.bucketKeysOrdered = (0, sortBy_1.default)(Object.keys(addedChart.buckets), key => (0, chartTools_1.computeChartBucketCardinality)(addedChart.buckets[key]));
288
+ addedChart.bucketKeysOrdered = (0, sortBy_1.default)([...addedChart.bucketKeysSet], key => (0, chartTools_1.computeChartBucketCardinality)(addedChart.buckets[key]));
211
289
  if (sortOrder == 'descValues') {
212
290
  addedChart.bucketKeysOrdered.reverse();
213
291
  }
@@ -226,10 +304,13 @@ class ChartProcessor {
226
304
  this.charts.push(addedChart);
227
305
  }
228
306
  this.groupPieOtherBuckets(addedChart);
307
+ addedChart.groups = [...addedChart.groupSet];
308
+ addedChart.bucketKeysSet = undefined;
309
+ addedChart.groupSet = undefined;
229
310
  }
230
311
  this.charts = [
231
312
  ...this.charts.filter(x => x.isGivenDefinition),
232
- ...(0, sortBy_1.default)(this.charts.filter(x => !x.isGivenDefinition), chart => -(0, chartScoring_1.getChartScore)(chart)),
313
+ ...(0, sortBy_1.default)(this.charts.filter(x => !x.isGivenDefinition && !x.errorMessage && x.definition.ydefs.length > 0), chart => -(0, chartScoring_1.getChartScore)(chart)),
233
314
  ];
234
315
  }
235
316
  groupPieOtherBuckets(chart) {
@@ -292,6 +373,15 @@ class ChartProcessor {
292
373
  return; // skip if date is invalid
293
374
  }
294
375
  const [bucketKey, bucketKeyParsed] = (0, chartTools_1.computeChartBucketKey)(dateParsed, chart, row);
376
+ const bucketGroup = chart.definition.groupingField
377
+ ? (0, chartTools_1.runTransformFunction)(row[chart.definition.groupingField], chart.definition.groupTransformFunction)
378
+ : null;
379
+ if (bucketGroup) {
380
+ chart.groupSet.add(bucketGroup);
381
+ }
382
+ if (chart.groupSet.size > chartDefinitions_1.ChartLimits.CHART_GROUP_LIMIT) {
383
+ chart.errorMessage = `Chart has too many groups, limit is ${chartDefinitions_1.ChartLimits.CHART_GROUP_LIMIT}.`;
384
+ }
295
385
  if (!bucketKey) {
296
386
  return; // skip if no bucket key
297
387
  }
@@ -304,13 +394,17 @@ class ChartProcessor {
304
394
  if (chart.maxX == null || bucketKey > chart.maxX) {
305
395
  chart.maxX = bucketKey;
306
396
  }
307
- if (!chart.buckets[bucketKey]) {
308
- chart.buckets[bucketKey] = {};
397
+ const groupedBucketKey = chart.definition.groupingField ? `${bucketGroup !== null && bucketGroup !== void 0 ? bucketGroup : ''}::${bucketKey}` : bucketKey;
398
+ if (!chart.buckets[groupedBucketKey]) {
399
+ chart.buckets[groupedBucketKey] = {};
400
+ }
401
+ if (!chart.bucketKeysSet.has(bucketKey)) {
402
+ chart.bucketKeysSet.add(bucketKey);
309
403
  if (chart.definition.xdef.sortOrder == 'natural') {
310
404
  chart.bucketKeysOrdered.push(bucketKey);
311
405
  }
312
406
  }
313
- (0, chartTools_1.aggregateChartNumericValuesFromSource)(chart, bucketKey, numericColumns, row);
407
+ (0, chartTools_1.aggregateChartNumericValuesFromSource)(chart, groupedBucketKey, numericColumns, row);
314
408
  chart.rowsAdded += 1;
315
409
  }
316
410
  }
package/lib/chartScoring.js CHANGED
@@ -8,11 +8,21 @@ const sortBy_1 = __importDefault(require("lodash/sortBy"));
8
8
  const sum_1 = __importDefault(require("lodash/sum"));
9
9
  const chartDefinitions_1 = require("./chartDefinitions");
10
10
  function getChartScore(chart) {
11
+ var _a, _b, _c;
12
+ if (chart.errorMessage) {
13
+ return -1; // negative score for charts with errors
14
+ }
11
15
  let res = 0;
12
16
  res += chart.rowsAdded * 5;
13
17
  const ydefScores = chart.definition.ydefs.map(yField => getChartYFieldScore(chart, yField));
14
18
  const sorted = (0, sortBy_1.default)(ydefScores).reverse();
15
19
  res += (0, sum_1.default)(sorted.slice(0, chartDefinitions_1.ChartLimits.AUTODETECT_MEASURES_LIMIT));
20
+ if (((_a = chart.groupSet) === null || _a === void 0 ? void 0 : _a.size) >= 2 && ((_b = chart.groupSet) === null || _b === void 0 ? void 0 : _b.size) <= 6) {
21
+ res += 50; // bonus for nice grouping
22
+ }
23
+ if (((_c = chart.groupSet) === null || _c === void 0 ? void 0 : _c.size) == 1) {
24
+ res -= 20; // penalty for single group
25
+ }
16
26
  return res;
17
27
  }
18
28
  exports.getChartScore = getChartScore;
package/lib/chartTools.d.ts CHANGED
@@ -1,8 +1,9 @@
1
- import { ChartDateParsed, ChartXTransformFunction, ProcessedChart } from './chartDefinitions';
1
+ import { ChartDateParsed, ChartXTransformFunction, ChartYFieldDefinition, ProcessedChart } from './chartDefinitions';
2
2
  export declare function getChartDebugPrint(chart: ProcessedChart): string;
3
3
  export declare function tryParseChartDate(dateInput: any): ChartDateParsed | null;
4
4
  export declare function stringifyChartDate(value: ChartDateParsed, transform: ChartXTransformFunction): string;
5
5
  export declare function incrementChartDate(value: ChartDateParsed, transform: ChartXTransformFunction): ChartDateParsed;
6
+ export declare function runTransformFunction(value: string, transformFunction: ChartXTransformFunction): string;
6
7
  export declare function computeChartBucketKey(dateParsed: ChartDateParsed, chart: ProcessedChart, row: any): [string, ChartDateParsed];
7
8
  export declare function computeDateBucketDistance(begin: ChartDateParsed, end: ChartDateParsed, transform: ChartXTransformFunction): number;
8
9
  export declare function compareChartDatesParsed(a: ChartDateParsed, b: ChartDateParsed, transform: ChartXTransformFunction): number;
@@ -17,3 +18,8 @@ export declare function fillChartTimelineBuckets(chart: ProcessedChart): void;
17
18
  export declare function computeChartBucketCardinality(bucket: {
18
19
  [key: string]: any;
19
20
  }): number;
21
+ export declare function getChartYRange(chart: ProcessedChart, ydef: ChartYFieldDefinition): {
22
+ min: any;
23
+ max: any;
24
+ };
25
+ export declare function chartsHaveSimilarRange(range1: number, range2: number): boolean;
package/lib/chartTools.js CHANGED
@@ -3,14 +3,16 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.computeChartBucketCardinality = exports.fillChartTimelineBuckets = exports.aggregateChartNumericValuesFromChild = exports.aggregateChartNumericValuesFromSource = exports.autoAggregateCompactTimelineChart = exports.compareChartDatesParsed = exports.computeDateBucketDistance = exports.computeChartBucketKey = exports.incrementChartDate = exports.stringifyChartDate = exports.tryParseChartDate = exports.getChartDebugPrint = void 0;
6
+ exports.chartsHaveSimilarRange = exports.getChartYRange = exports.computeChartBucketCardinality = exports.fillChartTimelineBuckets = exports.aggregateChartNumericValuesFromChild = exports.aggregateChartNumericValuesFromSource = exports.autoAggregateCompactTimelineChart = exports.compareChartDatesParsed = exports.computeDateBucketDistance = exports.computeChartBucketKey = exports.runTransformFunction = exports.incrementChartDate = exports.stringifyChartDate = exports.tryParseChartDate = exports.getChartDebugPrint = void 0;
7
7
  const toPairs_1 = __importDefault(require("lodash/toPairs"));
8
8
  const sumBy_1 = __importDefault(require("lodash/sumBy"));
9
9
  const chartDefinitions_1 = require("./chartDefinitions");
10
10
  const date_fns_1 = require("date-fns");
11
11
  function getChartDebugPrint(chart) {
12
12
  let res = '';
13
- res += `Chart: ${chart.definition.chartType} (${chart.definition.xdef.transformFunction})\n`;
13
+ res += `Chart: ${chart.definition.chartType} (${chart.definition.xdef.transformFunction}): (${chart.definition.ydefs
14
+ .map(yd => yd.field)
15
+ .join(', ')})\n`;
14
16
  for (const key of chart.bucketKeysOrdered) {
15
17
  res += `${key}: ${(0, toPairs_1.default)(chart.buckets[key])
16
18
  .map(([k, v]) => `${k}=${v}`)
@@ -33,19 +35,46 @@ function tryParseChartDate(dateInput) {
33
35
  }
34
36
  if (typeof dateInput !== 'string')
35
37
  return null;
36
- const m = dateInput.match(/^(\d{4})-(\d{2})-(\d{2})(?:[ T](\d{2}):(\d{2}):(\d{2})(?:\.(\d+))?(Z|[+-]\d{2}:\d{2})?)?$/);
37
- if (!m)
38
- return null;
39
- const [_notUsed, year, month, day, hour, minute, second, fraction] = m;
40
- return {
41
- year: parseInt(year, 10),
42
- month: parseInt(month, 10),
43
- day: parseInt(day, 10),
44
- hour: parseInt(hour, 10) || 0,
45
- minute: parseInt(minute, 10) || 0,
46
- second: parseInt(second, 10) || 0,
47
- fraction: fraction || undefined,
48
- };
38
+ const dateMatch = dateInput.match(/^(\d{4})-(\d{2})-(\d{2})(?:[ T](\d{2}):(\d{2}):(\d{2})(?:\.(\d+))?(Z|[+-]\d{2}:\d{2})?)?$/);
39
+ const monthMatch = dateInput.match(/^(\d{4})-(\d{2})$/);
40
+ // const yearMatch = dateInput.match(/^(\d{4})$/);
41
+ if (dateMatch) {
42
+ const [_notUsed, year, month, day, hour, minute, second, fraction] = dateMatch;
43
+ return {
44
+ year: parseInt(year, 10),
45
+ month: parseInt(month, 10),
46
+ day: parseInt(day, 10),
47
+ hour: parseInt(hour, 10) || 0,
48
+ minute: parseInt(minute, 10) || 0,
49
+ second: parseInt(second, 10) || 0,
50
+ fraction: fraction || undefined,
51
+ };
52
+ }
53
+ if (monthMatch) {
54
+ const [_notUsed, year, month] = monthMatch;
55
+ return {
56
+ year: parseInt(year, 10),
57
+ month: parseInt(month, 10),
58
+ day: 1,
59
+ hour: 0,
60
+ minute: 0,
61
+ second: 0,
62
+ fraction: undefined,
63
+ };
64
+ }
65
+ // if (yearMatch) {
66
+ // const [_notUsed, year] = yearMatch;
67
+ // return {
68
+ // year: parseInt(year, 10),
69
+ // month: 1,
70
+ // day: 1,
71
+ // hour: 0,
72
+ // minute: 0,
73
+ // second: 0,
74
+ // fraction: undefined,
75
+ // };
76
+ // }
77
+ return null;
49
78
  }
50
79
  exports.tryParseChartDate = tryParseChartDate;
51
80
  function pad2Digits(number) {
@@ -121,6 +150,29 @@ function incrementChartDate(value, transform) {
121
150
  }
122
151
  }
123
152
  exports.incrementChartDate = incrementChartDate;
153
+ function runTransformFunction(value, transformFunction) {
154
+ const dateParsed = tryParseChartDate(value);
155
+ switch (transformFunction) {
156
+ case 'date:year':
157
+ return dateParsed ? `${dateParsed.year}` : null;
158
+ case 'date:month':
159
+ return dateParsed ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}` : null;
160
+ case 'date:day':
161
+ return dateParsed ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}-${pad2Digits(dateParsed.day)}` : null;
162
+ case 'date:hour':
163
+ return dateParsed
164
+ ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}-${pad2Digits(dateParsed.day)} ${pad2Digits(dateParsed.hour)}`
165
+ : null;
166
+ case 'date:minute':
167
+ return dateParsed
168
+ ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}-${pad2Digits(dateParsed.day)} ${pad2Digits(dateParsed.hour)}:${pad2Digits(dateParsed.minute)}`
169
+ : null;
170
+ case 'identity':
171
+ default:
172
+ return value;
173
+ }
174
+ }
175
+ exports.runTransformFunction = runTransformFunction;
124
176
  function computeChartBucketKey(dateParsed, chart, row) {
125
177
  switch (chart.definition.xdef.transformFunction) {
126
178
  case 'date:year':
@@ -233,7 +285,21 @@ function compareChartDatesParsed(a, b, transform) {
233
285
  }
234
286
  }
235
287
  exports.compareChartDatesParsed = compareChartDatesParsed;
236
- function getParentDateBucketKey(bucketKey, transform) {
288
+ function extractBucketKeyWithoutGroup(bucketKey, definition) {
289
+ if (definition.groupingField) {
290
+ const [_group, key] = bucketKey.split('::', 2);
291
+ return key || bucketKey;
292
+ }
293
+ return bucketKey;
294
+ }
295
+ function getParentDateBucketKey(bucketKey, transform, isGrouped) {
296
+ if (isGrouped) {
297
+ const [group, key] = bucketKey.split('::', 2);
298
+ if (!key) {
299
+ return null; // no parent for grouped bucket
300
+ }
301
+ return `${group}::${getParentDateBucketKey(key, transform, false)}`;
302
+ }
237
303
  switch (transform) {
238
304
  case 'date:year':
239
305
  return null; // no parent for year
@@ -300,15 +366,21 @@ function createParentChartAggregation(chart) {
300
366
  validYRows: Object.assign({}, chart.validYRows),
301
367
  topDistinctValues: Object.assign({}, chart.topDistinctValues),
302
368
  availableColumns: chart.availableColumns,
369
+ groups: [...chart.groups],
370
+ groupSet: new Set(chart.groups),
371
+ bucketKeysSet: new Set(), // initialize empty set for bucket keys
303
372
  };
304
- for (const [bucketKey, bucketValues] of Object.entries(chart.buckets)) {
305
- const parentKey = getParentDateBucketKey(bucketKey, chart.definition.xdef.transformFunction);
306
- if (!parentKey) {
373
+ for (const bucketKey of chart.bucketKeysSet) {
374
+ res.bucketKeysSet.add(getParentDateBucketKey(bucketKey, chart.definition.xdef.transformFunction, false));
375
+ }
376
+ for (const [groupedBucketKey, bucketValues] of Object.entries(chart.buckets)) {
377
+ const groupedParentKey = getParentDateBucketKey(groupedBucketKey, chart.definition.xdef.transformFunction, !!chart.definition.groupingField);
378
+ if (!groupedParentKey) {
307
379
  // skip if the bucket is already a parent
308
380
  continue;
309
381
  }
310
- res.bucketKeyDateParsed[parentKey] = getParentKeyParsed(chart.bucketKeyDateParsed[bucketKey], chart.definition.xdef.transformFunction);
311
- aggregateChartNumericValuesFromChild(res, parentKey, bucketValues);
382
+ res.bucketKeyDateParsed[extractBucketKeyWithoutGroup(groupedParentKey, chart.definition)] = getParentKeyParsed(chart.bucketKeyDateParsed[extractBucketKeyWithoutGroup(groupedBucketKey, chart.definition)], chart.definition.xdef.transformFunction);
383
+ aggregateChartNumericValuesFromChild(res, groupedParentKey, bucketValues);
312
384
  }
313
385
  const bucketKeys = Object.keys(res.buckets).sort();
314
386
  res.minX = bucketKeys.length > 0 ? bucketKeys[0] : null;
@@ -341,7 +413,7 @@ function autoAggregateCompactTimelineChart(chart) {
341
413
  exports.autoAggregateCompactTimelineChart = autoAggregateCompactTimelineChart;
342
414
  function aggregateChartNumericValuesFromSource(chart, bucketKey, numericColumns, row) {
343
415
  for (const ydef of chart.definition.ydefs) {
344
- if (numericColumns[ydef.field] == null) {
416
+ if (numericColumns[ydef.field] == null && ydef.field != '__count') {
345
417
  if (row[ydef.field]) {
346
418
  chart.invalidYRows[ydef.field] = (chart.invalidYRows[ydef.field] || 0) + 1; // increment invalid row count if the field is not numeric
347
419
  }
@@ -460,8 +532,11 @@ function fillChartTimelineBuckets(chart) {
460
532
  const bucketKey = stringifyChartDate(currentParsed, transform);
461
533
  if (!chart.buckets[bucketKey]) {
462
534
  chart.buckets[bucketKey] = {};
535
+ }
536
+ if (!chart.bucketKeyDateParsed[bucketKey]) {
463
537
  chart.bucketKeyDateParsed[bucketKey] = currentParsed;
464
538
  }
539
+ chart.bucketKeysSet.add(bucketKey);
465
540
  currentParsed = incrementChartDate(currentParsed, transform);
466
541
  count++;
467
542
  if (count > chartDefinitions_1.ChartLimits.CHART_FILL_LIMIT) {
@@ -472,6 +547,33 @@ function fillChartTimelineBuckets(chart) {
472
547
  }
473
548
  exports.fillChartTimelineBuckets = fillChartTimelineBuckets;
474
549
  function computeChartBucketCardinality(bucket) {
475
- return (0, sumBy_1.default)(Object.keys(bucket), field => bucket[field]);
550
+ return (0, sumBy_1.default)(Object.keys(bucket !== null && bucket !== void 0 ? bucket : {}), field => bucket[field]);
476
551
  }
477
552
  exports.computeChartBucketCardinality = computeChartBucketCardinality;
553
+ function getChartYRange(chart, ydef) {
554
+ let min = null;
555
+ let max = null;
556
+ for (const obj of Object.values(chart.buckets)) {
557
+ const value = obj[ydef.field];
558
+ if (value != null) {
559
+ if (min === null || value < min) {
560
+ min = value;
561
+ }
562
+ if (max === null || value > max) {
563
+ max = value;
564
+ }
565
+ }
566
+ }
567
+ return { min, max };
568
+ }
569
+ exports.getChartYRange = getChartYRange;
570
+ function chartsHaveSimilarRange(range1, range2) {
571
+ if (range1 < 0 && range2 < 0) {
572
+ return Math.abs(range1 - range2) / Math.abs(range1) < 0.5;
573
+ }
574
+ if (range1 > 0 && range2 > 0) {
575
+ return Math.abs(range1 - range2) / Math.abs(range1) < 0.5;
576
+ }
577
+ return false;
578
+ }
579
+ exports.chartsHaveSimilarRange = chartsHaveSimilarRange;
package/lib/tests/chartProcessor.test.js CHANGED
@@ -53,7 +53,7 @@ const DS2 = [
53
53
  {
54
54
  ts1: '2023-10-03T07:10:00Z',
55
55
  ts2: '2024-10-03T07:10:00Z',
56
- price1: '13',
56
+ price1: '22',
57
57
  price2: '24',
58
58
  },
59
59
  {
@@ -111,22 +111,34 @@ describe('Chart processor', () => {
111
111
  const processor = new chartProcessor_1.ChartProcessor();
112
112
  processor.addRows(...DS1.slice(0, 3));
113
113
  processor.finalize();
114
- expect(processor.charts.length).toEqual(1);
115
- const chart = processor.charts[0];
116
- expect(chart.definition.xdef.transformFunction).toEqual('date:day');
117
- expect(chart.definition.ydefs).toEqual([
114
+ // console.log(getChartDebugPrint(processor.charts[0]));
115
+ expect(processor.charts.length).toEqual(6);
116
+ const chart1 = processor.charts.find(x => !x.definition.groupingField && x.definition.xdef.field === 'timestamp');
117
+ expect(chart1.definition.xdef.transformFunction).toEqual('date:day');
118
+ expect(chart1.definition.ydefs).toEqual([
118
119
  expect.objectContaining({
119
120
  field: 'value',
120
121
  }),
121
122
  ]);
122
- expect(chart.bucketKeysOrdered).toEqual(['2023-10-01', '2023-10-02', '2023-10-03']);
123
+ expect(chart1.bucketKeysOrdered).toEqual(['2023-10-01', '2023-10-02', '2023-10-03']);
124
+ const chart2 = processor.charts.find(x => x.definition.groupingField && x.definition.xdef.field === 'timestamp');
125
+ expect(chart2.definition.xdef.transformFunction).toEqual('date:day');
126
+ expect(chart2.bucketKeysOrdered).toEqual(['2023-10-01', '2023-10-02', '2023-10-03']);
127
+ expect(chart2.definition.groupingField).toEqual('category');
128
+ const chart3 = processor.charts.find(x => x.definition.xdef.field === 'category');
129
+ expect(chart3.bucketKeysOrdered).toEqual(['A', 'B']);
130
+ expect(chart3.definition.groupingField).toBeUndefined();
131
+ const countCharts = processor.charts.filter(x => x.definition.ydefs.length == 1 && x.definition.ydefs[0].field == '__count');
132
+ expect(countCharts.length).toEqual(3);
123
133
  });
124
134
  test('By month grouped, autedetected', () => {
125
135
  const processor = new chartProcessor_1.ChartProcessor();
126
136
  processor.addRows(...DS1.slice(0, 4));
127
137
  processor.finalize();
128
- expect(processor.charts.length).toEqual(1);
129
- const chart = processor.charts[0];
138
+ expect(processor.charts.length).toEqual(6);
139
+ const chart = processor.charts.find(x => !x.definition.groupingField &&
140
+ x.definition.xdef.field === 'timestamp' &&
141
+ !x.definition.ydefs.find(y => y.field === '__count'));
130
142
  expect(chart.definition.xdef.transformFunction).toEqual('date:month');
131
143
  expect(chart.bucketKeysOrdered).toEqual([
132
144
  '2023-10',
@@ -195,7 +207,7 @@ describe('Chart processor', () => {
195
207
  const processor = new chartProcessor_1.ChartProcessor();
196
208
  processor.addRows(...DS2);
197
209
  processor.finalize();
198
- expect(processor.charts.length).toEqual(2);
210
+ expect(processor.charts.length).toEqual(4);
199
211
  expect(processor.charts[0].definition).toEqual(expect.objectContaining({
200
212
  xdef: expect.objectContaining({
201
213
  field: 'ts1',
@@ -233,8 +245,8 @@ describe('Chart processor', () => {
233
245
  const processor = new chartProcessor_1.ChartProcessor();
234
246
  processor.addRows(...DS3);
235
247
  processor.finalize();
236
- expect(processor.charts.length).toEqual(1);
237
- const chart = processor.charts[0];
248
+ expect(processor.charts.length).toEqual(2);
249
+ const chart = processor.charts.find(x => !x.definition.ydefs.find(y => y.field === '__count'));
238
250
  expect(chart.definition.xdef.transformFunction).toEqual('date:day');
239
251
  expect(chart.definition.ydefs).toEqual([
240
252
  expect.objectContaining({
package/package.json CHANGED
@@ -1,5 +1,5 @@
1
1
  {
2
- "version": "6.5.4",
2
+ "version": "6.5.5",
3
3
  "name": "dbgate-datalib",
4
4
  "main": "lib/index.js",
5
5
  "typings": "lib/index.d.ts",
@@ -15,14 +15,14 @@
15
15
  ],
16
16
  "dependencies": {
17
17
  "date-fns": "^4.1.0",
18
- "dbgate-filterparser": "^6.5.4",
19
- "dbgate-sqltree": "^6.5.4",
20
- "dbgate-tools": "^6.5.4",
18
+ "dbgate-filterparser": "^6.5.5",
19
+ "dbgate-sqltree": "^6.5.5",
20
+ "dbgate-tools": "^6.5.5",
21
21
  "uuid": "^3.4.0"
22
22
  },
23
23
  "devDependencies": {
24
24
  "@types/node": "^13.7.0",
25
- "dbgate-types": "^6.5.4",
25
+ "dbgate-types": "^6.5.5",
26
26
  "jest": "^28.1.3",
27
27
  "ts-jest": "^28.0.7",
28
28
  "typescript": "^4.4.3"