@finos/legend-application-data-cube 0.2.6 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/index.css +1 -1
- package/lib/package.json +1 -1
- package/lib/stores/LegendDataCubeCacheManager.d.ts +1 -0
- package/lib/stores/LegendDataCubeCacheManager.d.ts.map +1 -1
- package/lib/stores/LegendDataCubeCacheManager.js +19 -42
- package/lib/stores/LegendDataCubeCacheManager.js.map +1 -1
- package/package.json +1 -1
- package/src/stores/LegendDataCubeCacheManager.ts +21 -45
package/lib/index.css
CHANGED
package/lib/package.json
CHANGED
|
@@ -18,6 +18,7 @@ import type { CachedDataCubeSource } from '@finos/legend-data-cube';
|
|
|
18
18
|
export declare class LegendDataCubeDataCubeCacheManager {
|
|
19
19
|
private static readonly DUCKDB_DEFAULT_SCHEMA_NAME;
|
|
20
20
|
private static readonly TABLE_NAME_PREFIX;
|
|
21
|
+
private static readonly CSV_FILE_NAME;
|
|
21
22
|
private static tableCounter;
|
|
22
23
|
private _database?;
|
|
23
24
|
private get database();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LegendDataCubeCacheManager.d.ts","sourceRoot":"","sources":["../../src/stores/LegendDataCubeCacheManager.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAKH,OAAO,EAIL,kBAAkB,EAGnB,MAAM,qBAAqB,CAAC;AAM7B,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAGpE,qBAAa,kCAAkC;IAC7C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,0BAA0B,CAAU;IAC5D,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAW;IACpD,OAAO,CAAC,MAAM,CAAC,YAAY,CAAK;IAEhC,OAAO,CAAC,SAAS,CAAC,CAAiC;IAEnD,OAAO,KAAK,QAAQ,GAKnB;IAEK,UAAU;IAmCV,KAAK,CAAC,MAAM,EAAE,kBAAkB;;;;;
|
|
1
|
+
{"version":3,"file":"LegendDataCubeCacheManager.d.ts","sourceRoot":"","sources":["../../src/stores/LegendDataCubeCacheManager.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAKH,OAAO,EAIL,kBAAkB,EAGnB,MAAM,qBAAqB,CAAC;AAM7B,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAGpE,qBAAa,kCAAkC;IAC7C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,0BAA0B,CAAU;IAC5D,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAW;IACpD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAU;IAC/C,OAAO,CAAC,MAAM,CAAC,YAAY,CAAK;IAEhC,OAAO,CAAC,SAAS,CAAC,CAAiC;IAEnD,OAAO,KAAK,QAAQ,GAKnB;IAEK,UAAU;IAmCV,KAAK,CAAC,MAAM,EAAE,kBAAkB;;;;;IA+ChC,WAAW,CAAC,GAAG,EAAE,MAAM;IAmFvB,YAAY,CAAC,MAAM,EAAE,oBAAoB;IAOzC,OAAO;CAId"}
|
|
@@ -22,6 +22,7 @@ import { Type } from 'apache-arrow';
|
|
|
22
22
|
export class LegendDataCubeDataCubeCacheManager {
|
|
23
23
|
static DUCKDB_DEFAULT_SCHEMA_NAME = 'main'; // See https://duckdb.org/docs/sql/statements/use.html
|
|
24
24
|
static TABLE_NAME_PREFIX = 'cache';
|
|
25
|
+
static CSV_FILE_NAME = 'data';
|
|
25
26
|
static tableCounter = 0;
|
|
26
27
|
_database;
|
|
27
28
|
get database() {
|
|
@@ -56,46 +57,12 @@ export class LegendDataCubeDataCubeCacheManager {
|
|
|
56
57
|
const schema = LegendDataCubeDataCubeCacheManager.DUCKDB_DEFAULT_SCHEMA_NAME;
|
|
57
58
|
LegendDataCubeDataCubeCacheManager.tableCounter += 1;
|
|
58
59
|
const table = `${LegendDataCubeDataCubeCacheManager.TABLE_NAME_PREFIX}${LegendDataCubeDataCubeCacheManager.tableCounter}`;
|
|
60
|
+
const csvFileName = `${LegendDataCubeDataCubeCacheManager.CSV_FILE_NAME}${LegendDataCubeDataCubeCacheManager.tableCounter}.csv`;
|
|
59
61
|
const connection = await this.database.connect();
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
switch (col.type) {
|
|
65
|
-
case PRIMITIVE_TYPE.BOOLEAN: {
|
|
66
|
-
colType = 'BIT';
|
|
67
|
-
break;
|
|
68
|
-
}
|
|
69
|
-
case PRIMITIVE_TYPE.INTEGER: {
|
|
70
|
-
colType = 'INTEGER';
|
|
71
|
-
break;
|
|
72
|
-
}
|
|
73
|
-
case PRIMITIVE_TYPE.NUMBER:
|
|
74
|
-
case PRIMITIVE_TYPE.DECIMAL:
|
|
75
|
-
case PRIMITIVE_TYPE.FLOAT: {
|
|
76
|
-
colType = 'FLOAT';
|
|
77
|
-
break;
|
|
78
|
-
}
|
|
79
|
-
// We don't use type DATE because DuckDB will automatically convert it to a TIMESTAMP
|
|
80
|
-
case PRIMITIVE_TYPE.STRICTDATE:
|
|
81
|
-
case PRIMITIVE_TYPE.DATETIME:
|
|
82
|
-
case PRIMITIVE_TYPE.DATE: {
|
|
83
|
-
colType = 'VARCHAR';
|
|
84
|
-
break;
|
|
85
|
-
}
|
|
86
|
-
case PRIMITIVE_TYPE.STRING: {
|
|
87
|
-
colType = 'VARCHAR';
|
|
88
|
-
break;
|
|
89
|
-
}
|
|
90
|
-
default: {
|
|
91
|
-
throw new UnsupportedOperationError(`Can't initialize cache: failed to find matching DuckDB type for Pure type '${col.type}'`);
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
columns.push(`"${col.name}" ${colType}`);
|
|
95
|
-
});
|
|
96
|
-
const CREATE_TABLE_SQL = `CREATE TABLE ${schema}.${table} (${columns.join(',')})`;
|
|
97
|
-
await connection.query(CREATE_TABLE_SQL);
|
|
98
|
-
const rowString = [];
|
|
62
|
+
const columnString = result.builder.columns
|
|
63
|
+
.map((col) => col.name)
|
|
64
|
+
.join(',');
|
|
65
|
+
const dataString = [columnString];
|
|
99
66
|
result.result.rows.forEach((row) => {
|
|
100
67
|
const updatedRows = row.values.map((val) => {
|
|
101
68
|
if (val !== null && typeof val === 'string') {
|
|
@@ -106,10 +73,20 @@ export class LegendDataCubeDataCubeCacheManager {
|
|
|
106
73
|
}
|
|
107
74
|
return val;
|
|
108
75
|
});
|
|
109
|
-
|
|
76
|
+
dataString.push(`${updatedRows.join(',')}`);
|
|
77
|
+
});
|
|
78
|
+
const csvString = dataString.join('\n');
|
|
79
|
+
await this._database?.registerFileText(csvFileName, csvString);
|
|
80
|
+
await connection.insertCSVFromPath(csvFileName, {
|
|
81
|
+
schema: schema,
|
|
82
|
+
name: table,
|
|
83
|
+
create: false,
|
|
84
|
+
header: true,
|
|
85
|
+
detect: true,
|
|
86
|
+
escape: `'`,
|
|
87
|
+
quote: `'`,
|
|
88
|
+
delimiter: ',',
|
|
110
89
|
});
|
|
111
|
-
const INSERT_TABLE_SQL = `INSERT INTO ${schema}.${table} VALUES ${rowString.join(',')}`;
|
|
112
|
-
await connection.query(INSERT_TABLE_SQL);
|
|
113
90
|
await connection.close();
|
|
114
91
|
return { table, schema, rowCount: result.result.rows.length };
|
|
115
92
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LegendDataCubeCacheManager.js","sourceRoot":"","sources":["../../src/stores/LegendDataCubeCacheManager.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,MAAM,MAAM,qBAAqB,CAAC;AAC9C,OAAO,WAAW,MAAM,0CAA0C,CAAC;AACnE,OAAO,gBAAgB,MAAM,yCAAyC,CAAC;AACvE,OAAO,EACL,mBAAmB,EACnB,cAAc,EACd,UAAU,EACV,kBAAkB,EAClB,MAAM,EACN,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,iBAAiB,EACjB,oBAAoB,EACpB,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEpC,MAAM,OAAO,kCAAkC;IACrC,MAAM,CAAU,0BAA0B,GAAG,MAAM,CAAC,CAAC,sDAAsD;IAC3G,MAAM,CAAU,iBAAiB,GAAG,OAAO,CAAC;IAC5C,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC;IAExB,SAAS,CAAkC;IAEnD,IAAY,QAAQ;QAClB,OAAO,oBAAoB,CACzB,IAAI,CAAC,SAAS,EACd,wCAAwC,CACzC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU;QACd,8BAA8B;QAC9B,2DAA2D;QAC3D,MAAM,cAAc,GAAyB;YAC3C,GAAG,EAAE;gBACH,mEAAmE;gBACnE,UAAU,EAAE,WAAW;gBACvB,UAAU,EAAE,IAAI,GAAG,CACjB,uDAAuD,EACvD,MAAM,CAAC,IAAI,CAAC,GAAG,CAChB,CAAC,QAAQ,EAAE;aACb;YACD,EAAE,EAAE;gBACF,mEAAmE;gBACnE,UAAU,EAAE,gBAAgB;gBAC5B,UAAU,EAAE,IAAI,GAAG,CACjB,sDAAsD,EACtD,MAAM,CAAC,IAAI,CAAC,GAAG,CAChB,CAAC,QAAQ,EAAE;aACb;SACF,CAAC;QACF,0CAA0C;QAC1C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC;QACzD,qDAAqD;QACrD,iBAAiB,CACf,MAAM,CAAC,UAAU,EACjB,oEAAoE,CACrE,CAAC;QACF,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,gCAAgC;QAClG,MAAM,QAAQ,GAAG,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACxD,MAAM,QAAQ,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;QACpE,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAA0B;QACpC,MAAM,MAAM,GACV,kCAAkC,CAAC,0BAA0B,CAAC;QAChE,kCAAkC,CAAC,YAAY,IAAI,CAAC,CAAC;QACrD,MAAM,KAAK,GAAG,GAAG,kCAAkC,CAAC,iBAAiB,GAAG,kCAAkC,CAAC,YAAY,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"LegendDataCubeCacheManager.js","sourceRoot":"","sources":["../../src/stores/LegendDataCubeCacheManager.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,MAAM,MAAM,qBAAqB,CAAC;AAC9C,OAAO,WAAW,MAAM,0CAA0C,CAAC;AACnE,OAAO,gBAAgB,MAAM,yCAAyC,CAAC;AACvE,OAAO,EACL,mBAAmB,EACnB,cAAc,EACd,UAAU,EACV,kBAAkB,EAClB,MAAM,EACN,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,iBAAiB,EACjB,oBAAoB,EACpB,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEpC,MAAM,OAAO,kCAAkC;IACrC,MAAM,CAAU,0BAA0B,GAAG,MAAM,CAAC,CAAC,sDAAsD;IAC3G,MAAM,CAAU,iBAAiB,GAAG,OAAO,CAAC;IAC5C,MAAM,CAAU,aAAa,GAAG,MAAM,CAAC;IACvC,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC;IAExB,SAAS,CAAkC;IAEnD,IAAY,QAAQ;QAClB,OAAO,oBAAoB,CACzB,IAAI,CAAC,SAAS,EACd,wCAAwC,CACzC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU;QACd,8BAA8B;QAC9B,2DAA2D;QAC3D,MAAM,cAAc,GAAyB;YAC3C,GAAG,EAAE;gBACH,mEAAmE;gBACnE,UAAU,EAAE,WAAW;gBACvB,UAAU,EAAE,IAAI,GAAG,CACjB,uDAAuD,EACvD,MAAM,CAAC,IAAI,CAAC,GAAG,CAChB,CAAC,QAAQ,EAAE;aACb;YACD,EAAE,EAAE;gBACF,mEAAmE;gBACnE,UAAU,EAAE,gBAAgB;gBAC5B,UAAU,EAAE,IAAI,GAAG,CACjB,sDAAsD,EACtD,MAAM,CAAC,IAAI,CAAC,GAAG,CAChB,CAAC,QAAQ,EAAE;aACb;SACF,CAAC;QACF,0CAA0C;QAC1C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC;QACzD,qDAAqD;QACrD,iBAAiB,CACf,MAAM,CAAC,UAAU,EACjB,oEAAoE,CACrE,CAAC;QACF,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,gCAAgC;QAClG,MAAM,QAAQ,GAAG,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACxD,MAAM,QAAQ,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;QACpE,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAA0B;QACpC,MAAM,MAAM,GACV,kCAAkC,CAAC,0BAA0B,CAAC;QAChE,kCAAkC,CAAC,YAAY,IAAI,CAAC,CAAC;QACrD,MAAM,KAAK,GAAG,GAAG,kCAAkC,CAAC,iBAAiB,GAAG,kCAAkC,CAAC,YAAY,EAAE,CAAC;QAC1H,MAAM,WAAW,GAAG,GAAG,kCAAkC,CAAC,aAAa,GAAG,kCAAkC,CAAC,YAAY,MAAM,CAAC;QAEhI,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QAEjD,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO;aACxC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC;aACtB,IAAI,CAAC,GAAG,CAAC,CAAC;QAEb,MAAM,UAAU,GAAa,CAAC,YAAY,CAAC,CAAC;QAE5C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YACjC,MAAM,WAAW,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;gBACzC,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;oBAC5C,OAAO,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC;gBAC1C,CAAC;qBAAM,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;oBACxB,OAAO,MAAM,CAAC;gBAChB,CAAC;gBACD,OAAO,GAAG,CAAC;YACb,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAExC,MAAM,IAAI,CAAC,SAAS,EAAE,gBAAgB,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QAE/D,MAAM,UAAU,CAAC,iBAAiB,CAAC,WAAW,EAAE;YAC9C,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,KAAK;YACX,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,GAAG;YACX,KAAK,EAAE,GAAG;YACV,SAAS,EAAE,GAAG;SACf,CAAC,CAAC;QAEH,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;QAEzB,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;IAChE,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,GAAW;QAC3B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3C,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;QAEzB,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;QAC9B,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACpE,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAC5B,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YAC5B,MAAM,CAAC,MAAM,GAAG,WAAW,CAAC,GAAG,CAC7B,CAAC,MAAM,EAAE,EAAE;YACT,gFAAgF;YAChF,qCAAqC;YACrC,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC9B,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE;gBACvB,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAqC,CACvD,CAAC;YACF,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC,CAAC;QACH,MAAM,kBAAkB,GAAG,IAAI,kBAAkB,EAAE,CAAC;QACpD,MAAM,GAAG,GAAG,IAAI,cAAc,EAAE,CAAC;QACjC,GAAG,CAAC,OAAO,GAAG,WAAW,CAAC;QAC1B,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;QAChB,kBAAkB,CAAC,MAAM,GAAG,GAAG,CAAC;QAChC,MAAM,OAAO,GAAG,IAAI,UAAU,EAAE,CAAC;QACjC,OAAO,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YACnD,MAAM,GAAG,GAAG,IAAI,mBAAmB,EAAE,CAAC;YACtC,GAAG,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;YACtB,QAAQ,KAAK,CAAC,MAAM,EAAE,CAAC;gBACrB,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;oBACjB,GAAG,CAAC,IAAI,GAAG,cAAc,CAAC,MAAM,CAAC;oBACjC,MAAM;gBACR,CAAC;gBACD,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;oBACf,GAAG,CAAC,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC;oBAClC,MAAM;gBACR,CAAC;gBACD,KAAK,IAAI,CAAC,IAAI,CAAC;gBACf,KAAK,IAAI,CAAC,OAAO,CAAC;gBAClB,KAAK,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC;oBAC1B,GAAG,CAAC,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC;oBAC/B,MAAM;gBACR,CAAC;gBACD,KAAK,IAAI,CAAC,IAAI,CAAC;gBACf,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;oBACpB,GAAG,CAAC,IAAI,GAAG,cAAc,CAAC,MAAM,CAAC;oBACjC,MAAM;gBACR,CAAC;gBACD,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;oBAClB,GAAG,CAAC,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC;oBAClC,MAAM;gBACR,CAAC;gBACD,KAAK,IAAI,CAAC,GAAG,CAAC;gBACd,KAAK,IAAI,CAAC,IAAI,CAAC;gBACf,KAAK,IAAI,CAAC,KAAK,CAAC;gBAChB,KAAK,IAAI,CAAC,KAAK,CAAC;gBAChB,KAAK,IAAI,CAAC,MAAM,CAAC;gBACjB,KAAK,IAAI,CAAC,KAAK,CAAC;gBAChB,KAAK,IAAI,CAAC,MAAM,CAAC;gBACjB,KAAK,IAAI,CAAC,KAAK,CAAC;gBAChB,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;oBACjB,GAAG,CAAC,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC;oBAClC,MAAM;gBACR,CAAC;gBACD,KAAK,IAAI,CAAC,KAAK,CAAC;gBAChB,KAAK,IAAI,CAAC,OAAO,CAAC;gBAClB,KAAK,IAAI,CAAC,OAAO,CAAC;gBAClB,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;oBAClB,GAAG,CAAC,IAAI,GAAG,cAAc,CAAC,KAAK,CAAC;oBAChC,MAAM;gBACR,CAAC;gBACD,OAAO,CAAC,CAAC,CAAC;oBACR,MAAM,IAAI,yBAAyB,CACjC,oDAAoD,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,IAAI,EAAE,CACjF,CAAC;gBACJ,CAAC;YACH,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC,CAAC,CAAC;QACH,kBAAkB,CAAC,OAAO,GAAG,OAAO,CAAC;QACrC,OAAO,kBAAkB,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAA4B;QAC7C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QACjD,MAAM,cAAc,GAAG,yBAAyB,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC;QACjF,MAAM,UAAU,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QACvC,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAO;QACX,MAAM,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,CAAC;QACnC,MAAM,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,CAAC;IACpC,CAAC"}
|
package/package.json
CHANGED
|
@@ -36,6 +36,7 @@ import { Type } from 'apache-arrow';
|
|
|
36
36
|
export class LegendDataCubeDataCubeCacheManager {
|
|
37
37
|
private static readonly DUCKDB_DEFAULT_SCHEMA_NAME = 'main'; // See https://duckdb.org/docs/sql/statements/use.html
|
|
38
38
|
private static readonly TABLE_NAME_PREFIX = 'cache';
|
|
39
|
+
private static readonly CSV_FILE_NAME = 'data';
|
|
39
40
|
private static tableCounter = 0;
|
|
40
41
|
|
|
41
42
|
private _database?: duckdb.AsyncDuckDB | undefined;
|
|
@@ -87,52 +88,15 @@ export class LegendDataCubeDataCubeCacheManager {
|
|
|
87
88
|
LegendDataCubeDataCubeCacheManager.DUCKDB_DEFAULT_SCHEMA_NAME;
|
|
88
89
|
LegendDataCubeDataCubeCacheManager.tableCounter += 1;
|
|
89
90
|
const table = `${LegendDataCubeDataCubeCacheManager.TABLE_NAME_PREFIX}${LegendDataCubeDataCubeCacheManager.tableCounter}`;
|
|
91
|
+
const csvFileName = `${LegendDataCubeDataCubeCacheManager.CSV_FILE_NAME}${LegendDataCubeDataCubeCacheManager.tableCounter}.csv`;
|
|
90
92
|
|
|
91
93
|
const connection = await this.database.connect();
|
|
92
94
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
let colType: string;
|
|
97
|
-
switch (col.type as string) {
|
|
98
|
-
case PRIMITIVE_TYPE.BOOLEAN: {
|
|
99
|
-
colType = 'BIT';
|
|
100
|
-
break;
|
|
101
|
-
}
|
|
102
|
-
case PRIMITIVE_TYPE.INTEGER: {
|
|
103
|
-
colType = 'INTEGER';
|
|
104
|
-
break;
|
|
105
|
-
}
|
|
106
|
-
case PRIMITIVE_TYPE.NUMBER:
|
|
107
|
-
case PRIMITIVE_TYPE.DECIMAL:
|
|
108
|
-
case PRIMITIVE_TYPE.FLOAT: {
|
|
109
|
-
colType = 'FLOAT';
|
|
110
|
-
break;
|
|
111
|
-
}
|
|
112
|
-
// We don't use type DATE because DuckDB will automatically convert it to a TIMESTAMP
|
|
113
|
-
case PRIMITIVE_TYPE.STRICTDATE:
|
|
114
|
-
case PRIMITIVE_TYPE.DATETIME:
|
|
115
|
-
case PRIMITIVE_TYPE.DATE: {
|
|
116
|
-
colType = 'VARCHAR';
|
|
117
|
-
break;
|
|
118
|
-
}
|
|
119
|
-
case PRIMITIVE_TYPE.STRING: {
|
|
120
|
-
colType = 'VARCHAR';
|
|
121
|
-
break;
|
|
122
|
-
}
|
|
123
|
-
default: {
|
|
124
|
-
throw new UnsupportedOperationError(
|
|
125
|
-
`Can't initialize cache: failed to find matching DuckDB type for Pure type '${col.type}'`,
|
|
126
|
-
);
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
columns.push(`"${col.name}" ${colType}`);
|
|
130
|
-
});
|
|
131
|
-
|
|
132
|
-
const CREATE_TABLE_SQL = `CREATE TABLE ${schema}.${table} (${columns.join(',')})`;
|
|
133
|
-
await connection.query(CREATE_TABLE_SQL);
|
|
95
|
+
const columnString = result.builder.columns
|
|
96
|
+
.map((col) => col.name)
|
|
97
|
+
.join(',');
|
|
134
98
|
|
|
135
|
-
const
|
|
99
|
+
const dataString: string[] = [columnString];
|
|
136
100
|
|
|
137
101
|
result.result.rows.forEach((row) => {
|
|
138
102
|
const updatedRows = row.values.map((val) => {
|
|
@@ -143,12 +107,24 @@ export class LegendDataCubeDataCubeCacheManager {
|
|
|
143
107
|
}
|
|
144
108
|
return val;
|
|
145
109
|
});
|
|
146
|
-
|
|
110
|
+
dataString.push(`${updatedRows.join(',')}`);
|
|
147
111
|
});
|
|
148
112
|
|
|
149
|
-
const
|
|
113
|
+
const csvString = dataString.join('\n');
|
|
114
|
+
|
|
115
|
+
await this._database?.registerFileText(csvFileName, csvString);
|
|
116
|
+
|
|
117
|
+
await connection.insertCSVFromPath(csvFileName, {
|
|
118
|
+
schema: schema,
|
|
119
|
+
name: table,
|
|
120
|
+
create: false,
|
|
121
|
+
header: true,
|
|
122
|
+
detect: true,
|
|
123
|
+
escape: `'`,
|
|
124
|
+
quote: `'`,
|
|
125
|
+
delimiter: ',',
|
|
126
|
+
});
|
|
150
127
|
|
|
151
|
-
await connection.query(INSERT_TABLE_SQL);
|
|
152
128
|
await connection.close();
|
|
153
129
|
|
|
154
130
|
return { table, schema, rowCount: result.result.rows.length };
|