@finos/legend-application-data-cube 0.2.6 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/index.css +1 -1
- package/lib/package.json +3 -3
- package/lib/stores/LegendDataCubeCacheManager.d.ts +1 -0
- package/lib/stores/LegendDataCubeCacheManager.d.ts.map +1 -1
- package/lib/stores/LegendDataCubeCacheManager.js +19 -52
- package/lib/stores/LegendDataCubeCacheManager.js.map +1 -1
- package/package.json +13 -13
- package/src/stores/LegendDataCubeCacheManager.ts +21 -54
package/lib/index.css
CHANGED
package/lib/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@finos/legend-application-data-cube",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.8",
|
|
4
4
|
"description": "Legend DataCube application core",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"legend",
|
|
@@ -65,11 +65,11 @@
|
|
|
65
65
|
"@finos/legend-dev-utils": "workspace:*",
|
|
66
66
|
"@jest/globals": "29.7.0",
|
|
67
67
|
"cross-env": "7.0.3",
|
|
68
|
-
"eslint": "9.
|
|
68
|
+
"eslint": "9.20.0",
|
|
69
69
|
"jest": "29.7.0",
|
|
70
70
|
"npm-run-all": "4.1.5",
|
|
71
71
|
"rimraf": "6.0.1",
|
|
72
|
-
"sass": "1.
|
|
72
|
+
"sass": "1.84.0",
|
|
73
73
|
"typescript": "5.7.3"
|
|
74
74
|
},
|
|
75
75
|
"peerDependencies": {
|
|
@@ -18,6 +18,7 @@ import type { CachedDataCubeSource } from '@finos/legend-data-cube';
|
|
|
18
18
|
export declare class LegendDataCubeDataCubeCacheManager {
|
|
19
19
|
private static readonly DUCKDB_DEFAULT_SCHEMA_NAME;
|
|
20
20
|
private static readonly TABLE_NAME_PREFIX;
|
|
21
|
+
private static readonly CSV_FILE_NAME;
|
|
21
22
|
private static tableCounter;
|
|
22
23
|
private _database?;
|
|
23
24
|
private get database();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LegendDataCubeCacheManager.d.ts","sourceRoot":"","sources":["../../src/stores/LegendDataCubeCacheManager.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAKH,OAAO,EAIL,kBAAkB,EAGnB,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"LegendDataCubeCacheManager.d.ts","sourceRoot":"","sources":["../../src/stores/LegendDataCubeCacheManager.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAKH,OAAO,EAIL,kBAAkB,EAGnB,MAAM,qBAAqB,CAAC;AAO7B,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAGpE,qBAAa,kCAAkC;IAC7C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,0BAA0B,CAAU;IAC5D,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAW;IACpD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAU;IAC/C,OAAO,CAAC,MAAM,CAAC,YAAY,CAAK;IAEhC,OAAO,CAAC,SAAS,CAAC,CAAiC;IAEnD,OAAO,KAAK,QAAQ,GAKnB;IAEK,UAAU;IAmCV,KAAK,CAAC,MAAM,EAAE,kBAAkB;;;;;IAqChC,WAAW,CAAC,GAAG,EAAE,MAAM;IAmFvB,YAAY,CAAC,MAAM,EAAE,oBAAoB;IAOzC,OAAO;CAId"}
|
|
@@ -17,11 +17,12 @@ import * as duckdb from '@duckdb/duckdb-wasm';
|
|
|
17
17
|
import duckdb_wasm from '@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm';
|
|
18
18
|
import duckdb_wasm_next from '@duckdb/duckdb-wasm/dist/duckdb-eh.wasm';
|
|
19
19
|
import { INTERNAL__TDSColumn, PRIMITIVE_TYPE, TDSBuilder, TDSExecutionResult, TDSRow, TabularDataSet, } from '@finos/legend-graph';
|
|
20
|
-
import { assertNonNullable, guaranteeNonNullable, UnsupportedOperationError, } from '@finos/legend-shared';
|
|
20
|
+
import { assertNonNullable, csvStringify, guaranteeNonNullable, UnsupportedOperationError, } from '@finos/legend-shared';
|
|
21
21
|
import { Type } from 'apache-arrow';
|
|
22
22
|
export class LegendDataCubeDataCubeCacheManager {
|
|
23
23
|
static DUCKDB_DEFAULT_SCHEMA_NAME = 'main'; // See https://duckdb.org/docs/sql/statements/use.html
|
|
24
24
|
static TABLE_NAME_PREFIX = 'cache';
|
|
25
|
+
static CSV_FILE_NAME = 'data';
|
|
25
26
|
static tableCounter = 0;
|
|
26
27
|
_database;
|
|
27
28
|
get database() {
|
|
@@ -56,60 +57,26 @@ export class LegendDataCubeDataCubeCacheManager {
|
|
|
56
57
|
const schema = LegendDataCubeDataCubeCacheManager.DUCKDB_DEFAULT_SCHEMA_NAME;
|
|
57
58
|
LegendDataCubeDataCubeCacheManager.tableCounter += 1;
|
|
58
59
|
const table = `${LegendDataCubeDataCubeCacheManager.TABLE_NAME_PREFIX}${LegendDataCubeDataCubeCacheManager.tableCounter}`;
|
|
60
|
+
const csvFileName = `${LegendDataCubeDataCubeCacheManager.CSV_FILE_NAME}${LegendDataCubeDataCubeCacheManager.tableCounter}.csv`;
|
|
59
61
|
const connection = await this.database.connect();
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
result.
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
colType = 'BIT';
|
|
67
|
-
break;
|
|
68
|
-
}
|
|
69
|
-
case PRIMITIVE_TYPE.INTEGER: {
|
|
70
|
-
colType = 'INTEGER';
|
|
71
|
-
break;
|
|
72
|
-
}
|
|
73
|
-
case PRIMITIVE_TYPE.NUMBER:
|
|
74
|
-
case PRIMITIVE_TYPE.DECIMAL:
|
|
75
|
-
case PRIMITIVE_TYPE.FLOAT: {
|
|
76
|
-
colType = 'FLOAT';
|
|
77
|
-
break;
|
|
78
|
-
}
|
|
79
|
-
// We don't use type DATE because DuckDB will automatically convert it to a TIMESTAMP
|
|
80
|
-
case PRIMITIVE_TYPE.STRICTDATE:
|
|
81
|
-
case PRIMITIVE_TYPE.DATETIME:
|
|
82
|
-
case PRIMITIVE_TYPE.DATE: {
|
|
83
|
-
colType = 'VARCHAR';
|
|
84
|
-
break;
|
|
85
|
-
}
|
|
86
|
-
case PRIMITIVE_TYPE.STRING: {
|
|
87
|
-
colType = 'VARCHAR';
|
|
88
|
-
break;
|
|
89
|
-
}
|
|
90
|
-
default: {
|
|
91
|
-
throw new UnsupportedOperationError(`Can't initialize cache: failed to find matching DuckDB type for Pure type '${col.type}'`);
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
columns.push(`"${col.name}" ${colType}`);
|
|
62
|
+
const columnNames = [];
|
|
63
|
+
result.builder.columns.forEach((col) => columnNames.push(col.name));
|
|
64
|
+
const data = result.result.rows.map((row) => row.values);
|
|
65
|
+
const csv = csvStringify([columnNames, ...data], {
|
|
66
|
+
escapeChar: `'`,
|
|
67
|
+
quoteChar: `'`,
|
|
95
68
|
});
|
|
96
|
-
|
|
97
|
-
await connection.
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
}
|
|
107
|
-
return val;
|
|
108
|
-
});
|
|
109
|
-
rowString.push(`(${updatedRows.join(',')})`);
|
|
69
|
+
await this._database?.registerFileText(csvFileName, csv);
|
|
70
|
+
await connection.insertCSVFromPath(csvFileName, {
|
|
71
|
+
schema: schema,
|
|
72
|
+
name: table,
|
|
73
|
+
create: true,
|
|
74
|
+
header: true,
|
|
75
|
+
detect: true,
|
|
76
|
+
escape: `'`,
|
|
77
|
+
quote: `'`,
|
|
78
|
+
delimiter: ',',
|
|
110
79
|
});
|
|
111
|
-
const INSERT_TABLE_SQL = `INSERT INTO ${schema}.${table} VALUES ${rowString.join(',')}`;
|
|
112
|
-
await connection.query(INSERT_TABLE_SQL);
|
|
113
80
|
await connection.close();
|
|
114
81
|
return { table, schema, rowCount: result.result.rows.length };
|
|
115
82
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LegendDataCubeCacheManager.js","sourceRoot":"","sources":["../../src/stores/LegendDataCubeCacheManager.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,MAAM,MAAM,qBAAqB,CAAC;AAC9C,OAAO,WAAW,MAAM,0CAA0C,CAAC;AACnE,OAAO,gBAAgB,MAAM,yCAAyC,CAAC;AACvE,OAAO,EACL,mBAAmB,EACnB,cAAc,EACd,UAAU,EACV,kBAAkB,EAClB,MAAM,EACN,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,iBAAiB,EACjB,oBAAoB,EACpB,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEpC,MAAM,OAAO,kCAAkC;IACrC,MAAM,CAAU,0BAA0B,GAAG,MAAM,CAAC,CAAC,sDAAsD;IAC3G,MAAM,CAAU,iBAAiB,GAAG,OAAO,CAAC;IAC5C,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC;IAExB,SAAS,CAAkC;IAEnD,IAAY,QAAQ;QAClB,OAAO,oBAAoB,CACzB,IAAI,CAAC,SAAS,EACd,wCAAwC,CACzC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU;QACd,8BAA8B;QAC9B,2DAA2D;QAC3D,MAAM,cAAc,GAAyB;YAC3C,GAAG,EAAE;gBACH,mEAAmE;gBACnE,UAAU,EAAE,WAAW;gBACvB,UAAU,EAAE,IAAI,GAAG,CACjB,uDAAuD,EACvD,MAAM,CAAC,IAAI,CAAC,GAAG,CAChB,CAAC,QAAQ,EAAE;aACb;YACD,EAAE,EAAE;gBACF,mEAAmE;gBACnE,UAAU,EAAE,gBAAgB;gBAC5B,UAAU,EAAE,IAAI,GAAG,CACjB,sDAAsD,EACtD,MAAM,CAAC,IAAI,CAAC,GAAG,CAChB,CAAC,QAAQ,EAAE;aACb;SACF,CAAC;QACF,0CAA0C;QAC1C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC;QACzD,qDAAqD;QACrD,iBAAiB,CACf,MAAM,CAAC,UAAU,EACjB,oEAAoE,CACrE,CAAC;QACF,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,gCAAgC;QAClG,MAAM,QAAQ,GAAG,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACxD,MAAM,QAAQ,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;QACpE,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAA0B;QACpC,MAAM,MAAM,GACV,kCAAkC,CAAC,0BAA0B,CAAC;QAChE,kCAAkC,CAAC,YAAY,IAAI,CAAC,CAAC;QACrD,MAAM,KAAK,GAAG,GAAG,kCAAkC,CAAC,iBAAiB,GAAG,kCAAkC,CAAC,YAAY,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"LegendDataCubeCacheManager.js","sourceRoot":"","sources":["../../src/stores/LegendDataCubeCacheManager.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,MAAM,MAAM,qBAAqB,CAAC;AAC9C,OAAO,WAAW,MAAM,0CAA0C,CAAC;AACnE,OAAO,gBAAgB,MAAM,yCAAyC,CAAC;AACvE,OAAO,EACL,mBAAmB,EACnB,cAAc,EACd,UAAU,EACV,kBAAkB,EAClB,MAAM,EACN,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,iBAAiB,EACjB,YAAY,EACZ,oBAAoB,EACpB,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEpC,MAAM,OAAO,kCAAkC;IACrC,MAAM,CAAU,0BAA0B,GAAG,MAAM,CAAC,CAAC,sDAAsD;IAC3G,MAAM,CAAU,iBAAiB,GAAG,OAAO,CAAC;IAC5C,MAAM,CAAU,aAAa,GAAG,MAAM,CAAC;IACvC,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC;IAExB,SAAS,CAAkC;IAEnD,IAAY,QAAQ;QAClB,OAAO,oBAAoB,CACzB,IAAI,CAAC,SAAS,EACd,wCAAwC,CACzC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU;QACd,8BAA8B;QAC9B,2DAA2D;QAC3D,MAAM,cAAc,GAAyB;YAC3C,GAAG,EAAE;gBACH,mEAAmE;gBACnE,UAAU,EAAE,WAAW;gBACvB,UAAU,EAAE,IAAI,GAAG,CACjB,uDAAuD,EACvD,MAAM,CAAC,IAAI,CAAC,GAAG,CAChB,CAAC,QAAQ,EAAE;aACb;YACD,EAAE,EAAE;gBACF,mEAAmE;gBACnE,UAAU,EAAE,gBAAgB;gBAC5B,UAAU,EAAE,IAAI,GAAG,CACjB,sDAAsD,EACtD,MAAM,CAAC,IAAI,CAAC,GAAG,CAChB,CAAC,QAAQ,EAAE;aACb;SACF,CAAC;QACF,0CAA0C;QAC1C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC;QACzD,qDAAqD;QACrD,iBAAiB,CACf,MAAM,CAAC,UAAU,EACjB,oEAAoE,CACrE,CAAC;QACF,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,gCAAgC;QAClG,MAAM,QAAQ,GAAG,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACxD,MAAM,QAAQ,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;QACpE,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAA0B;QACpC,MAAM,MAAM,GACV,kCAAkC,CAAC,0BAA0B,CAAC;QAChE,kCAAkC,CAAC,YAAY,IAAI,CAAC,CAAC;QACrD,MAAM,KAAK,GAAG,GAAG,kCAAkC,CAAC,iBAAiB,GAAG,kCAAkC,CAAC,YAAY,EAAE,CAAC;QAC1H,MAAM,WAAW,GAAG,GAAG,kCAAkC,CAAC,aAAa,GAAG,kCAAkC,CAAC,YAAY,MAAM,CAAC;QAEhI,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QAEjD,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QAEpE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAEzD,MAAM,GAAG,GAAG,YAAY,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,EAAE;YAC/C,UAAU,EAAE,GAAG;YACf,SAAS,EAAE,GAAG;SACf,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,SAAS,EAAE,gBAAgB,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QAEzD,MAAM,UAAU,CAAC,iBAAiB,CAAC,WAAW,EAAE;YAC9C,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,KAAK;YACX,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,GAAG;YACX,KAAK,EAAE,GAAG;YACV,SAAS,EAAE,GAAG;SACf,CAAC,CAAC;QAEH,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;QAEzB,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;IAChE,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,GAAW;QAC3B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3C,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;QAEzB,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;QAC9B,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACpE,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAC5B,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YAC5B,MAAM,CAAC,MAAM,GAAG,WAAW,CAAC,GAAG,CAC7B,CAAC,MAAM,EAAE,EAAE;YACT,gFAAgF;YAChF,qCAAqC;YACrC,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC9B,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE;gBACvB,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAqC,CACvD,CAAC;YACF,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC,CAAC;QACH,MAAM,kBAAkB,GAAG,IAAI,kBAAkB,EAAE,CAAC;QACpD,MAAM,GAAG,GAAG,IAAI,cAAc,EAAE,CAAC;QACjC,GAAG,CAAC,OAAO,GAAG,WAAW,CAAC;QAC1B,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;QAChB,kBAAkB,CAAC,MAAM,GAAG,GAAG,CAAC;QAChC,MAAM,OAAO,GAAG,IAAI,UAAU,EAAE,CAAC;QACjC,OAAO,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YACnD,MAAM,GAAG,GAAG,IAAI,mBAAmB,EAAE,CAAC;YACtC,GAAG,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;YACtB,QAAQ,KAAK,CAAC,MAAM,EAAE,CAAC;gBACrB,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;oBACjB,GAAG,CAAC,IAAI,GAAG,cAAc,CAAC,MAAM,CAAC;oBACjC,MAAM;gBACR,CAAC;gBACD,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;oBACf,GAAG,CAAC,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC;oBAClC,MAAM;gBACR,CAAC;gBACD,KAAK,IAAI,CAAC,IAAI,CAAC;gBACf,KAAK,IAAI,CAAC,OAAO,CAAC;gBAClB,KAAK,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC;oBAC1B,GAAG,CAAC,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC;oBAC/B,MAAM;gBACR,CAAC;gBACD,KAAK,IAAI,CAAC,IAAI,CAAC;gBACf,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;oBACpB,GAAG,CAAC,IAAI,GAAG,cAAc,CAAC,MAAM,CAAC;oBACjC,MAAM;gBACR,CAAC;gBACD,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;oBAClB,GAAG,CAAC,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC;oBAClC,MAAM;gBACR,CAAC;gBACD,KAAK,IAAI,CAAC,GAAG,CAAC;gBACd,KAAK,IAAI,CAAC,IAAI,CAAC;gBACf,KAAK,IAAI,CAAC,KAAK,CAAC;gBAChB,KAAK,IAAI,CAAC,KAAK,CAAC;gBAChB,KAAK,IAAI,CAAC,MAAM,CAAC;gBACjB,KAAK,IAAI,CAAC,KAAK,CAAC;gBAChB,KAAK,IAAI,CAAC,MAAM,CAAC;gBACjB,KAAK,IAAI,CAAC,KAAK,CAAC;gBAChB,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;oBACjB,GAAG,CAAC,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC;oBAClC,MAAM;gBACR,CAAC;gBACD,KAAK,IAAI,CAAC,KAAK,CAAC;gBAChB,KAAK,IAAI,CAAC,OAAO,CAAC;gBAClB,KAAK,IAAI,CAAC,OAAO,CAAC;gBAClB,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;oBAClB,GAAG,CAAC,IAAI,GAAG,cAAc,CAAC,KAAK,CAAC;oBAChC,MAAM;gBACR,CAAC;gBACD,OAAO,CAAC,CAAC,CAAC;oBACR,MAAM,IAAI,yBAAyB,CACjC,oDAAoD,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,IAAI,EAAE,CACjF,CAAC;gBACJ,CAAC;YACH,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC,CAAC,CAAC;QACH,kBAAkB,CAAC,OAAO,GAAG,OAAO,CAAC;QACrC,OAAO,kBAAkB,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAA4B;QAC7C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QACjD,MAAM,cAAc,GAAG,yBAAyB,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC;QACjF,MAAM,UAAU,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QACvC,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAO;QACX,MAAM,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,CAAC;QACnC,MAAM,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,CAAC;IACpC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@finos/legend-application-data-cube",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.8",
|
|
4
4
|
"description": "Legend DataCube application core",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"legend",
|
|
@@ -43,15 +43,15 @@
|
|
|
43
43
|
},
|
|
44
44
|
"dependencies": {
|
|
45
45
|
"@duckdb/duckdb-wasm": "1.29.0",
|
|
46
|
-
"@finos/legend-application": "16.0.
|
|
47
|
-
"@finos/legend-art": "7.1.
|
|
48
|
-
"@finos/legend-code-editor": "2.0.
|
|
49
|
-
"@finos/legend-data-cube": "0.1.
|
|
50
|
-
"@finos/legend-graph": "32.1.
|
|
51
|
-
"@finos/legend-query-builder": "4.16.
|
|
52
|
-
"@finos/legend-server-depot": "6.0.
|
|
53
|
-
"@finos/legend-shared": "11.0.
|
|
54
|
-
"@finos/legend-storage": "3.0.
|
|
46
|
+
"@finos/legend-application": "16.0.25",
|
|
47
|
+
"@finos/legend-art": "7.1.82",
|
|
48
|
+
"@finos/legend-code-editor": "2.0.46",
|
|
49
|
+
"@finos/legend-data-cube": "0.1.6",
|
|
50
|
+
"@finos/legend-graph": "32.1.4",
|
|
51
|
+
"@finos/legend-query-builder": "4.16.6",
|
|
52
|
+
"@finos/legend-server-depot": "6.0.81",
|
|
53
|
+
"@finos/legend-shared": "11.0.4",
|
|
54
|
+
"@finos/legend-storage": "3.0.123",
|
|
55
55
|
"@types/react": "19.0.8",
|
|
56
56
|
"@types/react-dom": "19.0.3",
|
|
57
57
|
"apache-arrow": "19.0.0",
|
|
@@ -62,14 +62,14 @@
|
|
|
62
62
|
"serializr": "3.0.3"
|
|
63
63
|
},
|
|
64
64
|
"devDependencies": {
|
|
65
|
-
"@finos/legend-dev-utils": "2.2.
|
|
65
|
+
"@finos/legend-dev-utils": "2.2.1",
|
|
66
66
|
"@jest/globals": "29.7.0",
|
|
67
67
|
"cross-env": "7.0.3",
|
|
68
|
-
"eslint": "9.
|
|
68
|
+
"eslint": "9.20.0",
|
|
69
69
|
"jest": "29.7.0",
|
|
70
70
|
"npm-run-all": "4.1.5",
|
|
71
71
|
"rimraf": "6.0.1",
|
|
72
|
-
"sass": "1.
|
|
72
|
+
"sass": "1.84.0",
|
|
73
73
|
"typescript": "5.7.3"
|
|
74
74
|
},
|
|
75
75
|
"peerDependencies": {
|
|
@@ -27,6 +27,7 @@ import {
|
|
|
27
27
|
} from '@finos/legend-graph';
|
|
28
28
|
import {
|
|
29
29
|
assertNonNullable,
|
|
30
|
+
csvStringify,
|
|
30
31
|
guaranteeNonNullable,
|
|
31
32
|
UnsupportedOperationError,
|
|
32
33
|
} from '@finos/legend-shared';
|
|
@@ -36,6 +37,7 @@ import { Type } from 'apache-arrow';
|
|
|
36
37
|
export class LegendDataCubeDataCubeCacheManager {
|
|
37
38
|
private static readonly DUCKDB_DEFAULT_SCHEMA_NAME = 'main'; // See https://duckdb.org/docs/sql/statements/use.html
|
|
38
39
|
private static readonly TABLE_NAME_PREFIX = 'cache';
|
|
40
|
+
private static readonly CSV_FILE_NAME = 'data';
|
|
39
41
|
private static tableCounter = 0;
|
|
40
42
|
|
|
41
43
|
private _database?: duckdb.AsyncDuckDB | undefined;
|
|
@@ -87,68 +89,33 @@ export class LegendDataCubeDataCubeCacheManager {
|
|
|
87
89
|
LegendDataCubeDataCubeCacheManager.DUCKDB_DEFAULT_SCHEMA_NAME;
|
|
88
90
|
LegendDataCubeDataCubeCacheManager.tableCounter += 1;
|
|
89
91
|
const table = `${LegendDataCubeDataCubeCacheManager.TABLE_NAME_PREFIX}${LegendDataCubeDataCubeCacheManager.tableCounter}`;
|
|
92
|
+
const csvFileName = `${LegendDataCubeDataCubeCacheManager.CSV_FILE_NAME}${LegendDataCubeDataCubeCacheManager.tableCounter}.csv`;
|
|
90
93
|
|
|
91
94
|
const connection = await this.database.connect();
|
|
92
95
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
result.builder.columns.forEach((col) => {
|
|
96
|
-
let colType: string;
|
|
97
|
-
switch (col.type as string) {
|
|
98
|
-
case PRIMITIVE_TYPE.BOOLEAN: {
|
|
99
|
-
colType = 'BIT';
|
|
100
|
-
break;
|
|
101
|
-
}
|
|
102
|
-
case PRIMITIVE_TYPE.INTEGER: {
|
|
103
|
-
colType = 'INTEGER';
|
|
104
|
-
break;
|
|
105
|
-
}
|
|
106
|
-
case PRIMITIVE_TYPE.NUMBER:
|
|
107
|
-
case PRIMITIVE_TYPE.DECIMAL:
|
|
108
|
-
case PRIMITIVE_TYPE.FLOAT: {
|
|
109
|
-
colType = 'FLOAT';
|
|
110
|
-
break;
|
|
111
|
-
}
|
|
112
|
-
// We don't use type DATE because DuckDB will automatically convert it to a TIMESTAMP
|
|
113
|
-
case PRIMITIVE_TYPE.STRICTDATE:
|
|
114
|
-
case PRIMITIVE_TYPE.DATETIME:
|
|
115
|
-
case PRIMITIVE_TYPE.DATE: {
|
|
116
|
-
colType = 'VARCHAR';
|
|
117
|
-
break;
|
|
118
|
-
}
|
|
119
|
-
case PRIMITIVE_TYPE.STRING: {
|
|
120
|
-
colType = 'VARCHAR';
|
|
121
|
-
break;
|
|
122
|
-
}
|
|
123
|
-
default: {
|
|
124
|
-
throw new UnsupportedOperationError(
|
|
125
|
-
`Can't initialize cache: failed to find matching DuckDB type for Pure type '${col.type}'`,
|
|
126
|
-
);
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
columns.push(`"${col.name}" ${colType}`);
|
|
130
|
-
});
|
|
131
|
-
|
|
132
|
-
const CREATE_TABLE_SQL = `CREATE TABLE ${schema}.${table} (${columns.join(',')})`;
|
|
133
|
-
await connection.query(CREATE_TABLE_SQL);
|
|
96
|
+
const columnNames: string[] = [];
|
|
97
|
+
result.builder.columns.forEach((col) => columnNames.push(col.name));
|
|
134
98
|
|
|
135
|
-
const
|
|
99
|
+
const data = result.result.rows.map((row) => row.values);
|
|
136
100
|
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
return `'${val.replaceAll(`'`, `''`)}'`;
|
|
141
|
-
} else if (val === null) {
|
|
142
|
-
return `NULL`;
|
|
143
|
-
}
|
|
144
|
-
return val;
|
|
145
|
-
});
|
|
146
|
-
rowString.push(`(${updatedRows.join(',')})`);
|
|
101
|
+
const csv = csvStringify([columnNames, ...data], {
|
|
102
|
+
escapeChar: `'`,
|
|
103
|
+
quoteChar: `'`,
|
|
147
104
|
});
|
|
148
105
|
|
|
149
|
-
|
|
106
|
+
await this._database?.registerFileText(csvFileName, csv);
|
|
107
|
+
|
|
108
|
+
await connection.insertCSVFromPath(csvFileName, {
|
|
109
|
+
schema: schema,
|
|
110
|
+
name: table,
|
|
111
|
+
create: true,
|
|
112
|
+
header: true,
|
|
113
|
+
detect: true,
|
|
114
|
+
escape: `'`,
|
|
115
|
+
quote: `'`,
|
|
116
|
+
delimiter: ',',
|
|
117
|
+
});
|
|
150
118
|
|
|
151
|
-
await connection.query(INSERT_TABLE_SQL);
|
|
152
119
|
await connection.close();
|
|
153
120
|
|
|
154
121
|
return { table, schema, rowCount: result.result.rows.length };
|