@finos/legend-application-data-cube 0.6.14 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/components/LegendDataCubeWebApplication.d.ts.map +1 -1
- package/lib/components/LegendDataCubeWebApplication.js +1 -0
- package/lib/components/LegendDataCubeWebApplication.js.map +1 -1
- package/lib/components/builder/source/LakehouseConsumerDataCubeSourceBuilder.d.ts.map +1 -1
- package/lib/components/builder/source/LakehouseConsumerDataCubeSourceBuilder.js +6 -14
- package/lib/components/builder/source/LakehouseConsumerDataCubeSourceBuilder.js.map +1 -1
- package/lib/components/builder/source/LakehouseProducerDataCubeSourceBuilder.d.ts.map +1 -1
- package/lib/components/builder/source/LakehouseProducerDataCubeSourceBuilder.js +13 -4
- package/lib/components/builder/source/LakehouseProducerDataCubeSourceBuilder.js.map +1 -1
- package/lib/components/builder/source/LakehouseProducerDataCubeSourceLoader.d.ts.map +1 -1
- package/lib/components/builder/source/LakehouseProducerDataCubeSourceLoader.js +3 -0
- package/lib/components/builder/source/LakehouseProducerDataCubeSourceLoader.js.map +1 -1
- package/lib/index.css +1 -1
- package/lib/package.json +3 -2
- package/lib/stores/LegendDataCubeDataCubeEngine.d.ts +6 -1
- package/lib/stores/LegendDataCubeDataCubeEngine.d.ts.map +1 -1
- package/lib/stores/LegendDataCubeDataCubeEngine.js +150 -71
- package/lib/stores/LegendDataCubeDataCubeEngine.js.map +1 -1
- package/lib/stores/LegendDataCubeDuckDBEngine.d.ts +4 -2
- package/lib/stores/LegendDataCubeDuckDBEngine.d.ts.map +1 -1
- package/lib/stores/LegendDataCubeDuckDBEngine.js +52 -0
- package/lib/stores/LegendDataCubeDuckDBEngine.js.map +1 -1
- package/lib/stores/builder/source/LakehouseConsumerDataCubeSourceBuilderState.d.ts +2 -8
- package/lib/stores/builder/source/LakehouseConsumerDataCubeSourceBuilderState.d.ts.map +1 -1
- package/lib/stores/builder/source/LakehouseConsumerDataCubeSourceBuilderState.js +21 -54
- package/lib/stores/builder/source/LakehouseConsumerDataCubeSourceBuilderState.js.map +1 -1
- package/lib/stores/builder/source/LakehouseProducerDataCubeSourceBuilderState.d.ts +14 -1
- package/lib/stores/builder/source/LakehouseProducerDataCubeSourceBuilderState.d.ts.map +1 -1
- package/lib/stores/builder/source/LakehouseProducerDataCubeSourceBuilderState.js +73 -8
- package/lib/stores/builder/source/LakehouseProducerDataCubeSourceBuilderState.js.map +1 -1
- package/lib/stores/builder/source/LakehouseProducerDataCubeSourceLoaderState.d.ts +6 -2
- package/lib/stores/builder/source/LakehouseProducerDataCubeSourceLoaderState.d.ts.map +1 -1
- package/lib/stores/builder/source/LakehouseProducerDataCubeSourceLoaderState.js +25 -5
- package/lib/stores/builder/source/LakehouseProducerDataCubeSourceLoaderState.js.map +1 -1
- package/lib/stores/model/LakehouseConsumerDataCubeSource.d.ts +2 -2
- package/lib/stores/model/LakehouseConsumerDataCubeSource.d.ts.map +1 -1
- package/lib/stores/model/LakehouseConsumerDataCubeSource.js.map +1 -1
- package/lib/stores/model/LakehouseProducerDataCubeSource.d.ts +13 -0
- package/lib/stores/model/LakehouseProducerDataCubeSource.d.ts.map +1 -1
- package/lib/stores/model/LakehouseProducerDataCubeSource.js +19 -2
- package/lib/stores/model/LakehouseProducerDataCubeSource.js.map +1 -1
- package/lib/stores/model/SecondaryOauthClient.d.ts +26 -0
- package/lib/stores/model/SecondaryOauthClient.d.ts.map +1 -0
- package/lib/stores/model/SecondaryOauthClient.js +48 -0
- package/lib/stores/model/SecondaryOauthClient.js.map +1 -0
- package/package.json +13 -12
- package/src/components/LegendDataCubeWebApplication.tsx +1 -0
- package/src/components/builder/source/LakehouseConsumerDataCubeSourceBuilder.tsx +7 -32
- package/src/components/builder/source/LakehouseProducerDataCubeSourceBuilder.tsx +40 -14
- package/src/components/builder/source/LakehouseProducerDataCubeSourceLoader.tsx +4 -0
- package/src/stores/LegendDataCubeDataCubeEngine.ts +201 -80
- package/src/stores/LegendDataCubeDuckDBEngine.ts +73 -1
- package/src/stores/builder/source/LakehouseConsumerDataCubeSourceBuilderState.ts +31 -74
- package/src/stores/builder/source/LakehouseProducerDataCubeSourceBuilderState.ts +151 -10
- package/src/stores/builder/source/LakehouseProducerDataCubeSourceLoaderState.ts +56 -11
- package/src/stores/model/LakehouseConsumerDataCubeSource.ts +2 -2
- package/src/stores/model/LakehouseProducerDataCubeSource.ts +26 -1
- package/src/stores/model/SecondaryOauthClient.ts +56 -0
- package/tsconfig.json +1 -0
|
@@ -19,7 +19,8 @@ import type { DataCubeAlertService } from '@finos/legend-data-cube';
|
|
|
19
19
|
import type { LegendDataCubeApplicationStore } from '../../LegendDataCubeBaseStore.js';
|
|
20
20
|
import type { LegendDataCubeDataCubeEngine } from '../../LegendDataCubeDataCubeEngine.js';
|
|
21
21
|
import { type LakehouseIngestServerClient, type LakehousePlatformServerClient } from '@finos/legend-server-lakehouse';
|
|
22
|
-
import type
|
|
22
|
+
import { type V1_IngestDefinition } from '@finos/legend-graph';
|
|
23
|
+
import type { UserManagerSettings } from 'oidc-client-ts';
|
|
23
24
|
export declare class LakehouseProducerDataCubeSourceBuilderState extends LegendDataCubeSourceBuilderState {
|
|
24
25
|
deploymentId: number | undefined;
|
|
25
26
|
warehouse: string | undefined;
|
|
@@ -31,6 +32,13 @@ export declare class LakehouseProducerDataCubeSourceBuilderState extends LegendD
|
|
|
31
32
|
ingestUrns: string[];
|
|
32
33
|
tables: string[];
|
|
33
34
|
datasetGroup: string | undefined;
|
|
35
|
+
icebergEnabled: boolean | undefined;
|
|
36
|
+
enableIceberg: boolean;
|
|
37
|
+
databaseName: string | undefined;
|
|
38
|
+
catalogUrl: string | undefined;
|
|
39
|
+
milestoning: boolean;
|
|
40
|
+
userManagerSettings: UserManagerSettings | undefined;
|
|
41
|
+
private LAKEHOUSE_SECTION;
|
|
34
42
|
readonly _platformServerClient: LakehousePlatformServerClient;
|
|
35
43
|
readonly _ingestServerClient: LakehouseIngestServerClient;
|
|
36
44
|
constructor(application: LegendDataCubeApplicationStore, engine: LegendDataCubeDataCubeEngine, platformServerClient: LakehousePlatformServerClient, ingestServerClient: LakehouseIngestServerClient, alertService: DataCubeAlertService);
|
|
@@ -41,10 +49,15 @@ export declare class LakehouseProducerDataCubeSourceBuilderState extends LegendD
|
|
|
41
49
|
setTables(tables: string[]): void;
|
|
42
50
|
setDatasetGroup(datasetGroup: string | undefined): void;
|
|
43
51
|
setSelectedTable(selectedTable: string | undefined): void;
|
|
52
|
+
setEnableIceberg(enable: boolean): void;
|
|
53
|
+
setUserManagerSettings(settings: UserManagerSettings): void;
|
|
44
54
|
fetchIngestUrns(access_token: string | undefined): Promise<void>;
|
|
55
|
+
private fetchProducerEnvironmentDetails;
|
|
45
56
|
fetchDatasets(access_token: string | undefined): Promise<void>;
|
|
46
57
|
decoratedIngest(ingestUrn: string): string | undefined;
|
|
58
|
+
fetchIcebergCatalogDetails(access_token: string | undefined): Promise<void>;
|
|
47
59
|
createPath(): void;
|
|
60
|
+
createIcebergPath(): void;
|
|
48
61
|
reset(): void;
|
|
49
62
|
resetDeployment(deploymentId: number | undefined): void;
|
|
50
63
|
get label(): LegendDataCubeSourceBuilderType;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LakehouseProducerDataCubeSourceBuilderState.d.ts","sourceRoot":"","sources":["../../../../src/stores/builder/source/LakehouseProducerDataCubeSourceBuilderState.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,EACL,gCAAgC,EAChC,+BAA+B,EAChC,MAAM,uCAAuC,CAAC;AAC/C,OAAO,
|
|
1
|
+
{"version":3,"file":"LakehouseProducerDataCubeSourceBuilderState.d.ts","sourceRoot":"","sources":["../../../../src/stores/builder/source/LakehouseProducerDataCubeSourceBuilderState.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,EACL,gCAAgC,EAChC,+BAA+B,EAChC,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EAGL,KAAK,WAAW,EACjB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AACpE,OAAO,KAAK,EAAE,8BAA8B,EAAE,MAAM,kCAAkC,CAAC;AACvF,OAAO,KAAK,EAAE,4BAA4B,EAAE,MAAM,uCAAuC,CAAC;AAK1F,OAAO,EAGL,KAAK,2BAA2B,EAChC,KAAK,6BAA6B,EACnC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAML,KAAK,mBAAmB,EAGzB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAG1D,qBAAa,2CAA4C,SAAQ,gCAAgC;IAC/F,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,gBAAgB,EAAE,WAAW,CAAC,mBAAmB,CAAC,GAAG,SAAS,CAAC;IAC/D,iBAAiB,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,aAAa,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,UAAU,EAAE,MAAM,EAAE,CAAM;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAM;IACtB,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,cAAc,EAAE,OAAO,GAAG,SAAS,CAAC;IACpC,aAAa,EAAE,OAAO,CAAC;IACvB,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,WAAW,EAAE,OAAO,CAAC;IAErB,mBAAmB,EAAE,mBAAmB,GAAG,SAAS,CAAC;IAErD,OAAO,CAAC,iBAAiB,CAAkB;IAE3C,QAAQ,CAAC,qBAAqB,EAAE,6BAA6B,CAAC;IAC9D,QAAQ,CAAC,mBAAmB,EAAE,2BAA2B,CAAC;gBAGxD,WAAW,EAAE,8BAA8B,EAC3C,MAAM,EAAE,4BAA4B,EACpC,oBAAoB,EAAE,6BAA6B,EACnD,kBAAkB,EAAE,2BAA2B,EAC/C,YAAY,EAAE,oBAAoB;IAmCpC,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAKvD,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAIjD,oBAAoB,CAAC,iBAAiB,EAAE,MAAM,GAAG,SAAS;IAI1D,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE;IAIlC,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE;IAI1B,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS;IAIhD,gBAAgB,CAAC,aAAa,EAAE,MAAM,GAAG,SAAS;IAIlD,gBAAgB,CAAC,MAAM,EAAE,OAAO;IAIhC,sBAAsB,CAAC,QAAQ,EAAE,mBAAmB;IAI9C,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS;YA0CxC,+BAA+B;IA0BvC,aAAa,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS;IAuCpD,eAAe,CAAC,SAAS,EAAE,MAAM;IAI3B,0BAA0B,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS;IAqBjE,UAAU;IAUV,iBAAiB;IAWjB,KAAK;IAUL,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS;IAShD,IAAa,KAAK,IAAI,+BAA+B,CAEpD;IAED,IAAa,OAAO,IAAI,OAAO,CAO9B;IAEc,kBAAkB,IAAI,OAAO,CAAC,WAAW,CAAC;CA0C1D"}
|
|
@@ -15,9 +15,11 @@
|
|
|
15
15
|
*/
|
|
16
16
|
import { action, makeObservable, observable } from 'mobx';
|
|
17
17
|
import { LegendDataCubeSourceBuilderState, LegendDataCubeSourceBuilderType, } from './LegendDataCubeSourceBuilderState.js';
|
|
18
|
-
import { guaranteeNonNullable } from '@finos/legend-shared';
|
|
19
|
-
import { RawLakehouseProducerDataCubeSource } from '../../model/LakehouseProducerDataCubeSource.js';
|
|
18
|
+
import { guaranteeNonNullable, guaranteeType, } from '@finos/legend-shared';
|
|
19
|
+
import { IcebergConfig, RawLakehouseProducerDataCubeSource, } from '../../model/LakehouseProducerDataCubeSource.js';
|
|
20
20
|
import { IngestDeploymentServerConfig, ProducerEnvironment, } from '@finos/legend-server-lakehouse';
|
|
21
|
+
import { V1_AWSSnowflakeIngestEnvironment, V1_AWSSnowflakeProducerEnvironment, V1_deserializeIngestEnvironment, V1_deserializeProducerEnvironment, V1_OpenCatalog, V1_deserializePureModelContext, V1_PureModelContextData, } from '@finos/legend-graph';
|
|
22
|
+
import { SecondaryOAuthClient } from '../../model/SecondaryOauthClient.js';
|
|
21
23
|
export class LakehouseProducerDataCubeSourceBuilderState extends LegendDataCubeSourceBuilderState {
|
|
22
24
|
deploymentId;
|
|
23
25
|
warehouse;
|
|
@@ -29,6 +31,13 @@ export class LakehouseProducerDataCubeSourceBuilderState extends LegendDataCubeS
|
|
|
29
31
|
ingestUrns = [];
|
|
30
32
|
tables = [];
|
|
31
33
|
datasetGroup;
|
|
34
|
+
icebergEnabled;
|
|
35
|
+
enableIceberg;
|
|
36
|
+
databaseName;
|
|
37
|
+
catalogUrl;
|
|
38
|
+
milestoning;
|
|
39
|
+
userManagerSettings;
|
|
40
|
+
LAKEHOUSE_SECTION = '###Lakehouse';
|
|
32
41
|
_platformServerClient;
|
|
33
42
|
_ingestServerClient;
|
|
34
43
|
constructor(application, engine, platformServerClient, ingestServerClient, alertService) {
|
|
@@ -43,6 +52,8 @@ export class LakehouseProducerDataCubeSourceBuilderState extends LegendDataCubeS
|
|
|
43
52
|
tables: observable,
|
|
44
53
|
datasetGroup: observable,
|
|
45
54
|
selectedTable: observable,
|
|
55
|
+
icebergEnabled: observable,
|
|
56
|
+
enableIceberg: observable,
|
|
46
57
|
setDeploymentId: action,
|
|
47
58
|
setSelectedIngestUrn: action,
|
|
48
59
|
setWarehouse: action,
|
|
@@ -50,11 +61,14 @@ export class LakehouseProducerDataCubeSourceBuilderState extends LegendDataCubeS
|
|
|
50
61
|
setTables: action,
|
|
51
62
|
setDatasetGroup: action,
|
|
52
63
|
setSelectedTable: action,
|
|
64
|
+
setEnableIceberg: action,
|
|
53
65
|
});
|
|
54
66
|
this.selectedIngestUrn = '';
|
|
55
67
|
this.selectedTable = '';
|
|
56
68
|
this.warehouse = undefined;
|
|
57
69
|
this.paths = [];
|
|
70
|
+
this.enableIceberg = false;
|
|
71
|
+
this.milestoning = false;
|
|
58
72
|
}
|
|
59
73
|
setDeploymentId(deploymentId) {
|
|
60
74
|
this.deploymentId = deploymentId;
|
|
@@ -78,6 +92,12 @@ export class LakehouseProducerDataCubeSourceBuilderState extends LegendDataCubeS
|
|
|
78
92
|
setSelectedTable(selectedTable) {
|
|
79
93
|
this.selectedTable = selectedTable;
|
|
80
94
|
}
|
|
95
|
+
setEnableIceberg(enable) {
|
|
96
|
+
this.enableIceberg = enable;
|
|
97
|
+
}
|
|
98
|
+
setUserManagerSettings(settings) {
|
|
99
|
+
this.userManagerSettings = settings;
|
|
100
|
+
}
|
|
81
101
|
async fetchIngestUrns(access_token) {
|
|
82
102
|
//TODO: we should retry this method if access token is invalid
|
|
83
103
|
this.resetDeployment(this.deploymentId);
|
|
@@ -87,6 +107,7 @@ export class LakehouseProducerDataCubeSourceBuilderState extends LegendDataCubeS
|
|
|
87
107
|
this.ingestionServerUrl = ingestServerUrl;
|
|
88
108
|
const producerUrn = await this._ingestServerClient.getProducerEnvironment(guaranteeNonNullable(this.deploymentId), ingestServerUrl, access_token);
|
|
89
109
|
const producer = ProducerEnvironment.serialization.fromJson(producerUrn);
|
|
110
|
+
await this.fetchProducerEnvironmentDetails(producer, ingestServerUrl, access_token);
|
|
90
111
|
const ingestDefinitions = await this._ingestServerClient.getIngestDefinitions(producer.producerEnvironmentUrn, ingestServerUrl, access_token);
|
|
91
112
|
this.setIngestUrns(ingestDefinitions);
|
|
92
113
|
return;
|
|
@@ -95,12 +116,24 @@ export class LakehouseProducerDataCubeSourceBuilderState extends LegendDataCubeS
|
|
|
95
116
|
throw error;
|
|
96
117
|
}
|
|
97
118
|
}
|
|
119
|
+
async fetchProducerEnvironmentDetails(producer, ingestServerUrl, access_token) {
|
|
120
|
+
const producerEnvPlainObject = await this._ingestServerClient.getProducerEnvironmentDetails(producer.producerEnvironmentUrn, ingestServerUrl, access_token);
|
|
121
|
+
const producerEnv = guaranteeType(V1_deserializeProducerEnvironment(producerEnvPlainObject), V1_AWSSnowflakeProducerEnvironment);
|
|
122
|
+
this.icebergEnabled = producerEnv.icebergEnabled;
|
|
123
|
+
if (this.icebergEnabled) {
|
|
124
|
+
this.setEnableIceberg(this.icebergEnabled);
|
|
125
|
+
await this.fetchIcebergCatalogDetails(access_token);
|
|
126
|
+
}
|
|
127
|
+
this.databaseName = producerEnv.databaseName;
|
|
128
|
+
}
|
|
98
129
|
async fetchDatasets(access_token) {
|
|
99
130
|
this.setTables([]);
|
|
100
131
|
this.setSelectedTable(undefined);
|
|
101
132
|
try {
|
|
102
|
-
const
|
|
103
|
-
|
|
133
|
+
const ingestGrammar = await this._ingestServerClient.getIngestDefinitionGrammar(guaranteeNonNullable(this.selectedIngestUrn), this.ingestionServerUrl, access_token);
|
|
134
|
+
const ingestPMCDPlainObject = await this._engine.parseCompatibleModel(`${this.LAKEHOUSE_SECTION}\n${ingestGrammar}`);
|
|
135
|
+
const ingestDefPMCD = guaranteeType(V1_deserializePureModelContext(ingestPMCDPlainObject), V1_PureModelContextData);
|
|
136
|
+
this.ingestDefinition = ingestDefPMCD.elements.at(0).content;
|
|
104
137
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
105
138
|
this.setDatasetGroup(this.ingestDefinition.datasetGroup);
|
|
106
139
|
this.setTables(
|
|
@@ -116,10 +149,27 @@ export class LakehouseProducerDataCubeSourceBuilderState extends LegendDataCubeS
|
|
|
116
149
|
decoratedIngest(ingestUrn) {
|
|
117
150
|
return ingestUrn.split('~').pop();
|
|
118
151
|
}
|
|
152
|
+
async fetchIcebergCatalogDetails(access_token) {
|
|
153
|
+
try {
|
|
154
|
+
const ingestEnvPlainObject = await this._ingestServerClient.getIngestEnvironment(this.ingestionServerUrl, access_token);
|
|
155
|
+
const ingestEnv = guaranteeType(V1_deserializeIngestEnvironment(ingestEnvPlainObject), V1_AWSSnowflakeIngestEnvironment);
|
|
156
|
+
this.warehouse = ingestEnv.iceberg.catalog.name;
|
|
157
|
+
this.catalogUrl = guaranteeType(ingestEnv.iceberg.catalog, V1_OpenCatalog).proxyUrl;
|
|
158
|
+
}
|
|
159
|
+
catch (error) {
|
|
160
|
+
throw error;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
119
163
|
createPath() {
|
|
120
164
|
this.paths = [];
|
|
121
165
|
this.paths.push(guaranteeNonNullable(this.decoratedIngest(guaranteeNonNullable(this.selectedIngestUrn))), guaranteeNonNullable(this.selectedTable));
|
|
122
166
|
}
|
|
167
|
+
createIcebergPath() {
|
|
168
|
+
this.paths = [];
|
|
169
|
+
this.paths.push(guaranteeNonNullable(this.databaseName), guaranteeNonNullable(this.datasetGroup), this.milestoning
|
|
170
|
+
? `${guaranteeNonNullable(this.selectedTable)}_MILESTONED`
|
|
171
|
+
: guaranteeNonNullable(this.selectedTable));
|
|
172
|
+
}
|
|
123
173
|
reset() {
|
|
124
174
|
this.setDeploymentId(undefined);
|
|
125
175
|
this.setIngestUrns([]);
|
|
@@ -146,12 +196,27 @@ export class LakehouseProducerDataCubeSourceBuilderState extends LegendDataCubeS
|
|
|
146
196
|
Boolean(this.selectedTable) &&
|
|
147
197
|
Boolean(this.deploymentId));
|
|
148
198
|
}
|
|
149
|
-
generateSourceData() {
|
|
199
|
+
async generateSourceData() {
|
|
150
200
|
// register ingest definition
|
|
151
|
-
this._engine.registerIngestDefinition(this.ingestDefinition);
|
|
152
|
-
// build data cube source
|
|
153
|
-
this.createPath();
|
|
154
201
|
const rawSource = new RawLakehouseProducerDataCubeSource();
|
|
202
|
+
// build data cube source
|
|
203
|
+
if (this.enableIceberg) {
|
|
204
|
+
const oauthClient = new SecondaryOAuthClient(guaranteeNonNullable(this.userManagerSettings));
|
|
205
|
+
this.milestoning =
|
|
206
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
207
|
+
this.ingestDefinition.writeMode._type === 'batch_milestoned';
|
|
208
|
+
this.createIcebergPath();
|
|
209
|
+
const icebergConfig = new IcebergConfig();
|
|
210
|
+
icebergConfig.catalogUrl = guaranteeNonNullable(this.catalogUrl);
|
|
211
|
+
const token = await oauthClient.getToken();
|
|
212
|
+
const refId = await this._engine.ingestIcebergTable(guaranteeNonNullable(this.warehouse), this.paths, guaranteeNonNullable(this.catalogUrl), undefined, token);
|
|
213
|
+
icebergConfig.icebergRef = refId.dbReference;
|
|
214
|
+
rawSource.icebergConfig = icebergConfig;
|
|
215
|
+
}
|
|
216
|
+
else {
|
|
217
|
+
this.createPath();
|
|
218
|
+
this._engine.registerIngestDefinition(this.ingestDefinition);
|
|
219
|
+
}
|
|
155
220
|
rawSource.ingestDefinitionUrn = guaranteeNonNullable(this.selectedIngestUrn);
|
|
156
221
|
rawSource.ingestServerUrl = guaranteeNonNullable(this.ingestionServerUrl);
|
|
157
222
|
rawSource.paths = this.paths;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LakehouseProducerDataCubeSourceBuilderState.js","sourceRoot":"","sources":["../../../../src/stores/builder/source/LakehouseProducerDataCubeSourceBuilderState.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,MAAM,CAAC;AAC1D,OAAO,EACL,gCAAgC,EAChC,+BAA+B,GAChC,MAAM,uCAAuC,CAAC;AAC/C,OAAO,
|
|
1
|
+
{"version":3,"file":"LakehouseProducerDataCubeSourceBuilderState.js","sourceRoot":"","sources":["../../../../src/stores/builder/source/LakehouseProducerDataCubeSourceBuilderState.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,MAAM,CAAC;AAC1D,OAAO,EACL,gCAAgC,EAChC,+BAA+B,GAChC,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EACL,oBAAoB,EACpB,aAAa,GAEd,MAAM,sBAAsB,CAAC;AAI9B,OAAO,EACL,aAAa,EACb,kCAAkC,GACnC,MAAM,gDAAgD,CAAC;AACxD,OAAO,EACL,4BAA4B,EAC5B,mBAAmB,GAGpB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EACL,gCAAgC,EAChC,kCAAkC,EAClC,+BAA+B,EAC/B,iCAAiC,EACjC,cAAc,EAEd,8BAA8B,EAC9B,uBAAuB,GACxB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,oBAAoB,EAAE,MAAM,qCAAqC,CAAC;AAE3E,MAAM,OAAO,2CAA4C,SAAQ,gCAAgC;IAC/F,YAAY,CAAqB;IACjC,SAAS,CAAqB;IAC9B,gBAAgB,CAA+C;IAC/D,iBAAiB,CAAqB;IACtC,kBAAkB,CAAqB;IACvC,aAAa,CAAqB;IAClC,KAAK,CAAW;IAChB,UAAU,GAAa,EAAE,CAAC;IAC1B,MAAM,GAAa,EAAE,CAAC;IACtB,YAAY,CAAqB;IACjC,cAAc,CAAsB;IACpC,aAAa,CAAU;IACvB,YAAY,CAAqB;IACjC,UAAU,CAAqB;IAC/B,WAAW,CAAU;IAErB,mBAAmB,CAAkC;IAE7C,iBAAiB,GAAG,cAAc,CAAC;IAElC,qBAAqB,CAAgC;IACrD,mBAAmB,CAA8B;IAE1D,YACE,WAA2C,EAC3C,MAAoC,EACpC,oBAAmD,EACnD,kBAA+C,EAC/C,YAAkC;QAElC,KAAK,CAAC,WAAW,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;QACzC,IAAI,CAAC,qBAAqB,GAAG,oBAAoB,CAAC;QAClD,IAAI,CAAC,mBAAmB,GAAG,kBAAkB,CAAC;QAE9C,cAAc,CAAC,IAAI,EAAE;YACnB,YAAY,EAAE,UAAU;YACxB,SAAS,EAAE,UAAU;YACrB,UAAU,EAAE,UAAU;YACtB,iBAAiB,EAAE,UAAU;YAC7B,MAAM,EAAE,UAAU;YAClB,YAAY,EAAE,UAAU;YACxB,aAAa,EAAE,UAAU;YACzB,cAAc,EAAE,UAAU;YAC1B,aAAa,EAAE,UAAU;YAEzB,eAAe,EAAE,MAAM;YACvB,oBAAoB,EAAE,MAAM;YAC5B,YAAY,EAAE,MAAM;YACpB,aAAa,EAAE,MAAM;YACrB,SAAS,EAAE,MAAM;YACjB,eAAe,EAAE,MAAM;YACvB,gBAAgB,EAAE,MAAM;YACxB,gBAAgB,EAAE,MAAM;SACzB,CAAC,CAAC;QAEH,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;QACxB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;QAC3B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED,eAAe,CAAC,YAAgC;QAC9C,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,YAAY,CAAC,sBAAsB,YAAY,WAAW,CAAC,CAAC;IACnE,CAAC;IAED,YAAY,CAAC,SAA6B;QACxC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED,oBAAoB,CAAC,iBAAqC;QACxD,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;IAC7C,CAAC;IAED,aAAa,CAAC,UAAoB;QAChC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED,SAAS,CAAC,MAAgB;QACxB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,eAAe,CAAC,YAAgC;QAC9C,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAED,gBAAgB,CAAC,aAAiC;QAChD,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACrC,CAAC;IAED,gBAAgB,CAAC,MAAe;QAC9B,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC;IAC9B,CAAC;IAED,sBAAsB,CAAC,QAA6B;QAClD,IAAI,CAAC,mBAAmB,GAAG,QAAQ,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,YAAgC;QACpD,8DAA8D;QAC9D,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACxC,IAAI,CAAC;YACH,MAAM,cAAc,GAClB,MAAM,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,CACjD,oBAAoB,CAAC,IAAI,CAAC,YAAY,CAAC,EACvC,YAAY,EACZ,YAAY,CACb,CAAC;YACJ,MAAM,eAAe,GACnB,4BAA4B,CAAC,aAAa,CAAC,QAAQ,CACjD,cAAc,CACf,CAAC,eAAe,CAAC;YACpB,IAAI,CAAC,kBAAkB,GAAG,eAAe,CAAC;YAE1C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,sBAAsB,CACvE,oBAAoB,CAAC,IAAI,CAAC,YAAY,CAAC,EACvC,eAAe,EACf,YAAY,CACb,CAAC;YACF,MAAM,QAAQ,GAAG,mBAAmB,CAAC,aAAa,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAEzE,MAAM,IAAI,CAAC,+BAA+B,CACxC,QAAQ,EACR,eAAe,EACf,YAAY,CACb,CAAC;YAEF,MAAM,iBAAiB,GACrB,MAAM,IAAI,CAAC,mBAAmB,CAAC,oBAAoB,CACjD,QAAQ,CAAC,sBAAsB,EAC/B,eAAe,EACf,YAAY,CACb,CAAC;YACJ,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC;YACtC,OAAO;QACT,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,+BAA+B,CAC3C,QAA6B,EAC7B,eAAuB,EACvB,YAAgC;QAEhC,MAAM,sBAAsB,GAC1B,MAAM,IAAI,CAAC,mBAAmB,CAAC,6BAA6B,CAC1D,QAAQ,CAAC,sBAAsB,EAC/B,eAAe,EACf,YAAY,CACb,CAAC;QAEJ,MAAM,WAAW,GAAG,aAAa,CAC/B,iCAAiC,CAAC,sBAAsB,CAAC,EACzD,kCAAkC,CACnC,CAAC;QACF,IAAI,CAAC,cAAc,GAAG,WAAW,CAAC,cAAc,CAAC;QAEjD,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAC3C,MAAM,IAAI,CAAC,0BAA0B,CAAC,YAAY,CAAC,CAAC;QACtD,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC,YAAY,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,YAAgC;QAClD,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACnB,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QACjC,IAAI,CAAC;YACH,MAAM,aAAa,GACjB,MAAM,IAAI,CAAC,mBAAmB,CAAC,0BAA0B,CACvD,oBAAoB,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAC5C,IAAI,CAAC,kBAAkB,EACvB,YAAY,CACb,CAAC;YAEJ,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,oBAAoB,CACnE,GAAG,IAAI,CAAC,iBAAiB,KAAK,aAAa,EAAE,CAC9C,CAAC;YAEF,MAAM,aAAa,GAAG,aAAa,CACjC,8BAA8B,CAAC,qBAAqB,CAAC,EACrD,uBAAuB,CACxB,CAAC;YAEF,IAAI,CAAC,gBAAgB,GACnB,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAC5B,CAAC,OAAO,CAAC;YAEV,8DAA8D;YAC9D,IAAI,CAAC,eAAe,CAAE,IAAI,CAAC,gBAAwB,CAAC,YAAY,CAAC,CAAC;YAElE,IAAI,CAAC,SAAS;YACZ,8DAA8D;YAC7D,IAAI,CAAC,gBAAwB,CAAC,QAAQ,CAAC,GAAG;YACzC,8DAA8D;YAC9D,CAAC,OAAY,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAC/B,CACF,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,eAAe,CAAC,SAAiB;QAC/B,OAAO,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,0BAA0B,CAAC,YAAgC;QAC/D,IAAI,CAAC;YACH,MAAM,oBAAoB,GACxB,MAAM,IAAI,CAAC,mBAAmB,CAAC,oBAAoB,CACjD,IAAI,CAAC,kBAAkB,EACvB,YAAY,CACb,CAAC;YACJ,MAAM,SAAS,GAAG,aAAa,CAC7B,+BAA+B,CAAC,oBAAoB,CAAC,EACrD,gCAAgC,CACjC,CAAC;YACF,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;YAChD,IAAI,CAAC,UAAU,GAAG,aAAa,CAC7B,SAAS,CAAC,OAAO,CAAC,OAAO,EACzB,cAAc,CACf,CAAC,QAAQ,CAAC;QACb,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,UAAU;QACR,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC,KAAK,CAAC,IAAI,CACb,oBAAoB,CAClB,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CACnE,EACD,oBAAoB,CAAC,IAAI,CAAC,aAAa,CAAC,CACzC,CAAC;IACJ,CAAC;IAED,iBAAiB;QACf,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC,KAAK,CAAC,IAAI,CACb,oBAAoB,CAAC,IAAI,CAAC,YAAY,CAAC,EACvC,oBAAoB,CAAC,IAAI,CAAC,YAAY,CAAC,EACvC,IAAI,CAAC,WAAW;YACd,CAAC,CAAC,GAAG,oBAAoB,CAAC,IAAI,CAAC,aAAa,CAAC,aAAa;YAC1D,CAAC,CAAC,oBAAoB,CAAC,IAAI,CAAC,aAAa,CAAC,CAC7C,CAAC;IACJ,CAAC;IAED,KAAK;QACH,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;QAChC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QACvB,IAAI,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;QACrC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACnB,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QACjC,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;QAChC,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;IAC/B,CAAC;IAED,eAAe,CAAC,YAAgC;QAC9C,IAAI,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;QACnC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QACvB,IAAI,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;QACrC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACnB,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;QAChC,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;IAED,IAAa,KAAK;QAChB,OAAO,+BAA+B,CAAC,kBAAkB,CAAC;IAC5D,CAAC;IAED,IAAa,OAAO;QAClB,OAAO,CACL,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC;YAC/B,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC;YAC3B,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAC3B,CAAC;IACJ,CAAC;IAEQ,KAAK,CAAC,kBAAkB;QAC/B,6BAA6B;QAE7B,MAAM,SAAS,GAAG,IAAI,kCAAkC,EAAE,CAAC;QAC3D,yBAAyB;QACzB,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,MAAM,WAAW,GAAG,IAAI,oBAAoB,CAC1C,oBAAoB,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAC/C,CAAC;YACF,IAAI,CAAC,WAAW;gBACd,8DAA8D;gBAC7D,IAAI,CAAC,gBAAwB,CAAC,SAAS,CAAC,KAAK,KAAK,kBAAkB,CAAC;YACxE,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACzB,MAAM,aAAa,GAAG,IAAI,aAAa,EAAE,CAAC;YAC1C,aAAa,CAAC,UAAU,GAAG,oBAAoB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAEjE,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,QAAQ,EAAE,CAAC;YAE3C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CACjD,oBAAoB,CAAC,IAAI,CAAC,SAAS,CAAC,EACpC,IAAI,CAAC,KAAK,EACV,oBAAoB,CAAC,IAAI,CAAC,UAAU,CAAC,EACrC,SAAS,EACT,KAAK,CACN,CAAC;YACF,aAAa,CAAC,UAAU,GAAG,KAAK,CAAC,WAAW,CAAC;YAC7C,SAAS,CAAC,aAAa,GAAG,aAAa,CAAC;QAC1C,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,IAAI,CAAC,OAAO,CAAC,wBAAwB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC/D,CAAC;QACD,SAAS,CAAC,mBAAmB,GAAG,oBAAoB,CAClD,IAAI,CAAC,iBAAiB,CACvB,CAAC;QACF,SAAS,CAAC,eAAe,GAAG,oBAAoB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC1E,SAAS,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QAC7B,SAAS,CAAC,SAAS,GAAG,oBAAoB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAE3D,OAAO,OAAO,CAAC,OAAO,CACpB,kCAAkC,CAAC,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,CACnE,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -18,17 +18,21 @@ import type { LegendDataCubeApplicationStore } from '../../LegendDataCubeBaseSto
|
|
|
18
18
|
import type { LegendDataCubeDataCubeEngine } from '../../LegendDataCubeDataCubeEngine.js';
|
|
19
19
|
import { LegendDataCubeSourceLoaderState } from './LegendDataCubeSourceLoaderState.js';
|
|
20
20
|
import type { DataCubeAlertService } from '@finos/legend-data-cube';
|
|
21
|
-
import type
|
|
21
|
+
import { type PersistentDataCube } from '@finos/legend-graph';
|
|
22
22
|
import type { LakehouseIngestServerClient } from '@finos/legend-server-lakehouse';
|
|
23
23
|
import { LegendDataCubeSourceBuilderType } from './LegendDataCubeSourceBuilderState.js';
|
|
24
|
+
import type { UserManagerSettings } from 'oidc-client-ts';
|
|
24
25
|
export declare class LakehouseProducerDataCubeSourceLoaderState extends LegendDataCubeSourceLoaderState {
|
|
25
26
|
readonly processState: ActionState;
|
|
26
27
|
ingestDefinition: PlainObject | undefined;
|
|
27
28
|
ingestDefinitionUrn: string;
|
|
28
29
|
ingestServerUrl: string;
|
|
30
|
+
private userManagerSettings;
|
|
31
|
+
private LAKEHOUSE_SECTION;
|
|
29
32
|
constructor(application: LegendDataCubeApplicationStore, engine: LegendDataCubeDataCubeEngine, alertService: DataCubeAlertService, sourceData: PlainObject, persistentDataCube: PersistentDataCube, onSuccess: () => Promise<void>, onError: (error: unknown) => Promise<void>);
|
|
30
|
-
|
|
33
|
+
setIngestDefinition(ingestDefinition: PlainObject | undefined): void;
|
|
31
34
|
setIngestDefinitionUrn(urn: string): void;
|
|
35
|
+
setUserManagerSettings(settings: UserManagerSettings | undefined): void;
|
|
32
36
|
get isValid(): boolean;
|
|
33
37
|
get label(): LegendDataCubeSourceBuilderType;
|
|
34
38
|
reset(): void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LakehouseProducerDataCubeSourceLoaderState.d.ts","sourceRoot":"","sources":["../../../../src/stores/builder/source/LakehouseProducerDataCubeSourceLoaderState.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EACL,WAAW,
|
|
1
|
+
{"version":3,"file":"LakehouseProducerDataCubeSourceLoaderState.d.ts","sourceRoot":"","sources":["../../../../src/stores/builder/source/LakehouseProducerDataCubeSourceLoaderState.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EACL,WAAW,EAGX,KAAK,WAAW,EACjB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,8BAA8B,EAAE,MAAM,kCAAkC,CAAC;AACvF,OAAO,KAAK,EAAE,4BAA4B,EAAE,MAAM,uCAAuC,CAAC;AAC1F,OAAO,EAAE,+BAA+B,EAAE,MAAM,sCAAsC,CAAC;AACvF,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AACpE,OAAO,EAGL,KAAK,kBAAkB,EAExB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,gCAAgC,CAAC;AAElF,OAAO,EAAE,+BAA+B,EAAE,MAAM,uCAAuC,CAAC;AACxF,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAG1D,qBAAa,0CAA2C,SAAQ,+BAA+B;IAC7F,QAAQ,CAAC,YAAY,cAAwB;IAE7C,gBAAgB,EAAE,WAAW,GAAG,SAAS,CAAC;IAC1C,mBAAmB,EAAE,MAAM,CAAC;IAC5B,eAAe,EAAE,MAAM,CAAC;IAExB,OAAO,CAAC,mBAAmB,CAAkC;IAE7D,OAAO,CAAC,iBAAiB,CAAkB;gBAGzC,WAAW,EAAE,8BAA8B,EAC3C,MAAM,EAAE,4BAA4B,EACpC,YAAY,EAAE,oBAAoB,EAClC,UAAU,EAAE,WAAW,EACvB,kBAAkB,EAAE,kBAAkB,EACtC,SAAS,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,EAC9B,OAAO,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC;IAwB5C,mBAAmB,CAAC,gBAAgB,EAAE,WAAW,GAAG,SAAS;IAI7D,sBAAsB,CAAC,GAAG,EAAE,MAAM;IAIlC,sBAAsB,CAAC,QAAQ,EAAE,mBAAmB,GAAG,SAAS;IAIhE,IAAa,OAAO,IAAI,OAAO,CAE9B;IAED,IAAa,KAAK,oCAEjB;IAED,KAAK;IAQC,oBAAoB,CACxB,YAAY,EAAE,MAAM,GAAG,SAAS,EAChC,2BAA2B,EAAE,2BAA2B;IAwB3C,IAAI,CAAC,MAAM,EAAE,WAAW,GAAG,SAAS;CAgCpD"}
|
|
@@ -13,33 +13,40 @@
|
|
|
13
13
|
* See the License for the specific language governing permissions and
|
|
14
14
|
* limitations under the License.
|
|
15
15
|
*/
|
|
16
|
-
import { ActionState, guaranteeNonNullable, } from '@finos/legend-shared';
|
|
16
|
+
import { ActionState, guaranteeNonNullable, guaranteeType, } from '@finos/legend-shared';
|
|
17
17
|
import { LegendDataCubeSourceLoaderState } from './LegendDataCubeSourceLoaderState.js';
|
|
18
|
+
import { V1_deserializePureModelContext, V1_PureModelContextData, } from '@finos/legend-graph';
|
|
18
19
|
import { RawLakehouseProducerDataCubeSource } from '../../model/LakehouseProducerDataCubeSource.js';
|
|
19
20
|
import { action, makeObservable, observable } from 'mobx';
|
|
20
21
|
import { LegendDataCubeSourceBuilderType } from './LegendDataCubeSourceBuilderState.js';
|
|
22
|
+
import { SecondaryOAuthClient } from '../../model/SecondaryOauthClient.js';
|
|
21
23
|
export class LakehouseProducerDataCubeSourceLoaderState extends LegendDataCubeSourceLoaderState {
|
|
22
24
|
processState = ActionState.create();
|
|
23
25
|
ingestDefinition;
|
|
24
26
|
ingestDefinitionUrn;
|
|
25
27
|
ingestServerUrl;
|
|
28
|
+
userManagerSettings;
|
|
29
|
+
LAKEHOUSE_SECTION = '###Lakehouse';
|
|
26
30
|
constructor(application, engine, alertService, sourceData, persistentDataCube, onSuccess, onError) {
|
|
27
31
|
super(application, engine, alertService, sourceData, persistentDataCube, onSuccess, onError);
|
|
28
32
|
this.ingestDefinitionUrn = '';
|
|
29
33
|
this.ingestServerUrl = '';
|
|
30
34
|
makeObservable(this, {
|
|
31
35
|
ingestDefinition: observable,
|
|
32
|
-
|
|
36
|
+
setIngestDefinition: action,
|
|
33
37
|
ingestDefinitionUrn: observable,
|
|
34
38
|
setIngestDefinitionUrn: action,
|
|
35
39
|
});
|
|
36
40
|
}
|
|
37
|
-
|
|
41
|
+
setIngestDefinition(ingestDefinition) {
|
|
38
42
|
this.ingestDefinition = ingestDefinition;
|
|
39
43
|
}
|
|
40
44
|
setIngestDefinitionUrn(urn) {
|
|
41
45
|
this.ingestDefinitionUrn = urn;
|
|
42
46
|
}
|
|
47
|
+
setUserManagerSettings(settings) {
|
|
48
|
+
this.userManagerSettings = settings;
|
|
49
|
+
}
|
|
43
50
|
get isValid() {
|
|
44
51
|
return Boolean(this.ingestDefinition);
|
|
45
52
|
}
|
|
@@ -52,11 +59,24 @@ export class LakehouseProducerDataCubeSourceLoaderState extends LegendDataCubeSo
|
|
|
52
59
|
this.ingestServerUrl = rawSource.ingestServerUrl;
|
|
53
60
|
}
|
|
54
61
|
async loadIngestDefinition(access_token, lakehouseIngestServerClient) {
|
|
55
|
-
|
|
62
|
+
const ingestGrammar = await lakehouseIngestServerClient.getIngestDefinitionGrammar(guaranteeNonNullable(this.ingestDefinitionUrn), this.ingestServerUrl, access_token);
|
|
63
|
+
const ingestPMCDPlainObject = await this._engine.parseCompatibleModel(`${this.LAKEHOUSE_SECTION}\n${ingestGrammar}`);
|
|
64
|
+
const ingestDefPMCD = guaranteeType(V1_deserializePureModelContext(ingestPMCDPlainObject), V1_PureModelContextData);
|
|
65
|
+
const protocolIngestDefinition = ingestDefPMCD.elements.at(0).content;
|
|
66
|
+
this.setIngestDefinition(protocolIngestDefinition);
|
|
56
67
|
}
|
|
57
68
|
async load(source) {
|
|
58
69
|
const deserializedSource = RawLakehouseProducerDataCubeSource.serialization.fromJson(guaranteeNonNullable(source));
|
|
59
|
-
|
|
70
|
+
if (deserializedSource.icebergConfig?.icebergRef &&
|
|
71
|
+
deserializedSource.icebergConfig.catalogUrl) {
|
|
72
|
+
const oauthClient = new SecondaryOAuthClient(guaranteeNonNullable(this.userManagerSettings));
|
|
73
|
+
const token = await oauthClient.getToken();
|
|
74
|
+
const refId = await this._engine.ingestIcebergTable(deserializedSource.warehouse, deserializedSource.paths, deserializedSource.icebergConfig.catalogUrl, deserializedSource.icebergConfig.icebergRef, token);
|
|
75
|
+
deserializedSource.icebergConfig.icebergRef = refId.dbReference;
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
this._engine.registerIngestDefinition(guaranteeNonNullable(this.ingestDefinition));
|
|
79
|
+
}
|
|
60
80
|
return RawLakehouseProducerDataCubeSource.serialization.toJson(deserializedSource);
|
|
61
81
|
}
|
|
62
82
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LakehouseProducerDataCubeSourceLoaderState.js","sourceRoot":"","sources":["../../../../src/stores/builder/source/LakehouseProducerDataCubeSourceLoaderState.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EACL,WAAW,EACX,oBAAoB,
|
|
1
|
+
{"version":3,"file":"LakehouseProducerDataCubeSourceLoaderState.js","sourceRoot":"","sources":["../../../../src/stores/builder/source/LakehouseProducerDataCubeSourceLoaderState.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EACL,WAAW,EACX,oBAAoB,EACpB,aAAa,GAEd,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EAAE,+BAA+B,EAAE,MAAM,sCAAsC,CAAC;AAEvF,OAAO,EACL,8BAA8B,EAC9B,uBAAuB,GAGxB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,kCAAkC,EAAE,MAAM,gDAAgD,CAAC;AAEpG,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,MAAM,CAAC;AAC1D,OAAO,EAAE,+BAA+B,EAAE,MAAM,uCAAuC,CAAC;AAExF,OAAO,EAAE,oBAAoB,EAAE,MAAM,qCAAqC,CAAC;AAE3E,MAAM,OAAO,0CAA2C,SAAQ,+BAA+B;IACpF,YAAY,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC;IAE7C,gBAAgB,CAA0B;IAC1C,mBAAmB,CAAS;IAC5B,eAAe,CAAS;IAEhB,mBAAmB,CAAkC;IAErD,iBAAiB,GAAG,cAAc,CAAC;IAE3C,YACE,WAA2C,EAC3C,MAAoC,EACpC,YAAkC,EAClC,UAAuB,EACvB,kBAAsC,EACtC,SAA8B,EAC9B,OAA0C;QAE1C,KAAK,CACH,WAAW,EACX,MAAM,EACN,YAAY,EACZ,UAAU,EACV,kBAAkB,EAClB,SAAS,EACT,OAAO,CACR,CAAC;QAEF,IAAI,CAAC,mBAAmB,GAAG,EAAE,CAAC;QAC9B,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;QAE1B,cAAc,CAAC,IAAI,EAAE;YACnB,gBAAgB,EAAE,UAAU;YAC5B,mBAAmB,EAAE,MAAM;YAE3B,mBAAmB,EAAE,UAAU;YAC/B,sBAAsB,EAAE,MAAM;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,mBAAmB,CAAC,gBAAyC;QAC3D,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;IAC3C,CAAC;IAED,sBAAsB,CAAC,GAAW;QAChC,IAAI,CAAC,mBAAmB,GAAG,GAAG,CAAC;IACjC,CAAC;IAED,sBAAsB,CAAC,QAAyC;QAC9D,IAAI,CAAC,mBAAmB,GAAG,QAAQ,CAAC;IACtC,CAAC;IAED,IAAa,OAAO;QAClB,OAAO,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IACxC,CAAC;IAED,IAAa,KAAK;QAChB,OAAO,+BAA+B,CAAC,kBAAkB,CAAC;IAC5D,CAAC;IAED,KAAK;QACH,MAAM,SAAS,GAAG,kCAAkC,CAAC,aAAa,CAAC,QAAQ,CACzE,IAAI,CAAC,UAAU,CAChB,CAAC;QACF,IAAI,CAAC,sBAAsB,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;QAC3D,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC,eAAe,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,oBAAoB,CACxB,YAAgC,EAChC,2BAAwD;QAExD,MAAM,aAAa,GACjB,MAAM,2BAA2B,CAAC,0BAA0B,CAC1D,oBAAoB,CAAC,IAAI,CAAC,mBAAmB,CAAC,EAC9C,IAAI,CAAC,eAAe,EACpB,YAAY,CACb,CAAC;QAEJ,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,oBAAoB,CACnE,GAAG,IAAI,CAAC,iBAAiB,KAAK,aAAa,EAAE,CAC9C,CAAC;QAEF,MAAM,aAAa,GAAG,aAAa,CACjC,8BAA8B,CAAC,qBAAqB,CAAC,EACrD,uBAAuB,CACxB,CAAC;QAEF,MAAM,wBAAwB,GAC5B,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAC5B,CAAC,OAAO,CAAC;QACV,IAAI,CAAC,mBAAmB,CAAC,wBAAwB,CAAC,CAAC;IACrD,CAAC;IAEQ,KAAK,CAAC,IAAI,CAAC,MAA+B;QACjD,MAAM,kBAAkB,GACtB,kCAAkC,CAAC,aAAa,CAAC,QAAQ,CACvD,oBAAoB,CAAC,MAAM,CAAC,CAC7B,CAAC;QAEJ,IACE,kBAAkB,CAAC,aAAa,EAAE,UAAU;YAC5C,kBAAkB,CAAC,aAAa,CAAC,UAAU,EAC3C,CAAC;YACD,MAAM,WAAW,GAAG,IAAI,oBAAoB,CAC1C,oBAAoB,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAC/C,CAAC;YACF,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,QAAQ,EAAE,CAAC;YAC3C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CACjD,kBAAkB,CAAC,SAAS,EAC5B,kBAAkB,CAAC,KAAK,EACxB,kBAAkB,CAAC,aAAa,CAAC,UAAU,EAC3C,kBAAkB,CAAC,aAAa,CAAC,UAAU,EAC3C,KAAK,CACN,CAAC;YACF,kBAAkB,CAAC,aAAa,CAAC,UAAU,GAAG,KAAK,CAAC,WAAW,CAAC;QAClE,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,OAAO,CAAC,wBAAwB,CACnC,oBAAoB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAC5C,CAAC;QACJ,CAAC;QAED,OAAO,kCAAkC,CAAC,aAAa,CAAC,MAAM,CAC5D,kBAAkB,CACnB,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -14,12 +14,12 @@
|
|
|
14
14
|
* limitations under the License.
|
|
15
15
|
*/
|
|
16
16
|
import { DataCubeSource } from '@finos/legend-data-cube';
|
|
17
|
-
import type {
|
|
17
|
+
import type { V1_PureModelContextData } from '@finos/legend-graph';
|
|
18
18
|
import { VersionedProjectData } from '@finos/legend-server-depot';
|
|
19
19
|
import { SerializationFactory, type PlainObject } from '@finos/legend-shared';
|
|
20
20
|
export declare const LAKEHOUSE_CONSUMER_DATA_CUBE_SOURCE_TYPE = "lakehouseConsumer";
|
|
21
21
|
export declare class LakehouseConsumerDataCubeSource extends DataCubeSource {
|
|
22
|
-
model: PlainObject<
|
|
22
|
+
model: PlainObject<V1_PureModelContextData>;
|
|
23
23
|
dpCoordinates: VersionedProjectData;
|
|
24
24
|
runtime: string;
|
|
25
25
|
warehouse: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LakehouseConsumerDataCubeSource.d.ts","sourceRoot":"","sources":["../../../src/stores/model/LakehouseConsumerDataCubeSource.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"LakehouseConsumerDataCubeSource.d.ts","sourceRoot":"","sources":["../../../src/stores/model/LakehouseConsumerDataCubeSource.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,EACL,oBAAoB,EAGpB,KAAK,WAAW,EACjB,MAAM,sBAAsB,CAAC;AAG9B,eAAO,MAAM,wCAAwC,sBAAsB,CAAC;AAE5E,qBAAa,+BAAgC,SAAQ,cAAc;IACjE,KAAK,EAAG,WAAW,CAAC,uBAAuB,CAAC,CAAC;IAC7C,aAAa,EAAG,oBAAoB,CAAC;IACrC,OAAO,EAAG,MAAM,CAAC;IACjB,SAAS,EAAG,MAAM,CAAC;IACnB,WAAW,EAAG,MAAM,CAAC;IACrB,KAAK,EAAG,MAAM,EAAE,CAAC;CAClB;AAED,qBAAa,kCAAkC;IAC7C,aAAa,EAAG,oBAAoB,CAAC;IACrC,SAAS,EAAG,MAAM,CAAC;IACnB,WAAW,EAAG,MAAM,CAAC;IACrB,KAAK,EAAG,MAAM,EAAE,CAAC;IAEjB,MAAM,CAAC,QAAQ,CAAC,aAAa,2DAU3B;CACH"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LakehouseConsumerDataCubeSource.js","sourceRoot":"","sources":["../../../src/stores/model/LakehouseConsumerDataCubeSource.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,EACL,oBAAoB,EACpB,wBAAwB,EACxB,gBAAgB,GAEjB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,iBAAiB,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAE/D,MAAM,CAAC,MAAM,wCAAwC,GAAG,mBAAmB,CAAC;AAE5E,MAAM,OAAO,+BAAgC,SAAQ,cAAc;IACjE,KAAK,
|
|
1
|
+
{"version":3,"file":"LakehouseConsumerDataCubeSource.js","sourceRoot":"","sources":["../../../src/stores/model/LakehouseConsumerDataCubeSource.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,EACL,oBAAoB,EACpB,wBAAwB,EACxB,gBAAgB,GAEjB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,iBAAiB,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAE/D,MAAM,CAAC,MAAM,wCAAwC,GAAG,mBAAmB,CAAC;AAE5E,MAAM,OAAO,+BAAgC,SAAQ,cAAc;IACjE,KAAK,CAAwC;IAC7C,aAAa,CAAwB;IACrC,OAAO,CAAU;IACjB,SAAS,CAAU;IACnB,WAAW,CAAU;IACrB,KAAK,CAAY;CAClB;AAED,MAAM,OAAO,kCAAkC;IAC7C,aAAa,CAAwB;IACrC,SAAS,CAAU;IACnB,WAAW,CAAU;IACrB,KAAK,CAAY;IAEjB,MAAM,CAAU,aAAa,GAAG,IAAI,oBAAoB,CACtD,iBAAiB,CAAC,kCAAkC,EAAE;QACpD,KAAK,EAAE,wBAAwB,CAAC,wCAAwC,CAAC;QACzE,aAAa,EAAE,gBAAgB,CAC7B,oBAAoB,CAAC,aAAa,CAAC,MAAM,CAC1C;QACD,SAAS,EAAE,SAAS,EAAE;QACtB,WAAW,EAAE,SAAS,EAAE;QACxB,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC;KACzB,CAAC,CACH,CAAC"}
|
|
@@ -21,11 +21,24 @@ export declare class LakehouseProducerDataCubeSource extends DataCubeSource {
|
|
|
21
21
|
model: PlainObject<V1_PureModelContextData>;
|
|
22
22
|
runtime: string;
|
|
23
23
|
}
|
|
24
|
+
export declare class LakehouseProducerIcebergCachedDataCubeSource extends DataCubeSource {
|
|
25
|
+
model: PlainObject<V1_PureModelContextData>;
|
|
26
|
+
runtime: string;
|
|
27
|
+
db: string;
|
|
28
|
+
schema: string;
|
|
29
|
+
table: string;
|
|
30
|
+
}
|
|
31
|
+
export declare class IcebergConfig {
|
|
32
|
+
catalogUrl: string;
|
|
33
|
+
icebergRef: string;
|
|
34
|
+
static readonly serialization: SerializationFactory<IcebergConfig>;
|
|
35
|
+
}
|
|
24
36
|
export declare class RawLakehouseProducerDataCubeSource {
|
|
25
37
|
ingestDefinitionUrn: string;
|
|
26
38
|
warehouse: string;
|
|
27
39
|
ingestServerUrl: string;
|
|
28
40
|
paths: string[];
|
|
41
|
+
icebergConfig?: IcebergConfig;
|
|
29
42
|
static readonly serialization: SerializationFactory<RawLakehouseProducerDataCubeSource>;
|
|
30
43
|
}
|
|
31
44
|
//# sourceMappingURL=LakehouseProducerDataCubeSource.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LakehouseProducerDataCubeSource.d.ts","sourceRoot":"","sources":["../../../src/stores/model/LakehouseProducerDataCubeSource.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AACnE,OAAO,EACL,oBAAoB,
|
|
1
|
+
{"version":3,"file":"LakehouseProducerDataCubeSource.d.ts","sourceRoot":"","sources":["../../../src/stores/model/LakehouseProducerDataCubeSource.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AACnE,OAAO,EACL,oBAAoB,EAGpB,KAAK,WAAW,EACjB,MAAM,sBAAsB,CAAC;AAG9B,eAAO,MAAM,wCAAwC,sBAAsB,CAAC;AAE5E,qBAAa,+BAAgC,SAAQ,cAAc;IACjE,KAAK,EAAG,WAAW,CAAC,uBAAuB,CAAC,CAAC;IAC7C,OAAO,EAAG,MAAM,CAAC;CAClB;AAED,qBAAa,4CAA6C,SAAQ,cAAc;IAC9E,KAAK,EAAG,WAAW,CAAC,uBAAuB,CAAC,CAAC;IAC7C,OAAO,EAAG,MAAM,CAAC;IACjB,EAAE,EAAG,MAAM,CAAC;IACZ,MAAM,EAAG,MAAM,CAAC;IAChB,KAAK,EAAG,MAAM,CAAC;CAChB;AAED,qBAAa,aAAa;IACxB,UAAU,EAAG,MAAM,CAAC;IACpB,UAAU,EAAG,MAAM,CAAC;IAEpB,MAAM,CAAC,QAAQ,CAAC,aAAa,sCAK3B;CACH;AAED,qBAAa,kCAAkC;IAC7C,mBAAmB,EAAG,MAAM,CAAC;IAC7B,SAAS,EAAG,MAAM,CAAC;IACnB,eAAe,EAAG,MAAM,CAAC;IACzB,KAAK,EAAG,MAAM,EAAE,CAAC;IACjB,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B,MAAM,CAAC,QAAQ,CAAC,aAAa,2DAW3B;CACH"}
|
|
@@ -14,24 +14,41 @@
|
|
|
14
14
|
* limitations under the License.
|
|
15
15
|
*/
|
|
16
16
|
import { DataCubeSource } from '@finos/legend-data-cube';
|
|
17
|
-
import { SerializationFactory, usingConstantValueSchema, } from '@finos/legend-shared';
|
|
18
|
-
import { createModelSchema, list, primitive } from 'serializr';
|
|
17
|
+
import { SerializationFactory, usingConstantValueSchema, usingModelSchema, } from '@finos/legend-shared';
|
|
18
|
+
import { createModelSchema, list, optional, primitive } from 'serializr';
|
|
19
19
|
export const LAKEHOUSE_PRODUCER_DATA_CUBE_SOURCE_TYPE = 'lakehouseProducer';
|
|
20
20
|
export class LakehouseProducerDataCubeSource extends DataCubeSource {
|
|
21
21
|
model;
|
|
22
22
|
runtime;
|
|
23
23
|
}
|
|
24
|
+
export class LakehouseProducerIcebergCachedDataCubeSource extends DataCubeSource {
|
|
25
|
+
model;
|
|
26
|
+
runtime;
|
|
27
|
+
db;
|
|
28
|
+
schema;
|
|
29
|
+
table;
|
|
30
|
+
}
|
|
31
|
+
export class IcebergConfig {
|
|
32
|
+
catalogUrl;
|
|
33
|
+
icebergRef;
|
|
34
|
+
static serialization = new SerializationFactory(createModelSchema(IcebergConfig, {
|
|
35
|
+
catalogUrl: primitive(),
|
|
36
|
+
icebergRef: primitive(),
|
|
37
|
+
}));
|
|
38
|
+
}
|
|
24
39
|
export class RawLakehouseProducerDataCubeSource {
|
|
25
40
|
ingestDefinitionUrn;
|
|
26
41
|
warehouse;
|
|
27
42
|
ingestServerUrl;
|
|
28
43
|
paths;
|
|
44
|
+
icebergConfig;
|
|
29
45
|
static serialization = new SerializationFactory(createModelSchema(RawLakehouseProducerDataCubeSource, {
|
|
30
46
|
_type: usingConstantValueSchema(LAKEHOUSE_PRODUCER_DATA_CUBE_SOURCE_TYPE),
|
|
31
47
|
ingestDefinitionUrn: primitive(),
|
|
32
48
|
warehouse: primitive(),
|
|
33
49
|
ingestServerUrl: primitive(),
|
|
34
50
|
paths: list(primitive()),
|
|
51
|
+
icebergConfig: optional(usingModelSchema(IcebergConfig.serialization.schema)),
|
|
35
52
|
}));
|
|
36
53
|
}
|
|
37
54
|
//# sourceMappingURL=LakehouseProducerDataCubeSource.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LakehouseProducerDataCubeSource.js","sourceRoot":"","sources":["../../../src/stores/model/LakehouseProducerDataCubeSource.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EACL,oBAAoB,EACpB,wBAAwB,
|
|
1
|
+
{"version":3,"file":"LakehouseProducerDataCubeSource.js","sourceRoot":"","sources":["../../../src/stores/model/LakehouseProducerDataCubeSource.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EACL,oBAAoB,EACpB,wBAAwB,EACxB,gBAAgB,GAEjB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,iBAAiB,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEzE,MAAM,CAAC,MAAM,wCAAwC,GAAG,mBAAmB,CAAC;AAE5E,MAAM,OAAO,+BAAgC,SAAQ,cAAc;IACjE,KAAK,CAAwC;IAC7C,OAAO,CAAU;CAClB;AAED,MAAM,OAAO,4CAA6C,SAAQ,cAAc;IAC9E,KAAK,CAAwC;IAC7C,OAAO,CAAU;IACjB,EAAE,CAAU;IACZ,MAAM,CAAU;IAChB,KAAK,CAAU;CAChB;AAED,MAAM,OAAO,aAAa;IACxB,UAAU,CAAU;IACpB,UAAU,CAAU;IAEpB,MAAM,CAAU,aAAa,GAAG,IAAI,oBAAoB,CACtD,iBAAiB,CAAC,aAAa,EAAE;QAC/B,UAAU,EAAE,SAAS,EAAE;QACvB,UAAU,EAAE,SAAS,EAAE;KACxB,CAAC,CACH,CAAC;;AAGJ,MAAM,OAAO,kCAAkC;IAC7C,mBAAmB,CAAU;IAC7B,SAAS,CAAU;IACnB,eAAe,CAAU;IACzB,KAAK,CAAY;IACjB,aAAa,CAAiB;IAE9B,MAAM,CAAU,aAAa,GAAG,IAAI,oBAAoB,CACtD,iBAAiB,CAAC,kCAAkC,EAAE;QACpD,KAAK,EAAE,wBAAwB,CAAC,wCAAwC,CAAC;QACzE,mBAAmB,EAAE,SAAS,EAAE;QAChC,SAAS,EAAE,SAAS,EAAE;QACtB,eAAe,EAAE,SAAS,EAAE;QAC5B,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC;QACxB,aAAa,EAAE,QAAQ,CACrB,gBAAgB,CAAC,aAAa,CAAC,aAAa,CAAC,MAAM,CAAC,CACrD;KACF,CAAC,CACH,CAAC"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2020-present, Goldman Sachs
|
|
3
|
+
*
|
|
4
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
* you may not use this file except in compliance with the License.
|
|
6
|
+
* You may obtain a copy of the License at
|
|
7
|
+
*
|
|
8
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
*
|
|
10
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
* See the License for the specific language governing permissions and
|
|
14
|
+
* limitations under the License.
|
|
15
|
+
*/
|
|
16
|
+
import { type UserManagerSettings } from 'oidc-client-ts';
|
|
17
|
+
export declare class SecondaryOAuthClient {
|
|
18
|
+
private userManager;
|
|
19
|
+
private user;
|
|
20
|
+
constructor(config: UserManagerSettings);
|
|
21
|
+
/** Login via popup */
|
|
22
|
+
loginWithPopup(): Promise<void>;
|
|
23
|
+
/** Get current token, refresh if needed */
|
|
24
|
+
getToken(): Promise<string>;
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=SecondaryOauthClient.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SecondaryOauthClient.d.ts","sourceRoot":"","sources":["../../../src/stores/model/SecondaryOauthClient.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAEL,KAAK,mBAAmB,EAGzB,MAAM,gBAAgB,CAAC;AAExB,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,IAAI,CAAqB;gBAErB,MAAM,EAAE,mBAAmB;IAQvC,sBAAsB;IAChB,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAIrC,2CAA2C;IAC9B,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC;CAczC"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2020-present, Goldman Sachs
|
|
3
|
+
*
|
|
4
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
* you may not use this file except in compliance with the License.
|
|
6
|
+
* You may obtain a copy of the License at
|
|
7
|
+
*
|
|
8
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
*
|
|
10
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
* See the License for the specific language governing permissions and
|
|
14
|
+
* limitations under the License.
|
|
15
|
+
*/
|
|
16
|
+
import { UserManager, WebStorageStateStore, } from 'oidc-client-ts';
|
|
17
|
+
export class SecondaryOAuthClient {
|
|
18
|
+
userManager;
|
|
19
|
+
user = null;
|
|
20
|
+
constructor(config) {
|
|
21
|
+
this.userManager = new UserManager({
|
|
22
|
+
...config,
|
|
23
|
+
userStore: new WebStorageStateStore({ store: window.localStorage }),
|
|
24
|
+
scope: config.extraQueryParams?.scope,
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
/** Login via popup */
|
|
28
|
+
async loginWithPopup() {
|
|
29
|
+
this.user = await this.userManager.signinPopup();
|
|
30
|
+
}
|
|
31
|
+
/** Get current token, refresh if needed */
|
|
32
|
+
async getToken() {
|
|
33
|
+
this.user = this.user ?? (await this.userManager.getUser());
|
|
34
|
+
if (this.user) {
|
|
35
|
+
const exp = this.user.expires_at ? this.user.expires_at * 1000 : 0;
|
|
36
|
+
if (Date.now() < exp - 10 * 60 * 1000) {
|
|
37
|
+
return this.user.access_token;
|
|
38
|
+
}
|
|
39
|
+
else {
|
|
40
|
+
this.user = await this.userManager.signinPopup();
|
|
41
|
+
return this.user.access_token;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
this.user = await this.userManager.signinPopup();
|
|
45
|
+
return this.user.access_token;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
//# sourceMappingURL=SecondaryOauthClient.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SecondaryOauthClient.js","sourceRoot":"","sources":["../../../src/stores/model/SecondaryOauthClient.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EACL,WAAW,EAGX,oBAAoB,GACrB,MAAM,gBAAgB,CAAC;AAExB,MAAM,OAAO,oBAAoB;IACvB,WAAW,CAAc;IACzB,IAAI,GAAgB,IAAI,CAAC;IAEjC,YAAY,MAA2B;QACrC,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAAC;YACjC,GAAG,MAAM;YACT,SAAS,EAAE,IAAI,oBAAoB,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC;YACnE,KAAK,EAAE,MAAM,CAAC,gBAAgB,EAAE,KAAe;SAChD,CAAC,CAAC;IACL,CAAC;IAED,sBAAsB;IACtB,KAAK,CAAC,cAAc;QAClB,IAAI,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;IACnD,CAAC;IAED,2CAA2C;IACpC,KAAK,CAAC,QAAQ;QACnB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAC5D,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACnE,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC;gBACtC,OAAO,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC;YAChC,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;gBACjD,OAAO,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC;YAChC,CAAC;QACH,CAAC;QACD,IAAI,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;QACjD,OAAO,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC;IAChC,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@finos/legend-application-data-cube",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"description": "Legend DataCube application core",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"legend",
|
|
@@ -42,22 +42,23 @@
|
|
|
42
42
|
"test:watch": "jest --watch"
|
|
43
43
|
},
|
|
44
44
|
"dependencies": {
|
|
45
|
-
"@duckdb/duckdb-wasm": "1.29.0",
|
|
46
|
-
"@finos/legend-application": "16.0.
|
|
47
|
-
"@finos/legend-art": "7.1.
|
|
48
|
-
"@finos/legend-code-editor": "2.0.
|
|
49
|
-
"@finos/legend-data-cube": "0.3.
|
|
50
|
-
"@finos/legend-graph": "32.
|
|
51
|
-
"@finos/legend-query-builder": "4.17.
|
|
52
|
-
"@finos/legend-server-depot": "6.1.
|
|
53
|
-
"@finos/legend-server-lakehouse": "0.
|
|
54
|
-
"@finos/legend-shared": "11.0.
|
|
55
|
-
"@finos/legend-storage": "3.0.
|
|
45
|
+
"@duckdb/duckdb-wasm": "1.29.1-dev283.0",
|
|
46
|
+
"@finos/legend-application": "16.0.71",
|
|
47
|
+
"@finos/legend-art": "7.1.124",
|
|
48
|
+
"@finos/legend-code-editor": "2.0.119",
|
|
49
|
+
"@finos/legend-data-cube": "0.3.29",
|
|
50
|
+
"@finos/legend-graph": "32.3.0",
|
|
51
|
+
"@finos/legend-query-builder": "4.17.42",
|
|
52
|
+
"@finos/legend-server-depot": "6.1.1",
|
|
53
|
+
"@finos/legend-server-lakehouse": "0.3.0",
|
|
54
|
+
"@finos/legend-shared": "11.0.20",
|
|
55
|
+
"@finos/legend-storage": "3.0.139",
|
|
56
56
|
"@types/react": "19.0.10",
|
|
57
57
|
"@types/react-dom": "19.0.4",
|
|
58
58
|
"apache-arrow": "19.0.1",
|
|
59
59
|
"mobx": "6.13.6",
|
|
60
60
|
"mobx-react-lite": "4.1.0",
|
|
61
|
+
"oidc-client-ts": "^3.2.0",
|
|
61
62
|
"react": "19.0.0",
|
|
62
63
|
"react-dom": "19.0.0",
|
|
63
64
|
"react-oidc-context": "^3.3.0",
|