@finos/legend-application-query 13.8.28 → 13.8.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/__lib__/LegendQueryNavigation.d.ts +12 -0
- package/lib/__lib__/LegendQueryNavigation.d.ts.map +1 -1
- package/lib/__lib__/LegendQueryNavigation.js +16 -0
- package/lib/__lib__/LegendQueryNavigation.js.map +1 -1
- package/lib/components/Core_LegendQueryApplicationPlugin.d.ts.map +1 -1
- package/lib/components/Core_LegendQueryApplicationPlugin.js +15 -0
- package/lib/components/Core_LegendQueryApplicationPlugin.js.map +1 -1
- package/lib/components/LegendQueryWebApplication.d.ts.map +1 -1
- package/lib/components/LegendQueryWebApplication.js +2 -1
- package/lib/components/LegendQueryWebApplication.js.map +1 -1
- package/lib/components/QueryEditor.d.ts.map +1 -1
- package/lib/components/QueryEditor.js +5 -1
- package/lib/components/QueryEditor.js.map +1 -1
- package/lib/components/data-product/LegendQueryDataProductQueryBuilder.d.ts.map +1 -1
- package/lib/components/data-product/LegendQueryDataProductQueryBuilder.js +6 -40
- package/lib/components/data-product/LegendQueryDataProductQueryBuilder.js.map +1 -1
- package/lib/components/ingest/IngestInfo.d.ts +32 -0
- package/lib/components/ingest/IngestInfo.d.ts.map +1 -0
- package/lib/components/ingest/IngestInfo.js +63 -0
- package/lib/components/ingest/IngestInfo.js.map +1 -0
- package/lib/components/ingest/IngestQueryBuilderSetupPanel.d.ts +18 -0
- package/lib/components/ingest/IngestQueryBuilderSetupPanel.d.ts.map +1 -0
- package/lib/components/ingest/IngestQueryBuilderSetupPanel.js +77 -0
- package/lib/components/ingest/IngestQueryBuilderSetupPanel.js.map +1 -0
- package/lib/components/ingest/IngestQueryCreator.d.ts +27 -0
- package/lib/components/ingest/IngestQueryCreator.d.ts.map +1 -0
- package/lib/components/ingest/IngestQueryCreator.js +45 -0
- package/lib/components/ingest/IngestQueryCreator.js.map +1 -0
- package/lib/components/shared/LakehouseRuntimeConfigModal.d.ts +29 -0
- package/lib/components/shared/LakehouseRuntimeConfigModal.d.ts.map +1 -0
- package/lib/components/shared/LakehouseRuntimeConfigModal.js +60 -0
- package/lib/components/shared/LakehouseRuntimeConfigModal.js.map +1 -0
- package/lib/index.css +1 -1
- package/lib/light-mode.css +1 -1
- package/lib/package.json +1 -1
- package/lib/stores/QueryEditorStore.d.ts +10 -0
- package/lib/stores/QueryEditorStore.d.ts.map +1 -1
- package/lib/stores/QueryEditorStore.js +81 -2
- package/lib/stores/QueryEditorStore.js.map +1 -1
- package/lib/stores/ingest/IngestLegendQueryBuilderState.d.ts +122 -0
- package/lib/stores/ingest/IngestLegendQueryBuilderState.d.ts.map +1 -0
- package/lib/stores/ingest/IngestLegendQueryBuilderState.js +191 -0
- package/lib/stores/ingest/IngestLegendQueryBuilderState.js.map +1 -0
- package/lib/stores/ingest/IngestQueryCreatorStore.d.ts +75 -0
- package/lib/stores/ingest/IngestQueryCreatorStore.d.ts.map +1 -0
- package/lib/stores/ingest/IngestQueryCreatorStore.js +169 -0
- package/lib/stores/ingest/IngestQueryCreatorStore.js.map +1 -0
- package/lib/stores/ingest/IngestQueryGraphHelper.d.ts +58 -0
- package/lib/stores/ingest/IngestQueryGraphHelper.d.ts.map +1 -0
- package/lib/stores/ingest/IngestQueryGraphHelper.js +72 -0
- package/lib/stores/ingest/IngestQueryGraphHelper.js.map +1 -0
- package/package.json +12 -12
- package/src/__lib__/LegendQueryNavigation.ts +35 -0
- package/src/components/Core_LegendQueryApplicationPlugin.tsx +20 -0
- package/src/components/LegendQueryWebApplication.tsx +14 -0
- package/src/components/QueryEditor.tsx +12 -0
- package/src/components/data-product/LegendQueryDataProductQueryBuilder.tsx +7 -127
- package/src/components/ingest/IngestInfo.tsx +197 -0
- package/src/components/ingest/IngestQueryBuilderSetupPanel.tsx +204 -0
- package/src/components/ingest/IngestQueryCreator.tsx +75 -0
- package/src/components/shared/LakehouseRuntimeConfigModal.tsx +148 -0
- package/src/stores/QueryEditorStore.ts +131 -1
- package/src/stores/ingest/IngestLegendQueryBuilderState.ts +259 -0
- package/src/stores/ingest/IngestQueryCreatorStore.ts +262 -0
- package/src/stores/ingest/IngestQueryGraphHelper.ts +120 -0
- package/tsconfig.json +7 -0
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2020-present, Goldman Sachs
|
|
3
|
+
*
|
|
4
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
* you may not use this file except in compliance with the License.
|
|
6
|
+
* You may obtain a copy of the License at
|
|
7
|
+
*
|
|
8
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
*
|
|
10
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
* See the License for the specific language governing permissions and
|
|
14
|
+
* limitations under the License.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import {
|
|
18
|
+
type Accessor,
|
|
19
|
+
type AccessorOwner,
|
|
20
|
+
type GraphManagerState,
|
|
21
|
+
type IngestDefinition,
|
|
22
|
+
type PackageableElement,
|
|
23
|
+
type PackageableRuntime,
|
|
24
|
+
type QueryExecutionContext,
|
|
25
|
+
LakehouseRuntime,
|
|
26
|
+
QueryIngestExecutionContext,
|
|
27
|
+
} from '@finos/legend-graph';
|
|
28
|
+
import {
|
|
29
|
+
AccessorQueryBuilderState,
|
|
30
|
+
type QueryBuilderActionConfig,
|
|
31
|
+
type QueryBuilderConfig,
|
|
32
|
+
type QueryBuilderWorkflowState,
|
|
33
|
+
} from '@finos/legend-query-builder';
|
|
34
|
+
import type { GenericLegendApplicationStore } from '@finos/legend-application';
|
|
35
|
+
import type {
|
|
36
|
+
ProjectGAVCoordinates,
|
|
37
|
+
QueryableSourceInfo,
|
|
38
|
+
} from '@finos/legend-storage';
|
|
39
|
+
import { action, makeObservable, observable, runInAction } from 'mobx';
|
|
40
|
+
import { assertErrorThrown } from '@finos/legend-shared';
|
|
41
|
+
import { renderIngestQueryBuilderSetupPanelContent } from '../../components/ingest/IngestQueryBuilderSetupPanel.js';
|
|
42
|
+
import { generateIngestQueryCreatorRoute } from '../../__lib__/LegendQueryNavigation.js';
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Query builder state for an ingest-backed query.
|
|
46
|
+
*
|
|
47
|
+
* Extends {@link AccessorQueryBuilderState} because an ingest definition is
|
|
48
|
+
* already an `AccessorOwner` (see `AccessorQueryBuilderState.accessorOwners`)
|
|
49
|
+
* and ingest queries don't require a mapping (`requiresMappingForExecution`
|
|
50
|
+
* is already `false` in the base class).
|
|
51
|
+
*
|
|
52
|
+
* The execution context produced here is a {@link QueryIngestExecutionContext}
|
|
53
|
+
* carrying `ingestDefinitionPath` and `dataSet` — the same two values the
|
|
54
|
+
* route handler resolved from the URL.
|
|
55
|
+
*/
|
|
56
|
+
export class IngestLegendQueryBuilderState extends AccessorQueryBuilderState {
|
|
57
|
+
/**
|
|
58
|
+
* Currently selected & graph-resolved ingest definition. Observable so the
|
|
59
|
+
* setup panel re-renders when the user picks a different ingest from the
|
|
60
|
+
* source dropdown.
|
|
61
|
+
*/
|
|
62
|
+
ingestDefinition: IngestDefinition;
|
|
63
|
+
/**
|
|
64
|
+
* All ingest definition paths discovered in the project (fetched up front
|
|
65
|
+
* via depot's classifier-scoped entities endpoint). Only `ingestDefinition`
|
|
66
|
+
* is actually built into the graph — the others are surfaced as paths so
|
|
67
|
+
* the user can switch without paying the cost of building them all.
|
|
68
|
+
*/
|
|
69
|
+
readonly allIngestPaths: string[];
|
|
70
|
+
/**
|
|
71
|
+
* Swap callback wired by {@link IngestQueryCreatorStore}. Deletes the
|
|
72
|
+
* currently built ingest from the graph, builds the target one from its
|
|
73
|
+
* cached entity, and returns the new metamodel instance.
|
|
74
|
+
*/
|
|
75
|
+
readonly swapIngest: (path: string) => Promise<IngestDefinition>;
|
|
76
|
+
readonly project: ProjectGAVCoordinates;
|
|
77
|
+
/**
|
|
78
|
+
* Adhoc lakehouse runtime built from the user's lakehouse environment +
|
|
79
|
+
* consumer warehouse, mirroring how `LegendQueryDataProductQueryBuilderState`
|
|
80
|
+
* builds its runtime for model-access / lakehouse access points. Created by
|
|
81
|
+
* the creator store (where the depot/lakehouse clients live) and injected
|
|
82
|
+
* here so the editor can execute against it without a packaged runtime.
|
|
83
|
+
*/
|
|
84
|
+
readonly adhocRuntime: PackageableRuntime;
|
|
85
|
+
/**
|
|
86
|
+
* Loading flag flipped while {@link changeIngestDefinition} is fetching /
|
|
87
|
+
* rebuilding the newly selected ingest. Lets the setup panel disable the
|
|
88
|
+
* source dropdown during the swap so we don't get overlapping changes.
|
|
89
|
+
*/
|
|
90
|
+
isSwappingIngest = false;
|
|
91
|
+
|
|
92
|
+
constructor(
|
|
93
|
+
applicationStore: GenericLegendApplicationStore,
|
|
94
|
+
accessor: Accessor | undefined,
|
|
95
|
+
graphManagerState: GraphManagerState,
|
|
96
|
+
workflow: QueryBuilderWorkflowState,
|
|
97
|
+
actionConfig: QueryBuilderActionConfig,
|
|
98
|
+
ingestDefinition: IngestDefinition,
|
|
99
|
+
allIngestPaths: string[],
|
|
100
|
+
swapIngest: (path: string) => Promise<IngestDefinition>,
|
|
101
|
+
adhocRuntime: PackageableRuntime,
|
|
102
|
+
project: ProjectGAVCoordinates,
|
|
103
|
+
config?: QueryBuilderConfig | undefined,
|
|
104
|
+
sourceInfo?: QueryableSourceInfo | undefined,
|
|
105
|
+
) {
|
|
106
|
+
super(
|
|
107
|
+
applicationStore,
|
|
108
|
+
accessor,
|
|
109
|
+
graphManagerState,
|
|
110
|
+
workflow,
|
|
111
|
+
actionConfig,
|
|
112
|
+
config,
|
|
113
|
+
sourceInfo,
|
|
114
|
+
);
|
|
115
|
+
makeObservable(this, {
|
|
116
|
+
ingestDefinition: observable,
|
|
117
|
+
isSwappingIngest: observable,
|
|
118
|
+
changeIngestDefinition: action,
|
|
119
|
+
});
|
|
120
|
+
this.ingestDefinition = ingestDefinition;
|
|
121
|
+
this.allIngestPaths = allIngestPaths;
|
|
122
|
+
this.swapIngest = swapIngest;
|
|
123
|
+
this.adhocRuntime = adhocRuntime;
|
|
124
|
+
this.project = project;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
override TEMPORARY__setupPanelContentRenderer = (): React.ReactNode =>
|
|
128
|
+
renderIngestQueryBuilderSetupPanelContent(this);
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* The currently selected data set. Derived from `sourceAccessor` so it
|
|
132
|
+
* stays in sync with `changeAccessor` (e.g. when the user picks a different
|
|
133
|
+
* data set, or when {@link changeIngestDefinition} re-points the source).
|
|
134
|
+
*/
|
|
135
|
+
get dataSet(): string {
|
|
136
|
+
return this.sourceAccessor?.accessor ?? '';
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
override getQueryExecutionContext(): QueryExecutionContext {
|
|
140
|
+
const exec = new QueryIngestExecutionContext();
|
|
141
|
+
exec.ingestDefinitionPath = this.ingestDefinition.path;
|
|
142
|
+
exec.dataSet = this.dataSet;
|
|
143
|
+
return exec;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* The deep-link flow opens the editor against a single, pre-resolved ingest
|
|
148
|
+
* definition. The user shouldn't be picking a different owner, but we still
|
|
149
|
+
* surface the current ingest so the source dropdown reflects the selection
|
|
150
|
+
* instead of rendering the empty "Choose a source..." placeholder.
|
|
151
|
+
*
|
|
152
|
+
* The full list of selectable ingests is exposed separately via
|
|
153
|
+
* {@link allIngestPaths} (the others aren't built into the graph yet — see
|
|
154
|
+
* {@link changeIngestDefinition}).
|
|
155
|
+
*/
|
|
156
|
+
override get accessorOwners(): AccessorOwner[] {
|
|
157
|
+
return [this.ingestDefinition];
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Execution always goes through {@link adhocRuntime} (selected by the
|
|
162
|
+
* creator store). Return an empty list so the runtime dropdown is hidden
|
|
163
|
+
* and the user can't override it.
|
|
164
|
+
*/
|
|
165
|
+
override get compatibleRuntimes(): PackageableRuntime[] {
|
|
166
|
+
return [];
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* The adhoc lakehouse runtime is registered on the graph under the
|
|
171
|
+
* `_internal_` package so the editor can resolve it, but it is not part
|
|
172
|
+
* of the published project. Surface it as a floating execution element so
|
|
173
|
+
* it's bundled into the execution payload at run time (mirrors what the
|
|
174
|
+
* data-product flow does for its lakehouse runtime).
|
|
175
|
+
*/
|
|
176
|
+
override get floatingExecutionElements(): PackageableElement[] | undefined {
|
|
177
|
+
return this.graphManagerState.graph.origin !== undefined
|
|
178
|
+
? [this.adhocRuntime]
|
|
179
|
+
: undefined;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Convenience accessor: the underlying {@link LakehouseRuntime} on the
|
|
184
|
+
* adhoc runtime, if any. Used by the setup panel to feed the runtime
|
|
185
|
+
* configuration modal.
|
|
186
|
+
*/
|
|
187
|
+
get lakehouseRuntime(): LakehouseRuntime | undefined {
|
|
188
|
+
return this.adhocRuntime.runtimeValue instanceof LakehouseRuntime
|
|
189
|
+
? this.adhocRuntime.runtimeValue
|
|
190
|
+
: undefined;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* Swap the active ingest definition to the one at `path`. Triggered by the
|
|
195
|
+
* source dropdown in {@link IngestQueryBuilderSetupPanel}. Delegates the
|
|
196
|
+
* graph mutation (delete current, build new) to {@link swapIngest} and then
|
|
197
|
+
* rewires the source accessor so the editor reflects the new ingest's
|
|
198
|
+
* datasets.
|
|
199
|
+
*/
|
|
200
|
+
async changeIngestDefinition(path: string): Promise<void> {
|
|
201
|
+
if (path === this.ingestDefinition.path || this.isSwappingIngest) {
|
|
202
|
+
return;
|
|
203
|
+
}
|
|
204
|
+
runInAction(() => {
|
|
205
|
+
this.isSwappingIngest = true;
|
|
206
|
+
});
|
|
207
|
+
try {
|
|
208
|
+
const next = await this.swapIngest(path);
|
|
209
|
+
runInAction(() => {
|
|
210
|
+
this.ingestDefinition = next;
|
|
211
|
+
});
|
|
212
|
+
// Re-derive `sourceAccessor` against the new ingest. The previously
|
|
213
|
+
// selected data set most likely doesn't exist on the new ingest, so
|
|
214
|
+
// pick the first available one (if any).
|
|
215
|
+
await this.changeAccessorOwner(next);
|
|
216
|
+
const firstAccessor = this.accessors[0];
|
|
217
|
+
if (firstAccessor) {
|
|
218
|
+
await this.changeAccessor(firstAccessor);
|
|
219
|
+
}
|
|
220
|
+
} catch (error) {
|
|
221
|
+
assertErrorThrown(error);
|
|
222
|
+
this.applicationStore.notificationService.notifyError(error);
|
|
223
|
+
} finally {
|
|
224
|
+
runInAction(() => {
|
|
225
|
+
this.isSwappingIngest = false;
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Copies the deep-link to this ingest query set up (GAV + ingest definition
|
|
232
|
+
* path + data set) to the user's clipboard. Mirrors
|
|
233
|
+
* `LegendQueryDataProductQueryBuilderState.copyDataProductLinkToClipBoard`.
|
|
234
|
+
*/
|
|
235
|
+
copyIngestQueryLinkToClipBoard(): void {
|
|
236
|
+
const route =
|
|
237
|
+
this.applicationStore.navigationService.navigator.generateAddress(
|
|
238
|
+
generateIngestQueryCreatorRoute(
|
|
239
|
+
this.project.groupId,
|
|
240
|
+
this.project.artifactId,
|
|
241
|
+
this.project.versionId,
|
|
242
|
+
this.ingestDefinition.path,
|
|
243
|
+
this.dataSet,
|
|
244
|
+
),
|
|
245
|
+
);
|
|
246
|
+
navigator.clipboard
|
|
247
|
+
.writeText(route)
|
|
248
|
+
.then(() =>
|
|
249
|
+
this.applicationStore.notificationService.notifySuccess(
|
|
250
|
+
'Copied ingest query set up link to clipboard',
|
|
251
|
+
),
|
|
252
|
+
)
|
|
253
|
+
.catch(() =>
|
|
254
|
+
this.applicationStore.notificationService.notifyError(
|
|
255
|
+
'Error copying ingest query set up link to clipboard',
|
|
256
|
+
),
|
|
257
|
+
);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2020-present, Goldman Sachs
|
|
3
|
+
*
|
|
4
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
* you may not use this file except in compliance with the License.
|
|
6
|
+
* You may obtain a copy of the License at
|
|
7
|
+
*
|
|
8
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
*
|
|
10
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
* See the License for the specific language governing permissions and
|
|
14
|
+
* limitations under the License.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import {
|
|
18
|
+
type GeneratorFn,
|
|
19
|
+
assertErrorThrown,
|
|
20
|
+
guaranteeType,
|
|
21
|
+
LogEvent,
|
|
22
|
+
StopWatch,
|
|
23
|
+
uuid,
|
|
24
|
+
} from '@finos/legend-shared';
|
|
25
|
+
import type { DepotServerClient } from '@finos/legend-server-depot';
|
|
26
|
+
import {
|
|
27
|
+
type Entity,
|
|
28
|
+
extractEntityNameFromPath,
|
|
29
|
+
type ProjectGAVCoordinates,
|
|
30
|
+
} from '@finos/legend-storage';
|
|
31
|
+
import {
|
|
32
|
+
type QueryBuilderState,
|
|
33
|
+
QueryBuilderDataBrowserWorkflow,
|
|
34
|
+
} from '@finos/legend-query-builder';
|
|
35
|
+
import {
|
|
36
|
+
type Query,
|
|
37
|
+
type RawLambda,
|
|
38
|
+
GRAPH_MANAGER_EVENT,
|
|
39
|
+
IngestDefinition,
|
|
40
|
+
} from '@finos/legend-graph';
|
|
41
|
+
import type { LegendQueryApplicationStore } from '../LegendQueryBaseStore.js';
|
|
42
|
+
import {
|
|
43
|
+
QueryEditorStore,
|
|
44
|
+
type QueryPersistConfiguration,
|
|
45
|
+
QueryBuilderActionConfig_QueryApplication,
|
|
46
|
+
} from '../QueryEditorStore.js';
|
|
47
|
+
import { LEGEND_QUERY_APP_EVENT } from '../../__lib__/LegendQueryEvent.js';
|
|
48
|
+
import { IngestLegendQueryBuilderState } from './IngestLegendQueryBuilderState.js';
|
|
49
|
+
import {
|
|
50
|
+
fetchIngestEntitiesByClassifier,
|
|
51
|
+
swapIngestInGraph,
|
|
52
|
+
} from './IngestQueryGraphHelper.js';
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Creator store backing the `INGEST_QUERY` route.
|
|
56
|
+
*
|
|
57
|
+
* Unlike the data-product creator (which builds a full graph to back the
|
|
58
|
+
* editor's element browsing), the ingest flow only needs the single ingest
|
|
59
|
+
* definition entity. We therefore override {@link QueryEditorStore.buildGraph}
|
|
60
|
+
* to fetch JUST that entity from Depot and build a minimal graph containing
|
|
61
|
+
* it — no dependency resolution, no full project entity fetch.
|
|
62
|
+
*/
|
|
63
|
+
export class IngestQueryCreatorStore extends QueryEditorStore {
|
|
64
|
+
readonly groupId: string;
|
|
65
|
+
readonly artifactId: string;
|
|
66
|
+
readonly versionId: string;
|
|
67
|
+
readonly ingestDefinitionPath: string;
|
|
68
|
+
readonly dataSet: string;
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* All ingest definition entities for this project version, keyed by path.
|
|
72
|
+
* Populated up front by {@link buildGraph} so the source dropdown can list
|
|
73
|
+
* every ingest without forcing us to build them all into the graph. The
|
|
74
|
+
* entity payloads are kept so {@link swapIngestDefinition} can build a
|
|
75
|
+
* different one on demand.
|
|
76
|
+
*/
|
|
77
|
+
private _ingestEntitiesByPath = new Map<string, Entity>();
|
|
78
|
+
|
|
79
|
+
declare queryBuilderState: IngestLegendQueryBuilderState | undefined;
|
|
80
|
+
|
|
81
|
+
constructor(
|
|
82
|
+
applicationStore: LegendQueryApplicationStore,
|
|
83
|
+
depotServerClient: DepotServerClient,
|
|
84
|
+
groupId: string,
|
|
85
|
+
artifactId: string,
|
|
86
|
+
versionId: string,
|
|
87
|
+
ingestDefinitionPath: string,
|
|
88
|
+
dataSet: string,
|
|
89
|
+
) {
|
|
90
|
+
super(applicationStore, depotServerClient);
|
|
91
|
+
this.groupId = groupId;
|
|
92
|
+
this.artifactId = artifactId;
|
|
93
|
+
this.versionId = versionId;
|
|
94
|
+
this.ingestDefinitionPath = ingestDefinitionPath;
|
|
95
|
+
this.dataSet = dataSet;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
getProjectInfo(): ProjectGAVCoordinates {
|
|
99
|
+
return {
|
|
100
|
+
groupId: this.groupId,
|
|
101
|
+
artifactId: this.artifactId,
|
|
102
|
+
versionId: this.versionId,
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Fetch every ingest definition entity in the project (so the source
|
|
108
|
+
* dropdown can show them all) but only build the requested one into the
|
|
109
|
+
* graph. The remaining entity payloads are cached on
|
|
110
|
+
* {@link _ingestEntitiesByPath} and built on demand when the user swaps
|
|
111
|
+
* ingests via {@link swapIngestDefinition}.
|
|
112
|
+
*
|
|
113
|
+
* TODO: revisit once `IngestDefinition` is properly modelled. Today the
|
|
114
|
+
* entity is self-contained, but materialized views can reference other
|
|
115
|
+
* elements (mappings, classes, runtimes, etc.) — at that point we'll need
|
|
116
|
+
* to either fetch those dependent entities too, or fall back to a full
|
|
117
|
+
* project build like the data-product creator does.
|
|
118
|
+
*/
|
|
119
|
+
override *buildGraph(): GeneratorFn<void> {
|
|
120
|
+
const stopWatch = new StopWatch();
|
|
121
|
+
|
|
122
|
+
yield this.graphManagerState.initializeSystem();
|
|
123
|
+
stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH_SYSTEM__SUCCESS);
|
|
124
|
+
|
|
125
|
+
this.initState.setMessage(`Fetching ingest definitions...`);
|
|
126
|
+
this._ingestEntitiesByPath = (yield fetchIngestEntitiesByClassifier(
|
|
127
|
+
this.depotServerClient,
|
|
128
|
+
this.groupId,
|
|
129
|
+
this.artifactId,
|
|
130
|
+
this.versionId,
|
|
131
|
+
)) as Map<string, Entity>;
|
|
132
|
+
this.initState.setMessage(undefined);
|
|
133
|
+
stopWatch.record(GRAPH_MANAGER_EVENT.FETCH_GRAPH_ENTITIES__SUCCESS);
|
|
134
|
+
|
|
135
|
+
const target = this._ingestEntitiesByPath.get(this.ingestDefinitionPath);
|
|
136
|
+
if (!target) {
|
|
137
|
+
throw new Error(
|
|
138
|
+
`Can't find ingest definition '${this.ingestDefinitionPath}' in project ${this.groupId}:${this.artifactId}:${this.versionId}`,
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
yield swapIngestInGraph(this.graphManagerState, target, {
|
|
143
|
+
groupId: this.groupId,
|
|
144
|
+
artifactId: this.artifactId,
|
|
145
|
+
versionId: this.versionId,
|
|
146
|
+
});
|
|
147
|
+
stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS);
|
|
148
|
+
|
|
149
|
+
this.applicationStore.logService.info(
|
|
150
|
+
LogEvent.create(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS),
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Remove the currently built ingest from the graph and build the entity at
|
|
156
|
+
* `path` in its place. Backs {@link IngestLegendQueryBuilderState.swapIngest}
|
|
157
|
+
* — see that callsite for why we don't pre-build every ingest.
|
|
158
|
+
*/
|
|
159
|
+
async swapIngestDefinition(path: string): Promise<IngestDefinition> {
|
|
160
|
+
const entity = this._ingestEntitiesByPath.get(path);
|
|
161
|
+
if (!entity) {
|
|
162
|
+
throw new Error(
|
|
163
|
+
`Can't find ingest definition '${path}' in project ${this.groupId}:${this.artifactId}:${this.versionId}`,
|
|
164
|
+
);
|
|
165
|
+
}
|
|
166
|
+
return swapIngestInGraph(this.graphManagerState, entity, {
|
|
167
|
+
currentPath: this.ingestDefinitionPath,
|
|
168
|
+
groupId: this.groupId,
|
|
169
|
+
artifactId: this.artifactId,
|
|
170
|
+
versionId: this.versionId,
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
async initializeQueryBuilderState(): Promise<QueryBuilderState> {
|
|
175
|
+
const ingestDefinition = guaranteeType(
|
|
176
|
+
this.graphManagerState.graph.getElement(this.ingestDefinitionPath),
|
|
177
|
+
IngestDefinition,
|
|
178
|
+
`Can't find ingest definition '${this.ingestDefinitionPath}'`,
|
|
179
|
+
);
|
|
180
|
+
|
|
181
|
+
const sourceInfo = {
|
|
182
|
+
groupId: this.groupId,
|
|
183
|
+
artifactId: this.artifactId,
|
|
184
|
+
versionId: this.versionId,
|
|
185
|
+
};
|
|
186
|
+
|
|
187
|
+
// Build an adhoc lakehouse runtime using the user's lakehouse env +
|
|
188
|
+
// consumer warehouse — same machinery used by the data product flow for
|
|
189
|
+
// model-access / lakehouse access points. Register it on the graph as an
|
|
190
|
+
// `_internal_` element so the editor can resolve it.
|
|
191
|
+
const adhocRuntime = await this.createLakehousePackageableRuntime(
|
|
192
|
+
this.ingestDefinitionPath,
|
|
193
|
+
{
|
|
194
|
+
groupId: this.groupId,
|
|
195
|
+
artifactId: this.artifactId,
|
|
196
|
+
versionId: this.versionId,
|
|
197
|
+
},
|
|
198
|
+
);
|
|
199
|
+
this.graphManagerState.graph.addElement(adhocRuntime, '_internal_');
|
|
200
|
+
|
|
201
|
+
const queryBuilderState = new IngestLegendQueryBuilderState(
|
|
202
|
+
this.applicationStore,
|
|
203
|
+
undefined,
|
|
204
|
+
this.graphManagerState,
|
|
205
|
+
QueryBuilderDataBrowserWorkflow.INSTANCE,
|
|
206
|
+
new QueryBuilderActionConfig_QueryApplication(this),
|
|
207
|
+
ingestDefinition,
|
|
208
|
+
Array.from(this._ingestEntitiesByPath.keys()),
|
|
209
|
+
(path) => this.swapIngestDefinition(path),
|
|
210
|
+
adhocRuntime,
|
|
211
|
+
{
|
|
212
|
+
groupId: this.groupId,
|
|
213
|
+
artifactId: this.artifactId,
|
|
214
|
+
versionId: this.versionId,
|
|
215
|
+
},
|
|
216
|
+
undefined,
|
|
217
|
+
sourceInfo,
|
|
218
|
+
);
|
|
219
|
+
|
|
220
|
+
try {
|
|
221
|
+
// Resolve the accessor for the requested data set on this ingest
|
|
222
|
+
// definition. This drives the editor source panel + compatible runtimes.
|
|
223
|
+
await queryBuilderState.changeAccessorOwner(ingestDefinition);
|
|
224
|
+
await queryBuilderState.changeAccessor({ tableName: this.dataSet });
|
|
225
|
+
// Select the adhoc lakehouse runtime so the editor can execute the
|
|
226
|
+
// query without the user having to pick one from `compatibleRuntimes`.
|
|
227
|
+
// Mirrors what `LegendQueryDataProductQueryBuilderState.prepareAccessForExecution`
|
|
228
|
+
// does for model-access / lakehouse access points in the data product flow.
|
|
229
|
+
queryBuilderState.changeSelectedRuntime(adhocRuntime);
|
|
230
|
+
} catch (error) {
|
|
231
|
+
assertErrorThrown(error);
|
|
232
|
+
this.applicationStore.logService.error(
|
|
233
|
+
LogEvent.create(LEGEND_QUERY_APP_EVENT.GENERIC_FAILURE),
|
|
234
|
+
error,
|
|
235
|
+
);
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
return queryBuilderState;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
getPersistConfiguration(
|
|
242
|
+
lambda: RawLambda,
|
|
243
|
+
options?: { update?: boolean | undefined },
|
|
244
|
+
): QueryPersistConfiguration {
|
|
245
|
+
const queryBuilderState = this.queryBuilderState;
|
|
246
|
+
const ingestPath =
|
|
247
|
+
queryBuilderState?.ingestDefinition.path ?? this.ingestDefinitionPath;
|
|
248
|
+
const dataSet = queryBuilderState?.dataSet ?? this.dataSet;
|
|
249
|
+
const ingestName = extractEntityNameFromPath(ingestPath);
|
|
250
|
+
return {
|
|
251
|
+
defaultName: options?.update
|
|
252
|
+
? `${ingestName}[${dataSet}]`
|
|
253
|
+
: `New Query for ${ingestName}[${dataSet}]`,
|
|
254
|
+
decorator: (query: Query): void => {
|
|
255
|
+
query.id = uuid();
|
|
256
|
+
query.groupId = this.groupId;
|
|
257
|
+
query.artifactId = this.artifactId;
|
|
258
|
+
query.versionId = this.versionId;
|
|
259
|
+
},
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2020-present, Goldman Sachs
|
|
3
|
+
*
|
|
4
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
* you may not use this file except in compliance with the License.
|
|
6
|
+
* You may obtain a copy of the License at
|
|
7
|
+
*
|
|
8
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
*
|
|
10
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
* See the License for the specific language governing permissions and
|
|
14
|
+
* limitations under the License.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import {
|
|
18
|
+
type DepotServerClient,
|
|
19
|
+
resolveVersion,
|
|
20
|
+
} from '@finos/legend-server-depot';
|
|
21
|
+
import type { Entity, EntityWithOrigin } from '@finos/legend-storage';
|
|
22
|
+
import {
|
|
23
|
+
type GraphManagerState,
|
|
24
|
+
CORE_PURE_PATH,
|
|
25
|
+
createGraphBuilderReport,
|
|
26
|
+
IngestDefinition,
|
|
27
|
+
LegendSDLC,
|
|
28
|
+
} from '@finos/legend-graph';
|
|
29
|
+
import { guaranteeType } from '@finos/legend-shared';
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Shared helpers for ingest-backed query flows (creator + existing-query
|
|
33
|
+
* loader). Centralizes how we list / cache / swap ingest definitions in the
|
|
34
|
+
* graph so both flows stay in lockstep.
|
|
35
|
+
*
|
|
36
|
+
* Why these live outside the stores: both
|
|
37
|
+
* {@link IngestQueryCreatorStore} (deep-link flow) and
|
|
38
|
+
* {@link ExistingQueryEditorStore} (saved-query flow) need to:
|
|
39
|
+
* 1. fetch every ingest entity in a project version (cheap, classifier-
|
|
40
|
+
* scoped depot call) so the source dropdown can list them all, and
|
|
41
|
+
* 2. build only the active ingest into the graph, swapping it out when the
|
|
42
|
+
* user picks a different one.
|
|
43
|
+
* Keeping the logic in one module avoids the two stores drifting apart.
|
|
44
|
+
*/
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Fetch every {@link IngestDefinition} entity for a project version via the
|
|
48
|
+
* classifier-scoped variant of `getVersionEntities`. The endpoint returns
|
|
49
|
+
* {@link EntityWithOrigin}-shaped payloads (entity wrapped with GAV) rather
|
|
50
|
+
* than raw entities, so we unwrap explicitly.
|
|
51
|
+
*
|
|
52
|
+
* Filters by `classifierPath` defensively in case the server returns extras.
|
|
53
|
+
*/
|
|
54
|
+
export const fetchIngestEntitiesByClassifier = async (
|
|
55
|
+
depotServerClient: DepotServerClient,
|
|
56
|
+
groupId: string,
|
|
57
|
+
artifactId: string,
|
|
58
|
+
versionId: string,
|
|
59
|
+
): Promise<Map<string, Entity>> => {
|
|
60
|
+
const wrapped = (await depotServerClient.getVersionEntities(
|
|
61
|
+
groupId,
|
|
62
|
+
artifactId,
|
|
63
|
+
resolveVersion(versionId),
|
|
64
|
+
CORE_PURE_PATH.INGEST_DEFINITION,
|
|
65
|
+
)) as unknown as EntityWithOrigin[];
|
|
66
|
+
const entities = wrapped
|
|
67
|
+
.map((w) => w.entity)
|
|
68
|
+
.filter((e) => e.classifierPath === CORE_PURE_PATH.INGEST_DEFINITION);
|
|
69
|
+
return new Map(entities.map((e) => [e.path, e]));
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Build the given ingest entity into `graph`. If `currentPath` is provided
|
|
74
|
+
* and resolves to an existing ingest on the graph, that ingest is removed
|
|
75
|
+
* first — this is the swap path used when the user picks a different ingest
|
|
76
|
+
* from the source dropdown.
|
|
77
|
+
*
|
|
78
|
+
* The SDLC origin is only set when the graph has none yet, so subsequent
|
|
79
|
+
* swaps (which run after the initial graph build set the origin) don't
|
|
80
|
+
* clobber it.
|
|
81
|
+
*/
|
|
82
|
+
export const swapIngestInGraph = async (
|
|
83
|
+
graphManagerState: GraphManagerState,
|
|
84
|
+
entity: Entity,
|
|
85
|
+
options: {
|
|
86
|
+
currentPath?: string | undefined;
|
|
87
|
+
groupId: string;
|
|
88
|
+
artifactId: string;
|
|
89
|
+
versionId: string;
|
|
90
|
+
},
|
|
91
|
+
): Promise<IngestDefinition> => {
|
|
92
|
+
if (options.currentPath !== undefined) {
|
|
93
|
+
const current = graphManagerState.graph.ingests.find(
|
|
94
|
+
(i) => i.path === options.currentPath,
|
|
95
|
+
);
|
|
96
|
+
if (current) {
|
|
97
|
+
graphManagerState.graph.deleteElement(current);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
await graphManagerState.graphManager.buildGraph(
|
|
101
|
+
graphManagerState.graph,
|
|
102
|
+
[entity],
|
|
103
|
+
graphManagerState.graphBuildState,
|
|
104
|
+
graphManagerState.graph.origin === undefined
|
|
105
|
+
? {
|
|
106
|
+
origin: new LegendSDLC(
|
|
107
|
+
options.groupId,
|
|
108
|
+
options.artifactId,
|
|
109
|
+
resolveVersion(options.versionId),
|
|
110
|
+
),
|
|
111
|
+
}
|
|
112
|
+
: undefined,
|
|
113
|
+
createGraphBuilderReport(),
|
|
114
|
+
);
|
|
115
|
+
return guaranteeType(
|
|
116
|
+
graphManagerState.graph.getElement(entity.path),
|
|
117
|
+
IngestDefinition,
|
|
118
|
+
`Can't find ingest definition '${entity.path}' after build`,
|
|
119
|
+
);
|
|
120
|
+
};
|
package/tsconfig.json
CHANGED
|
@@ -88,6 +88,9 @@
|
|
|
88
88
|
"./src/stores/data-space/DataSpaceTemplateQueryCreatorStore.ts",
|
|
89
89
|
"./src/stores/data-space/LegendQueryBareQueryBuilderState.ts",
|
|
90
90
|
"./src/stores/data-space/query-builder/LegendQueryDataSpaceQueryBuilderState.ts",
|
|
91
|
+
"./src/stores/ingest/IngestLegendQueryBuilderState.ts",
|
|
92
|
+
"./src/stores/ingest/IngestQueryCreatorStore.ts",
|
|
93
|
+
"./src/stores/ingest/IngestQueryGraphHelper.ts",
|
|
91
94
|
"./src/application/LegendQuery.tsx",
|
|
92
95
|
"./src/components/CloneQueryServiceSetup.tsx",
|
|
93
96
|
"./src/components/Core_LegendQueryApplicationPlugin.tsx",
|
|
@@ -116,6 +119,10 @@
|
|
|
116
119
|
"./src/components/data-space/DataSpaceQuerySetup.tsx",
|
|
117
120
|
"./src/components/data-space/DataSpaceTemplateQueryCreator.tsx",
|
|
118
121
|
"./src/components/data-space/LegendQueryDataSpaceQueryBuilder.tsx",
|
|
122
|
+
"./src/components/ingest/IngestInfo.tsx",
|
|
123
|
+
"./src/components/ingest/IngestQueryBuilderSetupPanel.tsx",
|
|
124
|
+
"./src/components/ingest/IngestQueryCreator.tsx",
|
|
125
|
+
"./src/components/shared/LakehouseRuntimeConfigModal.tsx",
|
|
119
126
|
"./src/components/shared/LegendQueryDataProductOptionLabel.tsx",
|
|
120
127
|
"./src/stores/LegendQueryApplicationPlugin.tsx"
|
|
121
128
|
],
|