@finos/legend-server-lakehouse 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/LICENSE +201 -0
  2. package/README.md +1 -0
  3. package/lib/LakehouseIngestServerClient.d.ts +38 -0
  4. package/lib/LakehouseIngestServerClient.d.ts.map +1 -0
  5. package/lib/LakehouseIngestServerClient.js +56 -0
  6. package/lib/LakehouseIngestServerClient.js.map +1 -0
  7. package/lib/LakehouseIngestionManager.d.ts +38 -0
  8. package/lib/LakehouseIngestionManager.d.ts.map +1 -0
  9. package/lib/LakehouseIngestionManager.js +112 -0
  10. package/lib/LakehouseIngestionManager.js.map +1 -0
  11. package/lib/LakehousePlatformServerClient.d.ts +29 -0
  12. package/lib/LakehousePlatformServerClient.d.ts.map +1 -0
  13. package/lib/LakehousePlatformServerClient.js +38 -0
  14. package/lib/LakehousePlatformServerClient.js.map +1 -0
  15. package/lib/index.d.ts +22 -0
  16. package/lib/index.d.ts.map +1 -0
  17. package/lib/index.js +22 -0
  18. package/lib/index.js.map +1 -0
  19. package/lib/models/AdhocDataProductDeployResponse.d.ts +21 -0
  20. package/lib/models/AdhocDataProductDeployResponse.d.ts.map +1 -0
  21. package/lib/models/AdhocDataProductDeployResponse.js +24 -0
  22. package/lib/models/AdhocDataProductDeployResponse.js.map +1 -0
  23. package/lib/models/IngestDeploymentServerConfig.d.ts +22 -0
  24. package/lib/models/IngestDeploymentServerConfig.d.ts.map +1 -0
  25. package/lib/models/IngestDeploymentServerConfig.js +26 -0
  26. package/lib/models/IngestDeploymentServerConfig.js.map +1 -0
  27. package/lib/models/LakehouseIngestionDeploymentResponse.d.ts +41 -0
  28. package/lib/models/LakehouseIngestionDeploymentResponse.d.ts.map +1 -0
  29. package/lib/models/LakehouseIngestionDeploymentResponse.js +56 -0
  30. package/lib/models/LakehouseIngestionDeploymentResponse.js.map +1 -0
  31. package/package.json +59 -0
  32. package/src/LakehouseIngestServerClient.ts +129 -0
  33. package/src/LakehouseIngestionManager.ts +216 -0
  34. package/src/LakehousePlatformServerClient.ts +65 -0
  35. package/src/index.ts +22 -0
  36. package/src/models/AdhocDataProductDeployResponse.ts +29 -0
  37. package/src/models/IngestDeploymentServerConfig.ts +30 -0
  38. package/src/models/LakehouseIngestionDeploymentResponse.ts +74 -0
  39. package/tsconfig.json +66 -0
@@ -0,0 +1,24 @@
1
+ /**
2
+ * Copyright (c) 2020-present, Goldman Sachs
3
+ *
4
+ * Licensed under the Apache License, Version 2.0 (the "License");
5
+ * you may not use this file except in compliance with the License.
6
+ * You may obtain a copy of the License at
7
+ *
8
+ * http://www.apache.org/licenses/LICENSE-2.0
9
+ *
10
+ * Unless required by applicable law or agreed to in writing, software
11
+ * distributed under the License is distributed on an "AS IS" BASIS,
12
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ * See the License for the specific language governing permissions and
14
+ * limitations under the License.
15
+ */
16
+ export class AdhocDataProductDeployResponse {
17
+ content;
18
+ }
19
+ export const createAdhocDataProductDeployResponse = (json) => {
20
+ const response = new AdhocDataProductDeployResponse();
21
+ response.content = json;
22
+ return response;
23
+ };
24
+ //# sourceMappingURL=AdhocDataProductDeployResponse.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"AdhocDataProductDeployResponse.js","sourceRoot":"","sources":["../../src/models/AdhocDataProductDeployResponse.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAIH,MAAM,OAAO,8BAA8B;IACzC,OAAO,CAAe;CACvB;AAED,MAAM,CAAC,MAAM,oCAAoC,GAAG,CAClD,IAAiD,EACjB,EAAE;IAClC,MAAM,QAAQ,GAAG,IAAI,8BAA8B,EAAE,CAAC;IACtD,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC;IACxB,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC"}
@@ -0,0 +1,22 @@
1
+ /**
2
+ * Copyright (c) 2020-present, Goldman Sachs
3
+ *
4
+ * Licensed under the Apache License, Version 2.0 (the "License");
5
+ * you may not use this file except in compliance with the License.
6
+ * You may obtain a copy of the License at
7
+ *
8
+ * http://www.apache.org/licenses/LICENSE-2.0
9
+ *
10
+ * Unless required by applicable law or agreed to in writing, software
11
+ * distributed under the License is distributed on an "AS IS" BASIS,
12
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ * See the License for the specific language governing permissions and
14
+ * limitations under the License.
15
+ */
16
+ import { SerializationFactory } from '@finos/legend-shared';
17
+ export declare class IngestDeploymentServerConfig {
18
+ environmentClassification: string;
19
+ ingestServerUrl: string;
20
+ static readonly serialization: SerializationFactory<IngestDeploymentServerConfig>;
21
+ }
22
+ //# sourceMappingURL=IngestDeploymentServerConfig.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"IngestDeploymentServerConfig.d.ts","sourceRoot":"","sources":["../../src/models/IngestDeploymentServerConfig.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAG5D,qBAAa,4BAA4B;IACvC,yBAAyB,EAAG,MAAM,CAAC;IACnC,eAAe,EAAG,MAAM,CAAC;IAEzB,MAAM,CAAC,QAAQ,CAAC,aAAa,qDAK3B;CACH"}
@@ -0,0 +1,26 @@
1
+ /**
2
+ * Copyright (c) 2020-present, Goldman Sachs
3
+ *
4
+ * Licensed under the Apache License, Version 2.0 (the "License");
5
+ * you may not use this file except in compliance with the License.
6
+ * You may obtain a copy of the License at
7
+ *
8
+ * http://www.apache.org/licenses/LICENSE-2.0
9
+ *
10
+ * Unless required by applicable law or agreed to in writing, software
11
+ * distributed under the License is distributed on an "AS IS" BASIS,
12
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ * See the License for the specific language governing permissions and
14
+ * limitations under the License.
15
+ */
16
+ import { SerializationFactory } from '@finos/legend-shared';
17
+ import { createModelSchema, primitive } from 'serializr';
18
+ export class IngestDeploymentServerConfig {
19
+ environmentClassification;
20
+ ingestServerUrl;
21
+ static serialization = new SerializationFactory(createModelSchema(IngestDeploymentServerConfig, {
22
+ environmentClassification: primitive(),
23
+ ingestServerUrl: primitive(),
24
+ }));
25
+ }
26
+ //# sourceMappingURL=IngestDeploymentServerConfig.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"IngestDeploymentServerConfig.js","sourceRoot":"","sources":["../../src/models/IngestDeploymentServerConfig.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEzD,MAAM,OAAO,4BAA4B;IACvC,yBAAyB,CAAU;IACnC,eAAe,CAAU;IAEzB,MAAM,CAAU,aAAa,GAAG,IAAI,oBAAoB,CACtD,iBAAiB,CAAC,4BAA4B,EAAE;QAC9C,yBAAyB,EAAE,SAAS,EAAE;QACtC,eAAe,EAAE,SAAS,EAAE;KAC7B,CAAC,CACH,CAAC"}
@@ -0,0 +1,41 @@
1
+ /**
2
+ * Copyright (c) 2020-present, Goldman Sachs
3
+ *
4
+ * Licensed under the Apache License, Version 2.0 (the "License");
5
+ * you may not use this file except in compliance with the License.
6
+ * You may obtain a copy of the License at
7
+ *
8
+ * http://www.apache.org/licenses/LICENSE-2.0
9
+ *
10
+ * Unless required by applicable law or agreed to in writing, software
11
+ * distributed under the License is distributed on an "AS IS" BASIS,
12
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ * See the License for the specific language governing permissions and
14
+ * limitations under the License.
15
+ */
16
+ import { SerializationFactory, type PlainObject } from '@finos/legend-shared';
17
+ export declare enum IngestDefinitionValidationResponseStatus {
18
+ NOT_CHECKED = "NOT_CHECKED",
19
+ SUCCESS = "SUCCESS",
20
+ FAILURE = "FAILURE"
21
+ }
22
+ export declare class IngestDefinitionValidationOutput {
23
+ status: IngestDefinitionValidationResponseStatus;
24
+ }
25
+ export declare class IngestDefinitionValidationResponse {
26
+ status: IngestDefinitionValidationResponseStatus;
27
+ content: PlainObject;
28
+ static readonly serialization: SerializationFactory<IngestDefinitionValidationResponse>;
29
+ }
30
+ export declare const createIngestDefinitionValidationResponse: (json: PlainObject<IngestDefinitionValidationResponse>) => IngestDefinitionValidationResponse;
31
+ export declare class IngestDefinitionDeploymentResponse {
32
+ ingestDefinitionUrn: string;
33
+ write_location: PlainObject | undefined;
34
+ static readonly serialization: SerializationFactory<IngestDefinitionDeploymentResponse>;
35
+ }
36
+ export declare class ValidateAndDeploymentResponse {
37
+ validationResponse: IngestDefinitionValidationResponse;
38
+ deploymentResponse: IngestDefinitionDeploymentResponse | undefined;
39
+ constructor(validationResponse: IngestDefinitionValidationResponse, deploymentResponse: IngestDefinitionDeploymentResponse | undefined);
40
+ }
41
+ //# sourceMappingURL=LakehouseIngestionDeploymentResponse.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"LakehouseIngestionDeploymentResponse.d.ts","sourceRoot":"","sources":["../../src/models/LakehouseIngestionDeploymentResponse.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,oBAAoB,EAAE,KAAK,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAG9E,oBAAY,wCAAwC;IAClD,WAAW,gBAAgB;IAC3B,OAAO,YAAY;IACnB,OAAO,YAAY;CACpB;AAED,qBAAa,gCAAgC;IAC3C,MAAM,EAAG,wCAAwC,CAAC;CACnD;AAED,qBAAa,kCAAkC;IAC7C,MAAM,EAAG,wCAAwC,CAAC;IAClD,OAAO,EAAG,WAAW,CAAC;IAEtB,MAAM,CAAC,QAAQ,CAAC,aAAa,2DAI3B;CACH;AAED,eAAO,MAAM,wCAAwC,GACnD,MAAM,WAAW,CAAC,kCAAkC,CAAC,KACpD,kCAOF,CAAC;AAEF,qBAAa,kCAAkC;IAC7C,mBAAmB,EAAG,MAAM,CAAC;IAC7B,cAAc,EAAE,WAAW,GAAG,SAAS,CAAC;IAExC,MAAM,CAAC,QAAQ,CAAC,aAAa,2DAI3B;CACH;AAED,qBAAa,6BAA6B;IACxC,kBAAkB,EAAE,kCAAkC,CAAC;IACvD,kBAAkB,EAAE,kCAAkC,GAAG,SAAS,CAAC;gBAGjE,kBAAkB,EAAE,kCAAkC,EACtD,kBAAkB,EAAE,kCAAkC,GAAG,SAAS;CAKrE"}
@@ -0,0 +1,56 @@
1
+ /**
2
+ * Copyright (c) 2020-present, Goldman Sachs
3
+ *
4
+ * Licensed under the Apache License, Version 2.0 (the "License");
5
+ * you may not use this file except in compliance with the License.
6
+ * You may obtain a copy of the License at
7
+ *
8
+ * http://www.apache.org/licenses/LICENSE-2.0
9
+ *
10
+ * Unless required by applicable law or agreed to in writing, software
11
+ * distributed under the License is distributed on an "AS IS" BASIS,
12
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ * See the License for the specific language governing permissions and
14
+ * limitations under the License.
15
+ */
16
+ import { SerializationFactory } from '@finos/legend-shared';
17
+ import { createModelSchema, primitive } from 'serializr';
18
+ export var IngestDefinitionValidationResponseStatus;
19
+ (function (IngestDefinitionValidationResponseStatus) {
20
+ IngestDefinitionValidationResponseStatus["NOT_CHECKED"] = "NOT_CHECKED";
21
+ IngestDefinitionValidationResponseStatus["SUCCESS"] = "SUCCESS";
22
+ IngestDefinitionValidationResponseStatus["FAILURE"] = "FAILURE";
23
+ })(IngestDefinitionValidationResponseStatus || (IngestDefinitionValidationResponseStatus = {}));
24
+ export class IngestDefinitionValidationOutput {
25
+ status;
26
+ }
27
+ export class IngestDefinitionValidationResponse {
28
+ status;
29
+ content;
30
+ static serialization = new SerializationFactory(createModelSchema(IngestDefinitionValidationResponse, {
31
+ status: primitive(),
32
+ }));
33
+ }
34
+ export const createIngestDefinitionValidationResponse = (json) => {
35
+ const ingestDefinitionValidationResponse = new IngestDefinitionValidationResponse();
36
+ ingestDefinitionValidationResponse.status =
37
+ json.status;
38
+ ingestDefinitionValidationResponse.content = json;
39
+ return ingestDefinitionValidationResponse;
40
+ };
41
+ export class IngestDefinitionDeploymentResponse {
42
+ ingestDefinitionUrn;
43
+ write_location;
44
+ static serialization = new SerializationFactory(createModelSchema(IngestDefinitionDeploymentResponse, {
45
+ ingestDefinitionUrn: primitive(),
46
+ }));
47
+ }
48
+ export class ValidateAndDeploymentResponse {
49
+ validationResponse;
50
+ deploymentResponse;
51
+ constructor(validationResponse, deploymentResponse) {
52
+ this.validationResponse = validationResponse;
53
+ this.deploymentResponse = deploymentResponse;
54
+ }
55
+ }
56
+ //# sourceMappingURL=LakehouseIngestionDeploymentResponse.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"LakehouseIngestionDeploymentResponse.js","sourceRoot":"","sources":["../../src/models/LakehouseIngestionDeploymentResponse.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,oBAAoB,EAAoB,MAAM,sBAAsB,CAAC;AAC9E,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEzD,MAAM,CAAN,IAAY,wCAIX;AAJD,WAAY,wCAAwC;IAClD,uEAA2B,CAAA;IAC3B,+DAAmB,CAAA;IACnB,+DAAmB,CAAA;AACrB,CAAC,EAJW,wCAAwC,KAAxC,wCAAwC,QAInD;AAED,MAAM,OAAO,gCAAgC;IAC3C,MAAM,CAA4C;CACnD;AAED,MAAM,OAAO,kCAAkC;IAC7C,MAAM,CAA4C;IAClD,OAAO,CAAe;IAEtB,MAAM,CAAU,aAAa,GAAG,IAAI,oBAAoB,CACtD,iBAAiB,CAAC,kCAAkC,EAAE;QACpD,MAAM,EAAE,SAAS,EAAE;KACpB,CAAC,CACH,CAAC;;AAGJ,MAAM,CAAC,MAAM,wCAAwC,GAAG,CACtD,IAAqD,EACjB,EAAE;IACtC,MAAM,kCAAkC,GACtC,IAAI,kCAAkC,EAAE,CAAC;IAC3C,kCAAkC,CAAC,MAAM;QACvC,IAAI,CAAC,MAAkD,CAAC;IAC1D,kCAAkC,CAAC,OAAO,GAAG,IAAI,CAAC;IAClD,OAAO,kCAAkC,CAAC;AAC5C,CAAC,CAAC;AAEF,MAAM,OAAO,kCAAkC;IAC7C,mBAAmB,CAAU;IAC7B,cAAc,CAA0B;IAExC,MAAM,CAAU,aAAa,GAAG,IAAI,oBAAoB,CACtD,iBAAiB,CAAC,kCAAkC,EAAE;QACpD,mBAAmB,EAAE,SAAS,EAAE;KACjC,CAAC,CACH,CAAC;;AAGJ,MAAM,OAAO,6BAA6B;IACxC,kBAAkB,CAAqC;IACvD,kBAAkB,CAAiD;IAEnE,YACE,kBAAsD,EACtD,kBAAkE;QAElE,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;IAC/C,CAAC;CACF"}
package/package.json ADDED
@@ -0,0 +1,59 @@
1
+ {
2
+ "name": "@finos/legend-server-lakehouse",
3
+ "version": "0.0.2",
4
+ "description": "Legend Lakehouse Server Client",
5
+ "keywords": [
6
+ "legend",
7
+ "legend-server",
8
+ "legend-server-lakehouse",
9
+ "storage",
10
+ "package-registry"
11
+ ],
12
+ "homepage": "https://github.com/finos/legend-studio/tree/master/packages/legend-server-lakehouse",
13
+ "bugs": {
14
+ "url": "https://github.com/finos/legend-studio/issues"
15
+ },
16
+ "repository": {
17
+ "type": "git",
18
+ "url": "https://github.com/finos/legend-studio.git",
19
+ "directory": "packages/legend-server-lakehouse"
20
+ },
21
+ "license": "Apache-2.0",
22
+ "sideEffects": false,
23
+ "type": "module",
24
+ "exports": "./lib/index.js",
25
+ "module": "lib/index.js",
26
+ "types": "lib/index.d.ts",
27
+ "scripts": {
28
+ "build": "yarn clean && yarn build:ts",
29
+ "build:ts": "tsc --project ./tsconfig.build.json",
30
+ "clean": "npm-run-all clean:cache clean:lib",
31
+ "clean:cache": "rimraf \"build\"",
32
+ "clean:lib": "rimraf \"lib\"",
33
+ "dev": "yarn dev:ts",
34
+ "dev:ts": "tsc --watch --preserveWatchOutput",
35
+ "lint:js": "cross-env NODE_ENV=production eslint --cache --cache-location ./build/.eslintcache --report-unused-disable-directives --parser-options=project:\"./tsconfig.json\" \"./src/**/*.{js,ts,tsx}\"",
36
+ "publish:prepare": "node ../../scripts/release/preparePublishContent.js",
37
+ "publish:snapshot": "node ../../scripts/release/publishDevSnapshot.js",
38
+ "test": "jest",
39
+ "test:watch": "jest --watch"
40
+ },
41
+ "dependencies": {
42
+ "@finos/legend-graph": "32.1.43",
43
+ "@finos/legend-shared": "11.0.13",
44
+ "serializr": "3.0.3"
45
+ },
46
+ "devDependencies": {
47
+ "@finos/legend-dev-utils": "2.2.4",
48
+ "@jest/globals": "29.7.0",
49
+ "cross-env": "7.0.3",
50
+ "eslint": "9.22.0",
51
+ "jest": "29.7.0",
52
+ "npm-run-all": "4.1.5",
53
+ "rimraf": "6.0.1",
54
+ "typescript": "5.8.2"
55
+ },
56
+ "publishConfig": {
57
+ "directory": "build/publishContent"
58
+ }
59
+ }
@@ -0,0 +1,129 @@
1
+ /**
2
+ * Copyright (c) 2020-present, Goldman Sachs
3
+ *
4
+ * Licensed under the Apache License, Version 2.0 (the "License");
5
+ * you may not use this file except in compliance with the License.
6
+ * You may obtain a copy of the License at
7
+ *
8
+ * http://www.apache.org/licenses/LICENSE-2.0
9
+ *
10
+ * Unless required by applicable law or agreed to in writing, software
11
+ * distributed under the License is distributed on an "AS IS" BASIS,
12
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ * See the License for the specific language governing permissions and
14
+ * limitations under the License.
15
+ */
16
+
17
+ import {
18
+ AbstractServerClient,
19
+ ContentType,
20
+ HttpHeader,
21
+ type PlainObject,
22
+ } from '@finos/legend-shared';
23
+ import type {
24
+ IngestDefinitionDeploymentResponse,
25
+ IngestDefinitionValidationResponse,
26
+ } from './models/LakehouseIngestionDeploymentResponse.js';
27
+ import type { IngestDeploymentServerConfig } from './models/IngestDeploymentServerConfig.js';
28
+ import type { AdhocDataProductDeployResponse } from './models/AdhocDataProductDeployResponse.js';
29
+ import type {
30
+ V1_IngestEnvironment,
31
+ V1_SandboxDataProductDeploymentResponse,
32
+ } from '@finos/legend-graph';
33
+
34
+ export class LakehouseIngestServerClient extends AbstractServerClient {
35
+ environmentClassification: string | undefined;
36
+
37
+ private DATA_PRODUCT_URL = 'data-product';
38
+ constructor(config: IngestDeploymentServerConfig | undefined) {
39
+ super({});
40
+ if (config) {
41
+ this.baseUrl = config.ingestServerUrl;
42
+ this.environmentClassification = config.environmentClassification;
43
+ }
44
+ }
45
+
46
+ private _token = (token?: string) => ({
47
+ Authorization: `Bearer ${token}`,
48
+ });
49
+
50
+ private _tokenWithTextPlain = (token?: string) => ({
51
+ [HttpHeader.CONTENT_TYPE]: ContentType.TEXT_PLAIN,
52
+ Authorization: `Bearer ${token}`,
53
+ });
54
+
55
+ private _dataProduct = (serverUrl?: string | undefined): string =>
56
+ `${serverUrl ?? this.baseUrl}/${this.DATA_PRODUCT_URL}/api/entitlements/sdlc/deploy/definitions`;
57
+
58
+ private _ingestDefinitions = (): string =>
59
+ `${this.baseUrl}/api/ingest/sdlc/deploy/definitions`;
60
+
61
+ private _ingest = (serverUrl?: string | undefined): string =>
62
+ `${serverUrl ?? this.baseUrl}/api/ingest`;
63
+
64
+ changeServer(serverConfig: IngestDeploymentServerConfig): void {
65
+ this.baseUrl = serverConfig.ingestServerUrl;
66
+ this.environmentClassification = serverConfig.environmentClassification;
67
+ }
68
+
69
+ validate(
70
+ validateGrammar: string,
71
+ token: string | undefined,
72
+ ): Promise<PlainObject<IngestDefinitionValidationResponse>> {
73
+ return this.post(
74
+ `${this._ingestDefinitions()}/validate`,
75
+ validateGrammar,
76
+ undefined,
77
+ this._tokenWithTextPlain(token),
78
+ );
79
+ }
80
+
81
+ deploy(
82
+ deployGrammar: string,
83
+ token: string | undefined,
84
+ ): Promise<IngestDefinitionDeploymentResponse> {
85
+ return this.post(
86
+ `${this._ingestDefinitions()}`,
87
+ deployGrammar,
88
+ undefined,
89
+ this._tokenWithTextPlain(token),
90
+ );
91
+ }
92
+
93
+ write_location(urn: string, token: string | undefined): Promise<PlainObject> {
94
+ return this.post(
95
+ `${this._ingest()}/${encodeURIComponent(urn)}/write-location`,
96
+ undefined,
97
+ undefined,
98
+ this._token(token),
99
+ );
100
+ }
101
+
102
+ deployDataProduct(
103
+ fullGrammar: string,
104
+ token: string | undefined,
105
+ ): Promise<PlainObject<AdhocDataProductDeployResponse>> {
106
+ return this.post(
107
+ `${this._dataProduct()}`,
108
+ fullGrammar,
109
+ undefined,
110
+ this._token(token),
111
+ );
112
+ }
113
+
114
+ getIngestEnvironment = (
115
+ ingestServerUrl: string | undefined,
116
+ token: string | undefined,
117
+ ): Promise<PlainObject<V1_IngestEnvironment>> =>
118
+ this.get(
119
+ `${this._ingest(ingestServerUrl)}/catalog-state/environment`,
120
+ {},
121
+ this._token(token),
122
+ );
123
+
124
+ getDeployedIngestDefinitions = (
125
+ ingestServerUrl: string | undefined,
126
+ token: string | undefined,
127
+ ): Promise<PlainObject<V1_SandboxDataProductDeploymentResponse>> =>
128
+ this.get(`${this._dataProduct(ingestServerUrl)}`, {}, this._token(token));
129
+ }
@@ -0,0 +1,216 @@
1
+ /**
2
+ * Copyright (c) 2020-present, Goldman Sachs
3
+ *
4
+ * Licensed under the Apache License, Version 2.0 (the "License");
5
+ * you may not use this file except in compliance with the License.
6
+ * You may obtain a copy of the License at
7
+ *
8
+ * http://www.apache.org/licenses/LICENSE-2.0
9
+ *
10
+ * Unless required by applicable law or agreed to in writing, software
11
+ * distributed under the License is distributed on an "AS IS" BASIS,
12
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ * See the License for the specific language governing permissions and
14
+ * limitations under the License.
15
+ */
16
+
17
+ import type { AppDirNode } from '@finos/legend-graph';
18
+ import {
19
+ NetworkClientError,
20
+ type PlainObject,
21
+ type TracerService,
22
+ assertErrorThrown,
23
+ } from '@finos/legend-shared';
24
+ import { IngestDeploymentServerConfig } from './models/IngestDeploymentServerConfig.js';
25
+ import { LakehousePlatformServerClient } from './LakehousePlatformServerClient.js';
26
+ import { LakehouseIngestServerClient } from './LakehouseIngestServerClient.js';
27
+ import {
28
+ createIngestDefinitionValidationResponse,
29
+ IngestDefinitionDeploymentResponse,
30
+ type IngestDefinitionValidationResponse,
31
+ IngestDefinitionValidationResponseStatus,
32
+ ValidateAndDeploymentResponse,
33
+ } from './models/LakehouseIngestionDeploymentResponse.js';
34
+ import {
35
+ createAdhocDataProductDeployResponse,
36
+ type AdhocDataProductDeployResponse,
37
+ } from './models/AdhocDataProductDeployResponse.js';
38
+
39
+ export class LakehouseIngestionManager {
40
+ private ingestDiscoveryServerClient: LakehousePlatformServerClient;
41
+ private ingestDeploymentServerClient: LakehouseIngestServerClient;
42
+ private _currentAppID: number | undefined;
43
+ private _currentLevel: string | undefined;
44
+ private useDefaultServer = false;
45
+
46
+ constructor(
47
+ discoveryUrl: string,
48
+ defaultServer: IngestDeploymentServerConfig,
49
+ useDefaultServer: boolean | undefined,
50
+ tracerService: TracerService,
51
+ ) {
52
+ this.ingestDiscoveryServerClient = new LakehousePlatformServerClient(
53
+ discoveryUrl,
54
+ );
55
+ this.ingestDeploymentServerClient = new LakehouseIngestServerClient(
56
+ defaultServer,
57
+ );
58
+ this.setTracerService(tracerService);
59
+ this.useDefaultServer = Boolean(useDefaultServer);
60
+ }
61
+
62
+ setTracerService(tracerService: TracerService): void {
63
+ this.ingestDiscoveryServerClient.setTracerService(tracerService);
64
+ this.ingestDeploymentServerClient.setTracerService(tracerService);
65
+ }
66
+
67
+ isCurrentAppDirNode(appDirNode: AppDirNode): boolean {
68
+ return (
69
+ this._currentAppID === appDirNode.appDirId &&
70
+ this._currentLevel === appDirNode.level
71
+ );
72
+ }
73
+
74
+ setCurrentAppDirNode(appDirNode: AppDirNode): void {
75
+ this._currentAppID = appDirNode.appDirId;
76
+ this._currentLevel = appDirNode.level;
77
+ }
78
+
79
+ async deploy(
80
+ ingestDefinition: string,
81
+ appDirNode: AppDirNode,
82
+ messageCallBack: (message: string) => void,
83
+ token: string | undefined,
84
+ ): Promise<ValidateAndDeploymentResponse> {
85
+ messageCallBack(
86
+ `Discovering associated ingest environment for DID ${appDirNode.appDirId}...`,
87
+ );
88
+ await this.identifyIngestDeploymentServer(appDirNode, token);
89
+ messageCallBack(
90
+ `Validating ingest with server ${this.ingestDeploymentServerClient.baseUrl ?? ''} for realm ${this.ingestDeploymentServerClient.environmentClassification}...`,
91
+ );
92
+ const validateResonse = await this._validate(
93
+ ingestDefinition,
94
+ undefined,
95
+ token,
96
+ );
97
+ const fullResponse = new ValidateAndDeploymentResponse(
98
+ validateResonse,
99
+ undefined,
100
+ );
101
+ if (
102
+ validateResonse.status !==
103
+ IngestDefinitionValidationResponseStatus.SUCCESS
104
+ ) {
105
+ return fullResponse;
106
+ }
107
+ messageCallBack(
108
+ `Validation Success. Deploying ingest with server ${this.ingestDeploymentServerClient.baseUrl ?? ''} for realm ${this.ingestDeploymentServerClient.environmentClassification}...`,
109
+ );
110
+ const deployResponse = await this._deploy(
111
+ ingestDefinition,
112
+ undefined,
113
+ token,
114
+ );
115
+ try {
116
+ messageCallBack(`Deployment Success. Fetching write location...`);
117
+ await this._fetchDeployLocations(deployResponse, token);
118
+ } catch (error) {
119
+ assertErrorThrown(error);
120
+ }
121
+
122
+ return fullResponse;
123
+ }
124
+
125
+ async deployDataProduct(
126
+ grammarText: string,
127
+ appDirNode: AppDirNode,
128
+ messageCallBack: (message: string) => void,
129
+ token: string | undefined,
130
+ ): Promise<AdhocDataProductDeployResponse> {
131
+ messageCallBack(
132
+ `Discovering associated data product environment for DID ${appDirNode.appDirId}...`,
133
+ );
134
+ await this.identifyIngestDeploymentServer(appDirNode, token);
135
+ messageCallBack(
136
+ `Deploying data product with server ${this.ingestDeploymentServerClient.baseUrl ?? ''} for realm ${this.ingestDeploymentServerClient.environmentClassification}...`,
137
+ );
138
+ const deployResponse =
139
+ await this.ingestDeploymentServerClient.deployDataProduct(
140
+ grammarText,
141
+ token,
142
+ );
143
+ return createAdhocDataProductDeployResponse(deployResponse);
144
+ }
145
+
146
+ private async _validate(
147
+ ingestDefinition: string,
148
+ appDirNode: AppDirNode | undefined,
149
+ token: string | undefined,
150
+ ): Promise<IngestDefinitionValidationResponse> {
151
+ if (appDirNode) {
152
+ await this.identifyIngestDeploymentServer(appDirNode, token);
153
+ }
154
+ try {
155
+ const response = await this.ingestDeploymentServerClient.validate(
156
+ ingestDefinition,
157
+ token,
158
+ );
159
+ return createIngestDefinitionValidationResponse(response);
160
+ } catch (error) {
161
+ assertErrorThrown(error);
162
+ if (error instanceof NetworkClientError) {
163
+ return createIngestDefinitionValidationResponse(
164
+ error.payload as PlainObject<IngestDefinitionValidationResponse>,
165
+ );
166
+ }
167
+ throw error;
168
+ }
169
+ }
170
+
171
+ private async _deploy(
172
+ ingestDefinition: string,
173
+ appDirNode: AppDirNode | undefined,
174
+ token: string | undefined,
175
+ ): Promise<IngestDefinitionDeploymentResponse> {
176
+ if (appDirNode) {
177
+ await this.identifyIngestDeploymentServer(appDirNode, token);
178
+ }
179
+ // validate
180
+ const response = (await this.ingestDeploymentServerClient.deploy(
181
+ ingestDefinition,
182
+ token,
183
+ )) as unknown as PlainObject<IngestDefinitionDeploymentResponse>;
184
+ return IngestDefinitionDeploymentResponse.serialization.fromJson(response);
185
+ }
186
+
187
+ private async _fetchDeployLocations(
188
+ depploymentResponse: IngestDefinitionDeploymentResponse,
189
+ token: string | undefined,
190
+ ): Promise<void> {
191
+ const response = (await this.ingestDeploymentServerClient.write_location(
192
+ depploymentResponse.ingestDefinitionUrn,
193
+ token,
194
+ )) as unknown as PlainObject;
195
+ depploymentResponse.write_location = response;
196
+ }
197
+
198
+ private async identifyIngestDeploymentServer(
199
+ appDirNode: AppDirNode,
200
+ token: string | undefined,
201
+ ): Promise<void> {
202
+ // we do not change if current appDirNode is the same as the one we are trying to set
203
+ if (this.useDefaultServer || this.isCurrentAppDirNode(appDirNode)) {
204
+ return;
205
+ }
206
+ const serverConfig = IngestDeploymentServerConfig.serialization.fromJson(
207
+ await this.ingestDiscoveryServerClient.findProducerServer(
208
+ appDirNode.appDirId,
209
+ appDirNode.level,
210
+ token,
211
+ ),
212
+ );
213
+ this.ingestDeploymentServerClient.changeServer(serverConfig);
214
+ this.setCurrentAppDirNode(appDirNode);
215
+ }
216
+ }
@@ -0,0 +1,65 @@
1
+ /**
2
+ * Copyright (c) 2020-present, Goldman Sachs
3
+ *
4
+ * Licensed under the Apache License, Version 2.0 (the "License");
5
+ * you may not use this file except in compliance with the License.
6
+ * You may obtain a copy of the License at
7
+ *
8
+ * http://www.apache.org/licenses/LICENSE-2.0
9
+ *
10
+ * Unless required by applicable law or agreed to in writing, software
11
+ * distributed under the License is distributed on an "AS IS" BASIS,
12
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ * See the License for the specific language governing permissions and
14
+ * limitations under the License.
15
+ */
16
+
17
+ import { type V1_LakehouseDiscoveryEnvironmentResponse } from '@finos/legend-graph';
18
+ import { AbstractServerClient, type PlainObject } from '@finos/legend-shared';
19
+ import type { IngestDeploymentServerConfig } from './models/IngestDeploymentServerConfig.js';
20
+
21
+ export class LakehousePlatformServerClient extends AbstractServerClient {
22
+ constructor(url: string) {
23
+ super({
24
+ baseUrl: url,
25
+ });
26
+ }
27
+ private _ingest = (): string =>
28
+ `${this.baseUrl}/ingest/discovery/environments/producers`;
29
+
30
+ private _discovery = (): string => `${this.baseUrl}/ingest/discovery`;
31
+
32
+ private _env = (): string => `${this._discovery()}/environments`;
33
+
34
+ private _token = (token?: string) => ({
35
+ Authorization: `Bearer ${token}`,
36
+ });
37
+
38
+ getIngestEnvironmentSummaries(
39
+ token?: string | undefined,
40
+ ): Promise<PlainObject<IngestDeploymentServerConfig>[]> {
41
+ return this.get(`${this._env()}`, {}, this._token(token));
42
+ }
43
+
44
+ findProducerServer(
45
+ id: number,
46
+ level: string,
47
+ token?: string | undefined,
48
+ ): Promise<PlainObject<IngestDeploymentServerConfig>> {
49
+ return this.get(
50
+ `${this._ingest()}/${id}/${level}/search`,
51
+ {},
52
+ this._token(token),
53
+ );
54
+ }
55
+
56
+ getIngestEnvironmentSummary = (
57
+ ingestEnvironmentUrn: string,
58
+ token: string | undefined,
59
+ ): Promise<PlainObject<V1_LakehouseDiscoveryEnvironmentResponse>> =>
60
+ this.get(
61
+ `${this._discovery()}/environments/${ingestEnvironmentUrn}`,
62
+ {},
63
+ this._token(token),
64
+ );
65
+ }