@dremio/js-sdk 0.4.1 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cloud/interfaces.d.ts +2 -1
- package/dist/cloud/interfaces.js.map +1 -1
- package/dist/enterprise/interfaces.d.ts +2 -1
- package/dist/enterprise/interfaces.js.map +1 -1
- package/dist/oss/interfaces.d.ts +2 -1
- package/dist/oss/interfaces.js.map +1 -1
- package/dist/oss/jobs/Job.d.ts +13 -2
- package/dist/oss/jobs/Job.js +27 -5
- package/dist/oss/jobs/Job.js.map +1 -1
- package/dist/oss/jobs/utils/JobResultsResponse.d.ts +1 -1
- package/dist/oss/jobs/utils/JobResultsResponse.js.map +1 -1
- package/package.json +1 -1
|
@@ -7,7 +7,8 @@ import type { EnterpriseScript } from "../enterprise/scripts/EnterpriseScript.ts
|
|
|
7
7
|
import type { ReflectionSummary } from "../enterprise/reflections/ReflectionSummary.ts";
|
|
8
8
|
import type { Role } from "../enterprise/roles/Role.ts";
|
|
9
9
|
import type { BranchHeadVersionReference, BareCommitVersionReference, TagVersionReference, VersionReference } from "../oss/catalog/VersionReference.ts";
|
|
10
|
+
import type { JobResultsSchema } from "../oss/jobs/utils/JobResultsResponse.ts";
|
|
10
11
|
export type { ArcticCatalog, CloudUser, Engine, Project };
|
|
11
|
-
export type { Job, Reflection, EnterpriseScript, ReflectionSummary, Role, BranchHeadVersionReference, BareCommitVersionReference, TagVersionReference, VersionReference, };
|
|
12
|
+
export type { Job, JobResultsSchema, Reflection, EnterpriseScript, ReflectionSummary, Role, BranchHeadVersionReference, BareCommitVersionReference, TagVersionReference, VersionReference, };
|
|
12
13
|
export * from "../enterprise/catalog/CatalogObjects/index.ts";
|
|
13
14
|
export * from "../enterprise/catalog/CatalogReferences/index.ts";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"interfaces.js","sourceRoot":"","sources":["../../src/cloud/interfaces.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;
|
|
1
|
+
{"version":3,"file":"interfaces.js","sourceRoot":"","sources":["../../src/cloud/interfaces.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAgCH,cAAc,+CAA+C,CAAC;AAC9D,cAAc,kDAAkD,CAAC","sourcesContent":["/*\n * Copyright (C) 2024-2025 Dremio Corporation\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport type { ArcticCatalog } from \"./arctic/ArcticCatalog.ts\";\nimport type { CloudUser } from \"./users/CloudUser.ts\";\nimport type { Engine } from \"./engines/Engine.ts\";\nimport type { Project } from \"./projects/Project.ts\";\n\nimport type { Job, Reflection } from \"../oss/interfaces.ts\";\nimport type { EnterpriseScript } from \"../enterprise/scripts/EnterpriseScript.ts\";\nimport type { ReflectionSummary } from \"../enterprise/reflections/ReflectionSummary.ts\";\nimport type { Role } from \"../enterprise/roles/Role.ts\";\nimport type {\n BranchHeadVersionReference,\n BareCommitVersionReference,\n TagVersionReference,\n VersionReference,\n} from \"../oss/catalog/VersionReference.ts\";\nimport type { JobResultsSchema } from \"../oss/jobs/utils/JobResultsResponse.ts\";\n\nexport type { ArcticCatalog, CloudUser, Engine, Project };\nexport type {\n Job,\n JobResultsSchema,\n Reflection,\n EnterpriseScript,\n ReflectionSummary,\n Role,\n BranchHeadVersionReference,\n BareCommitVersionReference,\n TagVersionReference,\n VersionReference,\n};\nexport * from \"../enterprise/catalog/CatalogObjects/index.ts\";\nexport * from \"../enterprise/catalog/CatalogReferences/index.ts\";\n"]}
|
|
@@ -7,8 +7,9 @@ import type { EnterpriseUser } from "./users/EnterpriseUser.ts";
|
|
|
7
7
|
import type { ReflectionSummary } from "./reflections/ReflectionSummary.ts";
|
|
8
8
|
import type { Role } from "./roles/Role.ts";
|
|
9
9
|
import type { BranchHeadVersionReference, BareCommitVersionReference, TagVersionReference, VersionReference } from "../oss/catalog/VersionReference.ts";
|
|
10
|
+
import type { JobResultsSchema } from "../oss/jobs/utils/JobResultsResponse.ts";
|
|
10
11
|
export * from "./catalog/CatalogObjects/index.ts";
|
|
11
12
|
export * from "./catalog/CatalogReferences/index.ts";
|
|
12
13
|
export * from "./Ownable.ts";
|
|
13
14
|
export * from "./Grantee.ts";
|
|
14
|
-
export type { Engine, EnterpriseCatalogObject, EnterpriseCatalogReference, EnterpriseScript, EnterpriseUser, Job, Reflection, ReflectionSummary, Role, BranchHeadVersionReference, BareCommitVersionReference, TagVersionReference, VersionReference, };
|
|
15
|
+
export type { Engine, EnterpriseCatalogObject, EnterpriseCatalogReference, EnterpriseScript, EnterpriseUser, Job, JobResultsSchema, Reflection, ReflectionSummary, Role, BranchHeadVersionReference, BareCommitVersionReference, TagVersionReference, VersionReference, };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"interfaces.js","sourceRoot":"","sources":["../../src/enterprise/interfaces.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAiBH,cAAc,mCAAmC,CAAC;AAClD,cAAc,sCAAsC,CAAC;AACrD,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC","sourcesContent":["/*\n * Copyright (C) 2024-2025 Dremio Corporation\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport type { Engine } from \"./engines/Engine.ts\";\nimport type { Job, Reflection } from \"../oss/interfaces.ts\";\nimport type { EnterpriseCatalogObject } from \"./catalog/CatalogObjects/index.ts\";\nimport type { EnterpriseCatalogReference } from \"./catalog/CatalogReferences/index.ts\";\nimport type { EnterpriseScript } from \"./scripts/EnterpriseScript.ts\";\nimport type { EnterpriseUser } from \"./users/EnterpriseUser.ts\";\nimport type { ReflectionSummary } from \"./reflections/ReflectionSummary.ts\";\nimport type { Role } from \"./roles/Role.ts\";\nimport type {\n BranchHeadVersionReference,\n BareCommitVersionReference,\n TagVersionReference,\n VersionReference,\n} from \"../oss/catalog/VersionReference.ts\";\
|
|
1
|
+
{"version":3,"file":"interfaces.js","sourceRoot":"","sources":["../../src/enterprise/interfaces.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAiBH,cAAc,mCAAmC,CAAC;AAClD,cAAc,sCAAsC,CAAC;AACrD,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC","sourcesContent":["/*\n * Copyright (C) 2024-2025 Dremio Corporation\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport type { Engine } from \"./engines/Engine.ts\";\nimport type { Job, Reflection } from \"../oss/interfaces.ts\";\nimport type { EnterpriseCatalogObject } from \"./catalog/CatalogObjects/index.ts\";\nimport type { EnterpriseCatalogReference } from \"./catalog/CatalogReferences/index.ts\";\nimport type { EnterpriseScript } from \"./scripts/EnterpriseScript.ts\";\nimport type { EnterpriseUser } from \"./users/EnterpriseUser.ts\";\nimport type { ReflectionSummary } from \"./reflections/ReflectionSummary.ts\";\nimport type { Role } from \"./roles/Role.ts\";\nimport type {\n BranchHeadVersionReference,\n BareCommitVersionReference,\n TagVersionReference,\n VersionReference,\n} from \"../oss/catalog/VersionReference.ts\";\nimport type { JobResultsSchema } from \"../oss/jobs/utils/JobResultsResponse.ts\";\nexport * from \"./catalog/CatalogObjects/index.ts\";\nexport * from \"./catalog/CatalogReferences/index.ts\";\nexport * from \"./Ownable.ts\";\nexport * from \"./Grantee.ts\";\nexport type {\n Engine,\n EnterpriseCatalogObject,\n EnterpriseCatalogReference,\n EnterpriseScript,\n EnterpriseUser,\n Job,\n JobResultsSchema,\n Reflection,\n ReflectionSummary,\n Role,\n BranchHeadVersionReference,\n BareCommitVersionReference,\n TagVersionReference,\n VersionReference,\n};\n"]}
|
package/dist/oss/interfaces.d.ts
CHANGED
|
@@ -3,6 +3,7 @@ import type { Reflection } from "./reflections/Reflection.ts";
|
|
|
3
3
|
import type { Script } from "./scripts/Script.ts";
|
|
4
4
|
import type { User } from "./users/User.ts";
|
|
5
5
|
import type { BranchHeadVersionReference, BareCommitVersionReference, TagVersionReference, VersionReference } from "./catalog/VersionReference.ts";
|
|
6
|
+
import type { JobResultsSchema } from "./jobs/utils/JobResultsResponse.ts";
|
|
6
7
|
export * from "./catalog/CatalogObjects/index.ts";
|
|
7
8
|
export * from "./catalog/CatalogReferences/index.ts";
|
|
8
|
-
export type { Job, Reflection, Script, User, VersionReference, BranchHeadVersionReference, BareCommitVersionReference, TagVersionReference, };
|
|
9
|
+
export type { Job, JobResultsSchema, Reflection, Script, User, VersionReference, BranchHeadVersionReference, BareCommitVersionReference, TagVersionReference, };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"interfaces.js","sourceRoot":"","sources":["../../src/oss/interfaces.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;
|
|
1
|
+
{"version":3,"file":"interfaces.js","sourceRoot":"","sources":["../../src/oss/interfaces.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAcH,cAAc,mCAAmC,CAAC;AAClD,cAAc,sCAAsC,CAAC","sourcesContent":["/*\n * Copyright (C) 2024-2025 Dremio Corporation\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport type { Job } from \"./jobs/Job.ts\";\nimport type { Reflection } from \"./reflections/Reflection.ts\";\nimport type { Script } from \"./scripts/Script.ts\";\nimport type { User } from \"./users/User.ts\";\nimport type {\n BranchHeadVersionReference,\n BareCommitVersionReference,\n TagVersionReference,\n VersionReference,\n} from \"./catalog/VersionReference.ts\";\nimport type { JobResultsSchema } from \"./jobs/utils/JobResultsResponse.ts\";\n\nexport * from \"./catalog/CatalogObjects/index.ts\";\nexport * from \"./catalog/CatalogReferences/index.ts\";\nexport type {\n Job,\n JobResultsSchema,\n Reflection,\n Script,\n User,\n VersionReference,\n BranchHeadVersionReference,\n BareCommitVersionReference,\n TagVersionReference,\n};\n"]}
|
package/dist/oss/jobs/Job.d.ts
CHANGED
|
@@ -2,10 +2,12 @@ import type { SonarV3Config } from "../../common/Config.ts";
|
|
|
2
2
|
import type { Problem } from "../../common/Problem.ts";
|
|
3
3
|
import { Err, Ok, Result } from "ts-results-es";
|
|
4
4
|
import { type Observable } from "rxjs";
|
|
5
|
+
import type { SignalParam } from "../../common/Params.ts";
|
|
5
6
|
export declare class Job {
|
|
6
7
|
#private;
|
|
7
8
|
readonly acceleration: JobProperties["acceleration"];
|
|
8
9
|
readonly cancellationReason: JobProperties["cancellationReason"];
|
|
10
|
+
readonly errorMessage?: string;
|
|
9
11
|
readonly endedAt: JobProperties["endedAt"];
|
|
10
12
|
readonly id: JobProperties["id"];
|
|
11
13
|
readonly problem: JobProperties["problem"];
|
|
@@ -22,7 +24,7 @@ export declare class Job {
|
|
|
22
24
|
jsonBatches: <T extends Record<string, unknown> = Record<string, unknown>>(args?: {
|
|
23
25
|
limit?: number;
|
|
24
26
|
offset?: number;
|
|
25
|
-
}) => AsyncGenerator<{
|
|
27
|
+
}, { signal }?: SignalParam) => AsyncGenerator<{
|
|
26
28
|
readonly columns: Record<Extract<keyof T, string>, unknown[] | Float64Array<ArrayBuffer> | Float32Array<ArrayBuffer> | Int32Array<ArrayBuffer>>;
|
|
27
29
|
readonly rows: T[];
|
|
28
30
|
readonly schema: {
|
|
@@ -30,8 +32,16 @@ export declare class Job {
|
|
|
30
32
|
};
|
|
31
33
|
readonly totalRows: number;
|
|
32
34
|
}, void, unknown>;
|
|
33
|
-
|
|
35
|
+
metadata: ({ signal }?: SignalParam) => Promise<Err<unknown> | Ok<{
|
|
36
|
+
schema: import("./utils/JobResultsResponse.ts").JobResultsSchema<string>;
|
|
37
|
+
totalRows: number;
|
|
38
|
+
}>>;
|
|
39
|
+
slice: <T extends Record<string, unknown> = Record<string, unknown>>(start?: number, end?: number, { signal }?: SignalParam) => Promise<Err<unknown> | Ok<T[]>>;
|
|
34
40
|
};
|
|
41
|
+
/**
|
|
42
|
+
* @returns A `Job` instance once it's reached a settled state
|
|
43
|
+
*/
|
|
44
|
+
retrieveSettled(): Promise<Job>;
|
|
35
45
|
cancel(): Promise<Err<any> | Ok<undefined>>;
|
|
36
46
|
}
|
|
37
47
|
export declare const jobEntityToProperties: (id: string, entity: {
|
|
@@ -52,6 +62,7 @@ export declare const jobEntityToProperties: (id: string, entity: {
|
|
|
52
62
|
resourceSchedulingEndedAt: string | null;
|
|
53
63
|
cancellationReason: string | null;
|
|
54
64
|
}) => {
|
|
65
|
+
readonly errorMessage?: string | undefined;
|
|
55
66
|
readonly acceleration: {
|
|
56
67
|
reflectionRelationships: {
|
|
57
68
|
datasetId: string;
|
package/dist/oss/jobs/Job.js
CHANGED
|
@@ -20,6 +20,7 @@ import { mapRowData } from "./utils/mapRowData.js";
|
|
|
20
20
|
export class Job {
|
|
21
21
|
acceleration;
|
|
22
22
|
cancellationReason;
|
|
23
|
+
errorMessage;
|
|
23
24
|
endedAt;
|
|
24
25
|
id;
|
|
25
26
|
problem;
|
|
@@ -57,6 +58,9 @@ export class Job {
|
|
|
57
58
|
return of(job);
|
|
58
59
|
}
|
|
59
60
|
}));
|
|
61
|
+
if (properties.errorMessage) {
|
|
62
|
+
this.errorMessage = properties.errorMessage;
|
|
63
|
+
}
|
|
60
64
|
}
|
|
61
65
|
get settled() {
|
|
62
66
|
return (this.state === "COMPLETED" ||
|
|
@@ -67,10 +71,17 @@ export class Job {
|
|
|
67
71
|
get results() {
|
|
68
72
|
return {
|
|
69
73
|
jsonBatches: this.#jsonBatches.bind(this),
|
|
74
|
+
metadata: this.#metadata.bind(this),
|
|
70
75
|
slice: this.#slice.bind(this),
|
|
71
76
|
};
|
|
72
77
|
}
|
|
73
|
-
|
|
78
|
+
/**
|
|
79
|
+
* @returns A `Job` instance once it's reached a settled state
|
|
80
|
+
*/
|
|
81
|
+
retrieveSettled() {
|
|
82
|
+
return lastValueFrom(this.observer);
|
|
83
|
+
}
|
|
84
|
+
async *#jsonBatches(args = {}, { signal } = {}) {
|
|
74
85
|
if (typeof args.limit === "number" && args.limit < 0) {
|
|
75
86
|
throw new Error("Limit cannot be negative");
|
|
76
87
|
}
|
|
@@ -79,7 +90,7 @@ export class Job {
|
|
|
79
90
|
}
|
|
80
91
|
// Wait for job to enter a settled state before attempting to fetch batches
|
|
81
92
|
if (!this.settled) {
|
|
82
|
-
await
|
|
93
|
+
await this.retrieveSettled();
|
|
83
94
|
}
|
|
84
95
|
const limitArg = args.limit ?? Infinity;
|
|
85
96
|
const offsetArg = args.offset || 0;
|
|
@@ -93,7 +104,7 @@ export class Job {
|
|
|
93
104
|
// Make batch_size dynamic to allow for requesting a smaller final page
|
|
94
105
|
const batch_size = Math.min(MAX_BATCH_SIZE, stopAfterOffset - offset);
|
|
95
106
|
const batch = await this.#config
|
|
96
|
-
.sonarV3Request(`job/${this.id}/results?offset=${offset}&limit=${batch_size}
|
|
107
|
+
.sonarV3Request(`job/${this.id}/results?offset=${offset}&limit=${batch_size}`, { signal })
|
|
97
108
|
.then((res) => res.json());
|
|
98
109
|
offset += batch.rows.length;
|
|
99
110
|
hasMore = batch.rows.length === batch_size;
|
|
@@ -111,10 +122,20 @@ export class Job {
|
|
|
111
122
|
}
|
|
112
123
|
}
|
|
113
124
|
}
|
|
114
|
-
async #
|
|
125
|
+
async #metadata({ signal } = {}) {
|
|
126
|
+
return this.#config
|
|
127
|
+
.sonarV3Request(`job/${this.id}/results?offset=0&limit=0`, { signal })
|
|
128
|
+
.then((res) => res.json())
|
|
129
|
+
.then((response) => Ok({
|
|
130
|
+
schema: response.schema,
|
|
131
|
+
totalRows: response.rowCount,
|
|
132
|
+
}))
|
|
133
|
+
.catch((e) => Err(e));
|
|
134
|
+
}
|
|
135
|
+
async #slice(start = 0, end = Infinity, { signal } = {}) {
|
|
115
136
|
try {
|
|
116
137
|
const rows = [];
|
|
117
|
-
for await (const batch of this.#jsonBatches({ limit: end - start, offset: start })) {
|
|
138
|
+
for await (const batch of this.#jsonBatches({ limit: end - start, offset: start }, { signal })) {
|
|
118
139
|
rows.push(...batch.rows);
|
|
119
140
|
}
|
|
120
141
|
return Ok(rows);
|
|
@@ -149,6 +170,7 @@ export const jobEntityToProperties = (id, entity) => ({
|
|
|
149
170
|
rowCount: entity.rowCount,
|
|
150
171
|
startedAt: entity.startedAt ? new Date(entity.startedAt) : null,
|
|
151
172
|
state: entity.jobState,
|
|
173
|
+
...(entity.errorMessage && { errorMessage: entity.errorMessage }),
|
|
152
174
|
});
|
|
153
175
|
const MAX_BATCH_SIZE = 500;
|
|
154
176
|
//# sourceMappingURL=Job.js.map
|
package/dist/oss/jobs/Job.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Job.js","sourceRoot":"","sources":["../../../src/oss/jobs/Job.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAIH,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,GAAG,EAAE,EAAE,EAAE,SAAS,EAAmB,MAAM,MAAM,CAAC;AAE1E,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAEnD,MAAM,OAAO,GAAG;IACL,YAAY,CAAgC;IAC5C,kBAAkB,CAAsC;IACxD,OAAO,CAA2B;IAClC,EAAE,CAAsB;IACxB,OAAO,CAA2B;IAClC,SAAS,CAA6B;IACtC,yBAAyB,CAA6C;IACtE,2BAA2B,CAA+C;IAC1E,QAAQ,CAA4B;IACpC,SAAS,CAA6B;IACtC,KAAK,CAAyB;IAEvB,QAAQ,CAAkB;IAC1C,OAAO,CAAgB;IAEvB,YACE,UAAyB,EACzB,MAAqB,EACrB,OAA8C;QAE9C,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,YAAY,CAAC;QAC5C,IAAI,CAAC,kBAAkB,GAAG,UAAU,CAAC,kBAAkB,CAAC;QACxD,IAAI,CAAC,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC;QAClC,IAAI,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;QACxB,IAAI,CAAC,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC;QAClC,IAAI,CAAC,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC;QACtC,IAAI,CAAC,yBAAyB,GAAG,UAAU,CAAC,yBAAyB,CAAC;QACtE,IAAI,CAAC,2BAA2B,GAAG,UAAU,CAAC,2BAA2B,CAAC;QAC1E,IAAI,CAAC,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC;QACpC,IAAI,CAAC,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC;QACtC,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,CAC3B,SAAS,CAAC,CAAC,GAAG,EAAE,EAAE;YAChB,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;gBACjB,OAAO,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAC1B,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;oBACb,IAAI,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;wBACnB,MAAM,MAAM,CAAC,KAAK,CAAC;oBACrB,CAAC;oBACD,OAAO,MAAM,CAAC,KAAK,CAAC;gBACtB,CAAC,CAAC,CACH,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC;YACjB,CAAC;QACH,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IAED,IAAI,OAAO;QACT,OAAO,CACL,IAAI,CAAC,KAAK,KAAK,WAAW;YAC1B,IAAI,CAAC,KAAK,KAAK,QAAQ;YACvB,IAAI,CAAC,KAAK,KAAK,eAAe;YAC9B,IAAI,CAAC,KAAK,KAAK,UAAU,CAC1B,CAAC;IACJ,CAAC;IAED,IAAI,OAAO;QACT,OAAO;YACL,WAAW,EAAE,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;YACzC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;SAC9B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,CAAC,YAAY,CACjB,OAA4C,EAAE;QAE9C,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvD,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QAED,2EAA2E;QAC3E,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrC,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;QAEnC,6EAA6E;QAC7E,IAAI,OAAO,GAAG,IAAI,CAAC;QAEnB,uGAAuG;QACvG,IAAI,MAAM,GAAG,SAAS,CAAC;QAEvB,iEAAiE;QACjE,MAAM,eAAe,GAAG,QAAQ,GAAG,SAAS,CAAC;QAE7C,OAAO,OAAO,IAAI,MAAM,GAAG,eAAe,EAAE,CAAC;YAC3C,uEAAuE;YACvE,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,eAAe,GAAG,MAAM,CAAC,CAAC;YAEtE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO;iBAC7B,cAAc,CAAC,OAAO,IAAI,CAAC,EAAE,mBAAmB,MAAM,UAAU,UAAU,EAAE,CAAC;iBAC7E,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAoC,CAAC,CAAC;YAE/D,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;YAC5B,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC;YAE3C,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACtB,MAAM,MAAM,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;gBACxC,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;gBACrC,MAAM;oBACJ,IAAI,OAAO;wBACT,OAAO,gBAAgB,CAAC,KAAK,CAAC,CAAC;oBACjC,CAAC;oBACD,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,MAAM;oBACN,SAAS,EAAE,KAAK,CAAC,QAAQ;iBACjB,CAAC;YACb,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CACV,QAAgB,CAAC,EACjB,MAAc,QAAQ;QAEtB,IAAI,CAAC;YACH,MAAM,IAAI,GAAQ,EAAE,CAAC;YACrB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,YAAY,CAAI,EAAE,KAAK,EAAE,GAAG,GAAG,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;gBACtF,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;YAC3B,CAAC;YACD,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC;QAClB,CAAC;QAAC,OAAO,CAAU,EAAE,CAAC;YACpB,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;QAChB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM;QACV,OAAO,IAAI,CAAC,OAAO;aAChB,cAAc,CAAC,OAAO,IAAI,CAAC,EAAE,SAAS,EAAE;YACvC,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,MAAM;SACf,CAAC;aACD,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;aACzB,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1B,CAAC;CACF;AAED,MAAM,CAAC,MAAM,qBAAqB,GAAG,CACnC,EAAU,EACV,MA6CC,EACD,EAAE,CACF,CAAC;IACC,YAAY,EAAE,MAAM,CAAC,YAAY;IACjC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;IAC7C,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI;IACzD,EAAE;IACF,OAAO,EAAE,IAAsB;IAC/B,SAAS,EAAE,MAAM,CAAC,SAAS;IAC3B,yBAAyB,EAAE,MAAM,CAAC,yBAAyB;QACzD,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,yBAAyB,CAAC;QAC5C,CAAC,CAAC,IAAI;IACR,2BAA2B,EAAE,MAAM,CAAC,2BAA2B;QAC7D,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,2BAA2B,CAAC;QAC9C,CAAC,CAAC,IAAI;IACR,QAAQ,EAAE,MAAM,CAAC,QAAQ;IACzB,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI;IAC/D,KAAK,EAAE,MAAM,CAAC,QAAQ;CACvB,CAAU,CAAC;AAMd,MAAM,cAAc,GAAG,GAAG,CAAC","sourcesContent":["/*\n * Copyright (C) 2024-2025 Dremio Corporation\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport type { SonarV3Config } from \"../../common/Config.ts\";\nimport type { Problem } from \"../../common/Problem.ts\";\nimport { Err, Ok, Result } from \"ts-results-es\";\nimport { lastValueFrom, map, of, switchMap, type Observable } from \"rxjs\";\nimport type { JobResultsResponse } from \"./utils/JobResultsResponse.ts\";\nimport { mapRowsToColumns } from \"./utils/mapRowsToColumns.ts\";\nimport { mapRowData } from \"./utils/mapRowData.ts\";\n\nexport class Job {\n readonly acceleration: JobProperties[\"acceleration\"];\n readonly cancellationReason: JobProperties[\"cancellationReason\"];\n readonly endedAt: JobProperties[\"endedAt\"];\n readonly id: JobProperties[\"id\"];\n readonly problem: JobProperties[\"problem\"];\n readonly queryType: JobProperties[\"queryType\"];\n readonly resourceSchedulingEndedAt: JobProperties[\"resourceSchedulingEndedAt\"];\n readonly resourceSchedulingStartedAt: JobProperties[\"resourceSchedulingStartedAt\"];\n readonly rowCount: JobProperties[\"rowCount\"];\n readonly startedAt: JobProperties[\"startedAt\"];\n readonly state: JobProperties[\"state\"];\n\n public readonly observer: Observable<Job>;\n #config: SonarV3Config;\n\n constructor(\n properties: JobProperties,\n config: SonarV3Config,\n observe: (id: string) => Observable<JobResult>,\n ) {\n this.acceleration = properties.acceleration;\n this.cancellationReason = properties.cancellationReason;\n this.endedAt = properties.endedAt;\n this.id = properties.id;\n this.problem = properties.problem;\n this.queryType = properties.queryType;\n this.resourceSchedulingEndedAt = properties.resourceSchedulingEndedAt;\n this.resourceSchedulingStartedAt = properties.resourceSchedulingStartedAt;\n this.rowCount = properties.rowCount;\n this.startedAt = properties.startedAt;\n this.state = properties.state;\n this.#config = config;\n this.observer = of(this).pipe(\n switchMap((job) => {\n if (!job.settled) {\n return observe(this.id).pipe(\n map((result) => {\n if (result.isErr()) {\n throw result.error;\n }\n return result.value;\n }),\n );\n } else {\n return of(job);\n }\n }),\n );\n }\n\n get settled() {\n return (\n this.state === \"COMPLETED\" ||\n this.state === \"FAILED\" ||\n this.state === \"INVALID_STATE\" ||\n this.state === \"CANCELED\"\n );\n }\n\n get results() {\n return {\n jsonBatches: this.#jsonBatches.bind(this),\n slice: this.#slice.bind(this),\n };\n }\n\n async *#jsonBatches<T extends Record<string, unknown> = Record<string, unknown>>(\n args: { limit?: number; offset?: number } = {},\n ) {\n if (typeof args.limit === \"number\" && args.limit < 0) {\n throw new Error(\"Limit cannot be negative\");\n }\n\n if (typeof args.offset === \"number\" && args.offset < 0) {\n throw new Error(\"Offset cannot be negative\");\n }\n\n // Wait for job to enter a settled state before attempting to fetch batches\n if (!this.settled) {\n await lastValueFrom(this.observer);\n }\n\n const limitArg = args.limit ?? Infinity;\n const offsetArg = args.offset || 0;\n\n // Tracks whether there are more rows available from the job results endpoint\n let hasMore = true;\n\n // Tracks the currently requested offset. If the offset arg is provided, start from there instead of 0.\n let offset = offsetArg;\n\n // Keeps track of the total number of rows that need to be loaded\n const stopAfterOffset = limitArg + offsetArg;\n\n while (hasMore && offset < stopAfterOffset) {\n // Make batch_size dynamic to allow for requesting a smaller final page\n const batch_size = Math.min(MAX_BATCH_SIZE, stopAfterOffset - offset);\n\n const batch = await this.#config\n .sonarV3Request(`job/${this.id}/results?offset=${offset}&limit=${batch_size}`)\n .then((res) => res.json() as Promise<JobResultsResponse<T>>);\n\n offset += batch.rows.length;\n hasMore = batch.rows.length === batch_size;\n\n if (batch.rows.length) {\n const schema = { fields: batch.schema };\n mapRowData(batch.rows, batch.schema);\n yield {\n get columns() {\n return mapRowsToColumns(batch);\n },\n rows: batch.rows,\n schema,\n totalRows: batch.rowCount,\n } as const;\n }\n }\n }\n\n async #slice<T extends Record<string, unknown> = Record<string, unknown>>(\n start: number = 0,\n end: number = Infinity,\n ) {\n try {\n const rows: T[] = [];\n for await (const batch of this.#jsonBatches<T>({ limit: end - start, offset: start })) {\n rows.push(...batch.rows);\n }\n return Ok(rows);\n } catch (e: unknown) {\n return Err(e);\n }\n }\n\n async cancel() {\n return this.#config\n .sonarV3Request(`job/${this.id}/cancel`, {\n keepalive: true,\n method: \"POST\",\n })\n .then(() => Ok(undefined))\n .catch((e) => Err(e));\n }\n}\n\nexport const jobEntityToProperties = (\n id: string,\n entity: {\n jobState:\n | \"NOT_SUBMITTED\"\n | \"STARTING\"\n | \"RUNNING\"\n | \"COMPLETED\"\n | \"CANCELED\"\n | \"FAILED\"\n | \"CANCELLATION_REQUESTED\"\n | \"PLANNING\"\n | \"PENDING\"\n | \"METADATA_RETRIEVAL\"\n | \"QUEUED\"\n | \"ENGINE_START\"\n | \"EXECUTION_PLANNING\"\n | \"INVALID_STATE\";\n rowCount: number | null;\n errorMessage: string;\n startedAt: string | null;\n endedAt: string | null;\n acceleration: {\n reflectionRelationships: {\n datasetId: string;\n reflectionId: string;\n relationship: string;\n }[];\n };\n queryType:\n | \"UI_RUN\"\n | \"UI_PREVIEW\"\n | \"UI_INTERNAL_PREVIEW\"\n | \"UI_INTERNAL_RUN\"\n | \"UI_EXPORT\"\n | \"ODBC\"\n | \"JDBC\"\n | \"REST\"\n | \"ACCELERATOR_CREATE\"\n | \"ACCELERATOR_DROP\"\n | \"UNKNOWN\"\n | \"PREPARE_INTERNAL\"\n | \"ACCELERATOR_EXPLAIN\"\n | \"UI_INITIAL_PREVIEW\";\n resourceSchedulingStartedAt: string | null;\n resourceSchedulingEndedAt: string | null;\n cancellationReason: string | null;\n },\n) =>\n ({\n acceleration: entity.acceleration,\n cancellationReason: entity.cancellationReason,\n endedAt: entity.endedAt ? new Date(entity.endedAt) : null,\n id,\n problem: null as Problem | null,\n queryType: entity.queryType,\n resourceSchedulingEndedAt: entity.resourceSchedulingEndedAt\n ? new Date(entity.resourceSchedulingEndedAt)\n : null,\n resourceSchedulingStartedAt: entity.resourceSchedulingStartedAt\n ? new Date(entity.resourceSchedulingStartedAt)\n : null,\n rowCount: entity.rowCount,\n startedAt: entity.startedAt ? new Date(entity.startedAt) : null,\n state: entity.jobState,\n }) as const;\n\ntype JobProperties = ReturnType<typeof jobEntityToProperties>;\n\nexport type JobResult = Result<Job, unknown>;\n\nconst MAX_BATCH_SIZE = 500;\n"]}
|
|
1
|
+
{"version":3,"file":"Job.js","sourceRoot":"","sources":["../../../src/oss/jobs/Job.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAIH,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,GAAG,EAAE,EAAE,EAAE,SAAS,EAAmB,MAAM,MAAM,CAAC;AAE1E,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAGnD,MAAM,OAAO,GAAG;IACL,YAAY,CAAgC;IAC5C,kBAAkB,CAAsC;IACxD,YAAY,CAAU;IACtB,OAAO,CAA2B;IAClC,EAAE,CAAsB;IACxB,OAAO,CAA2B;IAClC,SAAS,CAA6B;IACtC,yBAAyB,CAA6C;IACtE,2BAA2B,CAA+C;IAC1E,QAAQ,CAA4B;IACpC,SAAS,CAA6B;IACtC,KAAK,CAAyB;IAEvB,QAAQ,CAAkB;IAC1C,OAAO,CAAgB;IAEvB,YACE,UAAyB,EACzB,MAAqB,EACrB,OAA8C;QAE9C,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,YAAY,CAAC;QAC5C,IAAI,CAAC,kBAAkB,GAAG,UAAU,CAAC,kBAAkB,CAAC;QACxD,IAAI,CAAC,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC;QAClC,IAAI,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;QACxB,IAAI,CAAC,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC;QAClC,IAAI,CAAC,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC;QACtC,IAAI,CAAC,yBAAyB,GAAG,UAAU,CAAC,yBAAyB,CAAC;QACtE,IAAI,CAAC,2BAA2B,GAAG,UAAU,CAAC,2BAA2B,CAAC;QAC1E,IAAI,CAAC,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC;QACpC,IAAI,CAAC,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC;QACtC,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,CAC3B,SAAS,CAAC,CAAC,GAAG,EAAE,EAAE;YAChB,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;gBACjB,OAAO,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAC1B,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;oBACb,IAAI,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;wBACnB,MAAM,MAAM,CAAC,KAAK,CAAC;oBACrB,CAAC;oBACD,OAAO,MAAM,CAAC,KAAK,CAAC;gBACtB,CAAC,CAAC,CACH,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC;YACjB,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QAEF,IAAI,UAAU,CAAC,YAAY,EAAE,CAAC;YAC5B,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,YAAY,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,IAAI,OAAO;QACT,OAAO,CACL,IAAI,CAAC,KAAK,KAAK,WAAW;YAC1B,IAAI,CAAC,KAAK,KAAK,QAAQ;YACvB,IAAI,CAAC,KAAK,KAAK,eAAe;YAC9B,IAAI,CAAC,KAAK,KAAK,UAAU,CAC1B,CAAC;IACJ,CAAC;IAED,IAAI,OAAO;QACT,OAAO;YACL,WAAW,EAAE,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;YACzC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;YACnC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;SAC9B,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,eAAe;QACb,OAAO,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,CAAC,YAAY,CACjB,OAA4C,EAAE,EAC9C,EAAE,MAAM,KAAkB,EAAE;QAE5B,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvD,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QAED,2EAA2E;QAC3E,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC/B,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;QAEnC,6EAA6E;QAC7E,IAAI,OAAO,GAAG,IAAI,CAAC;QAEnB,uGAAuG;QACvG,IAAI,MAAM,GAAG,SAAS,CAAC;QAEvB,iEAAiE;QACjE,MAAM,eAAe,GAAG,QAAQ,GAAG,SAAS,CAAC;QAE7C,OAAO,OAAO,IAAI,MAAM,GAAG,eAAe,EAAE,CAAC;YAC3C,uEAAuE;YACvE,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,eAAe,GAAG,MAAM,CAAC,CAAC;YAEtE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO;iBAC7B,cAAc,CAAC,OAAO,IAAI,CAAC,EAAE,mBAAmB,MAAM,UAAU,UAAU,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC;iBACzF,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAoC,CAAC,CAAC;YAE/D,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;YAC5B,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC;YAE3C,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACtB,MAAM,MAAM,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;gBACxC,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;gBACrC,MAAM;oBACJ,IAAI,OAAO;wBACT,OAAO,gBAAgB,CAAC,KAAK,CAAC,CAAC;oBACjC,CAAC;oBACD,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,MAAM;oBACN,SAAS,EAAE,KAAK,CAAC,QAAQ;iBACjB,CAAC;YACb,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,EAAE,MAAM,KAAkB,EAAE;QAC1C,OAAO,IAAI,CAAC,OAAO;aAChB,cAAc,CAAC,OAAO,IAAI,CAAC,EAAE,2BAA2B,EAAE,EAAE,MAAM,EAAE,CAAC;aACrE,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAiC,CAAC;aACxD,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CACjB,EAAE,CAAC;YACD,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,SAAS,EAAE,QAAQ,CAAC,QAAQ;SAC7B,CAAC,CACH;aACA,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,MAAM,CACV,QAAgB,CAAC,EACjB,MAAc,QAAQ,EACtB,EAAE,MAAM,KAAkB,EAAE;QAE5B,IAAI,CAAC;YACH,MAAM,IAAI,GAAQ,EAAE,CAAC;YACrB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,YAAY,CACzC,EAAE,KAAK,EAAE,GAAG,GAAG,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,EACrC,EAAE,MAAM,EAAE,CACX,EAAE,CAAC;gBACF,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;YAC3B,CAAC;YACD,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC;QAClB,CAAC;QAAC,OAAO,CAAU,EAAE,CAAC;YACpB,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;QAChB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM;QACV,OAAO,IAAI,CAAC,OAAO;aAChB,cAAc,CAAC,OAAO,IAAI,CAAC,EAAE,SAAS,EAAE;YACvC,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,MAAM;SACf,CAAC;aACD,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;aACzB,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1B,CAAC;CACF;AAED,MAAM,CAAC,MAAM,qBAAqB,GAAG,CACnC,EAAU,EACV,MA6CC,EACD,EAAE,CACF,CAAC;IACC,YAAY,EAAE,MAAM,CAAC,YAAY;IACjC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;IAC7C,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI;IACzD,EAAE;IACF,OAAO,EAAE,IAAsB;IAC/B,SAAS,EAAE,MAAM,CAAC,SAAS;IAC3B,yBAAyB,EAAE,MAAM,CAAC,yBAAyB;QACzD,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,yBAAyB,CAAC;QAC5C,CAAC,CAAC,IAAI;IACR,2BAA2B,EAAE,MAAM,CAAC,2BAA2B;QAC7D,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,2BAA2B,CAAC;QAC9C,CAAC,CAAC,IAAI;IACR,QAAQ,EAAE,MAAM,CAAC,QAAQ;IACzB,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI;IAC/D,KAAK,EAAE,MAAM,CAAC,QAAQ;IACtB,GAAG,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC;CAClE,CAAU,CAAC;AAMd,MAAM,cAAc,GAAG,GAAG,CAAC","sourcesContent":["/*\n * Copyright (C) 2024-2025 Dremio Corporation\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport type { SonarV3Config } from \"../../common/Config.ts\";\nimport type { Problem } from \"../../common/Problem.ts\";\nimport { Err, Ok, Result } from \"ts-results-es\";\nimport { lastValueFrom, map, of, switchMap, type Observable } from \"rxjs\";\nimport type { JobResultsResponse } from \"./utils/JobResultsResponse.ts\";\nimport { mapRowsToColumns } from \"./utils/mapRowsToColumns.ts\";\nimport { mapRowData } from \"./utils/mapRowData.ts\";\nimport type { SignalParam } from \"../../common/Params.ts\";\n\nexport class Job {\n readonly acceleration: JobProperties[\"acceleration\"];\n readonly cancellationReason: JobProperties[\"cancellationReason\"];\n readonly errorMessage?: string;\n readonly endedAt: JobProperties[\"endedAt\"];\n readonly id: JobProperties[\"id\"];\n readonly problem: JobProperties[\"problem\"];\n readonly queryType: JobProperties[\"queryType\"];\n readonly resourceSchedulingEndedAt: JobProperties[\"resourceSchedulingEndedAt\"];\n readonly resourceSchedulingStartedAt: JobProperties[\"resourceSchedulingStartedAt\"];\n readonly rowCount: JobProperties[\"rowCount\"];\n readonly startedAt: JobProperties[\"startedAt\"];\n readonly state: JobProperties[\"state\"];\n\n public readonly observer: Observable<Job>;\n #config: SonarV3Config;\n\n constructor(\n properties: JobProperties,\n config: SonarV3Config,\n observe: (id: string) => Observable<JobResult>,\n ) {\n this.acceleration = properties.acceleration;\n this.cancellationReason = properties.cancellationReason;\n this.endedAt = properties.endedAt;\n this.id = properties.id;\n this.problem = properties.problem;\n this.queryType = properties.queryType;\n this.resourceSchedulingEndedAt = properties.resourceSchedulingEndedAt;\n this.resourceSchedulingStartedAt = properties.resourceSchedulingStartedAt;\n this.rowCount = properties.rowCount;\n this.startedAt = properties.startedAt;\n this.state = properties.state;\n this.#config = config;\n this.observer = of(this).pipe(\n switchMap((job) => {\n if (!job.settled) {\n return observe(this.id).pipe(\n map((result) => {\n if (result.isErr()) {\n throw result.error;\n }\n return result.value;\n }),\n );\n } else {\n return of(job);\n }\n }),\n );\n\n if (properties.errorMessage) {\n this.errorMessage = properties.errorMessage;\n }\n }\n\n get settled() {\n return (\n this.state === \"COMPLETED\" ||\n this.state === \"FAILED\" ||\n this.state === \"INVALID_STATE\" ||\n this.state === \"CANCELED\"\n );\n }\n\n get results() {\n return {\n jsonBatches: this.#jsonBatches.bind(this),\n metadata: this.#metadata.bind(this),\n slice: this.#slice.bind(this),\n };\n }\n\n /**\n * @returns A `Job` instance once it's reached a settled state\n */\n retrieveSettled() {\n return lastValueFrom(this.observer);\n }\n\n async *#jsonBatches<T extends Record<string, unknown> = Record<string, unknown>>(\n args: { limit?: number; offset?: number } = {},\n { signal }: SignalParam = {},\n ) {\n if (typeof args.limit === \"number\" && args.limit < 0) {\n throw new Error(\"Limit cannot be negative\");\n }\n\n if (typeof args.offset === \"number\" && args.offset < 0) {\n throw new Error(\"Offset cannot be negative\");\n }\n\n // Wait for job to enter a settled state before attempting to fetch batches\n if (!this.settled) {\n await this.retrieveSettled();\n }\n\n const limitArg = args.limit ?? Infinity;\n const offsetArg = args.offset || 0;\n\n // Tracks whether there are more rows available from the job results endpoint\n let hasMore = true;\n\n // Tracks the currently requested offset. If the offset arg is provided, start from there instead of 0.\n let offset = offsetArg;\n\n // Keeps track of the total number of rows that need to be loaded\n const stopAfterOffset = limitArg + offsetArg;\n\n while (hasMore && offset < stopAfterOffset) {\n // Make batch_size dynamic to allow for requesting a smaller final page\n const batch_size = Math.min(MAX_BATCH_SIZE, stopAfterOffset - offset);\n\n const batch = await this.#config\n .sonarV3Request(`job/${this.id}/results?offset=${offset}&limit=${batch_size}`, { signal })\n .then((res) => res.json() as Promise<JobResultsResponse<T>>);\n\n offset += batch.rows.length;\n hasMore = batch.rows.length === batch_size;\n\n if (batch.rows.length) {\n const schema = { fields: batch.schema };\n mapRowData(batch.rows, batch.schema);\n yield {\n get columns() {\n return mapRowsToColumns(batch);\n },\n rows: batch.rows,\n schema,\n totalRows: batch.rowCount,\n } as const;\n }\n }\n }\n\n async #metadata({ signal }: SignalParam = {}) {\n return this.#config\n .sonarV3Request(`job/${this.id}/results?offset=0&limit=0`, { signal })\n .then((res) => res.json() as Promise<JobResultsResponse>)\n .then((response) =>\n Ok({\n schema: response.schema,\n totalRows: response.rowCount,\n }),\n )\n .catch((e: unknown) => Err(e));\n }\n\n async #slice<T extends Record<string, unknown> = Record<string, unknown>>(\n start: number = 0,\n end: number = Infinity,\n { signal }: SignalParam = {},\n ) {\n try {\n const rows: T[] = [];\n for await (const batch of this.#jsonBatches<T>(\n { limit: end - start, offset: start },\n { signal },\n )) {\n rows.push(...batch.rows);\n }\n return Ok(rows);\n } catch (e: unknown) {\n return Err(e);\n }\n }\n\n async cancel() {\n return this.#config\n .sonarV3Request(`job/${this.id}/cancel`, {\n keepalive: true,\n method: \"POST\",\n })\n .then(() => Ok(undefined))\n .catch((e) => Err(e));\n }\n}\n\nexport const jobEntityToProperties = (\n id: string,\n entity: {\n jobState:\n | \"NOT_SUBMITTED\"\n | \"STARTING\"\n | \"RUNNING\"\n | \"COMPLETED\"\n | \"CANCELED\"\n | \"FAILED\"\n | \"CANCELLATION_REQUESTED\"\n | \"PLANNING\"\n | \"PENDING\"\n | \"METADATA_RETRIEVAL\"\n | \"QUEUED\"\n | \"ENGINE_START\"\n | \"EXECUTION_PLANNING\"\n | \"INVALID_STATE\";\n rowCount: number | null;\n errorMessage: string;\n startedAt: string | null;\n endedAt: string | null;\n acceleration: {\n reflectionRelationships: {\n datasetId: string;\n reflectionId: string;\n relationship: string;\n }[];\n };\n queryType:\n | \"UI_RUN\"\n | \"UI_PREVIEW\"\n | \"UI_INTERNAL_PREVIEW\"\n | \"UI_INTERNAL_RUN\"\n | \"UI_EXPORT\"\n | \"ODBC\"\n | \"JDBC\"\n | \"REST\"\n | \"ACCELERATOR_CREATE\"\n | \"ACCELERATOR_DROP\"\n | \"UNKNOWN\"\n | \"PREPARE_INTERNAL\"\n | \"ACCELERATOR_EXPLAIN\"\n | \"UI_INITIAL_PREVIEW\";\n resourceSchedulingStartedAt: string | null;\n resourceSchedulingEndedAt: string | null;\n cancellationReason: string | null;\n },\n) =>\n ({\n acceleration: entity.acceleration,\n cancellationReason: entity.cancellationReason,\n endedAt: entity.endedAt ? new Date(entity.endedAt) : null,\n id,\n problem: null as Problem | null,\n queryType: entity.queryType,\n resourceSchedulingEndedAt: entity.resourceSchedulingEndedAt\n ? new Date(entity.resourceSchedulingEndedAt)\n : null,\n resourceSchedulingStartedAt: entity.resourceSchedulingStartedAt\n ? new Date(entity.resourceSchedulingStartedAt)\n : null,\n rowCount: entity.rowCount,\n startedAt: entity.startedAt ? new Date(entity.startedAt) : null,\n state: entity.jobState,\n ...(entity.errorMessage && { errorMessage: entity.errorMessage }),\n }) as const;\n\ntype JobProperties = ReturnType<typeof jobEntityToProperties>;\n\nexport type JobResult = Result<Job, unknown>;\n\nconst MAX_BATCH_SIZE = 500;\n"]}
|
|
@@ -73,7 +73,7 @@ type StructType = {
|
|
|
73
73
|
}[];
|
|
74
74
|
};
|
|
75
75
|
type ComplexTypes = ListType | StructType;
|
|
76
|
-
export type JobResultsSchema<TName extends string> = {
|
|
76
|
+
export type JobResultsSchema<TName extends string = string> = {
|
|
77
77
|
name: TName;
|
|
78
78
|
type: SimpleTypes | ComplexTypes;
|
|
79
79
|
}[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"JobResultsResponse.js","sourceRoot":"","sources":["../../../../src/oss/jobs/utils/JobResultsResponse.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG","sourcesContent":["/*\n * Copyright (C) 2024-2025 Dremio Corporation\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\ntype VarcharType = {\n name: \"VARCHAR\";\n};\n\ntype TextType = {\n name: \"TEXT\";\n};\n\ntype BinaryType = {\n name: \"BINARY\";\n};\n\ntype VarbinaryType = {\n name: \"VARBINARY\";\n};\n\ntype FloatType = {\n name: \"FLOAT\";\n};\n\ntype DecimalType = {\n name: \"DECIMAL\";\n};\n\ntype IntegerType = {\n name: \"INTEGER\";\n};\n\ntype MixedType = {\n name: \"MIXED\";\n};\n\ntype UnionType = {\n name: \"UNION\";\n};\n\ntype DateType = {\n name: \"DATE\";\n};\n\ntype TimeType = {\n name: \"TIME\";\n};\n\ntype DatetimeType = {\n name: \"DATETIME\";\n};\n\ntype MapType = {\n name: \"MAP\";\n};\n\ntype GeoType = {\n name: \"GEO\";\n};\n\ntype OtherType = {\n name: \"OTHER\";\n};\n\ntype AnyType = {\n name: \"ANY\";\n};\n\ntype DoubleType = {\n name: \"DOUBLE\";\n};\n\ntype BigintType = {\n name: \"BIGINT\";\n};\n\ntype BooleanType = {\n name: \"BOOLEAN\";\n};\n\ntype TimestampType = {\n name: \"TIMESTAMP\";\n};\n\ntype SimpleTypes =\n | VarcharType\n | DoubleType\n | BigintType\n | BooleanType\n | TimestampType\n | TextType\n | BinaryType\n | VarbinaryType\n | FloatType\n | DecimalType\n | IntegerType\n | MixedType\n | UnionType\n | DateType\n | TimeType\n | DatetimeType\n | MapType\n | GeoType\n | OtherType\n | AnyType;\n\ntype ListType = {\n name: \"LIST\";\n subSchema: {\n type: SimpleTypes | ComplexTypes;\n }[];\n};\n\ntype StructType = {\n name: \"STRUCT\";\n subSchema: {\n name: string;\n type: SimpleTypes | ComplexTypes;\n }[];\n};\n\ntype ComplexTypes = ListType | StructType;\n\nexport type JobResultsSchema<TName extends string> = {\n name: TName;\n type: SimpleTypes | ComplexTypes;\n}[];\n\nexport type JobResultsResponse<T extends Record<string, unknown> = Record<string, unknown>> = {\n rowCount: number;\n rows: T[];\n schema: JobResultsSchema<Extract<keyof T, string>>;\n};\n"]}
|
|
1
|
+
{"version":3,"file":"JobResultsResponse.js","sourceRoot":"","sources":["../../../../src/oss/jobs/utils/JobResultsResponse.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG","sourcesContent":["/*\n * Copyright (C) 2024-2025 Dremio Corporation\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\ntype VarcharType = {\n name: \"VARCHAR\";\n};\n\ntype TextType = {\n name: \"TEXT\";\n};\n\ntype BinaryType = {\n name: \"BINARY\";\n};\n\ntype VarbinaryType = {\n name: \"VARBINARY\";\n};\n\ntype FloatType = {\n name: \"FLOAT\";\n};\n\ntype DecimalType = {\n name: \"DECIMAL\";\n};\n\ntype IntegerType = {\n name: \"INTEGER\";\n};\n\ntype MixedType = {\n name: \"MIXED\";\n};\n\ntype UnionType = {\n name: \"UNION\";\n};\n\ntype DateType = {\n name: \"DATE\";\n};\n\ntype TimeType = {\n name: \"TIME\";\n};\n\ntype DatetimeType = {\n name: \"DATETIME\";\n};\n\ntype MapType = {\n name: \"MAP\";\n};\n\ntype GeoType = {\n name: \"GEO\";\n};\n\ntype OtherType = {\n name: \"OTHER\";\n};\n\ntype AnyType = {\n name: \"ANY\";\n};\n\ntype DoubleType = {\n name: \"DOUBLE\";\n};\n\ntype BigintType = {\n name: \"BIGINT\";\n};\n\ntype BooleanType = {\n name: \"BOOLEAN\";\n};\n\ntype TimestampType = {\n name: \"TIMESTAMP\";\n};\n\ntype SimpleTypes =\n | VarcharType\n | DoubleType\n | BigintType\n | BooleanType\n | TimestampType\n | TextType\n | BinaryType\n | VarbinaryType\n | FloatType\n | DecimalType\n | IntegerType\n | MixedType\n | UnionType\n | DateType\n | TimeType\n | DatetimeType\n | MapType\n | GeoType\n | OtherType\n | AnyType;\n\ntype ListType = {\n name: \"LIST\";\n subSchema: {\n type: SimpleTypes | ComplexTypes;\n }[];\n};\n\ntype StructType = {\n name: \"STRUCT\";\n subSchema: {\n name: string;\n type: SimpleTypes | ComplexTypes;\n }[];\n};\n\ntype ComplexTypes = ListType | StructType;\n\nexport type JobResultsSchema<TName extends string = string> = {\n name: TName;\n type: SimpleTypes | ComplexTypes;\n}[];\n\nexport type JobResultsResponse<T extends Record<string, unknown> = Record<string, unknown>> = {\n rowCount: number;\n rows: T[];\n schema: JobResultsSchema<Extract<keyof T, string>>;\n};\n"]}
|