@owox/backend 0.23.0-next-20260410055152 → 0.23.0-next-20260413072944
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/data-marts/data-storage-types/databricks/adapters/databricks-api.adapter.d.ts +12 -0
- package/dist/src/data-marts/data-storage-types/databricks/adapters/databricks-api.adapter.js +112 -17
- package/dist/src/data-marts/data-storage-types/databricks/services/databricks-data-mart-schema.provider.js +1 -2
- package/dist/src/data-marts/data-storage-types/databricks/services/databricks-query.builder.d.ts +2 -2
- package/dist/src/data-marts/data-storage-types/databricks/services/databricks-query.builder.js +10 -4
- package/dist/src/data-marts/data-storage-types/databricks/services/databricks-report-reader.service.d.ts +4 -1
- package/dist/src/data-marts/data-storage-types/databricks/services/databricks-report-reader.service.js +63 -17
- package/dist/src/data-marts/data-storage-types/databricks/services/databricks-sql-run.executor.d.ts +1 -0
- package/dist/src/data-marts/data-storage-types/databricks/services/databricks-sql-run.executor.js +56 -14
- package/package.json +5 -5
package/dist/src/data-marts/data-storage-types/databricks/adapters/databricks-api.adapter.d.ts
CHANGED
|
@@ -2,6 +2,13 @@ import { DatabricksConfig } from '../schemas/databricks-config.schema';
|
|
|
2
2
|
import { DatabricksCredentials } from '../schemas/databricks-credentials.schema';
|
|
3
3
|
import { DatabricksQueryMetadata } from '../interfaces/databricks-query-metadata';
|
|
4
4
|
import { DatabricksQueryExplainJsonResponse } from '../interfaces/databricks-query-explain-json-response';
|
|
5
|
+
export interface DatabricksQueryCursor {
|
|
6
|
+
queryId: string;
|
|
7
|
+
fetchChunk(maxRows?: number): Promise<Record<string, unknown>[]>;
|
|
8
|
+
hasMoreRows(): Promise<boolean>;
|
|
9
|
+
getColumns(): Promise<string[] | null>;
|
|
10
|
+
close(): Promise<void>;
|
|
11
|
+
}
|
|
5
12
|
export declare class DatabricksApiAdapter {
|
|
6
13
|
private readonly credentials;
|
|
7
14
|
private readonly config;
|
|
@@ -26,6 +33,11 @@ export declare class DatabricksApiAdapter {
|
|
|
26
33
|
getTableSchemaWithPath(catalogName: string | undefined, schemaName: string | undefined, tableName: string): Promise<Record<string, unknown>[]>;
|
|
27
34
|
createView(viewName: string, query: string): Promise<void>;
|
|
28
35
|
fetchResults(query: string, maxRows?: number): Promise<Record<string, unknown>[]>;
|
|
36
|
+
openQueryCursor(query: string): Promise<DatabricksQueryCursor>;
|
|
37
|
+
private createCursor;
|
|
38
|
+
private resolveColumns;
|
|
39
|
+
private assertCursorIsConsistent;
|
|
40
|
+
private closeCursorSafely;
|
|
29
41
|
getTableExtendedInfo(tableName: string): Promise<Record<string, unknown>[]>;
|
|
30
42
|
getPrimaryKeyColumns(fullyQualifiedName: string): Promise<string[]>;
|
|
31
43
|
}
|
package/dist/src/data-marts/data-storage-types/databricks/adapters/databricks-api.adapter.js
CHANGED
|
@@ -6,6 +6,7 @@ const sql_1 = require("@databricks/sql");
|
|
|
6
6
|
const IDBSQLLogger_1 = require("@databricks/sql/dist/contracts/IDBSQLLogger");
|
|
7
7
|
const internal_helpers_1 = require("@owox/internal-helpers");
|
|
8
8
|
const databricks_identifier_utils_1 = require("../utils/databricks-identifier.utils");
|
|
9
|
+
const DEFAULT_FETCH_CHUNK_MAX_ROWS = 1000;
|
|
9
10
|
class SilentDatabricksLogger {
|
|
10
11
|
logger = new common_1.Logger(SilentDatabricksLogger.name);
|
|
11
12
|
log(level, message) {
|
|
@@ -74,23 +75,38 @@ class DatabricksApiAdapter {
|
|
|
74
75
|
return this.session;
|
|
75
76
|
}
|
|
76
77
|
async executeQuery(query) {
|
|
77
|
-
const
|
|
78
|
+
const cursor = await this.openQueryCursor(query);
|
|
79
|
+
const rows = [];
|
|
80
|
+
let queryError = null;
|
|
78
81
|
try {
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
82
|
+
while (true) {
|
|
83
|
+
const chunk = await cursor.fetchChunk(DEFAULT_FETCH_CHUNK_MAX_ROWS);
|
|
84
|
+
if (chunk.length > 0) {
|
|
85
|
+
rows.push(...chunk);
|
|
86
|
+
}
|
|
87
|
+
if (chunk.length === 0) {
|
|
88
|
+
await this.assertCursorIsConsistent(cursor);
|
|
89
|
+
break;
|
|
90
|
+
}
|
|
91
|
+
const hasMoreRows = await cursor.hasMoreRows();
|
|
92
|
+
if (!hasMoreRows) {
|
|
93
|
+
break;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
84
96
|
this.logger.debug(`Query executed: ${query}`);
|
|
85
97
|
this.logger.debug(`Query returned ${rows?.length || 0} rows`);
|
|
86
98
|
const metadata = {
|
|
87
|
-
queryId,
|
|
88
|
-
statementId: queryId,
|
|
99
|
+
queryId: cursor.queryId,
|
|
100
|
+
statementId: cursor.queryId,
|
|
89
101
|
};
|
|
90
|
-
return { queryId, rows, metadata };
|
|
102
|
+
return { queryId: cursor.queryId, rows, metadata };
|
|
91
103
|
}
|
|
92
104
|
catch (error) {
|
|
93
|
-
|
|
105
|
+
queryError = new Error(`${DatabricksApiAdapter.DATABRICKS_QUERY_ERROR_PREFIX} ${(0, internal_helpers_1.castError)(error).message}, ${query}`);
|
|
106
|
+
throw queryError;
|
|
107
|
+
}
|
|
108
|
+
finally {
|
|
109
|
+
await this.closeCursorSafely(cursor, 'query execution', queryError);
|
|
94
110
|
}
|
|
95
111
|
}
|
|
96
112
|
async executeQueryAndFetchAll(query) {
|
|
@@ -160,17 +176,96 @@ class DatabricksApiAdapter {
|
|
|
160
176
|
await this.executeQuery(createViewQuery);
|
|
161
177
|
}
|
|
162
178
|
async fetchResults(query, maxRows = 1000) {
|
|
179
|
+
if (maxRows <= 0) {
|
|
180
|
+
return [];
|
|
181
|
+
}
|
|
182
|
+
const cursor = await this.openQueryCursor(query);
|
|
183
|
+
const rows = [];
|
|
184
|
+
let fetchError = null;
|
|
185
|
+
try {
|
|
186
|
+
while (rows.length < maxRows) {
|
|
187
|
+
const remainingRows = maxRows - rows.length;
|
|
188
|
+
const chunk = await cursor.fetchChunk(remainingRows);
|
|
189
|
+
if (chunk.length > 0) {
|
|
190
|
+
rows.push(...chunk);
|
|
191
|
+
}
|
|
192
|
+
if (chunk.length === 0) {
|
|
193
|
+
await this.assertCursorIsConsistent(cursor);
|
|
194
|
+
break;
|
|
195
|
+
}
|
|
196
|
+
const hasMoreRows = await cursor.hasMoreRows();
|
|
197
|
+
if (!hasMoreRows) {
|
|
198
|
+
break;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
return rows;
|
|
202
|
+
}
|
|
203
|
+
catch (error) {
|
|
204
|
+
fetchError = new Error(`Failed to fetch results: ${(0, internal_helpers_1.castError)(error).message}`);
|
|
205
|
+
throw fetchError;
|
|
206
|
+
}
|
|
207
|
+
finally {
|
|
208
|
+
await this.closeCursorSafely(cursor, 'result fetch', fetchError);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
async openQueryCursor(query) {
|
|
163
212
|
const session = await this.ensureConnection();
|
|
164
213
|
try {
|
|
165
|
-
const operation = await session.executeStatement(query
|
|
166
|
-
|
|
167
|
-
});
|
|
168
|
-
|
|
169
|
-
await operation.close();
|
|
170
|
-
return result;
|
|
214
|
+
const operation = await session.executeStatement(query);
|
|
215
|
+
const queryId = operation.id || '';
|
|
216
|
+
this.logger.debug(`Query cursor opened, queryId: ${queryId}`);
|
|
217
|
+
return this.createCursor(operation, queryId);
|
|
171
218
|
}
|
|
172
219
|
catch (error) {
|
|
173
|
-
throw new Error(
|
|
220
|
+
throw new Error(`${DatabricksApiAdapter.DATABRICKS_QUERY_ERROR_PREFIX} ${(0, internal_helpers_1.castError)(error).message}, ${query}`);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
createCursor(operation, queryId) {
|
|
224
|
+
let isClosed = false;
|
|
225
|
+
return {
|
|
226
|
+
queryId,
|
|
227
|
+
fetchChunk: async (maxRows) => {
|
|
228
|
+
const chunk = await operation.fetchChunk({
|
|
229
|
+
maxRows: maxRows ?? DEFAULT_FETCH_CHUNK_MAX_ROWS,
|
|
230
|
+
});
|
|
231
|
+
return chunk;
|
|
232
|
+
},
|
|
233
|
+
hasMoreRows: async () => operation.hasMoreRows(),
|
|
234
|
+
getColumns: async () => this.resolveColumns(operation),
|
|
235
|
+
close: async () => {
|
|
236
|
+
if (isClosed) {
|
|
237
|
+
return;
|
|
238
|
+
}
|
|
239
|
+
isClosed = true;
|
|
240
|
+
await operation.close();
|
|
241
|
+
},
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
async resolveColumns(operation) {
|
|
245
|
+
const schema = await operation.getSchema();
|
|
246
|
+
if (!schema?.columns?.length) {
|
|
247
|
+
return null;
|
|
248
|
+
}
|
|
249
|
+
const columns = schema.columns.map(column => column.columnName).filter(Boolean);
|
|
250
|
+
return columns.length > 0 ? columns : null;
|
|
251
|
+
}
|
|
252
|
+
async assertCursorIsConsistent(cursor) {
|
|
253
|
+
const hasMoreRows = await cursor.hasMoreRows();
|
|
254
|
+
if (hasMoreRows) {
|
|
255
|
+
throw new Error('Databricks cursor returned an empty chunk while indicating more rows');
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
async closeCursorSafely(cursor, context, primaryError) {
|
|
259
|
+
try {
|
|
260
|
+
await cursor.close();
|
|
261
|
+
}
|
|
262
|
+
catch (error) {
|
|
263
|
+
const closeError = (0, internal_helpers_1.castError)(error);
|
|
264
|
+
if (primaryError) {
|
|
265
|
+
this.logger.warn(`Failed to close Databricks cursor after ${context}: ${closeError.message}. Preserving original error: ${primaryError.message}`);
|
|
266
|
+
return;
|
|
267
|
+
}
|
|
268
|
+
throw new Error(`Failed to close Databricks cursor after ${context}: ${closeError.message}`);
|
|
174
269
|
}
|
|
175
270
|
}
|
|
176
271
|
async getTableExtendedInfo(tableName) {
|
|
@@ -69,8 +69,7 @@ let DatabricksDataMartSchemaProvider = DatabricksDataMartSchemaProvider_1 = clas
|
|
|
69
69
|
};
|
|
70
70
|
}
|
|
71
71
|
this.logger.debug('Table does not exist yet, deriving schema from query');
|
|
72
|
-
const
|
|
73
|
-
const query = `${baseQuery} LIMIT 0`;
|
|
72
|
+
const query = this.queryBuilder.buildQuery(definition, { limit: 0 });
|
|
74
73
|
const { rows: _rows } = await adapter.executeQuery(query);
|
|
75
74
|
const describeQuery = `DESCRIBE QUERY ${query}`;
|
|
76
75
|
const describeResult = await adapter.executeQuery(describeQuery);
|
package/dist/src/data-marts/data-storage-types/databricks/services/databricks-query.builder.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { DataMartDefinition } from '../../../dto/schemas/data-mart-table-definitions/data-mart-definition';
|
|
2
2
|
import { DataStorageType } from '../../enums/data-storage-type.enum';
|
|
3
|
-
import { DataMartQueryBuilder } from '../../interfaces/data-mart-query-builder.interface';
|
|
3
|
+
import { DataMartQueryBuilder, DataMartQueryOptions } from '../../interfaces/data-mart-query-builder.interface';
|
|
4
4
|
export declare class DatabricksQueryBuilder implements DataMartQueryBuilder {
|
|
5
5
|
readonly type = DataStorageType.DATABRICKS;
|
|
6
|
-
buildQuery(definition: DataMartDefinition): string;
|
|
6
|
+
buildQuery(definition: DataMartDefinition, queryOptions?: DataMartQueryOptions): string;
|
|
7
7
|
}
|
package/dist/src/data-marts/data-storage-types/databricks/services/databricks-query.builder.js
CHANGED
|
@@ -13,17 +13,18 @@ const data_storage_type_enum_1 = require("../../enums/data-storage-type.enum");
|
|
|
13
13
|
const databricks_identifier_utils_1 = require("../utils/databricks-identifier.utils");
|
|
14
14
|
let DatabricksQueryBuilder = class DatabricksQueryBuilder {
|
|
15
15
|
type = data_storage_type_enum_1.DataStorageType.DATABRICKS;
|
|
16
|
-
buildQuery(definition) {
|
|
16
|
+
buildQuery(definition, queryOptions) {
|
|
17
|
+
let query;
|
|
17
18
|
if ((0, data_mart_definition_guards_1.isTableDefinition)(definition) || (0, data_mart_definition_guards_1.isViewDefinition)(definition)) {
|
|
18
19
|
const parts = definition.fullyQualifiedName.split('.');
|
|
19
|
-
|
|
20
|
+
query = `SELECT * FROM ${(0, databricks_identifier_utils_1.escapeFullyQualifiedIdentifier)(parts)}`;
|
|
20
21
|
}
|
|
21
22
|
else if ((0, data_mart_definition_guards_1.isConnectorDefinition)(definition)) {
|
|
22
23
|
const parts = definition.connector.storage.fullyQualifiedName.split('.');
|
|
23
|
-
|
|
24
|
+
query = `SELECT * FROM ${(0, databricks_identifier_utils_1.escapeFullyQualifiedIdentifier)(parts)}`;
|
|
24
25
|
}
|
|
25
26
|
else if ((0, data_mart_definition_guards_1.isSqlDefinition)(definition)) {
|
|
26
|
-
|
|
27
|
+
query = definition.sqlQuery.trim();
|
|
27
28
|
}
|
|
28
29
|
else if ((0, data_mart_definition_guards_1.isTablePatternDefinition)(definition)) {
|
|
29
30
|
throw new Error('Table pattern definitions are not supported for Databricks');
|
|
@@ -31,6 +32,11 @@ let DatabricksQueryBuilder = class DatabricksQueryBuilder {
|
|
|
31
32
|
else {
|
|
32
33
|
throw new Error('Invalid data mart definition');
|
|
33
34
|
}
|
|
35
|
+
if (queryOptions?.limit !== undefined) {
|
|
36
|
+
const cleanQuery = query.endsWith(';') ? query.slice(0, -1) : query;
|
|
37
|
+
query = `SELECT * FROM (${cleanQuery}) LIMIT ${queryOptions.limit}`;
|
|
38
|
+
}
|
|
39
|
+
return query;
|
|
34
40
|
}
|
|
35
41
|
};
|
|
36
42
|
exports.DatabricksQueryBuilder = DatabricksQueryBuilder;
|
|
@@ -16,11 +16,13 @@ export declare class DatabricksReportReader implements DataStorageReportReader {
|
|
|
16
16
|
private readonly logger;
|
|
17
17
|
readonly type = DataStorageType.DATABRICKS;
|
|
18
18
|
private adapter;
|
|
19
|
+
private queryCursor;
|
|
19
20
|
private reportDataHeaders;
|
|
20
21
|
private reportConfig;
|
|
21
22
|
private queryId?;
|
|
22
23
|
private currentRowIndex;
|
|
23
|
-
private
|
|
24
|
+
private hasMoreRows;
|
|
25
|
+
private pendingRowsToSkip;
|
|
24
26
|
constructor(adapterFactory: DatabricksApiAdapterFactory, queryBuilder: DatabricksQueryBuilder, headersGenerator: DatabricksReportHeadersGenerator);
|
|
25
27
|
prepareReportData(report: Report): Promise<ReportDataDescription>;
|
|
26
28
|
readReportDataBatch(_batchId?: string, maxDataRows?: number): Promise<ReportDataBatch>;
|
|
@@ -29,4 +31,5 @@ export declare class DatabricksReportReader implements DataStorageReportReader {
|
|
|
29
31
|
private serializeValue;
|
|
30
32
|
getState(): DatabricksReaderState | null;
|
|
31
33
|
initFromState(state: DataStorageReportReaderState, reportDataHeaders: ReportDataHeader[]): Promise<void>;
|
|
34
|
+
private skipPendingRows;
|
|
32
35
|
}
|
|
@@ -12,6 +12,7 @@ var DatabricksReportReader_1;
|
|
|
12
12
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
13
|
exports.DatabricksReportReader = void 0;
|
|
14
14
|
const common_1 = require("@nestjs/common");
|
|
15
|
+
const internal_helpers_1 = require("@owox/internal-helpers");
|
|
15
16
|
const data_storage_type_enum_1 = require("../../enums/data-storage-type.enum");
|
|
16
17
|
const report_data_batch_dto_1 = require("../../../dto/domain/report-data-batch.dto");
|
|
17
18
|
const report_data_description_dto_1 = require("../../../dto/domain/report-data-description.dto");
|
|
@@ -27,11 +28,13 @@ let DatabricksReportReader = DatabricksReportReader_1 = class DatabricksReportRe
|
|
|
27
28
|
logger = new common_1.Logger(DatabricksReportReader_1.name);
|
|
28
29
|
type = data_storage_type_enum_1.DataStorageType.DATABRICKS;
|
|
29
30
|
adapter;
|
|
31
|
+
queryCursor = null;
|
|
30
32
|
reportDataHeaders;
|
|
31
33
|
reportConfig;
|
|
32
34
|
queryId;
|
|
33
35
|
currentRowIndex = 0;
|
|
34
|
-
|
|
36
|
+
hasMoreRows = false;
|
|
37
|
+
pendingRowsToSkip = 0;
|
|
35
38
|
constructor(adapterFactory, queryBuilder, headersGenerator) {
|
|
36
39
|
this.adapterFactory = adapterFactory;
|
|
37
40
|
this.queryBuilder = queryBuilder;
|
|
@@ -53,33 +56,51 @@ let DatabricksReportReader = DatabricksReportReader_1 = class DatabricksReportRe
|
|
|
53
56
|
this.adapter = await this.adapterFactory.createFromStorage(storage);
|
|
54
57
|
const query = this.queryBuilder.buildQuery(definition);
|
|
55
58
|
this.logger.debug(`Executing query: ${query}`);
|
|
56
|
-
|
|
57
|
-
this.queryId =
|
|
58
|
-
this.allRows = result.rows || [];
|
|
59
|
+
this.queryCursor = await this.adapter.openQueryCursor(query);
|
|
60
|
+
this.queryId = this.queryCursor.queryId;
|
|
59
61
|
this.currentRowIndex = 0;
|
|
60
|
-
this.
|
|
62
|
+
this.pendingRowsToSkip = 0;
|
|
63
|
+
this.hasMoreRows = true;
|
|
64
|
+
this.logger.debug(`Query cursor opened, queryId: ${this.queryId}`);
|
|
61
65
|
return new report_data_description_dto_1.ReportDataDescription(this.reportDataHeaders);
|
|
62
66
|
}
|
|
63
67
|
async readReportDataBatch(_batchId, maxDataRows = 1000) {
|
|
64
|
-
if (!this.reportDataHeaders || !this.queryId) {
|
|
68
|
+
if (!this.reportDataHeaders || !this.queryCursor || !this.queryId) {
|
|
65
69
|
throw new Error('Report data must be prepared before read');
|
|
66
70
|
}
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
71
|
+
if (this.pendingRowsToSkip > 0) {
|
|
72
|
+
await this.skipPendingRows();
|
|
73
|
+
}
|
|
74
|
+
const rows = await this.queryCursor.fetchChunk(maxDataRows);
|
|
70
75
|
const mappedRows = this.mapRowsToHeaders(rows);
|
|
71
|
-
this.currentRowIndex
|
|
72
|
-
const hasMoreData =
|
|
76
|
+
this.currentRowIndex += rows.length;
|
|
77
|
+
const hasMoreData = rows.length > 0 ? await this.queryCursor.hasMoreRows() : false;
|
|
78
|
+
this.hasMoreRows = hasMoreData;
|
|
73
79
|
const nextBatchId = hasMoreData ? this.currentRowIndex.toString() : null;
|
|
74
|
-
this.logger.debug(`Returning batch with ${rows.length} rows (
|
|
80
|
+
this.logger.debug(`Returning batch with ${rows.length} rows (rows read total: ${this.currentRowIndex})`);
|
|
75
81
|
return new report_data_batch_dto_1.ReportDataBatch(mappedRows, nextBatchId);
|
|
76
82
|
}
|
|
77
83
|
async finalize() {
|
|
78
84
|
this.logger.debug('Finalizing report read');
|
|
85
|
+
let closeError = null;
|
|
86
|
+
if (this.queryCursor) {
|
|
87
|
+
try {
|
|
88
|
+
await this.queryCursor.close();
|
|
89
|
+
}
|
|
90
|
+
catch (error) {
|
|
91
|
+
closeError = (0, internal_helpers_1.castError)(error);
|
|
92
|
+
this.logger.warn(`Failed to close Databricks cursor during finalize: ${closeError.message}`);
|
|
93
|
+
}
|
|
94
|
+
finally {
|
|
95
|
+
this.queryCursor = null;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
79
98
|
if (this.adapter) {
|
|
80
99
|
await this.adapter.destroy();
|
|
81
100
|
}
|
|
82
|
-
|
|
101
|
+
if (closeError) {
|
|
102
|
+
throw closeError;
|
|
103
|
+
}
|
|
83
104
|
}
|
|
84
105
|
mapRowsToHeaders(rows) {
|
|
85
106
|
const headerNames = this.reportDataHeaders.map(h => h.name);
|
|
@@ -147,17 +168,42 @@ let DatabricksReportReader = DatabricksReportReader_1 = class DatabricksReportRe
|
|
|
147
168
|
type: data_storage_type_enum_1.DataStorageType.DATABRICKS,
|
|
148
169
|
queryId: this.queryId,
|
|
149
170
|
rowsRead: this.currentRowIndex,
|
|
150
|
-
hasMore: this.
|
|
171
|
+
hasMore: this.hasMoreRows,
|
|
151
172
|
};
|
|
152
173
|
}
|
|
153
174
|
async initFromState(state, reportDataHeaders) {
|
|
154
175
|
if (!(0, databricks_reader_state_interface_1.isDatabricksReaderState)(state)) {
|
|
155
176
|
throw new Error('Invalid state type for Databricks reader');
|
|
156
177
|
}
|
|
157
|
-
this.
|
|
158
|
-
|
|
178
|
+
if (!this.queryCursor || !this.queryId) {
|
|
179
|
+
throw new Error('Report data must be prepared before state restore');
|
|
180
|
+
}
|
|
181
|
+
this.currentRowIndex = 0;
|
|
182
|
+
this.pendingRowsToSkip = state.rowsRead;
|
|
183
|
+
this.hasMoreRows = state.hasMore;
|
|
159
184
|
this.reportDataHeaders = reportDataHeaders;
|
|
160
|
-
this.logger.debug(`Restored from state: queryId=${this.queryId},
|
|
185
|
+
this.logger.debug(`Restored from state: cachedQueryId=${state.queryId}, activeQueryId=${this.queryId}, rowsToSkip=${this.pendingRowsToSkip}`);
|
|
186
|
+
}
|
|
187
|
+
async skipPendingRows() {
|
|
188
|
+
if (!this.queryCursor) {
|
|
189
|
+
throw new Error('Report data must be prepared before skip');
|
|
190
|
+
}
|
|
191
|
+
while (this.pendingRowsToSkip > 0) {
|
|
192
|
+
const chunkSize = Math.min(this.pendingRowsToSkip, 1000);
|
|
193
|
+
const skippedRows = await this.queryCursor.fetchChunk(chunkSize);
|
|
194
|
+
if (!skippedRows.length) {
|
|
195
|
+
this.pendingRowsToSkip = 0;
|
|
196
|
+
this.hasMoreRows = false;
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
this.pendingRowsToSkip -= skippedRows.length;
|
|
200
|
+
this.currentRowIndex += skippedRows.length;
|
|
201
|
+
this.hasMoreRows = await this.queryCursor.hasMoreRows();
|
|
202
|
+
if (!this.hasMoreRows) {
|
|
203
|
+
this.pendingRowsToSkip = 0;
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
161
207
|
}
|
|
162
208
|
};
|
|
163
209
|
exports.DatabricksReportReader = DatabricksReportReader;
|
package/dist/src/data-marts/data-storage-types/databricks/services/databricks-sql-run.executor.d.ts
CHANGED
|
@@ -10,6 +10,7 @@ export declare class DatabricksSqlRunExecutor implements SqlRunExecutor {
|
|
|
10
10
|
private readonly adapterFactory;
|
|
11
11
|
private readonly queryBuilder;
|
|
12
12
|
readonly type = DataStorageType.DATABRICKS;
|
|
13
|
+
private readonly logger;
|
|
13
14
|
constructor(adapterFactory: DatabricksApiAdapterFactory, queryBuilder: DatabricksQueryBuilder);
|
|
14
15
|
execute<Row = Record<string, unknown>>(credentials: DataStorageCredentials, config: DataStorageConfig, definition: DataMartDefinition, sql: string | undefined, options?: {
|
|
15
16
|
maxRowsPerBatch?: number;
|
package/dist/src/data-marts/data-storage-types/databricks/services/databricks-sql-run.executor.js
CHANGED
|
@@ -8,19 +8,22 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
|
|
|
8
8
|
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
9
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
10
|
};
|
|
11
|
+
var DatabricksSqlRunExecutor_1;
|
|
11
12
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
13
|
exports.DatabricksSqlRunExecutor = void 0;
|
|
13
14
|
const common_1 = require("@nestjs/common");
|
|
15
|
+
const internal_helpers_1 = require("@owox/internal-helpers");
|
|
14
16
|
const data_storage_type_enum_1 = require("../../enums/data-storage-type.enum");
|
|
15
17
|
const data_storage_credentials_guards_1 = require("../../data-storage-credentials.guards");
|
|
16
18
|
const data_storage_config_guards_1 = require("../../data-storage-config.guards");
|
|
17
19
|
const databricks_api_adapter_factory_1 = require("../adapters/databricks-api-adapter.factory");
|
|
18
20
|
const sql_run_batch_dto_1 = require("../../../dto/domain/sql-run-batch.dto");
|
|
19
21
|
const databricks_query_builder_1 = require("./databricks-query.builder");
|
|
20
|
-
let DatabricksSqlRunExecutor = class DatabricksSqlRunExecutor {
|
|
22
|
+
let DatabricksSqlRunExecutor = DatabricksSqlRunExecutor_1 = class DatabricksSqlRunExecutor {
|
|
21
23
|
adapterFactory;
|
|
22
24
|
queryBuilder;
|
|
23
25
|
type = data_storage_type_enum_1.DataStorageType.DATABRICKS;
|
|
26
|
+
logger = new common_1.Logger(DatabricksSqlRunExecutor_1.name);
|
|
24
27
|
constructor(adapterFactory, queryBuilder) {
|
|
25
28
|
this.adapterFactory = adapterFactory;
|
|
26
29
|
this.queryBuilder = queryBuilder;
|
|
@@ -36,28 +39,67 @@ let DatabricksSqlRunExecutor = class DatabricksSqlRunExecutor {
|
|
|
36
39
|
if (!sql) {
|
|
37
40
|
sql = this.queryBuilder.buildQuery(definition);
|
|
38
41
|
}
|
|
42
|
+
let cursor = null;
|
|
43
|
+
let executionError = null;
|
|
44
|
+
let closeError = null;
|
|
39
45
|
try {
|
|
40
|
-
const { rows } = await adapter.executeQuery(sql);
|
|
41
|
-
const columns = rows && rows.length > 0 ? Object.keys(rows[0]) : undefined;
|
|
42
|
-
if (!rows || rows.length === 0) {
|
|
43
|
-
yield new sql_run_batch_dto_1.SqlRunBatch([], null, columns ?? null);
|
|
44
|
-
return;
|
|
45
|
-
}
|
|
46
46
|
const maxRowsPerBatch = options?.maxRowsPerBatch || 1000;
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
47
|
+
cursor = await adapter.openQueryCursor(sql);
|
|
48
|
+
let rowsRead = 0;
|
|
49
|
+
let hasEmittedRows = false;
|
|
50
|
+
let columns = await cursor.getColumns();
|
|
51
|
+
let columnsResolvedFromRowData = false;
|
|
52
|
+
while (true) {
|
|
53
|
+
const rows = (await cursor.fetchChunk(maxRowsPerBatch));
|
|
54
|
+
if (!rows.length) {
|
|
55
|
+
if (!hasEmittedRows) {
|
|
56
|
+
yield new sql_run_batch_dto_1.SqlRunBatch([], null, columns ?? null);
|
|
57
|
+
}
|
|
58
|
+
break;
|
|
59
|
+
}
|
|
60
|
+
hasEmittedRows = true;
|
|
61
|
+
rowsRead += rows.length;
|
|
62
|
+
if (!columnsResolvedFromRowData) {
|
|
63
|
+
columns = Object.keys(rows[0]);
|
|
64
|
+
columnsResolvedFromRowData = true;
|
|
65
|
+
}
|
|
66
|
+
const hasMoreRows = await cursor.hasMoreRows();
|
|
67
|
+
const nextBatchId = hasMoreRows ? String(rowsRead) : null;
|
|
68
|
+
yield new sql_run_batch_dto_1.SqlRunBatch(rows, nextBatchId, columns);
|
|
69
|
+
if (!hasMoreRows) {
|
|
70
|
+
break;
|
|
71
|
+
}
|
|
52
72
|
}
|
|
53
73
|
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
executionError = (0, internal_helpers_1.castError)(error);
|
|
76
|
+
}
|
|
54
77
|
finally {
|
|
55
|
-
|
|
78
|
+
try {
|
|
79
|
+
if (cursor) {
|
|
80
|
+
await cursor.close();
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
catch (error) {
|
|
84
|
+
closeError = (0, internal_helpers_1.castError)(error);
|
|
85
|
+
if (executionError) {
|
|
86
|
+
this.logger.warn(`Failed to close Databricks cursor after SQL run error: ${closeError.message}. Preserving original error: ${executionError.message}`);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
finally {
|
|
90
|
+
await adapter.destroy();
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
if (executionError) {
|
|
94
|
+
throw executionError;
|
|
95
|
+
}
|
|
96
|
+
if (closeError) {
|
|
97
|
+
throw new Error(`Failed to close Databricks cursor after SQL run: ${closeError.message}`);
|
|
56
98
|
}
|
|
57
99
|
}
|
|
58
100
|
};
|
|
59
101
|
exports.DatabricksSqlRunExecutor = DatabricksSqlRunExecutor;
|
|
60
|
-
exports.DatabricksSqlRunExecutor = DatabricksSqlRunExecutor = __decorate([
|
|
102
|
+
exports.DatabricksSqlRunExecutor = DatabricksSqlRunExecutor = DatabricksSqlRunExecutor_1 = __decorate([
|
|
61
103
|
(0, common_1.Injectable)(),
|
|
62
104
|
__metadata("design:paramtypes", [databricks_api_adapter_factory_1.DatabricksApiAdapterFactory,
|
|
63
105
|
databricks_query_builder_1.DatabricksQueryBuilder])
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@owox/backend",
|
|
3
|
-
"version": "0.23.0-next-
|
|
3
|
+
"version": "0.23.0-next-20260413072944",
|
|
4
4
|
"description": "OWOX Data Marts Backend - Full-stack data orchestration platform",
|
|
5
5
|
"author": "OWOX",
|
|
6
6
|
"license": "ELv2",
|
|
@@ -78,9 +78,9 @@
|
|
|
78
78
|
"@nestjs/schedule": "^6.0.0",
|
|
79
79
|
"@nestjs/swagger": "^11.2.0",
|
|
80
80
|
"@nestjs/typeorm": "^11.0.0",
|
|
81
|
-
"@owox/connectors": "0.23.0-next-
|
|
82
|
-
"@owox/idp-protocol": "0.23.0-next-
|
|
83
|
-
"@owox/internal-helpers": "0.23.0-next-
|
|
81
|
+
"@owox/connectors": "0.23.0-next-20260413072944",
|
|
82
|
+
"@owox/idp-protocol": "0.23.0-next-20260413072944",
|
|
83
|
+
"@owox/internal-helpers": "0.23.0-next-20260413072944",
|
|
84
84
|
"better-sqlite3": "^12.2.0",
|
|
85
85
|
"class-transformer": "^0.5.1",
|
|
86
86
|
"class-validator": "^0.14.2",
|
|
@@ -139,7 +139,7 @@
|
|
|
139
139
|
"ts-node": "^10.9.2",
|
|
140
140
|
"tsconfig-paths": "^4.2.0",
|
|
141
141
|
"typeorm-ts-node-commonjs": "^0.3.20",
|
|
142
|
-
"@owox/test-utils": "2.0.0-next-
|
|
142
|
+
"@owox/test-utils": "2.0.0-next-20260413072944"
|
|
143
143
|
},
|
|
144
144
|
"jest": {
|
|
145
145
|
"moduleFileExtensions": [
|