@malloydata/db-trino 0.0.149-dev240704171255 → 0.0.149
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/test.nonspec.d.ts +1 -0
- package/dist/test.nonspec.js +78 -0
- package/dist/trino_connection.d.ts +34 -11
- package/dist/trino_connection.js +216 -135
- package/dist/trino_connection.spec.js +1 -1
- package/dist/trino_executor.d.ts +1 -1
- package/dist/trino_executor.js +32 -17
- package/package.json +2 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
/*import {Trino, BasicAuth} from 'trino-client';
|
|
4
|
+
import PrestoClient from '@prestodb/presto-js-client';
|
|
5
|
+
|
|
6
|
+
// function CallPresto(client: Client, query: string ) {
|
|
7
|
+
// return new Promise (resolve => {
|
|
8
|
+
// client.execute({query, data => resolve(response)})
|
|
9
|
+
// });
|
|
10
|
+
// }
|
|
11
|
+
|
|
12
|
+
describe('Trino connection', () => {
|
|
13
|
+
console.log('hello');
|
|
14
|
+
|
|
15
|
+
test('says hello1', async () => {
|
|
16
|
+
const trino: Trino = Trino.create({
|
|
17
|
+
server: 'http://localhost:8090',
|
|
18
|
+
catalog: 'bigquery',
|
|
19
|
+
schema: 'malloytest',
|
|
20
|
+
auth: new BasicAuth('test'),
|
|
21
|
+
});
|
|
22
|
+
const limit = 50;
|
|
23
|
+
const result = await trino.query(
|
|
24
|
+
// 'explain SELECT 1 as one'
|
|
25
|
+
'explain SELECT * FROM malloytest.ga_sample limit 2'
|
|
26
|
+
);
|
|
27
|
+
let queryResult = await result.next();
|
|
28
|
+
const columns = queryResult.value.columns;
|
|
29
|
+
|
|
30
|
+
const outputRows: unknown[] = [];
|
|
31
|
+
while (queryResult !== null && outputRows.length < limit) {
|
|
32
|
+
const rows = queryResult.value.data ?? [];
|
|
33
|
+
for (const row of rows) {
|
|
34
|
+
if (outputRows.length < limit) {
|
|
35
|
+
outputRows.push(row);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
if (!queryResult.done) {
|
|
39
|
+
queryResult = await result.next();
|
|
40
|
+
} else {
|
|
41
|
+
break;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const d = outputRows![0]![0];
|
|
46
|
+
console.log(d);
|
|
47
|
+
|
|
48
|
+
// console.log(outputRows);
|
|
49
|
+
// console.log(columns);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
test('says hello presto', async () => {
|
|
53
|
+
const client = new PrestoClient({
|
|
54
|
+
catalog: 'bigquery',
|
|
55
|
+
host: 'http://localhost',
|
|
56
|
+
port: 8080,
|
|
57
|
+
schema: 'malloytest',
|
|
58
|
+
timezone: 'America/Costa_Rica',
|
|
59
|
+
user: 'root',
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
try {
|
|
63
|
+
const ret = await client.query(
|
|
64
|
+
'explain SELECT totals FROM malloytest.ga_sample limit 2'
|
|
65
|
+
// 'explain select 1 as one, 2 as two'
|
|
66
|
+
);
|
|
67
|
+
const d = ret.data![0][0];
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
// console.log(ret);
|
|
71
|
+
console.log(d);
|
|
72
|
+
} catch (error) {
|
|
73
|
+
console.log(error);
|
|
74
|
+
}
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
*/
|
|
78
|
+
//# sourceMappingURL=test.nonspec.js.map
|
|
@@ -10,27 +10,37 @@ export interface TrinoManagerOptions {
|
|
|
10
10
|
}
|
|
11
11
|
export interface TrinoConnectionConfiguration {
|
|
12
12
|
server?: string;
|
|
13
|
+
port?: number;
|
|
13
14
|
catalog?: string;
|
|
14
15
|
schema?: string;
|
|
15
16
|
user?: string;
|
|
16
17
|
password?: string;
|
|
17
18
|
}
|
|
18
|
-
type TrinoConnectionOptions = ConnectionConfig;
|
|
19
|
-
export
|
|
19
|
+
export type TrinoConnectionOptions = ConnectionConfig;
|
|
20
|
+
export interface BaseConnection {
|
|
21
|
+
runSQL(sql: string, limit: number | undefined): Promise<{
|
|
22
|
+
rows: unknown[][];
|
|
23
|
+
columns: {
|
|
24
|
+
name: string;
|
|
25
|
+
type: string;
|
|
26
|
+
error?: string;
|
|
27
|
+
}[];
|
|
28
|
+
error?: string;
|
|
29
|
+
}>;
|
|
30
|
+
}
|
|
31
|
+
export declare abstract class TrinoPrestoConnection implements Connection, PersistSQLResults {
|
|
20
32
|
trinoToMalloyTypes: {
|
|
21
33
|
[key: string]: FieldAtomicTypeDef;
|
|
22
34
|
};
|
|
23
35
|
private sqlToMalloyType;
|
|
24
|
-
|
|
36
|
+
name: string;
|
|
25
37
|
private readonly dialect;
|
|
26
38
|
static DEFAULT_QUERY_OPTIONS: RunSQLOptions;
|
|
27
39
|
private schemaCache;
|
|
28
40
|
private sqlSchemaCache;
|
|
29
41
|
private queryOptions?;
|
|
30
|
-
private
|
|
31
|
-
|
|
32
|
-
constructor(option: TrinoConnectionOptions, queryOptions?: QueryOptionsReader);
|
|
33
|
-
constructor(name: string, queryOptions?: QueryOptionsReader, config?: TrinoConnectionConfiguration);
|
|
42
|
+
private client;
|
|
43
|
+
constructor(name: string, queryOptions?: QueryOptionsReader, pConfig?: TrinoConnectionConfiguration);
|
|
34
44
|
get dialectName(): string;
|
|
35
45
|
private readQueryOptions;
|
|
36
46
|
isPool(): this is PooledConnection;
|
|
@@ -38,8 +48,9 @@ export declare class TrinoConnection implements Connection, PersistSQLResults {
|
|
|
38
48
|
canStream(): this is StreamingConnection;
|
|
39
49
|
get supportsNesting(): boolean;
|
|
40
50
|
manifestTemporaryTable(_sqlCommand: string): Promise<string>;
|
|
51
|
+
unpackArray(data: unknown): unknown[];
|
|
41
52
|
convertRow(structDef: StructDef, _row: unknown): {};
|
|
42
|
-
convertNest(structDef: StructDef,
|
|
53
|
+
convertNest(structDef: StructDef, _data: unknown): {};
|
|
43
54
|
runSQL(sqlCommand: string, options?: RunSQLOptions, _rowIndex?: number): Promise<MalloyQueryData>;
|
|
44
55
|
runSQLBlockAndFetchResultSchema(_sqlBlock: SQLBlock, _options?: RunSQLOptions): Promise<{
|
|
45
56
|
data: MalloyQueryData;
|
|
@@ -58,14 +69,26 @@ export declare class TrinoConnection implements Connection, PersistSQLResults {
|
|
|
58
69
|
structDef?: undefined;
|
|
59
70
|
}>;
|
|
60
71
|
private structDefFromSqlBlock;
|
|
61
|
-
|
|
72
|
+
protected abstract fillStructDefForSqlBlockSchema(sql: string, structDef: StructDef): Promise<void>;
|
|
73
|
+
protected executeAndWait(sqlBlock: string): Promise<void>;
|
|
62
74
|
splitColumns(s: string): string[];
|
|
63
75
|
malloyTypeFromTrinoType(name: string, trinoType: string): FieldAtomicTypeDef | StructDef;
|
|
64
76
|
structDefFromSchema(rows: string[][], structDef: StructDef): void;
|
|
65
|
-
|
|
77
|
+
protected loadSchemaForSqlBlock(sqlBlock: string, structDef: StructDef, element: string): Promise<StructDef>;
|
|
66
78
|
estimateQueryCost(_sqlCommand: string): Promise<QueryRunStats>;
|
|
67
79
|
executeSQLRaw(_sqlCommand: string): Promise<QueryData>;
|
|
68
80
|
test(): Promise<void>;
|
|
69
81
|
close(): Promise<void>;
|
|
70
82
|
}
|
|
71
|
-
export {
|
|
83
|
+
export declare class PrestoConnection extends TrinoPrestoConnection {
|
|
84
|
+
constructor(name: string, queryOptions?: QueryOptionsReader, config?: TrinoConnectionConfiguration);
|
|
85
|
+
constructor(option: TrinoConnectionOptions, queryOptions?: QueryOptionsReader);
|
|
86
|
+
protected fillStructDefForSqlBlockSchema(sql: string, structDef: StructDef): Promise<void>;
|
|
87
|
+
private schemaFromExplain;
|
|
88
|
+
unpackArray(data: unknown): unknown[];
|
|
89
|
+
}
|
|
90
|
+
export declare class TrinoConnection extends TrinoPrestoConnection {
|
|
91
|
+
constructor(name: string, queryOptions?: QueryOptionsReader, config?: TrinoConnectionConfiguration);
|
|
92
|
+
constructor(option: TrinoConnectionOptions, queryOptions?: QueryOptionsReader);
|
|
93
|
+
protected fillStructDefForSqlBlockSchema(sql: string, structDef: StructDef): Promise<void>;
|
|
94
|
+
}
|
package/dist/trino_connection.js
CHANGED
|
@@ -22,12 +22,86 @@
|
|
|
22
22
|
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
23
23
|
*/
|
|
24
24
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
25
|
-
exports.TrinoConnection = void 0;
|
|
25
|
+
exports.TrinoConnection = exports.PrestoConnection = exports.TrinoPrestoConnection = void 0;
|
|
26
26
|
const malloy_1 = require("@malloydata/malloy");
|
|
27
|
+
const presto_js_client_1 = require("@prestodb/presto-js-client");
|
|
27
28
|
const crypto_1 = require("crypto");
|
|
28
29
|
const trino_client_1 = require("trino-client");
|
|
30
|
+
class PrestoBase {
|
|
31
|
+
constructor(config) {
|
|
32
|
+
this.client = new presto_js_client_1.PrestoClient({
|
|
33
|
+
catalog: config.catalog,
|
|
34
|
+
host: config.server,
|
|
35
|
+
port: config.port,
|
|
36
|
+
schema: config.schema,
|
|
37
|
+
timezone: 'America/Costa_Rica',
|
|
38
|
+
user: config.user || 'anyone',
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
async runSQL(sql, limit) {
|
|
42
|
+
let ret = undefined;
|
|
43
|
+
const q = limit ? `SELECT * FROM(${sql}) LIMIT ${limit}` : sql;
|
|
44
|
+
let error = undefined;
|
|
45
|
+
try {
|
|
46
|
+
ret = (await this.client.query(q)) || [];
|
|
47
|
+
// console.log(ret);
|
|
48
|
+
}
|
|
49
|
+
catch (errorObj) {
|
|
50
|
+
// console.log(error);
|
|
51
|
+
error = errorObj.toString();
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
rows: ret && ret.data ? ret.data : [],
|
|
55
|
+
columns: ret && ret.columns
|
|
56
|
+
? ret.columns
|
|
57
|
+
: [],
|
|
58
|
+
error,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
class TrinooBase {
|
|
63
|
+
constructor(config) {
|
|
64
|
+
this.client = trino_client_1.Trino.create({
|
|
65
|
+
catalog: config.catalog,
|
|
66
|
+
server: config.server,
|
|
67
|
+
schema: config.schema,
|
|
68
|
+
auth: new trino_client_1.BasicAuth(config.user, config.password || ''),
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
async runSQL(sql, limit) {
|
|
72
|
+
var _a;
|
|
73
|
+
const result = await this.client.query(sql);
|
|
74
|
+
let queryResult = await result.next();
|
|
75
|
+
if (queryResult.value.error) {
|
|
76
|
+
return {
|
|
77
|
+
rows: [],
|
|
78
|
+
columns: [],
|
|
79
|
+
error: JSON.stringify(queryResult.value.error),
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
const columns = queryResult.value.columns;
|
|
83
|
+
const outputRows = [];
|
|
84
|
+
while (queryResult !== null && (!limit || outputRows.length < limit)) {
|
|
85
|
+
const rows = (_a = queryResult.value.data) !== null && _a !== void 0 ? _a : [];
|
|
86
|
+
for (const row of rows) {
|
|
87
|
+
if (!limit || outputRows.length < limit) {
|
|
88
|
+
outputRows.push(row);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
if (!queryResult.done) {
|
|
92
|
+
queryResult = await result.next();
|
|
93
|
+
}
|
|
94
|
+
else {
|
|
95
|
+
break;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
// console.log(outputRows);
|
|
99
|
+
// console.log(columns);
|
|
100
|
+
return { rows: outputRows, columns };
|
|
101
|
+
}
|
|
102
|
+
}
|
|
29
103
|
// manage access to BQ, control costs, enforce global data/API limits
|
|
30
|
-
class
|
|
104
|
+
class TrinoPrestoConnection {
|
|
31
105
|
sqlToMalloyType(sqlType) {
|
|
32
106
|
var _a, _b;
|
|
33
107
|
const baseSqlType = (_b = (_a = sqlType.match(/^(\w+)/)) === null || _a === void 0 ? void 0 : _a.at(0)) !== null && _b !== void 0 ? _b : sqlType;
|
|
@@ -36,7 +110,7 @@ class TrinoConnection {
|
|
|
36
110
|
}
|
|
37
111
|
return undefined;
|
|
38
112
|
}
|
|
39
|
-
constructor(
|
|
113
|
+
constructor(name, queryOptions, pConfig) {
|
|
40
114
|
this.trinoToMalloyTypes = {
|
|
41
115
|
'varchar': { type: 'string' },
|
|
42
116
|
'integer': { type: 'number', numberType: 'integer' },
|
|
@@ -66,32 +140,19 @@ class TrinoConnection {
|
|
|
66
140
|
this.dialect = new malloy_1.StandardSQLDialect();
|
|
67
141
|
this.schemaCache = new Map();
|
|
68
142
|
this.sqlSchemaCache = new Map();
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
config.credentials = {
|
|
78
|
-
client_email,
|
|
79
|
-
private_key,
|
|
80
|
-
};
|
|
81
|
-
}
|
|
82
|
-
}*/
|
|
83
|
-
// TODO: check user is set.
|
|
84
|
-
this.trino = trino_client_1.Trino.create({
|
|
85
|
-
server: config.server,
|
|
86
|
-
catalog: 'malloy_demo', //config.catalog,
|
|
87
|
-
schema: config.schema,
|
|
88
|
-
auth: new trino_client_1.BasicAuth(config.user, config.password),
|
|
89
|
-
});
|
|
143
|
+
const config = pConfig || {};
|
|
144
|
+
this.name = name;
|
|
145
|
+
if (name === 'trino') {
|
|
146
|
+
this.client = new TrinooBase(config);
|
|
147
|
+
}
|
|
148
|
+
else {
|
|
149
|
+
this.client = new PrestoBase(config);
|
|
150
|
+
}
|
|
90
151
|
this.queryOptions = queryOptions;
|
|
91
|
-
this.config = config;
|
|
152
|
+
//this.config = config;
|
|
92
153
|
}
|
|
93
154
|
get dialectName() {
|
|
94
|
-
return
|
|
155
|
+
return this.name;
|
|
95
156
|
}
|
|
96
157
|
readQueryOptions() {
|
|
97
158
|
const options = TrinoConnection.DEFAULT_QUERY_OPTIONS;
|
|
@@ -122,53 +183,12 @@ class TrinoConnection {
|
|
|
122
183
|
async manifestTemporaryTable(_sqlCommand) {
|
|
123
184
|
throw new Error('not implemented 1');
|
|
124
185
|
}
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
rowIndex = 0
|
|
129
|
-
): Promise<{
|
|
130
|
-
data: MalloyQueryData;
|
|
131
|
-
schema: Trino.ITableFieldSchema | undefined;
|
|
132
|
-
}> {
|
|
133
|
-
const defaultOptions = this.readQueryOptions();
|
|
134
|
-
const pageSize = rowLimit ?? defaultOptions.rowLimit;
|
|
135
|
-
|
|
136
|
-
try {
|
|
137
|
-
const queryResultsOptions: QueryResultsOptions = {
|
|
138
|
-
maxResults: pageSize,
|
|
139
|
-
startIndex: rowIndex.toString(),
|
|
140
|
-
};
|
|
141
|
-
|
|
142
|
-
const jobResult = await this.createTrinoJobAndGetResults(
|
|
143
|
-
sqlCommand,
|
|
144
|
-
undefined,
|
|
145
|
-
queryResultsOptions,
|
|
146
|
-
abortSignal
|
|
147
|
-
);
|
|
148
|
-
|
|
149
|
-
const totalRows = +(jobResult[2]?.totalRows
|
|
150
|
-
? jobResult[2].totalRows
|
|
151
|
-
: '0');
|
|
152
|
-
|
|
153
|
-
// TODO even though we have 10 minute timeout limit, we still should confirm that resulting metadata has "jobComplete: true"
|
|
154
|
-
const queryCostBytes = jobResult[2]?.totalBytesProcessed;
|
|
155
|
-
const data: MalloyQueryData = {
|
|
156
|
-
rows: jobResult[0],
|
|
157
|
-
totalRows,
|
|
158
|
-
runStats: {
|
|
159
|
-
queryCostBytes: queryCostBytes ? +queryCostBytes : undefined,
|
|
160
|
-
},
|
|
161
|
-
};
|
|
162
|
-
const schema = jobResult[2]?.schema;
|
|
163
|
-
|
|
164
|
-
return {data, schema};
|
|
165
|
-
} catch (e) {
|
|
166
|
-
throw maybeRewriteError(e);
|
|
167
|
-
}
|
|
168
|
-
}*/
|
|
186
|
+
unpackArray(data) {
|
|
187
|
+
return data;
|
|
188
|
+
}
|
|
169
189
|
convertRow(structDef, _row) {
|
|
170
190
|
const retRow = {};
|
|
171
|
-
const row = _row;
|
|
191
|
+
const row = this.unpackArray(_row);
|
|
172
192
|
for (let i = 0; i < structDef.fields.length; i++) {
|
|
173
193
|
const field = structDef.fields[i];
|
|
174
194
|
if (field.type === 'struct') {
|
|
@@ -187,7 +207,8 @@ class TrinoConnection {
|
|
|
187
207
|
//console.log(retRow);
|
|
188
208
|
return retRow;
|
|
189
209
|
}
|
|
190
|
-
convertNest(structDef,
|
|
210
|
+
convertNest(structDef, _data) {
|
|
211
|
+
const data = this.unpackArray(_data);
|
|
191
212
|
const ret = [];
|
|
192
213
|
//console.log(
|
|
193
214
|
// `${JSON.stringify(structDef, null, 2)} ${JSON.stringify(data, null, 2)} `
|
|
@@ -204,64 +225,61 @@ class TrinoConnection {
|
|
|
204
225
|
async runSQL(sqlCommand, options = {},
|
|
205
226
|
// TODO(figutierrez): Use.
|
|
206
227
|
_rowIndex = 0) {
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
228
|
+
// const result = await this.trino.query(sqlCommand);
|
|
229
|
+
// let queryResult = await result.next();
|
|
230
|
+
// if (queryResult.value.error) {
|
|
231
|
+
// // TODO: handle.
|
|
232
|
+
// const {failureInfo: _, ...error} = queryResult.value.error;
|
|
233
|
+
// throw new Error(
|
|
234
|
+
// `Failed to execute sql: ${sqlCommand}. \n Error: ${JSON.stringify(
|
|
235
|
+
// error
|
|
236
|
+
// )}`
|
|
237
|
+
// );
|
|
238
|
+
// }
|
|
239
|
+
const r = await this.client.runSQL(sqlCommand, options.rowLimit);
|
|
240
|
+
const inputRows = r.rows;
|
|
241
|
+
const columns = r.columns;
|
|
242
|
+
const malloyColumns = columns.map(c => this.malloyTypeFromTrinoType(c.name, c.type));
|
|
216
243
|
// Debugging types
|
|
217
244
|
// const _x = queryResult.value.columns.map(c => console.log(c.type));
|
|
218
245
|
// console.log(JSON.stringify(malloyColumns, null, 2));
|
|
219
246
|
// console.log(JSON.stringify(queryResult.value.data, null, 2));
|
|
220
|
-
let maxRows = (_a = options.rowLimit) !== null && _a !== void 0 ? _a : 50;
|
|
221
247
|
const malloyRows = [];
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
malloyRow[column.name] = this.convertRow(structDef, row[i]);
|
|
232
|
-
}
|
|
233
|
-
else {
|
|
234
|
-
malloyRow[column.name] = this.convertNest(structDef, row[i]);
|
|
235
|
-
}
|
|
236
|
-
// console.log(
|
|
237
|
-
// column.name,
|
|
238
|
-
// JSON.stringify(malloyColumns[i], null, 2),
|
|
239
|
-
// JSON.stringify(row[i]),
|
|
240
|
-
// JSON.stringify(malloyRow[column.name])
|
|
241
|
-
// );
|
|
242
|
-
}
|
|
243
|
-
else if (malloyColumns[i].type === 'number' &&
|
|
244
|
-
typeof row[i] === 'string') {
|
|
245
|
-
// decimal numbers come back as strings
|
|
246
|
-
malloyRow[column.name] = +row[i];
|
|
247
|
-
}
|
|
248
|
-
else if (malloyColumns[i].type === 'timestamp' &&
|
|
249
|
-
typeof row[i] === 'string') {
|
|
250
|
-
// timestamps come back as strings
|
|
251
|
-
malloyRow[column.name] = new Date(row[i]);
|
|
248
|
+
const rows = inputRows !== null && inputRows !== void 0 ? inputRows : [];
|
|
249
|
+
for (const row of rows) {
|
|
250
|
+
const malloyRow = {};
|
|
251
|
+
for (let i = 0; i < columns.length; i++) {
|
|
252
|
+
const column = columns[i];
|
|
253
|
+
if (malloyColumns[i].type === 'struct') {
|
|
254
|
+
const structDef = malloyColumns[i];
|
|
255
|
+
if (structDef.structSource.type === 'inline') {
|
|
256
|
+
malloyRow[column.name] = this.convertRow(structDef, row[i]);
|
|
252
257
|
}
|
|
253
258
|
else {
|
|
254
|
-
malloyRow[column.name] = row[i];
|
|
259
|
+
malloyRow[column.name] = this.convertNest(structDef, row[i]);
|
|
255
260
|
}
|
|
261
|
+
// console.log(
|
|
262
|
+
// column.name,
|
|
263
|
+
// JSON.stringify(malloyColumns[i], null, 2),
|
|
264
|
+
// JSON.stringify(row[i]),
|
|
265
|
+
// JSON.stringify(malloyRow[column.name])
|
|
266
|
+
// );
|
|
267
|
+
}
|
|
268
|
+
else if (malloyColumns[i].type === 'number' &&
|
|
269
|
+
typeof row[i] === 'string') {
|
|
270
|
+
// decimal numbers come back as strings
|
|
271
|
+
malloyRow[column.name] = Number(row[i]);
|
|
272
|
+
}
|
|
273
|
+
else if (malloyColumns[i].type === 'timestamp' &&
|
|
274
|
+
typeof row[i] === 'string') {
|
|
275
|
+
// timestamps come back as strings
|
|
276
|
+
malloyRow[column.name] = new Date(row[i]);
|
|
277
|
+
}
|
|
278
|
+
else {
|
|
279
|
+
malloyRow[column.name] = row[i];
|
|
256
280
|
}
|
|
257
|
-
malloyRows.push(malloyRow);
|
|
258
|
-
}
|
|
259
|
-
if (!queryResult.done) {
|
|
260
|
-
queryResult = await result.next();
|
|
261
|
-
}
|
|
262
|
-
else {
|
|
263
|
-
break;
|
|
264
281
|
}
|
|
282
|
+
malloyRows.push(malloyRow);
|
|
265
283
|
}
|
|
266
284
|
// TODO(figutierrez): Remove.
|
|
267
285
|
// eslint-disable-next-line no-console
|
|
@@ -370,15 +388,13 @@ class TrinoConnection {
|
|
|
370
388
|
},
|
|
371
389
|
fields: [],
|
|
372
390
|
};
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
return await this.loadSchemaForSqlBlock(`DESCRIBE OUTPUT ${tmpQueryName}`, structDef, `query ${sqlRef.selectStr.substring(0, 50)}`);
|
|
391
|
+
await this.fillStructDefForSqlBlockSchema(sqlRef.selectStr, structDef);
|
|
392
|
+
return structDef;
|
|
376
393
|
}
|
|
377
394
|
async executeAndWait(sqlBlock) {
|
|
378
|
-
|
|
395
|
+
await this.client.runSQL(sqlBlock, undefined);
|
|
379
396
|
// TODO: make sure failure is handled correctly.
|
|
380
|
-
while (!(await result.next()).done)
|
|
381
|
-
;
|
|
397
|
+
//while (!(await result.next()).done);
|
|
382
398
|
}
|
|
383
399
|
splitColumns(s) {
|
|
384
400
|
const columns = [];
|
|
@@ -471,7 +487,8 @@ class TrinoConnection {
|
|
|
471
487
|
parts = innerType.match(/^(.+)\s(\S+)$/);
|
|
472
488
|
}
|
|
473
489
|
if (parts) {
|
|
474
|
-
|
|
490
|
+
// remove quotes from the name
|
|
491
|
+
const innerName = parts[1].replace(/^"(.+(?="$))"$/, '$1');
|
|
475
492
|
const innerTrinoType = parts[2];
|
|
476
493
|
const innerMalloyType = this.malloyTypeFromTrinoType(innerName, innerTrinoType);
|
|
477
494
|
malloyType.fields.push({ ...innerMalloyType, name: innerName });
|
|
@@ -506,17 +523,17 @@ class TrinoConnection {
|
|
|
506
523
|
async loadSchemaForSqlBlock(sqlBlock, structDef, element) {
|
|
507
524
|
var _a;
|
|
508
525
|
try {
|
|
509
|
-
const
|
|
510
|
-
|
|
511
|
-
if (queryResult.value.error) {
|
|
526
|
+
const queryResult = await this.client.runSQL(sqlBlock, undefined);
|
|
527
|
+
if (queryResult.error) {
|
|
512
528
|
// TODO: handle.
|
|
513
|
-
throw new Error(`Failed to grab schema for ${
|
|
529
|
+
throw new Error(`Failed to grab schema for ${queryResult.error}
|
|
530
|
+
)}`);
|
|
514
531
|
}
|
|
515
|
-
const rows = (_a = queryResult.
|
|
532
|
+
const rows = (_a = queryResult.rows) !== null && _a !== void 0 ? _a : [];
|
|
516
533
|
this.structDefFromSchema(rows, structDef);
|
|
517
534
|
}
|
|
518
535
|
catch (e) {
|
|
519
|
-
throw new Error(`Could not fetch schema for ${element} ${e}`);
|
|
536
|
+
throw new Error(`Could not fetch schema for ${element} ${JSON.stringify(e)}`);
|
|
520
537
|
}
|
|
521
538
|
return structDef;
|
|
522
539
|
}
|
|
@@ -545,8 +562,72 @@ class TrinoConnection {
|
|
|
545
562
|
return;
|
|
546
563
|
}
|
|
547
564
|
}
|
|
548
|
-
exports.
|
|
549
|
-
|
|
565
|
+
exports.TrinoPrestoConnection = TrinoPrestoConnection;
|
|
566
|
+
TrinoPrestoConnection.DEFAULT_QUERY_OPTIONS = {
|
|
550
567
|
rowLimit: 10,
|
|
551
568
|
};
|
|
569
|
+
class PrestoConnection extends TrinoPrestoConnection {
|
|
570
|
+
constructor(arg, queryOptions, config = {}) {
|
|
571
|
+
super('presto', queryOptions, config);
|
|
572
|
+
}
|
|
573
|
+
async fillStructDefForSqlBlockSchema(sql, structDef) {
|
|
574
|
+
const explainResult = await this.runSQL(`EXPLAIN ${sql}`, {});
|
|
575
|
+
this.schemaFromExplain(explainResult, structDef);
|
|
576
|
+
}
|
|
577
|
+
schemaFromExplain(explainResult, structDef) {
|
|
578
|
+
if (explainResult.rows.length === 0) {
|
|
579
|
+
throw new Error('Received empty explain result when trying to fetch schema.');
|
|
580
|
+
}
|
|
581
|
+
const resultFirstRow = explainResult.rows[0];
|
|
582
|
+
if (resultFirstRow['Query Plan'] === undefined) {
|
|
583
|
+
throw new Error("Explain result has rows but column 'Query Plan' is not present.");
|
|
584
|
+
}
|
|
585
|
+
const expResult = resultFirstRow['Query Plan'];
|
|
586
|
+
const lines = expResult.split('\n');
|
|
587
|
+
if ((lines === null || lines === void 0 ? void 0 : lines.length) === 0) {
|
|
588
|
+
throw new Error('Received invalid explain result when trying to fetch schema.');
|
|
589
|
+
}
|
|
590
|
+
let outputLine = lines[0];
|
|
591
|
+
const namesIndex = outputLine.indexOf('][');
|
|
592
|
+
outputLine = outputLine.substring(namesIndex + 2);
|
|
593
|
+
const lineParts = outputLine.split('] => [');
|
|
594
|
+
if (lineParts.length !== 2) {
|
|
595
|
+
throw new Error('There was a problem parsing schema from Explain.');
|
|
596
|
+
}
|
|
597
|
+
const fieldNamesPart = lineParts[0];
|
|
598
|
+
const fieldNames = fieldNamesPart.split(',').map(e => e.trim());
|
|
599
|
+
let schemaData = lineParts[1];
|
|
600
|
+
schemaData = schemaData.substring(0, schemaData.length - 1);
|
|
601
|
+
const rawFieldsTarget = schemaData
|
|
602
|
+
.split(',')
|
|
603
|
+
.map(e => e.trim())
|
|
604
|
+
.map(e => e.split(':'));
|
|
605
|
+
if (rawFieldsTarget.length !== fieldNames.length) {
|
|
606
|
+
throw new Error('There was a problem parsing schema from Explain. Field names size do not match target fields with types.');
|
|
607
|
+
}
|
|
608
|
+
for (let index = 0; index < fieldNames.length; index++) {
|
|
609
|
+
const name = fieldNames[index];
|
|
610
|
+
const type = rawFieldsTarget[index][1];
|
|
611
|
+
structDef.fields.push({
|
|
612
|
+
name,
|
|
613
|
+
...this.malloyTypeFromTrinoType(name, type),
|
|
614
|
+
});
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
unpackArray(data) {
|
|
618
|
+
return JSON.parse(data);
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
exports.PrestoConnection = PrestoConnection;
|
|
622
|
+
class TrinoConnection extends TrinoPrestoConnection {
|
|
623
|
+
constructor(arg, queryOptions, config = {}) {
|
|
624
|
+
super('trino', queryOptions, config);
|
|
625
|
+
}
|
|
626
|
+
async fillStructDefForSqlBlockSchema(sql, structDef) {
|
|
627
|
+
const tmpQueryName = `myMalloyQuery${(0, crypto_1.randomUUID)().replace(/-/g, '')}`;
|
|
628
|
+
await this.executeAndWait(`PREPARE ${tmpQueryName} FROM ${sql}`);
|
|
629
|
+
await this.loadSchemaForSqlBlock(`DESCRIBE OUTPUT ${tmpQueryName}`, structDef, `query ${sql.substring(0, 50)}`);
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
exports.TrinoConnection = TrinoConnection;
|
|
552
633
|
//# sourceMappingURL=trino_connection.js.map
|
|
@@ -34,7 +34,7 @@ const DEEP_SCHEMA = 'array(row(a double, b array(row(c integer, d varchar(60))))
|
|
|
34
34
|
describe('Trino connection', () => {
|
|
35
35
|
let connection;
|
|
36
36
|
beforeAll(() => {
|
|
37
|
-
connection = new _1.TrinoConnection('trino', {}, _1.TrinoExecutor.getConnectionOptionsFromEnv());
|
|
37
|
+
connection = new _1.TrinoConnection('trino', {}, _1.TrinoExecutor.getConnectionOptionsFromEnv('trino'));
|
|
38
38
|
});
|
|
39
39
|
afterAll(() => {
|
|
40
40
|
connection.close();
|
package/dist/trino_executor.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import { TrinoConnectionConfiguration } from './trino_connection';
|
|
2
2
|
export declare class TrinoExecutor {
|
|
3
|
-
static getConnectionOptionsFromEnv(): TrinoConnectionConfiguration | undefined;
|
|
3
|
+
static getConnectionOptionsFromEnv(dialectName: 'trino' | 'presto'): TrinoConnectionConfiguration | undefined;
|
|
4
4
|
}
|
package/dist/trino_executor.js
CHANGED
|
@@ -23,27 +23,42 @@
|
|
|
23
23
|
*/
|
|
24
24
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
25
25
|
exports.TrinoExecutor = void 0;
|
|
26
|
+
// Differences:
|
|
27
|
+
// Trino uses TRINO_SERVER
|
|
28
|
+
// Presto users PRESTO_HOST/PRESTO_PORT
|
|
29
|
+
// Trino requires TRINO_USER
|
|
26
30
|
class TrinoExecutor {
|
|
27
|
-
static getConnectionOptionsFromEnv() {
|
|
28
|
-
const
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
31
|
+
static getConnectionOptionsFromEnv(dialectName) {
|
|
32
|
+
const envPrefix = dialectName.toUpperCase();
|
|
33
|
+
const user = process.env[`${envPrefix}_USER`];
|
|
34
|
+
let server;
|
|
35
|
+
let port = undefined;
|
|
36
|
+
if (dialectName === 'trino') {
|
|
37
|
+
server = process.env['TRINO_SERVER'];
|
|
38
|
+
if (!user && server) {
|
|
32
39
|
throw Error('Trino server specified but no user was provided. Set TRINO_USER and TRINO_PASSWORD environment variables');
|
|
33
40
|
}
|
|
34
|
-
const password = process.env['TRINO_PASSWORD'];
|
|
35
|
-
// TODO(figutierrez): We may not need to support these.
|
|
36
|
-
const catalog = process.env['TRINO_CATALOG'];
|
|
37
|
-
const schema = process.env['TRINO_SCHEMA'];
|
|
38
|
-
return {
|
|
39
|
-
server,
|
|
40
|
-
user,
|
|
41
|
-
password,
|
|
42
|
-
catalog,
|
|
43
|
-
schema,
|
|
44
|
-
};
|
|
45
41
|
}
|
|
46
|
-
|
|
42
|
+
else {
|
|
43
|
+
server = process.env['PRESTO_HOST'];
|
|
44
|
+
port = Number(process.env['PRESTO_PORT']) || 8080;
|
|
45
|
+
}
|
|
46
|
+
if (!server) {
|
|
47
|
+
return undefined;
|
|
48
|
+
}
|
|
49
|
+
const password = process.env[`${envPrefix}_PASSWORD`];
|
|
50
|
+
// TODO(figutierrez): We may not need to support these.
|
|
51
|
+
const catalog = process.env[`${envPrefix}_CATALOG`];
|
|
52
|
+
const schema = process.env[`${envPrefix}_SCHEMA`];
|
|
53
|
+
const ret = {
|
|
54
|
+
server,
|
|
55
|
+
user,
|
|
56
|
+
port,
|
|
57
|
+
password,
|
|
58
|
+
catalog,
|
|
59
|
+
schema,
|
|
60
|
+
};
|
|
61
|
+
return ret;
|
|
47
62
|
}
|
|
48
63
|
}
|
|
49
64
|
exports.TrinoExecutor = TrinoExecutor;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@malloydata/db-trino",
|
|
3
|
-
"version": "0.0.149
|
|
3
|
+
"version": "0.0.149",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
},
|
|
24
24
|
"dependencies": {
|
|
25
25
|
"@malloydata/malloy": "^0.0.130",
|
|
26
|
+
"@prestodb/presto-js-client": "^1.0.0",
|
|
26
27
|
"gaxios": "^4.2.0",
|
|
27
28
|
"trino-client": "^0.2.2"
|
|
28
29
|
},
|