@malloydata/malloy-tests 0.0.426 → 0.0.427
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +10 -10
- package/src/databases/all/streaming.spec.ts +159 -0
- package/src/core/streaming.spec.ts +0 -163
package/package.json
CHANGED
|
@@ -20,15 +20,15 @@
|
|
|
20
20
|
"malloyc": "ts-node ../scripts/malloy-to-json"
|
|
21
21
|
},
|
|
22
22
|
"dependencies": {
|
|
23
|
-
"@malloydata/db-bigquery": "0.0.
|
|
24
|
-
"@malloydata/db-duckdb": "0.0.
|
|
25
|
-
"@malloydata/db-postgres": "0.0.
|
|
26
|
-
"@malloydata/db-snowflake": "0.0.
|
|
27
|
-
"@malloydata/db-trino": "0.0.
|
|
28
|
-
"@malloydata/malloy": "0.0.
|
|
29
|
-
"@malloydata/malloy-tag": "0.0.
|
|
30
|
-
"@malloydata/render": "0.0.
|
|
31
|
-
"@malloydata/render-validator": "0.0.
|
|
23
|
+
"@malloydata/db-bigquery": "0.0.427",
|
|
24
|
+
"@malloydata/db-duckdb": "0.0.427",
|
|
25
|
+
"@malloydata/db-postgres": "0.0.427",
|
|
26
|
+
"@malloydata/db-snowflake": "0.0.427",
|
|
27
|
+
"@malloydata/db-trino": "0.0.427",
|
|
28
|
+
"@malloydata/malloy": "0.0.427",
|
|
29
|
+
"@malloydata/malloy-tag": "0.0.427",
|
|
30
|
+
"@malloydata/render": "0.0.427",
|
|
31
|
+
"@malloydata/render-validator": "0.0.427",
|
|
32
32
|
"events": "^3.3.0",
|
|
33
33
|
"jsdom": "^22.1.0",
|
|
34
34
|
"luxon": "^3.7.2",
|
|
@@ -42,5 +42,5 @@
|
|
|
42
42
|
"overrides": {
|
|
43
43
|
"typescript": "^6.0.3"
|
|
44
44
|
},
|
|
45
|
-
"version": "0.0.
|
|
45
|
+
"version": "0.0.427"
|
|
46
46
|
}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright Contributors to the Malloy project
|
|
3
|
+
* SPDX-License-Identifier: MIT
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import type {DataRecord, WriteStream} from '@malloydata/malloy';
|
|
7
|
+
import {CSVWriter, JSONWriter} from '@malloydata/malloy';
|
|
8
|
+
import {RuntimeList, allDatabases} from '../../runtimes';
|
|
9
|
+
import {databasesFromEnvironmentOr} from '../../util';
|
|
10
|
+
import {TestSelect} from '../../test-select';
|
|
11
|
+
|
|
12
|
+
class StringAccumulator implements WriteStream {
|
|
13
|
+
public accumulatedValue = '';
|
|
14
|
+
|
|
15
|
+
write(text: string) {
|
|
16
|
+
this.accumulatedValue += text;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
close() {
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const runtimes = new RuntimeList(databasesFromEnvironmentOr(allDatabases));
|
|
25
|
+
|
|
26
|
+
afterAll(async () => {
|
|
27
|
+
await runtimes.closeAll();
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
runtimes.runtimeMap.forEach((runtime, databaseName) => {
|
|
31
|
+
// Streaming is an optional connection capability (StreamingConnection).
|
|
32
|
+
// Dialects which do not implement it skip rather than fail.
|
|
33
|
+
const streams = runtime.connection.canStream();
|
|
34
|
+
|
|
35
|
+
// TestSelect quotes the column alias per dialect, so the name survives
|
|
36
|
+
// Snowflake's folding of unquoted identifiers to upper case.
|
|
37
|
+
const ts = new TestSelect(runtime.dialect);
|
|
38
|
+
const bigintSQL = ts.generate({big_id: ts.mk_bigint(19999)});
|
|
39
|
+
|
|
40
|
+
test.when(streams)(`basic stream test - ${databaseName}`, async () => {
|
|
41
|
+
const stream = runtime
|
|
42
|
+
.loadModel(
|
|
43
|
+
`source: airports is ${databaseName}.table('malloytest.airports')`
|
|
44
|
+
)
|
|
45
|
+
.loadQuery('run: airports -> { select: code; order_by: code }')
|
|
46
|
+
.runStream({rowLimit: 10});
|
|
47
|
+
const rows: DataRecord[] = [];
|
|
48
|
+
for await (const row of stream) {
|
|
49
|
+
rows.push(row);
|
|
50
|
+
}
|
|
51
|
+
expect(rows.length).toBe(10);
|
|
52
|
+
expect(rows[0].cell('code').string.value).toBe('00A');
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test.when(streams)(`stream to JSON - ${databaseName}`, async () => {
|
|
56
|
+
const stream = runtime
|
|
57
|
+
.loadModel(
|
|
58
|
+
`source: airports is ${databaseName}.table('malloytest.airports')`
|
|
59
|
+
)
|
|
60
|
+
.loadQuery('run: airports -> { select: code; order_by: code }')
|
|
61
|
+
.runStream({rowLimit: 1});
|
|
62
|
+
const accummulator = new StringAccumulator();
|
|
63
|
+
const jsonWriter = new JSONWriter(accummulator);
|
|
64
|
+
await jsonWriter.process(stream);
|
|
65
|
+
expect(accummulator.accumulatedValue).toBe(
|
|
66
|
+
`[
|
|
67
|
+
{
|
|
68
|
+
"code": "00A"
|
|
69
|
+
}
|
|
70
|
+
]
|
|
71
|
+
`
|
|
72
|
+
);
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
test.when(streams)(`stream to CSV - ${databaseName}`, async () => {
|
|
76
|
+
const stream = runtime
|
|
77
|
+
.loadModel(
|
|
78
|
+
`source: airports is ${databaseName}.table('malloytest.airports')`
|
|
79
|
+
)
|
|
80
|
+
.loadQuery('run: airports -> { select: code; order_by: code }')
|
|
81
|
+
.runStream({rowLimit: 1});
|
|
82
|
+
const accummulator = new StringAccumulator();
|
|
83
|
+
const csvWriter = new CSVWriter(accummulator);
|
|
84
|
+
await csvWriter.process(stream);
|
|
85
|
+
expect(accummulator.accumulatedValue).toBe('code\n00A\n');
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
test.when(streams)(`JSON with timestamp - ${databaseName}`, async () => {
|
|
89
|
+
const stream = runtime
|
|
90
|
+
.loadModel(
|
|
91
|
+
`source: flights is ${databaseName}.table('malloytest.flights')`
|
|
92
|
+
)
|
|
93
|
+
.loadQuery(
|
|
94
|
+
"run: flights -> { select: dep_time; where: carrier = 'WN' and origin = 'SJC'; order_by: dep_time; limit: 1 }"
|
|
95
|
+
)
|
|
96
|
+
.runStream({rowLimit: 1});
|
|
97
|
+
const accummulator = new StringAccumulator();
|
|
98
|
+
const jsonWriter = new JSONWriter(accummulator);
|
|
99
|
+
await jsonWriter.process(stream);
|
|
100
|
+
const result = JSON.parse(accummulator.accumulatedValue);
|
|
101
|
+
expect(result.length).toBe(1);
|
|
102
|
+
// Should be an ISO date string
|
|
103
|
+
expect(typeof result[0].dep_time).toBe('string');
|
|
104
|
+
expect(result[0].dep_time).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/);
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
test.when(streams)(`CSV with timestamp - ${databaseName}`, async () => {
|
|
108
|
+
const stream = runtime
|
|
109
|
+
.loadModel(
|
|
110
|
+
`source: flights is ${databaseName}.table('malloytest.flights')`
|
|
111
|
+
)
|
|
112
|
+
.loadQuery(
|
|
113
|
+
"run: flights -> { select: dep_time; where: carrier = 'WN' and origin = 'SJC'; order_by: dep_time; limit: 1 }"
|
|
114
|
+
)
|
|
115
|
+
.runStream({rowLimit: 1});
|
|
116
|
+
const accummulator = new StringAccumulator();
|
|
117
|
+
const csvWriter = new CSVWriter(accummulator);
|
|
118
|
+
await csvWriter.process(stream);
|
|
119
|
+
const lines = accummulator.accumulatedValue.trim().split('\n');
|
|
120
|
+
expect(lines.length).toBe(2);
|
|
121
|
+
expect(lines[0]).toBe('dep_time');
|
|
122
|
+
// Should be formatted as a date string
|
|
123
|
+
expect(lines[1]).toMatch(/\d{4}-\d{2}-\d{2}/);
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
test.when(streams)(`JSON with bigint - ${databaseName}`, async () => {
|
|
127
|
+
const stream = runtime
|
|
128
|
+
.loadModel(
|
|
129
|
+
`source: bigint_test is ${databaseName}.sql("""${bigintSQL}""")`
|
|
130
|
+
)
|
|
131
|
+
.loadQuery('run: bigint_test -> { select: * }')
|
|
132
|
+
.runStream({rowLimit: 1});
|
|
133
|
+
const accummulator = new StringAccumulator();
|
|
134
|
+
const jsonWriter = new JSONWriter(accummulator);
|
|
135
|
+
await jsonWriter.process(stream);
|
|
136
|
+
const result = JSON.parse(accummulator.accumulatedValue);
|
|
137
|
+
expect(result.length).toBe(1);
|
|
138
|
+
// Bigint should be serialized as a string to preserve precision
|
|
139
|
+
expect(typeof result[0].big_id).toBe('string');
|
|
140
|
+
expect(result[0].big_id).toBe('19999');
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
test.when(streams)(`CSV with bigint - ${databaseName}`, async () => {
|
|
144
|
+
const stream = runtime
|
|
145
|
+
.loadModel(
|
|
146
|
+
`source: bigint_test is ${databaseName}.sql("""${bigintSQL}""")`
|
|
147
|
+
)
|
|
148
|
+
.loadQuery('run: bigint_test -> { select: * }')
|
|
149
|
+
.runStream({rowLimit: 1});
|
|
150
|
+
const accummulator = new StringAccumulator();
|
|
151
|
+
const csvWriter = new CSVWriter(accummulator);
|
|
152
|
+
await csvWriter.process(stream);
|
|
153
|
+
const lines = accummulator.accumulatedValue.trim().split('\n');
|
|
154
|
+
expect(lines.length).toBe(2);
|
|
155
|
+
expect(lines[0]).toBe('big_id');
|
|
156
|
+
// Should be a number without quotes
|
|
157
|
+
expect(lines[1]).toBe('19999');
|
|
158
|
+
});
|
|
159
|
+
});
|
|
@@ -1,163 +0,0 @@
|
|
|
1
|
-
/*
|
|
2
|
-
* Copyright Contributors to the Malloy project
|
|
3
|
-
* SPDX-License-Identifier: MIT
|
|
4
|
-
*/
|
|
5
|
-
|
|
6
|
-
import type {DataRecord, WriteStream} from '@malloydata/malloy';
|
|
7
|
-
import {CSVWriter, JSONWriter} from '@malloydata/malloy';
|
|
8
|
-
import {RuntimeList} from '../runtimes';
|
|
9
|
-
import {describeIfDatabaseAvailable} from '../util';
|
|
10
|
-
|
|
11
|
-
class StringAccumulator implements WriteStream {
|
|
12
|
-
public accumulatedValue = '';
|
|
13
|
-
|
|
14
|
-
write(text: string) {
|
|
15
|
-
this.accumulatedValue += text;
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
close() {
|
|
19
|
-
return;
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
const [describe, databases] = describeIfDatabaseAvailable([
|
|
24
|
-
'bigquery',
|
|
25
|
-
'postgres',
|
|
26
|
-
'duckdb',
|
|
27
|
-
'duckdb_wasm',
|
|
28
|
-
]);
|
|
29
|
-
|
|
30
|
-
describe('Streaming tests', () => {
|
|
31
|
-
if (!databases.length) {
|
|
32
|
-
it.skip('skipped', () => {});
|
|
33
|
-
}
|
|
34
|
-
const runtimes = new RuntimeList(databases);
|
|
35
|
-
|
|
36
|
-
afterAll(async () => {
|
|
37
|
-
await runtimes.closeAll();
|
|
38
|
-
});
|
|
39
|
-
|
|
40
|
-
runtimes.runtimeMap.forEach((runtime, databaseName) => {
|
|
41
|
-
it(`basic stream test - ${databaseName}`, async () => {
|
|
42
|
-
const stream = runtime
|
|
43
|
-
.loadModel(
|
|
44
|
-
`source: airports is ${databaseName}.table('malloytest.airports')`
|
|
45
|
-
)
|
|
46
|
-
.loadQuery('run: airports -> { select: code; order_by: code }')
|
|
47
|
-
.runStream({rowLimit: 10});
|
|
48
|
-
const rows: DataRecord[] = [];
|
|
49
|
-
for await (const row of stream) {
|
|
50
|
-
rows.push(row);
|
|
51
|
-
}
|
|
52
|
-
expect(rows.length).toBe(10);
|
|
53
|
-
expect(rows[0].cell('code').string.value).toBe('00A');
|
|
54
|
-
});
|
|
55
|
-
|
|
56
|
-
it(`stream to JSON - ${databaseName}`, async () => {
|
|
57
|
-
const stream = runtime
|
|
58
|
-
.loadModel(
|
|
59
|
-
`source: airports is ${databaseName}.table('malloytest.airports')`
|
|
60
|
-
)
|
|
61
|
-
.loadQuery('run: airports -> { select: code; order_by: code }')
|
|
62
|
-
.runStream({rowLimit: 1});
|
|
63
|
-
const accummulator = new StringAccumulator();
|
|
64
|
-
const jsonWriter = new JSONWriter(accummulator);
|
|
65
|
-
await jsonWriter.process(stream);
|
|
66
|
-
expect(accummulator.accumulatedValue).toBe(
|
|
67
|
-
`[
|
|
68
|
-
{
|
|
69
|
-
"code": "00A"
|
|
70
|
-
}
|
|
71
|
-
]
|
|
72
|
-
`
|
|
73
|
-
);
|
|
74
|
-
});
|
|
75
|
-
|
|
76
|
-
it(`stream to CSV - ${databaseName}`, async () => {
|
|
77
|
-
const stream = runtime
|
|
78
|
-
.loadModel(
|
|
79
|
-
`source: airports is ${databaseName}.table('malloytest.airports')`
|
|
80
|
-
)
|
|
81
|
-
.loadQuery('run: airports -> { select: code; order_by: code }')
|
|
82
|
-
.runStream({rowLimit: 1});
|
|
83
|
-
const accummulator = new StringAccumulator();
|
|
84
|
-
const csvWriter = new CSVWriter(accummulator);
|
|
85
|
-
await csvWriter.process(stream);
|
|
86
|
-
expect(accummulator.accumulatedValue).toBe('code\n00A\n');
|
|
87
|
-
});
|
|
88
|
-
|
|
89
|
-
it(`JSON with timestamp - ${databaseName}`, async () => {
|
|
90
|
-
const stream = runtime
|
|
91
|
-
.loadModel(
|
|
92
|
-
`source: flights is ${databaseName}.table('malloytest.flights')`
|
|
93
|
-
)
|
|
94
|
-
.loadQuery(
|
|
95
|
-
"run: flights -> { select: dep_time; where: carrier = 'WN' and origin = 'SJC'; order_by: dep_time; limit: 1 }"
|
|
96
|
-
)
|
|
97
|
-
.runStream({rowLimit: 1});
|
|
98
|
-
const accummulator = new StringAccumulator();
|
|
99
|
-
const jsonWriter = new JSONWriter(accummulator);
|
|
100
|
-
await jsonWriter.process(stream);
|
|
101
|
-
const result = JSON.parse(accummulator.accumulatedValue);
|
|
102
|
-
expect(result.length).toBe(1);
|
|
103
|
-
// Should be an ISO date string
|
|
104
|
-
expect(typeof result[0].dep_time).toBe('string');
|
|
105
|
-
expect(result[0].dep_time).toMatch(
|
|
106
|
-
/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/
|
|
107
|
-
);
|
|
108
|
-
});
|
|
109
|
-
|
|
110
|
-
it(`CSV with timestamp - ${databaseName}`, async () => {
|
|
111
|
-
const stream = runtime
|
|
112
|
-
.loadModel(
|
|
113
|
-
`source: flights is ${databaseName}.table('malloytest.flights')`
|
|
114
|
-
)
|
|
115
|
-
.loadQuery(
|
|
116
|
-
"run: flights -> { select: dep_time; where: carrier = 'WN' and origin = 'SJC'; order_by: dep_time; limit: 1 }"
|
|
117
|
-
)
|
|
118
|
-
.runStream({rowLimit: 1});
|
|
119
|
-
const accummulator = new StringAccumulator();
|
|
120
|
-
const csvWriter = new CSVWriter(accummulator);
|
|
121
|
-
await csvWriter.process(stream);
|
|
122
|
-
const lines = accummulator.accumulatedValue.trim().split('\n');
|
|
123
|
-
expect(lines.length).toBe(2);
|
|
124
|
-
expect(lines[0]).toBe('dep_time');
|
|
125
|
-
// Should be formatted as a date string
|
|
126
|
-
expect(lines[1]).toMatch(/\d{4}-\d{2}-\d{2}/);
|
|
127
|
-
});
|
|
128
|
-
|
|
129
|
-
it(`JSON with bigint - ${databaseName}`, async () => {
|
|
130
|
-
const stream = runtime
|
|
131
|
-
.loadModel(
|
|
132
|
-
`source: bigint_test is ${databaseName}.sql("SELECT CAST(19999 AS BIGINT) as big_id")`
|
|
133
|
-
)
|
|
134
|
-
.loadQuery('run: bigint_test -> { select: * }')
|
|
135
|
-
.runStream({rowLimit: 1});
|
|
136
|
-
const accummulator = new StringAccumulator();
|
|
137
|
-
const jsonWriter = new JSONWriter(accummulator);
|
|
138
|
-
await jsonWriter.process(stream);
|
|
139
|
-
const result = JSON.parse(accummulator.accumulatedValue);
|
|
140
|
-
expect(result.length).toBe(1);
|
|
141
|
-
// Bigint should be serialized as a string to preserve precision
|
|
142
|
-
expect(typeof result[0].big_id).toBe('string');
|
|
143
|
-
expect(result[0].big_id).toBe('19999');
|
|
144
|
-
});
|
|
145
|
-
|
|
146
|
-
it(`CSV with bigint - ${databaseName}`, async () => {
|
|
147
|
-
const stream = runtime
|
|
148
|
-
.loadModel(
|
|
149
|
-
`source: bigint_test is ${databaseName}.sql("SELECT CAST(19999 AS BIGINT) as big_id")`
|
|
150
|
-
)
|
|
151
|
-
.loadQuery('run: bigint_test -> { select: * }')
|
|
152
|
-
.runStream({rowLimit: 1});
|
|
153
|
-
const accummulator = new StringAccumulator();
|
|
154
|
-
const csvWriter = new CSVWriter(accummulator);
|
|
155
|
-
await csvWriter.process(stream);
|
|
156
|
-
const lines = accummulator.accumulatedValue.trim().split('\n');
|
|
157
|
-
expect(lines.length).toBe(2);
|
|
158
|
-
expect(lines[0]).toBe('big_id');
|
|
159
|
-
// Should be a number without quotes
|
|
160
|
-
expect(lines[1]).toBe('19999');
|
|
161
|
-
});
|
|
162
|
-
});
|
|
163
|
-
});
|