@hypequery/clickhouse 1.3.2 → 1.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/features/executor.d.ts.map +1 -1
- package/dist/core/features/executor.js +3 -1
- package/dist/core/tests/integration/pagination-test-tbc.js +1 -0
- package/dist/core/tests/integration/setup.d.ts.map +1 -1
- package/dist/core/tests/integration/setup.js +54 -22
- package/dist/core/tests/integration/test-config.d.ts +2 -2
- package/dist/core/tests/integration/test-config.d.ts.map +1 -1
- package/dist/core/tests/integration/test-config.js +3 -4
- package/dist/core/tests/integration/test-data.json +190 -0
- package/dist/core/utils/streaming-helpers.d.ts +2 -0
- package/dist/core/utils/streaming-helpers.d.ts.map +1 -0
- package/dist/core/utils/streaming-helpers.js +137 -0
- package/package.json +7 -10
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/core/features/executor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAChF,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/core/features/executor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAChF,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAMnD,qBAAa,eAAe,CAC1B,MAAM,SAAS,gBAAgB,CAAC,MAAM,CAAC,EACvC,KAAK,SAAS,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,MAAM,CAAC,CAAC,CAAC;IAExF,OAAO,CAAC,OAAO;gBAAP,OAAO,EAAE,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC;IAExD,eAAe,IAAI;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,GAAG,EAAE,CAAA;KAAE;IAOrD,KAAK,IAAI,MAAM;IAKT,OAAO,IAAI,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;IAgDrC,MAAM,IAAI,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAgD1D,OAAO,CAAC,sBAAsB;CA0C/B"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { ClickHouseConnection } from '../connection.js';
|
|
2
2
|
import { substituteParameters } from '../utils.js';
|
|
3
3
|
import { logger } from '../utils/logger.js';
|
|
4
|
+
import { createJsonEachRowStream } from '../utils/streaming-helpers.js';
|
|
4
5
|
export class ExecutorFeature {
|
|
5
6
|
builder;
|
|
6
7
|
constructor(builder) {
|
|
@@ -76,6 +77,7 @@ export class ExecutorFeature {
|
|
|
76
77
|
format: 'JSONEachRow'
|
|
77
78
|
});
|
|
78
79
|
const stream = result.stream();
|
|
80
|
+
const webStream = createJsonEachRowStream(stream);
|
|
79
81
|
const endTime = Date.now();
|
|
80
82
|
logger.logQuery({
|
|
81
83
|
query: finalSQL,
|
|
@@ -85,7 +87,7 @@ export class ExecutorFeature {
|
|
|
85
87
|
duration: endTime - startTime,
|
|
86
88
|
status: 'completed'
|
|
87
89
|
});
|
|
88
|
-
return
|
|
90
|
+
return webStream;
|
|
89
91
|
}
|
|
90
92
|
catch (error) {
|
|
91
93
|
const endTime = Date.now();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"setup.d.ts","sourceRoot":"","sources":["../../../../src/core/tests/integration/setup.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"setup.d.ts","sourceRoot":"","sources":["../../../../src/core/tests/integration/setup.ts"],"names":[],"mappings":"AAgDA,eAAO,MAAM,wBAAwB;;EAyBpC,CAAC;AAGF,eAAO,MAAM,2BAA2B,yGAevC,CAAC;AAGF,eAAO,MAAM,iBAAiB,QAAa,OAAO,CAAC,OAAO,CAOzD,CAAC;AAGF,eAAO,MAAM,wBAAwB,QAAa,OAAO,CAAC,OAAO,CAahE,CAAC;AAGF,eAAO,MAAM,kBAAkB,GAAU,eAAe,MAAM,KAAG,OAAO,CAAC,OAAO,CAO/E,CAAC;AAGF,eAAO,MAAM,iBAAiB,QAAa,OAAO,CAAC,OAAO,CAQzD,CAAC;AAGF,eAAO,MAAM,wBAAwB,QAAa,OAAO,CAAC,IAAI,CAwC7D,CAAC;AAGF,eAAO,MAAM,iBAAiB,GAC5B,oBAAgB,EAChB,sBAAoB,KACnB,OAAO,CAAC,IAAI,CAad,CAAC;AAGF,eAAO,MAAM,uBAAuB,QAAa,OAAO,CAAC,IAAI,CA0B5D,CAAC;AAGF,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,KAAK,CAAC;QAChB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,OAAO,CAAC;KACpB,CAAC,CAAC;IACH,KAAK,EAAE,KAAK,CAAC;QACX,EAAE,EAAE,MAAM,CAAC;QACX,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;IACH,MAAM,EAAE,KAAK,CAAC;QACZ,EAAE,EAAE,MAAM,CAAC;QACX,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;CACJ;AAkDD,eAAO,MAAM,SAAS,EAAE,cAAoC,CAAC;AAK7D,eAAO,MAAM,iBAAiB,QAAa,OAAO,CAAC,IAAI,CAkGtD,CAAC"}
|
|
@@ -4,6 +4,8 @@ import { ClickHouseConnection } from '../../connection.js';
|
|
|
4
4
|
import { exec } from 'child_process';
|
|
5
5
|
import { promisify } from 'util';
|
|
6
6
|
import { logger as hypeQueryLogger } from '../../utils/logger.js';
|
|
7
|
+
//@ts-expect-error
|
|
8
|
+
import rawTestData from './test-data.json';
|
|
7
9
|
// Disable the hypequery logger to prevent "logs after tests" errors
|
|
8
10
|
// This must be done early in the setup, before any queries run
|
|
9
11
|
hypeQueryLogger.configure({ enabled: false });
|
|
@@ -30,8 +32,9 @@ const __filename = fileURLToPath(import.meta.url);
|
|
|
30
32
|
const __dirname = path.dirname(__filename);
|
|
31
33
|
const projectRoot = path.resolve(__dirname, '../../../../../');
|
|
32
34
|
// Connection configuration (with defaults that can be overridden by env variables)
|
|
35
|
+
const CLICKHOUSE_TEST_PORT = process.env.CLICKHOUSE_TEST_PORT || '8123';
|
|
33
36
|
const config = {
|
|
34
|
-
host: process.env.CLICKHOUSE_TEST_HOST ||
|
|
37
|
+
host: process.env.CLICKHOUSE_TEST_HOST || `http://localhost:${CLICKHOUSE_TEST_PORT}`,
|
|
35
38
|
user: process.env.CLICKHOUSE_TEST_USER || 'default',
|
|
36
39
|
password: process.env.CLICKHOUSE_TEST_PASSWORD || 'hypequery_test',
|
|
37
40
|
database: process.env.CLICKHOUSE_TEST_DB || 'test_db',
|
|
@@ -210,30 +213,57 @@ export const stopClickHouseContainer = async () => {
|
|
|
210
213
|
}
|
|
211
214
|
}
|
|
212
215
|
};
|
|
216
|
+
function normalizeDateValue(value) {
|
|
217
|
+
if (!value) {
|
|
218
|
+
return value;
|
|
219
|
+
}
|
|
220
|
+
if (value.includes('T')) {
|
|
221
|
+
return value.split('T')[0];
|
|
222
|
+
}
|
|
223
|
+
if (value.includes(' ')) {
|
|
224
|
+
return value.split(' ')[0];
|
|
225
|
+
}
|
|
226
|
+
return value;
|
|
227
|
+
}
|
|
228
|
+
function normalizeTestData() {
|
|
229
|
+
const testTable = (rawTestData.test_table ?? []).map(row => ({
|
|
230
|
+
id: row.id,
|
|
231
|
+
name: row.name,
|
|
232
|
+
category: row.category,
|
|
233
|
+
price: row.price,
|
|
234
|
+
created_at: normalizeDateValue(row.created_at),
|
|
235
|
+
is_active: row.is_active
|
|
236
|
+
}));
|
|
237
|
+
const users = (rawTestData.users ?? []).map(row => ({
|
|
238
|
+
id: row.id,
|
|
239
|
+
user_name: row.user_name,
|
|
240
|
+
email: row.email,
|
|
241
|
+
status: row.status,
|
|
242
|
+
created_at: normalizeDateValue(row.created_at)
|
|
243
|
+
}));
|
|
244
|
+
const orders = (rawTestData.orders ?? []).map(row => ({
|
|
245
|
+
id: row.id,
|
|
246
|
+
user_id: row.user_id,
|
|
247
|
+
product_id: row.product_id,
|
|
248
|
+
quantity: row.quantity,
|
|
249
|
+
total: row.total,
|
|
250
|
+
status: row.status,
|
|
251
|
+
created_at: normalizeDateValue(row.created_at)
|
|
252
|
+
}));
|
|
253
|
+
return { test_table: testTable, users, orders };
|
|
254
|
+
}
|
|
213
255
|
// Test data
|
|
214
|
-
export const TEST_DATA =
|
|
215
|
-
|
|
216
|
-
{ id: 1, name: 'Product A', category: 'A', price: 10.5, created_at: '2023-01-01', is_active: true },
|
|
217
|
-
{ id: 2, name: 'Product B', category: 'B', price: 20.75, created_at: '2023-01-02', is_active: true },
|
|
218
|
-
{ id: 3, name: 'Product C', category: 'A', price: 15.0, created_at: '2023-01-03', is_active: false },
|
|
219
|
-
{ id: 4, name: 'Product D', category: 'C', price: 8.25, created_at: '2023-01-04', is_active: true },
|
|
220
|
-
{ id: 5, name: 'Product E', category: 'B', price: 30.0, created_at: '2023-01-05', is_active: true },
|
|
221
|
-
],
|
|
222
|
-
users: [
|
|
223
|
-
{ id: 1, user_name: 'john_doe', email: 'john@example.com', status: 'active', created_at: '2023-01-01' },
|
|
224
|
-
{ id: 2, user_name: 'jane_smith', email: 'jane@example.com', status: 'active', created_at: '2023-01-02' },
|
|
225
|
-
{ id: 3, user_name: 'bob_jones', email: 'bob@example.com', status: 'inactive', created_at: '2023-01-03' },
|
|
226
|
-
],
|
|
227
|
-
orders: [
|
|
228
|
-
{ id: 1, user_id: 1, product_id: 1, quantity: 2, total: 21.0, status: 'completed', created_at: '2023-01-10' },
|
|
229
|
-
{ id: 2, user_id: 1, product_id: 3, quantity: 1, total: 15.0, status: 'completed', created_at: '2023-01-11' },
|
|
230
|
-
{ id: 3, user_id: 2, product_id: 2, quantity: 3, total: 62.25, status: 'pending', created_at: '2023-01-12' },
|
|
231
|
-
{ id: 4, user_id: 2, product_id: 5, quantity: 1, total: 30.0, status: 'completed', created_at: '2023-01-13' },
|
|
232
|
-
{ id: 5, user_id: 3, product_id: 4, quantity: 2, total: 16.5, status: 'cancelled', created_at: '2023-01-14' },
|
|
233
|
-
],
|
|
234
|
-
};
|
|
256
|
+
export const TEST_DATA = normalizeTestData();
|
|
257
|
+
let hasSetupRun = false;
|
|
235
258
|
// Setup the test database
|
|
236
259
|
export const setupTestDatabase = async () => {
|
|
260
|
+
if (process.env.HYPEQUERY_SKIP_TEST_DB_SETUP === 'true') {
|
|
261
|
+
logger.info('Skipping test database setup because HYPEQUERY_SKIP_TEST_DB_SETUP is true.');
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
if (hasSetupRun) {
|
|
265
|
+
return;
|
|
266
|
+
}
|
|
237
267
|
// Make sure connection is initialized before getting client
|
|
238
268
|
const client = ensureConnectionInitialized();
|
|
239
269
|
try {
|
|
@@ -306,9 +336,11 @@ export const setupTestDatabase = async () => {
|
|
|
306
336
|
format: 'JSONEachRow'
|
|
307
337
|
});
|
|
308
338
|
}
|
|
339
|
+
hasSetupRun = true;
|
|
309
340
|
logger.info('Test database setup complete');
|
|
310
341
|
}
|
|
311
342
|
catch (error) {
|
|
343
|
+
hasSetupRun = false;
|
|
312
344
|
logger.error('Failed to set up test database:', error);
|
|
313
345
|
throw error;
|
|
314
346
|
}
|
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
*/
|
|
5
5
|
/**
|
|
6
6
|
* Whether to skip integration tests:
|
|
7
|
-
* -
|
|
8
|
-
*
|
|
7
|
+
* - Only skip when SKIP_INTEGRATION_TESTS is explicitly set to 'true'
|
|
8
|
+
* (used locally when ClickHouse is unavailable)
|
|
9
9
|
*/
|
|
10
10
|
export declare const SKIP_INTEGRATION_TESTS: boolean;
|
|
11
11
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"test-config.d.ts","sourceRoot":"","sources":["../../../../src/core/tests/integration/test-config.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;GAIG;AACH,eAAO,MAAM,sBAAsB,
|
|
1
|
+
{"version":3,"file":"test-config.d.ts","sourceRoot":"","sources":["../../../../src/core/tests/integration/test-config.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;GAIG;AACH,eAAO,MAAM,sBAAsB,SAAgD,CAAC;AAEpF;;GAEG;AACH,eAAO,MAAM,aAAa,QAAQ,CAAC"}
|
|
@@ -4,11 +4,10 @@
|
|
|
4
4
|
*/
|
|
5
5
|
/**
|
|
6
6
|
* Whether to skip integration tests:
|
|
7
|
-
* -
|
|
8
|
-
*
|
|
7
|
+
* - Only skip when SKIP_INTEGRATION_TESTS is explicitly set to 'true'
|
|
8
|
+
* (used locally when ClickHouse is unavailable)
|
|
9
9
|
*/
|
|
10
|
-
export const SKIP_INTEGRATION_TESTS = process.env.SKIP_INTEGRATION_TESTS === 'true'
|
|
11
|
-
(process.env.CI === 'true' && process.env.ENABLE_CI_INTEGRATION_TESTS !== 'true');
|
|
10
|
+
export const SKIP_INTEGRATION_TESTS = process.env.SKIP_INTEGRATION_TESTS === 'true';
|
|
12
11
|
/**
|
|
13
12
|
* Default timeout for test setup (in milliseconds)
|
|
14
13
|
*/
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
{
|
|
2
|
+
"test_table": [
|
|
3
|
+
{
|
|
4
|
+
"id": 1,
|
|
5
|
+
"name": "Product A",
|
|
6
|
+
"category": "A",
|
|
7
|
+
"price": 10.5,
|
|
8
|
+
"created_at": "2023-01-01 10:00:00",
|
|
9
|
+
"is_active": true,
|
|
10
|
+
"tags": [
|
|
11
|
+
"new",
|
|
12
|
+
"sale"
|
|
13
|
+
],
|
|
14
|
+
"attributes": {
|
|
15
|
+
"color": "red",
|
|
16
|
+
"size": "M"
|
|
17
|
+
},
|
|
18
|
+
"optional_note": "Popular item",
|
|
19
|
+
"sku": "A-100",
|
|
20
|
+
"delivery_dates": [
|
|
21
|
+
"2023-01-05",
|
|
22
|
+
"2023-01-10"
|
|
23
|
+
]
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
"id": 2,
|
|
27
|
+
"name": "Product B",
|
|
28
|
+
"category": "B",
|
|
29
|
+
"price": 20.75,
|
|
30
|
+
"created_at": "2023-01-02 12:30:00",
|
|
31
|
+
"is_active": true,
|
|
32
|
+
"tags": [
|
|
33
|
+
"featured"
|
|
34
|
+
],
|
|
35
|
+
"attributes": {
|
|
36
|
+
"color": "blue",
|
|
37
|
+
"size": "L"
|
|
38
|
+
},
|
|
39
|
+
"optional_note": null,
|
|
40
|
+
"sku": "B-200",
|
|
41
|
+
"delivery_dates": [
|
|
42
|
+
"2023-01-06"
|
|
43
|
+
]
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
"id": 3,
|
|
47
|
+
"name": "Product C",
|
|
48
|
+
"category": "A",
|
|
49
|
+
"price": 15.0,
|
|
50
|
+
"created_at": "2023-01-03 09:45:00",
|
|
51
|
+
"is_active": false,
|
|
52
|
+
"tags": [],
|
|
53
|
+
"attributes": {
|
|
54
|
+
"color": "green",
|
|
55
|
+
"size": "S"
|
|
56
|
+
},
|
|
57
|
+
"optional_note": "Backordered",
|
|
58
|
+
"sku": "C-300",
|
|
59
|
+
"delivery_dates": []
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
"id": 4,
|
|
63
|
+
"name": "Product D",
|
|
64
|
+
"category": "C",
|
|
65
|
+
"price": 8.25,
|
|
66
|
+
"created_at": "2023-01-04 15:15:00",
|
|
67
|
+
"is_active": true,
|
|
68
|
+
"tags": [
|
|
69
|
+
"clearance"
|
|
70
|
+
],
|
|
71
|
+
"attributes": {
|
|
72
|
+
"color": "yellow",
|
|
73
|
+
"size": "XL"
|
|
74
|
+
},
|
|
75
|
+
"optional_note": null,
|
|
76
|
+
"sku": "D-400",
|
|
77
|
+
"delivery_dates": [
|
|
78
|
+
"2023-01-08"
|
|
79
|
+
]
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
"id": 5,
|
|
83
|
+
"name": "Product E",
|
|
84
|
+
"category": "B",
|
|
85
|
+
"price": 30.0,
|
|
86
|
+
"created_at": "2023-01-05 11:20:00",
|
|
87
|
+
"is_active": true,
|
|
88
|
+
"tags": [
|
|
89
|
+
"premium",
|
|
90
|
+
"gift"
|
|
91
|
+
],
|
|
92
|
+
"attributes": {
|
|
93
|
+
"color": "black",
|
|
94
|
+
"size": "M"
|
|
95
|
+
},
|
|
96
|
+
"optional_note": "Limited stock",
|
|
97
|
+
"sku": "E-500",
|
|
98
|
+
"delivery_dates": [
|
|
99
|
+
"2023-01-07",
|
|
100
|
+
"2023-01-09"
|
|
101
|
+
]
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
"id": 6,
|
|
105
|
+
"name": "Product F",
|
|
106
|
+
"category": "D",
|
|
107
|
+
"price": 12.5,
|
|
108
|
+
"created_at": "2023-01-06 14:40:00",
|
|
109
|
+
"is_active": false,
|
|
110
|
+
"tags": [],
|
|
111
|
+
"attributes": {
|
|
112
|
+
"color": "white",
|
|
113
|
+
"size": "M"
|
|
114
|
+
},
|
|
115
|
+
"optional_note": null,
|
|
116
|
+
"sku": "F-600",
|
|
117
|
+
"delivery_dates": []
|
|
118
|
+
}
|
|
119
|
+
],
|
|
120
|
+
"users": [
|
|
121
|
+
{
|
|
122
|
+
"id": 1,
|
|
123
|
+
"user_name": "john_doe",
|
|
124
|
+
"email": "john@example.com",
|
|
125
|
+
"status": "active",
|
|
126
|
+
"created_at": "2023-01-01"
|
|
127
|
+
},
|
|
128
|
+
{
|
|
129
|
+
"id": 2,
|
|
130
|
+
"user_name": "jane_smith",
|
|
131
|
+
"email": "jane@example.com",
|
|
132
|
+
"status": "active",
|
|
133
|
+
"created_at": "2023-01-02"
|
|
134
|
+
},
|
|
135
|
+
{
|
|
136
|
+
"id": 3,
|
|
137
|
+
"user_name": "bob_jones",
|
|
138
|
+
"email": "bob@example.com",
|
|
139
|
+
"status": "inactive",
|
|
140
|
+
"created_at": "2023-01-03"
|
|
141
|
+
}
|
|
142
|
+
],
|
|
143
|
+
"orders": [
|
|
144
|
+
{
|
|
145
|
+
"id": 1,
|
|
146
|
+
"user_id": 1,
|
|
147
|
+
"product_id": 1,
|
|
148
|
+
"quantity": 2,
|
|
149
|
+
"total": 21.0,
|
|
150
|
+
"status": "completed",
|
|
151
|
+
"created_at": "2023-01-10"
|
|
152
|
+
},
|
|
153
|
+
{
|
|
154
|
+
"id": 2,
|
|
155
|
+
"user_id": 1,
|
|
156
|
+
"product_id": 3,
|
|
157
|
+
"quantity": 1,
|
|
158
|
+
"total": 15.0,
|
|
159
|
+
"status": "completed",
|
|
160
|
+
"created_at": "2023-01-11"
|
|
161
|
+
},
|
|
162
|
+
{
|
|
163
|
+
"id": 3,
|
|
164
|
+
"user_id": 2,
|
|
165
|
+
"product_id": 2,
|
|
166
|
+
"quantity": 3,
|
|
167
|
+
"total": 62.25,
|
|
168
|
+
"status": "pending",
|
|
169
|
+
"created_at": "2023-01-12"
|
|
170
|
+
},
|
|
171
|
+
{
|
|
172
|
+
"id": 4,
|
|
173
|
+
"user_id": 2,
|
|
174
|
+
"product_id": 5,
|
|
175
|
+
"quantity": 1,
|
|
176
|
+
"total": 30.0,
|
|
177
|
+
"status": "completed",
|
|
178
|
+
"created_at": "2023-01-13"
|
|
179
|
+
},
|
|
180
|
+
{
|
|
181
|
+
"id": 5,
|
|
182
|
+
"user_id": 3,
|
|
183
|
+
"product_id": 4,
|
|
184
|
+
"quantity": 2,
|
|
185
|
+
"total": 16.5,
|
|
186
|
+
"status": "cancelled",
|
|
187
|
+
"created_at": "2023-01-14"
|
|
188
|
+
}
|
|
189
|
+
]
|
|
190
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"streaming-helpers.d.ts","sourceRoot":"","sources":["../../../src/core/utils/streaming-helpers.ts"],"names":[],"mappings":"AAoIA,wBAAgB,uBAAuB,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,cAAc,GAAG,cAAc,CAAC,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,CAAC,EAAE,CAAC,CAwCnH"}
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { Readable } from 'stream';
|
|
2
|
+
function createBufferFlusher() {
|
|
3
|
+
let buffer = '';
|
|
4
|
+
const flush = () => {
|
|
5
|
+
if (!buffer.length) {
|
|
6
|
+
return [];
|
|
7
|
+
}
|
|
8
|
+
const lines = buffer.split('\n');
|
|
9
|
+
buffer = lines.pop() ?? '';
|
|
10
|
+
const rows = [];
|
|
11
|
+
for (const line of lines) {
|
|
12
|
+
const trimmed = line.trim();
|
|
13
|
+
if (!trimmed.length) {
|
|
14
|
+
continue;
|
|
15
|
+
}
|
|
16
|
+
rows.push(JSON.parse(trimmed));
|
|
17
|
+
}
|
|
18
|
+
return rows;
|
|
19
|
+
};
|
|
20
|
+
const append = (value) => {
|
|
21
|
+
buffer += typeof value === 'string' ? value : value.toString('utf8');
|
|
22
|
+
};
|
|
23
|
+
return { flush, append };
|
|
24
|
+
}
|
|
25
|
+
async function normalizeChunk(chunk, flush, append) {
|
|
26
|
+
if (chunk == null) {
|
|
27
|
+
return [];
|
|
28
|
+
}
|
|
29
|
+
if (Array.isArray(chunk)) {
|
|
30
|
+
const rows = [];
|
|
31
|
+
for (const item of chunk) {
|
|
32
|
+
rows.push(...await normalizeChunk(item, flush, append));
|
|
33
|
+
}
|
|
34
|
+
return rows;
|
|
35
|
+
}
|
|
36
|
+
if (typeof chunk.json === 'function') {
|
|
37
|
+
return [await chunk.json()];
|
|
38
|
+
}
|
|
39
|
+
if (typeof chunk.text === 'function') {
|
|
40
|
+
const text = await chunk.text();
|
|
41
|
+
return [JSON.parse(text)];
|
|
42
|
+
}
|
|
43
|
+
if (typeof chunk.text === 'string') {
|
|
44
|
+
return [JSON.parse(chunk.text)];
|
|
45
|
+
}
|
|
46
|
+
if (Buffer.isBuffer(chunk)) {
|
|
47
|
+
append(chunk);
|
|
48
|
+
return flush();
|
|
49
|
+
}
|
|
50
|
+
if (typeof chunk === 'string') {
|
|
51
|
+
append(chunk);
|
|
52
|
+
return flush();
|
|
53
|
+
}
|
|
54
|
+
if (typeof chunk === 'object') {
|
|
55
|
+
return [chunk];
|
|
56
|
+
}
|
|
57
|
+
return [];
|
|
58
|
+
}
|
|
59
|
+
async function createChunkReader(nodeStream) {
|
|
60
|
+
const iterator = nodeStream[Symbol.asyncIterator]?.();
|
|
61
|
+
let webReader;
|
|
62
|
+
const readNext = async () => {
|
|
63
|
+
if (iterator) {
|
|
64
|
+
const result = await iterator.next();
|
|
65
|
+
return { done: Boolean(result.done), value: result.value };
|
|
66
|
+
}
|
|
67
|
+
if (!webReader) {
|
|
68
|
+
const webStream = Readable.toWeb(nodeStream);
|
|
69
|
+
webReader = webStream.getReader();
|
|
70
|
+
}
|
|
71
|
+
const result = await webReader.read();
|
|
72
|
+
return { done: Boolean(result.done), value: result.value };
|
|
73
|
+
};
|
|
74
|
+
const close = async () => {
|
|
75
|
+
if (iterator && typeof iterator.return === 'function') {
|
|
76
|
+
try {
|
|
77
|
+
await iterator.return();
|
|
78
|
+
}
|
|
79
|
+
catch { }
|
|
80
|
+
}
|
|
81
|
+
if (typeof nodeStream.destroy === 'function') {
|
|
82
|
+
nodeStream.destroy();
|
|
83
|
+
}
|
|
84
|
+
};
|
|
85
|
+
return { readNext, close };
|
|
86
|
+
}
|
|
87
|
+
async function createWebStreamReader(webStream) {
|
|
88
|
+
const reader = webStream.getReader();
|
|
89
|
+
const readNext = async () => {
|
|
90
|
+
const result = await reader.read();
|
|
91
|
+
return { done: Boolean(result.done), value: result.value };
|
|
92
|
+
};
|
|
93
|
+
const close = async () => {
|
|
94
|
+
try {
|
|
95
|
+
await reader.cancel();
|
|
96
|
+
}
|
|
97
|
+
catch { }
|
|
98
|
+
};
|
|
99
|
+
return { readNext, close };
|
|
100
|
+
}
|
|
101
|
+
export function createJsonEachRowStream(stream) {
|
|
102
|
+
const { flush, append } = createBufferFlusher();
|
|
103
|
+
let readerPromise;
|
|
104
|
+
const ensureReader = () => {
|
|
105
|
+
if (!readerPromise) {
|
|
106
|
+
if (typeof stream?.getReader === 'function') {
|
|
107
|
+
readerPromise = createWebStreamReader(stream);
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
readerPromise = createChunkReader(stream);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return readerPromise;
|
|
114
|
+
};
|
|
115
|
+
return new ReadableStream({
|
|
116
|
+
async pull(controller) {
|
|
117
|
+
const reader = await ensureReader();
|
|
118
|
+
const { done, value } = await reader.readNext();
|
|
119
|
+
if (done) {
|
|
120
|
+
const remaining = flush();
|
|
121
|
+
if (remaining.length) {
|
|
122
|
+
controller.enqueue(remaining);
|
|
123
|
+
}
|
|
124
|
+
controller.close();
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
const rows = await normalizeChunk(value, flush, append);
|
|
128
|
+
if (rows.length) {
|
|
129
|
+
controller.enqueue(rows);
|
|
130
|
+
}
|
|
131
|
+
},
|
|
132
|
+
async cancel() {
|
|
133
|
+
const reader = await ensureReader();
|
|
134
|
+
await reader.close();
|
|
135
|
+
}
|
|
136
|
+
});
|
|
137
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hypequery/clickhouse",
|
|
3
|
-
"version": "1.3.
|
|
3
|
+
"version": "1.3.3",
|
|
4
4
|
"description": "ClickHouse typescript query builder",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -25,11 +25,11 @@
|
|
|
25
25
|
"diagnose-ci": "node scripts/diagnose-ci.js",
|
|
26
26
|
"dev": "tsc --watch",
|
|
27
27
|
"test": "npm run test:types && npm run test:unit",
|
|
28
|
-
"test:unit": "
|
|
28
|
+
"test:unit": "vitest run --config vitest.config.ts",
|
|
29
29
|
"test:types": "tsc --project tsconfig.type-tests.json",
|
|
30
30
|
"test:integration": "node scripts/run-integration-tests.js",
|
|
31
|
-
"test:watch": "
|
|
32
|
-
"test:coverage": "
|
|
31
|
+
"test:watch": "vitest watch --config vitest.config.ts",
|
|
32
|
+
"test:coverage": "vitest run --coverage --config vitest.config.ts",
|
|
33
33
|
"test:cli": "node scripts/test-cli-integration.js",
|
|
34
34
|
"lint": "eslint src/**/*.ts",
|
|
35
35
|
"semantic-release": "npx semantic-release",
|
|
@@ -57,7 +57,6 @@
|
|
|
57
57
|
}
|
|
58
58
|
},
|
|
59
59
|
"devDependencies": {
|
|
60
|
-
"@babel/plugin-transform-modules-commonjs": "^7.26.3",
|
|
61
60
|
"@clickhouse/client": "^1.11.2",
|
|
62
61
|
"@clickhouse/client-common": "^1.11.2",
|
|
63
62
|
"@clickhouse/client-web": "^1.11.2",
|
|
@@ -67,17 +66,15 @@
|
|
|
67
66
|
"@semantic-release/github": "^9.2.6",
|
|
68
67
|
"@semantic-release/npm": "^11.0.2",
|
|
69
68
|
"@semantic-release/release-notes-generator": "^12.1.0",
|
|
70
|
-
"@types/jest": "^29.5.11",
|
|
71
69
|
"@types/node": "^18.19.80",
|
|
72
70
|
"glob": "^11.0.3",
|
|
73
|
-
"jest": "^29.7.0",
|
|
74
|
-
"jest-esbuild": "^0.3.0",
|
|
75
71
|
"semantic-release": "^23.0.2",
|
|
76
|
-
"ts-jest": "^29.1.1",
|
|
77
72
|
"ts-node": "^10.9.0",
|
|
78
73
|
"typedoc": "^0.28.1",
|
|
79
74
|
"typedoc-plugin-markdown": "^4.6.0",
|
|
80
|
-
"typescript": "^5.7.3"
|
|
75
|
+
"typescript": "^5.7.3",
|
|
76
|
+
"@vitest/coverage-v8": "^2.1.6",
|
|
77
|
+
"vitest": "^2.1.6"
|
|
81
78
|
},
|
|
82
79
|
"ts-node": {
|
|
83
80
|
"esm": true,
|