@clickhouse/client 1.22.0 → 1.23.0-head.287977a.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/client.d.ts +2 -2
- package/dist/client.js +11 -4
- package/dist/client.js.map +1 -1
- package/dist/common/clickhouse_types.d.ts +98 -0
- package/dist/common/clickhouse_types.js +30 -0
- package/dist/common/clickhouse_types.js.map +1 -0
- package/dist/common/client.d.ts +233 -0
- package/dist/common/client.js +414 -0
- package/dist/common/client.js.map +1 -0
- package/dist/common/config.d.ts +234 -0
- package/dist/common/config.js +364 -0
- package/dist/common/config.js.map +1 -0
- package/dist/common/connection.d.ts +124 -0
- package/dist/common/connection.js +3 -0
- package/dist/common/connection.js.map +1 -0
- package/dist/common/data_formatter/format_query_params.d.ts +11 -0
- package/dist/common/data_formatter/format_query_params.js +128 -0
- package/dist/common/data_formatter/format_query_params.js.map +1 -0
- package/dist/common/data_formatter/format_query_settings.d.ts +2 -0
- package/dist/common/data_formatter/format_query_settings.js +20 -0
- package/dist/common/data_formatter/format_query_settings.js.map +1 -0
- package/dist/common/data_formatter/formatter.d.ts +41 -0
- package/dist/common/data_formatter/formatter.js +78 -0
- package/dist/common/data_formatter/formatter.js.map +1 -0
- package/dist/common/data_formatter/index.d.ts +3 -0
- package/dist/common/data_formatter/index.js +24 -0
- package/dist/common/data_formatter/index.js.map +1 -0
- package/dist/common/error/error.d.ts +20 -0
- package/dist/common/error/error.js +73 -0
- package/dist/common/error/error.js.map +1 -0
- package/dist/common/error/index.d.ts +1 -0
- package/dist/common/error/index.js +18 -0
- package/dist/common/error/index.js.map +1 -0
- package/dist/common/index.d.ts +67 -0
- package/dist/common/index.js +97 -0
- package/dist/common/index.js.map +1 -0
- package/dist/common/logger.d.ts +80 -0
- package/dist/common/logger.js +154 -0
- package/dist/common/logger.js.map +1 -0
- package/dist/common/parse/column_types.d.ts +155 -0
- package/dist/common/parse/column_types.js +594 -0
- package/dist/common/parse/column_types.js.map +1 -0
- package/dist/common/parse/index.d.ts +2 -0
- package/dist/common/parse/index.js +19 -0
- package/dist/common/parse/index.js.map +1 -0
- package/dist/common/parse/json_handling.d.ts +19 -0
- package/dist/common/parse/json_handling.js +8 -0
- package/dist/common/parse/json_handling.js.map +1 -0
- package/dist/common/result.d.ts +90 -0
- package/dist/common/result.js +3 -0
- package/dist/common/result.js.map +1 -0
- package/dist/common/settings.d.ts +2007 -0
- package/dist/common/settings.js +19 -0
- package/dist/common/settings.js.map +1 -0
- package/dist/common/tracing.d.ts +146 -0
- package/dist/common/tracing.js +76 -0
- package/dist/common/tracing.js.map +1 -0
- package/dist/common/ts_utils.d.ts +4 -0
- package/dist/common/ts_utils.js +3 -0
- package/dist/common/ts_utils.js.map +1 -0
- package/dist/common/utils/connection.d.ts +21 -0
- package/dist/common/utils/connection.js +43 -0
- package/dist/common/utils/connection.js.map +1 -0
- package/dist/common/utils/index.d.ts +5 -0
- package/dist/common/utils/index.js +22 -0
- package/dist/common/utils/index.js.map +1 -0
- package/dist/common/utils/multipart.d.ts +34 -0
- package/dist/common/utils/multipart.js +81 -0
- package/dist/common/utils/multipart.js.map +1 -0
- package/dist/common/utils/sleep.d.ts +4 -0
- package/dist/common/utils/sleep.js +12 -0
- package/dist/common/utils/sleep.js.map +1 -0
- package/dist/common/utils/stream.d.ts +15 -0
- package/dist/common/utils/stream.js +50 -0
- package/dist/common/utils/stream.js.map +1 -0
- package/dist/common/utils/url.d.ts +20 -0
- package/dist/common/utils/url.js +67 -0
- package/dist/common/utils/url.js.map +1 -0
- package/dist/common/version.d.ts +2 -0
- package/dist/common/version.js +4 -0
- package/dist/common/version.js.map +1 -0
- package/dist/config.d.ts +22 -2
- package/dist/config.js +2 -2
- package/dist/config.js.map +1 -1
- package/dist/connection/compression.d.ts +2 -2
- package/dist/connection/compression.js +4 -4
- package/dist/connection/compression.js.map +1 -1
- package/dist/connection/create_connection.d.ts +1 -1
- package/dist/connection/node_base_connection.d.ts +3 -3
- package/dist/connection/node_base_connection.js +22 -22
- package/dist/connection/node_base_connection.js.map +1 -1
- package/dist/connection/node_custom_agent_connection.js +2 -2
- package/dist/connection/node_custom_agent_connection.js.map +1 -1
- package/dist/connection/node_http_connection.js +2 -2
- package/dist/connection/node_http_connection.js.map +1 -1
- package/dist/connection/node_https_connection.d.ts +1 -1
- package/dist/connection/node_https_connection.js +3 -3
- package/dist/connection/node_https_connection.js.map +1 -1
- package/dist/connection/socket_pool.d.ts +1 -1
- package/dist/connection/socket_pool.js +30 -30
- package/dist/connection/socket_pool.js.map +1 -1
- package/dist/connection/stream.d.ts +1 -1
- package/dist/connection/stream.js +9 -9
- package/dist/connection/stream.js.map +1 -1
- package/dist/index.d.ts +9 -7
- package/dist/index.js +26 -24
- package/dist/index.js.map +1 -1
- package/dist/result_set.d.ts +1 -1
- package/dist/result_set.js +10 -10
- package/dist/result_set.js.map +1 -1
- package/dist/utils/encoder.d.ts +1 -1
- package/dist/utils/encoder.js +5 -5
- package/dist/utils/encoder.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/version.js.map +1 -1
- package/package.json +7 -5
- package/skills/clickhouse-js-node-rowbinary-parser/EXAMPLES.md +48 -0
- package/skills/clickhouse-js-node-rowbinary-parser/README.md +255 -0
- package/skills/clickhouse-js-node-rowbinary-parser/SKILL.md +206 -0
- package/skills/clickhouse-js-node-rowbinary-parser/case-studies/iot-rowbinary-vs-json.md +83 -0
- package/skills/clickhouse-js-node-rowbinary-parser/case-studies/ledger-rowbinary-vs-json.md +103 -0
- package/skills/clickhouse-js-node-rowbinary-parser/case-studies/logs-json-wins.md +86 -0
- package/skills/clickhouse-js-node-rowbinary-parser/case-studies/wasm-vs-js.md +172 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/aggregateFunction.ts +34 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/bool.ts +10 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/columnar.ts +125 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/compile.ts +318 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/composite.ts +181 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/core.ts +77 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/datetime.ts +113 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/decimals.ts +57 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/dynamic.ts +328 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/enums.ts +28 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/examples/carts.ts +71 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/examples/events.ts +51 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/examples/iot.ts +158 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/examples/ledger.ts +98 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/examples/logs.ts +73 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/examples/observability.ts +142 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/examples/orders.ts +65 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/examples/profiles.ts +60 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/examples/telemetry.ts +102 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/floats.ts +32 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/geo.ts +109 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/header.ts +29 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/integers.ts +95 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/interval.ts +54 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/ip.ts +93 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/json.ts +33 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/lowCardinality.ts +18 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/nested.ts +23 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/nothing.ts +29 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/reader.ts +68 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/rowBinaryWithNamesAndTypes.ts +155 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/rows.ts +58 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/simpleAggregateFunction.ts +20 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/stream.ts +276 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/strings.ts +55 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/time.ts +61 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/uuid.ts +153 -0
- package/skills/clickhouse-js-node-rowbinary-parser/src/varint.ts +70 -0
package/dist/result_set.js
CHANGED
|
@@ -34,8 +34,8 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
36
|
exports.ResultSet = void 0;
|
|
37
|
-
const
|
|
38
|
-
const
|
|
37
|
+
const index_1 = require("./common/index");
|
|
38
|
+
const index_2 = require("./common/index");
|
|
39
39
|
const buffer_1 = require("buffer");
|
|
40
40
|
const stream_1 = __importStar(require("stream"));
|
|
41
41
|
const utils_1 = require("./utils");
|
|
@@ -71,14 +71,14 @@ class ResultSet {
|
|
|
71
71
|
this.query_id = query_id;
|
|
72
72
|
this.span = span;
|
|
73
73
|
this.jsonHandling = {
|
|
74
|
-
...
|
|
74
|
+
...index_1.defaultJSONHandling,
|
|
75
75
|
...jsonHandling,
|
|
76
76
|
};
|
|
77
77
|
// eslint-disable-next-line no-console
|
|
78
78
|
this.log_error = log_error ?? ((err) => console.error(err));
|
|
79
79
|
if (_response_headers !== undefined) {
|
|
80
80
|
this.response_headers = Object.freeze(_response_headers);
|
|
81
|
-
this.exceptionTag = _response_headers[
|
|
81
|
+
this.exceptionTag = _response_headers[index_1.EXCEPTION_TAG_HEADER_NAME];
|
|
82
82
|
}
|
|
83
83
|
}
|
|
84
84
|
consume() {
|
|
@@ -110,7 +110,7 @@ class ResultSet {
|
|
|
110
110
|
}
|
|
111
111
|
this.span.setAttributes(attributes);
|
|
112
112
|
if (err) {
|
|
113
|
-
(0,
|
|
113
|
+
(0, index_1.recordSpanError)(this.span, err);
|
|
114
114
|
}
|
|
115
115
|
this.span.end();
|
|
116
116
|
}
|
|
@@ -131,7 +131,7 @@ class ResultSet {
|
|
|
131
131
|
/** See {@link BaseResultSet.json}. */
|
|
132
132
|
async json() {
|
|
133
133
|
// JSONEachRow, etc.
|
|
134
|
-
if ((0,
|
|
134
|
+
if ((0, index_2.isStreamableJSONFamily)(this.format)) {
|
|
135
135
|
const result = [];
|
|
136
136
|
// Using the stream() instead of _stream directly to leverage the existing logic
|
|
137
137
|
// for handling incomplete chunks and exception tags.
|
|
@@ -146,7 +146,7 @@ class ResultSet {
|
|
|
146
146
|
return result;
|
|
147
147
|
}
|
|
148
148
|
// JSON, JSONObjectEachRow, etc.
|
|
149
|
-
if ((0,
|
|
149
|
+
if ((0, index_2.isNotStreamableJSONFamily)(this.format)) {
|
|
150
150
|
const stream = this.consume();
|
|
151
151
|
try {
|
|
152
152
|
const text = await (0, utils_1.getAsText)(stream);
|
|
@@ -164,7 +164,7 @@ class ResultSet {
|
|
|
164
164
|
}
|
|
165
165
|
/** See {@link BaseResultSet.stream}. */
|
|
166
166
|
stream() {
|
|
167
|
-
(0,
|
|
167
|
+
(0, index_2.validateStreamFormat)(this.format);
|
|
168
168
|
const incompleteChunks = [];
|
|
169
169
|
const logError = this.log_error;
|
|
170
170
|
const exceptionTag = this.exceptionTag;
|
|
@@ -193,8 +193,8 @@ class ResultSet {
|
|
|
193
193
|
// Check for exception in the chunk (only after 25.11)
|
|
194
194
|
if (exceptionTag !== undefined &&
|
|
195
195
|
idx >= 1 &&
|
|
196
|
-
chunk[idx - 1] ===
|
|
197
|
-
return callback((0,
|
|
196
|
+
chunk[idx - 1] === index_1.CARET_RETURN) {
|
|
197
|
+
return callback((0, index_1.extractErrorAtTheEndOfChunk)(chunk, exceptionTag));
|
|
198
198
|
}
|
|
199
199
|
if (incompleteChunks.length > 0) {
|
|
200
200
|
incompleteChunks.push(chunk.subarray(lastIdx, idx));
|
package/dist/result_set.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"result_set.js","sourceRoot":"","sources":["../src/result_set.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAWA,
|
|
1
|
+
{"version":3,"file":"result_set.js","sourceRoot":"","sources":["../src/result_set.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAWA,0CAMwB;AACxB,0CAIwB;AACxB,mCAAgC;AAEhC,iDAA2C;AAC3C,mCAAoC;AAEpC,MAAM,OAAO,GAAG,IAAa,CAAC;AAqC9B,MAAa,SAAS;IAGJ,gBAAgB,GAAoB,EAAE,CAAC;IAEtC,YAAY,GAAuB,SAAS,CAAC;IAC7C,SAAS,CAAyB;IAClC,YAAY,CAAe;IACpC,SAAS,GAAG,KAAK,CAAC;IAC1B;;;;;OAKG;IACK,OAAO,CAAkB;IAChB,MAAM,CAAS;IAChC;;wEAEoE;IACnD,IAAI,CAA6B;IAClD,oEAAoE;IAC5D,UAAU,GAAG,CAAC,CAAC;IACvB,oDAAoD;IAC5C,SAAS,GAAG,CAAC,CAAC;IACd,iBAAiB,GAAG,KAAK,CAAC;IAC1B,aAAa,GAAG,KAAK,CAAC;IACd,QAAQ,CAAS;IAEjC,YACE,OAAwB,EACxB,MAAc,EACd,QAAgB,EAChB,SAAkC,EAClC,iBAAmC,EACnC,YAA2B,EAC3B,IAAqB;QAErB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,YAAY,GAAG;YAClB,GAAG,2BAAmB;YACtB,GAAG,YAAY;SAChB,CAAC;QACF,sCAAsC;QACtC,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,CAAC,CAAC,GAAU,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QAEnE,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;YACpC,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;YACzD,IAAI,CAAC,YAAY,GAAG,iBAAiB,CAAC,iCAAyB,CAElD,CAAC;QAChB,CAAC;IACH,CAAC;IAEO,OAAO;QACb,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAChD,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,+DAA+D;IACvD,WAAW,CAAC,KAAa;QAC/B,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAC9B,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;IAC1B,CAAC;IAED;;;+BAG2B;IACnB,UAAU,CAAC,GAAa;QAC9B,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrC,OAAO;QACT,CAAC;QACD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,MAAM,UAAU,GAA6B;YAC3C,mCAAmC,EAAE,IAAI,CAAC,UAAU;SACrD,CAAC;QACF,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,UAAU,CAAC,2BAA2B,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC;QAC3D,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QACpC,IAAI,GAAG,EAAE,CAAC;YACR,IAAA,uBAAe,EAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAClC,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;IAClB,CAAC;IAED,sCAAsC;IACtC,KAAK,CAAC,IAAI;QACR,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAC9B,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAA,iBAAS,EAAC,MAAM,CAAC,CAAC;YACrC,IAAI,CAAC,UAAU,IAAI,eAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YAC3C,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YACrB,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED,sCAAsC;IACtC,KAAK,CAAC,IAAI;QACR,oBAAoB;QACpB,IAAI,IAAA,8BAAsB,EAAC,IAAI,CAAC,MAAoB,CAAC,EAAE,CAAC;YACtD,MAAM,MAAM,GAAQ,EAAE,CAAC;YACvB,gFAAgF;YAChF,qDAAqD;YACrD,kFAAkF;YAClF,uFAAuF;YACvF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAK,CAAC;YAChC,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;gBAChC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACvB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAO,CAAC,CAAC;gBAC/B,CAAC;YACH,CAAC;YACD,OAAO,MAAmC,CAAC;QAC7C,CAAC;QACD,gCAAgC;QAChC,IAAI,IAAA,iCAAyB,EAAC,IAAI,CAAC,MAAoB,CAAC,EAAE,CAAC;YACzD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,IAAA,iBAAS,EAAC,MAAM,CAAC,CAAC;gBACrC,IAAI,CAAC,UAAU,IAAI,eAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;gBAC3C,IAAI,CAAC,UAAU,EAAE,CAAC;gBAClB,OAAO,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;gBACrB,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QACD,qCAAqC;QACrC,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,CAAC,MAAM,UAAU,CAAC,CAAC;IAC1D,CAAC;IAED,wCAAwC;IACxC,MAAM;QACJ,IAAA,4BAAoB,EAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAElC,MAAM,gBAAgB,GAAa,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACvC,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACvC,4DAA4D;QAC5D,MAAM,SAAS,GAAG,IAAI,CAAC;QACvB,MAAM,MAAM,GAAG,IAAI,kBAAS,CAAC;YAC3B,SAAS,CACP,KAAa,EACb,SAAyB,EACzB,QAA2B;gBAE3B,SAAS,CAAC,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC;gBACrC,MAAM,IAAI,GAAU,EAAE,CAAC;gBAEvB,IAAI,OAAO,GAAG,CAAC,CAAC;gBAChB,IAAI,gBAAwB,CAAC;gBAE7B,OAAO,IAAI,EAAE,CAAC;oBACZ,2DAA2D;oBAC3D,oDAAoD;oBACpD,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;oBAC5C,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;wBACf,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;wBAC/C,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BACpB,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;4BACnC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;wBAClB,CAAC;wBACD,MAAM;oBACR,CAAC;yBAAM,CAAC;wBACN,sDAAsD;wBACtD,IACE,YAAY,KAAK,SAAS;4BAC1B,GAAG,IAAI,CAAC;4BACR,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,oBAAY,EAC/B,CAAC;4BACD,OAAO,QAAQ,CAAC,IAAA,mCAA2B,EAAC,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC;wBACpE,CAAC;wBAED,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BAChC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;4BACpD,gBAAgB,GAAG,eAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;4BACnD,iEAAiE;4BACjE,yBAAyB;4BACzB,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC;wBAC9B,CAAC;6BAAM,CAAC;4BACN,gBAAgB,GAAG,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;wBAClD,CAAC;wBAED,MAAM,IAAI,GAAG,gBAAgB,CAAC,QAAQ,EAAE,CAAC;wBACzC,IAAI,CAAC,IAAI,CAAC;4BACR,IAAI;4BACJ,IAAI;gCACF,OAAO,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;4BAClC,CAAC;yBACF,CAAC,CAAC;wBACH,OAAO,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,6BAA6B;oBAClD,CAAC;gBACH,CAAC;gBACD,QAAQ,EAAE,CAAC;YACb,CAAC;YACD,WAAW,EAAE,IAAI;YACjB,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,gBAAM,CAAC,QAAQ,CAC9B,IAAI,CAAC,OAAO,EAAE,EACd,MAAM,EACN,SAAS,UAAU,CAAC,GAAG;YACrB,IACE,GAAG;gBACH,GAAG,CAAC,IAAI,KAAK,YAAY;gBACzB,GAAG,CAAC,OAAO,KAAK,sBAAsB,EACtC,CAAC;gBACD,QAAQ,CAAC,GAAG,CAAC,CAAC;gBACd,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YAC5B,CAAC;iBAAM,CAAC;gBACN,SAAS,CAAC,UAAU,EAAE,CAAC;YACzB,CAAC;QACH,CAAC,CACF,CAAC;QACF,OAAO,QAAe,CAAC;IACzB,CAAC;IAED,uCAAuC;IACvC,KAAK;QACH,uEAAuE;QACvE,0EAA0E;QAC1E,wEAAwE;QACxE,2EAA2E;QAC3E,6EAA6E;QAC7E,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC;QACxD,IAAI,CAAC,UAAU,EAAE,CAAC;IACpB,CAAC;IAED;;;;;;OAMG;IACH,CAAC,MAAM,CAAC,OAAO,CAAC;QACd,IAAI,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAED,MAAM,CAAC,QAAQ,CAA4B,EACzC,MAAM,EACN,MAAM,EACN,QAAQ,EACR,SAAS,EACT,gBAAgB,EAChB,YAAY,EACZ,IAAI,GACqB;QACzB,OAAO,IAAI,SAAS,CAClB,MAAM,EACN,MAAM,EACN,QAAQ,EACR,SAAS,EACT,gBAAgB,EAChB,YAAY,EACZ,IAAI,CACL,CAAC;IACJ,CAAC;CACF;AA9QD,8BA8QC;AAED,MAAM,4BAA4B,GAAG,kCAAkC,CAAC;AACxE,MAAM,sBAAsB,GAAG,2BAA2B,CAAC"}
|
package/dist/utils/encoder.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { DataFormat, InsertValues, JSONHandling, ValuesEncoder } from "
|
|
1
|
+
import type { DataFormat, InsertValues, JSONHandling, ValuesEncoder } from "../common/index";
|
|
2
2
|
import Stream from "stream";
|
|
3
3
|
export declare class NodeValuesEncoder implements ValuesEncoder<Stream.Readable> {
|
|
4
4
|
private readonly json;
|
package/dist/utils/encoder.js
CHANGED
|
@@ -4,7 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.NodeValuesEncoder = void 0;
|
|
7
|
-
const
|
|
7
|
+
const index_1 = require("../common/index");
|
|
8
8
|
const stream_1 = __importDefault(require("stream"));
|
|
9
9
|
const stream_2 = require("./stream");
|
|
10
10
|
class NodeValuesEncoder {
|
|
@@ -19,17 +19,17 @@ class NodeValuesEncoder {
|
|
|
19
19
|
return values;
|
|
20
20
|
}
|
|
21
21
|
// JSON* formats streams
|
|
22
|
-
return stream_1.default.pipeline(values, (0, stream_2.mapStream)((value) => (0,
|
|
22
|
+
return stream_1.default.pipeline(values, (0, stream_2.mapStream)((value) => (0, index_1.encodeJSON)(value, format, this.json.stringify)), pipelineCb);
|
|
23
23
|
}
|
|
24
24
|
// JSON* arrays
|
|
25
25
|
if (Array.isArray(values)) {
|
|
26
26
|
return values
|
|
27
|
-
.map((value) => (0,
|
|
27
|
+
.map((value) => (0, index_1.encodeJSON)(value, format, this.json.stringify))
|
|
28
28
|
.join("");
|
|
29
29
|
}
|
|
30
30
|
// JSON & JSONObjectEachRow format input
|
|
31
31
|
if (typeof values === "object") {
|
|
32
|
-
return (0,
|
|
32
|
+
return (0, index_1.encodeJSON)(values, format, this.json.stringify);
|
|
33
33
|
}
|
|
34
34
|
throw new Error(`Cannot encode values of type ${typeof values} with ${format} format`);
|
|
35
35
|
}
|
|
@@ -41,7 +41,7 @@ class NodeValuesEncoder {
|
|
|
41
41
|
`got: ${typeof values}`);
|
|
42
42
|
}
|
|
43
43
|
if ((0, stream_2.isStream)(values)) {
|
|
44
|
-
if ((0,
|
|
44
|
+
if ((0, index_1.isSupportedRawFormat)(format)) {
|
|
45
45
|
if (values.readableObjectMode) {
|
|
46
46
|
throw new Error(`Insert for ${format} expected Readable Stream with disabled object mode.`);
|
|
47
47
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"encoder.js","sourceRoot":"","sources":["../../src/utils/encoder.ts"],"names":[],"mappings":";;;;;;AAMA,
|
|
1
|
+
{"version":3,"file":"encoder.js","sourceRoot":"","sources":["../../src/utils/encoder.ts"],"names":[],"mappings":";;;;;;AAMA,2CAAmE;AACnE,oDAA4B;AAC5B,qCAA+C;AAE/C,MAAa,iBAAiB;IACX,IAAI,CAAe;IAEpC,YAAY,gBAA8B;QACxC,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;IAC/B,CAAC;IAED,YAAY,CACV,MAAwC,EACxC,MAAkB;QAElB,IAAI,IAAA,iBAAQ,EAAC,MAAM,CAAC,EAAE,CAAC;YACrB,yEAAyE;YACzE,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC;gBAC/B,OAAO,MAAM,CAAC;YAChB,CAAC;YACD,wBAAwB;YACxB,OAAO,gBAAM,CAAC,QAAQ,CACpB,MAAM,EACN,IAAA,kBAAS,EAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAA,kBAAU,EAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EACpE,UAAU,CACX,CAAC;QACJ,CAAC;QACD,eAAe;QACf,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,OAAO,MAAM;iBACV,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAA,kBAAU,EAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;iBAC9D,IAAI,CAAC,EAAE,CAAC,CAAC;QACd,CAAC;QACD,wCAAwC;QACxC,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC/B,OAAO,IAAA,kBAAU,EAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACzD,CAAC;QACD,MAAM,IAAI,KAAK,CACb,gCAAgC,OAAO,MAAM,SAAS,MAAM,SAAS,CACtE,CAAC;IACJ,CAAC;IAED,oBAAoB,CAClB,MAAwC,EACxC,MAAkB;QAElB,IACE,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YACtB,CAAC,IAAA,iBAAQ,EAAC,MAAM,CAAC;YACjB,OAAO,MAAM,KAAK,QAAQ,EAC1B,CAAC;YACD,MAAM,IAAI,KAAK,CACb,gFAAgF;gBAC9E,QAAQ,OAAO,MAAM,EAAE,CAC1B,CAAC;QACJ,CAAC;QAED,IAAI,IAAA,iBAAQ,EAAC,MAAM,CAAC,EAAE,CAAC;YACrB,IAAI,IAAA,4BAAoB,EAAC,MAAM,CAAC,EAAE,CAAC;gBACjC,IAAI,MAAM,CAAC,kBAAkB,EAAE,CAAC;oBAC9B,MAAM,IAAI,KAAK,CACb,cAAc,MAAM,sDAAsD,CAC3E,CAAC;gBACJ,CAAC;YACH,CAAC;iBAAM,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC;gBACtC,MAAM,IAAI,KAAK,CACb,cAAc,MAAM,qDAAqD,CAC1E,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAnED,8CAmEC;AAED,SAAS,UAAU,CAAC,GAAiC;IACnD,IAAI,GAAG,EAAE,CAAC;QACR,4BAA4B;QAC5B,sCAAsC;QACtC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;AACH,CAAC"}
|
package/dist/version.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
declare const _default: "1.
|
|
1
|
+
declare const _default: "1.23.0-head.287977a.1";
|
|
2
2
|
export default _default;
|
package/dist/version.js
CHANGED
package/dist/version.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"version.js","sourceRoot":"","sources":["../src/version.ts"],"names":[],"mappings":";;AAAA,kBAAe,
|
|
1
|
+
{"version":3,"file":"version.js","sourceRoot":"","sources":["../src/version.ts"],"names":[],"mappings":";;AAAA,kBAAe,uBAAuB,CAAC"}
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "@clickhouse/client",
|
|
3
3
|
"description": "Official JS client for ClickHouse DB - Node.js implementation",
|
|
4
4
|
"homepage": "https://clickhouse.com",
|
|
5
|
-
"version": "1.
|
|
5
|
+
"version": "1.23.0-head.287977a.1",
|
|
6
6
|
"license": "Apache-2.0",
|
|
7
7
|
"keywords": [
|
|
8
8
|
"clickhouse",
|
|
@@ -32,20 +32,22 @@
|
|
|
32
32
|
{
|
|
33
33
|
"name": "clickhouse-js-node-troubleshooting",
|
|
34
34
|
"path": "./skills/clickhouse-js-node-troubleshooting"
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
"name": "clickhouse-js-node-rowbinary-parser",
|
|
38
|
+
"path": "./skills/clickhouse-js-node-rowbinary-parser"
|
|
35
39
|
}
|
|
36
40
|
]
|
|
37
41
|
},
|
|
38
42
|
"scripts": {
|
|
39
43
|
"pack": "npm pack",
|
|
40
|
-
"prepack": "rm -rf skills && cp ../../README.md ../../LICENSE . && cp -r ../../skills .",
|
|
44
|
+
"prepack": "rm -rf skills && cp ../../README.md ../../LICENSE . && cp -r ../../skills . && RBP=skills/clickhouse-js-node-rowbinary-parser && rm -rf $RBP/tests $RBP/node_modules $RBP/dist $RBP/package.json $RBP/package-lock.json $RBP/tsconfig.json $RBP/tsconfig.build.json $RBP/vitest.config.ts $RBP/.gitignore $RBP/LICENSE $RBP/eval_result*.md",
|
|
41
45
|
"typecheck": "tsc --noEmit",
|
|
42
46
|
"lint": "eslint --max-warnings=0 .",
|
|
43
47
|
"lint:fix": "eslint . --fix",
|
|
44
48
|
"build": "rm -rf dist; tsc"
|
|
45
49
|
},
|
|
46
|
-
"dependencies": {
|
|
47
|
-
"@clickhouse/client-common": "1.22.0"
|
|
48
|
-
},
|
|
50
|
+
"dependencies": {},
|
|
49
51
|
"devDependencies": {
|
|
50
52
|
"simdjson": "^0.9.2"
|
|
51
53
|
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# RowBinary reader examples
|
|
2
|
+
|
|
3
|
+
Six end-to-end examples. Each `src/examples/*.ts` exports TWO readers for the
|
|
4
|
+
same row — `readXRow` (built from the generic combinator API; easiest to read)
|
|
5
|
+
and `readXRowFast` (the optimized, monomorphized form: leaf reads inlined,
|
|
6
|
+
combinators flattened to straight-line loops, `advance()` coalesced over
|
|
7
|
+
fixed-width runs — still streaming-safe). The matching `tests/X.example.test.ts`
|
|
8
|
+
runs the full create → populate → read-back round trip against a live ClickHouse
|
|
9
|
+
server (verified, not illustrative), and `tests/X.bench.ts` decodes a large
|
|
10
|
+
`numbers()`-generated buffer with both readers (equivalence-checked before
|
|
11
|
+
timing) to measure the speedup.
|
|
12
|
+
|
|
13
|
+
To use one: find the example whose column types match your result, open its
|
|
14
|
+
reader, and adapt it. `readRows(readXRow)` drives a row reader over a whole
|
|
15
|
+
result; `streamRowBatches(chunks, readXRow)` drives it over a chunked HTTP stream.
|
|
16
|
+
|
|
17
|
+
| Example | SQL schema (the trigger) | Speedup | Reader · Test |
|
|
18
|
+
| ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
19
|
+
| **orders** | `id UInt8, uid UUID, price Decimal64(2), status Enum8(...)` | **~3.4x** | [`src/examples/orders.ts`](src/examples/orders.ts) · [`tests/orders.example.test.ts`](tests/orders.example.test.ts) |
|
|
20
|
+
| **carts** | `cart_id UInt32, items Array(Tuple(sku String, qty UInt16)), discounts Array(Nullable(Int32))` | **~2.0x** | [`src/examples/carts.ts`](src/examples/carts.ts) · [`tests/carts.example.test.ts`](tests/carts.example.test.ts) |
|
|
21
|
+
| **telemetry** | `host String, tags Map(String,String), cpu Array(Float64), region Nullable(String), window Tuple(start UInt32, count UInt16)` | **~1.4x** | [`src/examples/telemetry.ts`](src/examples/telemetry.ts) · [`tests/telemetry.example.test.ts`](tests/telemetry.example.test.ts) |
|
|
22
|
+
| **observability** | `id UInt64, ts DateTime64(3), level Enum8, trace_id UUID, payload Variant(String,Int64,Float64), tags Map(LowCardinality(String),String), metrics Array(Tuple(LowCardinality(String),Float64)), attrs Array(Nullable(Int64))` | **~1.4x** | [`src/examples/observability.ts`](src/examples/observability.ts) · [`tests/observability.example.test.ts`](tests/observability.example.test.ts) |
|
|
23
|
+
| **profiles** | `id UInt32, tags Array(String), score Nullable(Int32)` | **~1.3x** | [`src/examples/profiles.ts`](src/examples/profiles.ts) · [`tests/profiles.example.test.ts`](tests/profiles.example.test.ts) |
|
|
24
|
+
| **events** | `id UInt64, name String, ts DateTime('UTC')` | **~1.05x — on par** | [`src/examples/events.ts`](src/examples/events.ts) · [`tests/events.example.test.ts`](tests/events.example.test.ts) |
|
|
25
|
+
|
|
26
|
+
Speedups: Node 24 / V8, decoding a 20k-row buffer — read the ratio, not the
|
|
27
|
+
absolute hz, and run `npm run bench` for your own numbers. Two independent levers
|
|
28
|
+
drive them: **composite monomorphization** (removes per-row combinator closures —
|
|
29
|
+
`carts` / `telemetry` / `observability`) and **per-row formatting** (`orders` is
|
|
30
|
+
all-scalar yet the biggest win, almost entirely from the `formatUUIDTable` swap).
|
|
31
|
+
A flat scalar row with no hot formatter (`events`) is within noise, so prefer the
|
|
32
|
+
clearer API reader there. When in doubt, benchmark — the `*.bench.ts` files are
|
|
33
|
+
the template.
|
|
34
|
+
|
|
35
|
+
The readers live under `src/examples/` and are excluded from the published build
|
|
36
|
+
(`tsconfig.build.json`): reference material and test fixtures, type-checked by the
|
|
37
|
+
base `tsconfig.json` and run by the suite, not part of the package's public API.
|
|
38
|
+
|
|
39
|
+
## Columnar decode (struct-of-arrays) — the ~4x numeric path
|
|
40
|
+
|
|
41
|
+
The examples above produce one object per row (array-of-structs). For a
|
|
42
|
+
**numeric, fixed-width result the consumer reads column-wise** (aggregate / scan
|
|
43
|
+
/ filter / plot, or hand off to a Worker / WASM kernel), decode the same
|
|
44
|
+
row-major bytes directly into **one typed array per column** in the same single
|
|
45
|
+
pass — no per-row object, no `Date`, no number boxing. That removes the
|
|
46
|
+
allocation that dominates a numeric decode for a **measured ~4.2x**.
|
|
47
|
+
|
|
48
|
+
See example: [`decodeIotColumnar`](src/examples/iot.ts).
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
# ClickHouse Node.js RowBinary Parser Generator
|
|
2
|
+
|
|
3
|
+
**If JS had a -O3 compiler flag, this skill would be it.** (for RowBinary parsing)
|
|
4
|
+
|
|
5
|
+
A skill and a library that lets a coding agent generate bespoke RowBinary parsers on the first pass from the column type definitions of a ClickHouse response. The [spirit](#the-spirit) behind the approach.
|
|
6
|
+
|
|
7
|
+
**Reader only** for now. Today this covers reading (decoding) RowBinary streams. A matching RowBinary writer (encoding) is planned.
|
|
8
|
+
|
|
9
|
+
## Status
|
|
10
|
+
|
|
11
|
+
- ✅ Sonnet 4.6: 60% -> 94.0% pass rate
|
|
12
|
+
- ✅ Opus 4.8: 71% -> 94.7% pass rate
|
|
13
|
+
- ✅ Haiku 4.5: 52% -> 86.0% pass rate
|
|
14
|
+
- ✅ Composer 2.5 Fast: 3x parser performance
|
|
15
|
+
- ✅ 469/469 tests
|
|
16
|
+
- ✅ type-checked
|
|
17
|
+
- ✅ benchmarked
|
|
18
|
+
|
|
19
|
+
## Example
|
|
20
|
+
|
|
21
|
+
Take a small orders result:
|
|
22
|
+
|
|
23
|
+
```sql
|
|
24
|
+
SELECT id, uid, price, status FROM orders
|
|
25
|
+
-- id UInt8
|
|
26
|
+
-- uid UUID
|
|
27
|
+
-- price Decimal64(2)
|
|
28
|
+
-- status Enum8('new' = 1, 'shipped' = 2, 'done' = 3)
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
**The API-only reader** — what you write by composing the library's combinators. Correct, clear, and a fine default:
|
|
32
|
+
|
|
33
|
+
```ts
|
|
34
|
+
const readOrderRow: Reader<OrderRow> = (s) => ({
|
|
35
|
+
id: readUInt8(s),
|
|
36
|
+
uid: formatUUID(readUUID(s)),
|
|
37
|
+
price: readDecimal64(2)(s),
|
|
38
|
+
status: readEnum8(s),
|
|
39
|
+
});
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
**The optimized reader the skill generates** — same row, monomorphized to
|
|
43
|
+
straight-line code. The whole row is fixed-width (1 + 16 + 8 + 1 = 26 bytes), so
|
|
44
|
+
the four separate bounds checks coalesce into one `advance(s, 26)` and every leaf
|
|
45
|
+
read happens at a constant offset; the per-field combinators are gone:
|
|
46
|
+
|
|
47
|
+
```ts
|
|
48
|
+
const readOrderRowFast: Reader<OrderRow> = (s) => {
|
|
49
|
+
const { buf, view } = s;
|
|
50
|
+
const o = advance(s, 26); // one bounds check for the whole 26-byte row
|
|
51
|
+
const id = buf[o]!;
|
|
52
|
+
const uid = formatUUIDTable(buf.subarray(o + 1, o + 17));
|
|
53
|
+
const price: DecimalValue = [view.getBigInt64(o + 17, true), 2];
|
|
54
|
+
const status = view.getInt8(o + 25);
|
|
55
|
+
return { id, uid, price, status };
|
|
56
|
+
};
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Same values, same streaming-safety — **~3.4x** faster.
|
|
60
|
+
|
|
61
|
+
## How to use
|
|
62
|
+
|
|
63
|
+
As a library (comes with the skill):
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
npm install @clickhouse/rowbinary
|
|
67
|
+
npx skills-npm setup
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
As a skill only:
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
npx skills add ClickHouse/clickhouse-js/skills/clickhouse-js-node-rowbinary-parser
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
```console
|
|
77
|
+
> Hey, Claude, tell me what the rowbinary parser skill can do for me.
|
|
78
|
+
> A lot! It generates custom, high-performance RowBinary parsers…
|
|
79
|
+
> Super, generate a parser for the queries in app/src/model.ts.
|
|
80
|
+
< Reading skill clickhouse-js-node-rowbinary-parser…
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## Why it's worth it
|
|
84
|
+
|
|
85
|
+
Four pillars — speed, correctness, judgment, and lifting smaller models:
|
|
86
|
+
|
|
87
|
+
- **~2–3x faster code than the straightforward decoder.** The skill emits
|
|
88
|
+
monomorphized, flattened, straight-line code — inlined reads, bounds checks
|
|
89
|
+
coalesced across adjacent fixed-width columns, the right array layout — measured
|
|
90
|
+
at ~1.3–3.4x over the _same logic written with the plain combinator API_
|
|
91
|
+
(`npm run bench`). This is why
|
|
92
|
+
- inlined JIT friendly code
|
|
93
|
+
- benchmarked hot paths
|
|
94
|
+
- minimal allocations
|
|
95
|
+
- v8 and Node.js specific optimizations
|
|
96
|
+
- **Correct on the gotchas that otherwise quietly break.** UUID byte
|
|
97
|
+
order, `Variant`'s sort-by-type-name discriminant, `DateTime64` sub-second
|
|
98
|
+
precision, signed-high-word wide integers, faithful decimals, `Dynamic`/`JSON`
|
|
99
|
+
self-description, transparent wrappers, opaque `AggregateFunction` — each
|
|
100
|
+
encoded with a live, server-verified test ([details below](#correctness-on-the-gotcha-heavy-types)).
|
|
101
|
+
- **Judgment, not just code.** The skill carries the working knowledge to make
|
|
102
|
+
the right call _before_ writing a line, so the agent neither over- nor
|
|
103
|
+
under-engineers:
|
|
104
|
+
- **Is RowBinary even right?** For string-heavy results read as text, a `JSON*`
|
|
105
|
+
format + V8's native `JSON.parse` (plus `gzip`/`zstd`) can beat a JS RowBinary
|
|
106
|
+
decoder — reach for RowBinary when the data is numeric / wide-integer /
|
|
107
|
+
binary-blob heavy.
|
|
108
|
+
- **Whole buffer or stream?** Drop the `advance()` bounds checks for a complete
|
|
109
|
+
in-memory buffer (faster); keep them for a chunked HTTP response that must
|
|
110
|
+
survive rows straddling chunk boundaries.
|
|
111
|
+
- **Drop the portability scaffolding.** RowBinary is little-endian and the
|
|
112
|
+
target is x86/ARM, so the skill steers away from big-endian / byte-swap
|
|
113
|
+
"portability" code a cautious one-shot pass tends to add.
|
|
114
|
+
- **Improves smaller models' performance.** Because the skill hands over the
|
|
115
|
+
hard-won answers up front, it lifts a weaker model the most. In a 24-eval
|
|
116
|
+
with-skill vs no-skill benchmark, the skill [raised](eval_result_sonnet.md) **Sonnet 4.6** from 60.4% to
|
|
117
|
+
**94.0%** (+34pp) — bringing it level with skill-equipped **Opus 4.8** (94.7%),
|
|
118
|
+
which itself [gained](eval_result.md) +23pp (71.5% → 94.7%). Composer 2.5 Fast
|
|
119
|
+
[got](eval_result_composer.md) a 3x parser performance boost, Haiku 4.5
|
|
120
|
+
[raised](eval_result_haiku.md) from 52% to 86% — the skill closes
|
|
121
|
+
most of the model-capability gap on this task.
|
|
122
|
+
|
|
123
|
+
## What it does
|
|
124
|
+
|
|
125
|
+
Given the columns of a query result — their names and ClickHouse type
|
|
126
|
+
definitions (as returned by `RowBinaryWithNamesAndTypes`, or supplied by the
|
|
127
|
+
user) — the skill generates parser code tailored to exactly those types. Rather
|
|
128
|
+
than shipping a generic, runtime-driven decoder, it emits straight-line code
|
|
129
|
+
that reads each column in order, so the parser only contains the logic the
|
|
130
|
+
specific result shape needs.
|
|
131
|
+
|
|
132
|
+
**Schema only known at runtime?** `compileRowBinaryWithNamesAndTypes(cursor)`
|
|
133
|
+
reads the `RowBinaryWithNamesAndTypes` header and folds each column type into a
|
|
134
|
+
reader on the fly (type strings parsed by `@clickhouse/datatype-parser`),
|
|
135
|
+
returning a `readRows` driver for the rest of the stream — a generic, no-codegen
|
|
136
|
+
path for dynamic schemas. The specialized codegen above stays the fast path when
|
|
137
|
+
the types are fixed.
|
|
138
|
+
|
|
139
|
+
## Correctness on the gotcha-heavy types
|
|
140
|
+
|
|
141
|
+
For a plain `UInt64, String, DateTime` result a strong model already writes fast,
|
|
142
|
+
correct code on its own. The skill earns its keep on the **long tail of RowBinary
|
|
143
|
+
traps** — the encodings where a from-scratch decoder is quietly wrong — each one
|
|
144
|
+
captured here with a live, server-verified test:
|
|
145
|
+
|
|
146
|
+
- **UUID** — two little-endian `UInt64` halves, each byte-reversed vs. the text
|
|
147
|
+
form (not 16 bytes in order).
|
|
148
|
+
- **`Variant(...)`** — the 1-byte discriminant indexes the alternatives sorted by
|
|
149
|
+
**type name** (ClickHouse globally sorts them), NOT declaration order; `0xFF`
|
|
150
|
+
is NULL.
|
|
151
|
+
- **`DateTime64(P)`** — returned as `[Date, nanoseconds]` so the sub-second part
|
|
152
|
+
isn't lost to a `Date`'s millisecond resolution; `Time`/`Time64` are durations,
|
|
153
|
+
not instants.
|
|
154
|
+
- **Wide integers** — `Int128`/`Int256` compose from 64-bit words with the **high
|
|
155
|
+
word read signed**; 64-bit values stay `bigint`, never a lossy `number`.
|
|
156
|
+
- **Decimals** — kept as the exact `[unscaled, scale]` pair, not a lossy float.
|
|
157
|
+
- **`Dynamic` / `JSON`** — self-describing: a per-value binary type encoding, then
|
|
158
|
+
the value; declared typed `JSON` paths are written without a tag (need the
|
|
159
|
+
schema). Wrappers are erased (`Nullable`/`Variant` → concrete type).
|
|
160
|
+
- **Transparent wrappers** — `LowCardinality(T)` / `SimpleAggregateFunction(f, T)`
|
|
161
|
+
decode as the inner `T` (no dictionary layer in RowBinary); `Nested(...)` is
|
|
162
|
+
`Array(Tuple(...))` with no wire of its own.
|
|
163
|
+
- **`AggregateFunction(...)`** — opaque, unframed state: not decodable or even
|
|
164
|
+
skippable; finalize server-side instead.
|
|
165
|
+
- **`FixedString`** preserves trailing NUL padding; **`Enum`** decodes to the
|
|
166
|
+
underlying int (the name map is metadata); **`BFloat16`** is the top 16 bits of
|
|
167
|
+
a `Float32`.
|
|
168
|
+
|
|
169
|
+
This is also where a raw model is most likely to go wrong. In a clean-room test
|
|
170
|
+
on a `Variant` / `UUID` / `DateTime64` / `LowCardinality` schema, a no-skill
|
|
171
|
+
Sonnet produced a **silently wrong UUID** (treated the bytes as plain, missing
|
|
172
|
+
the two-reversed-halves layout), and a no-skill Opus got it right only after
|
|
173
|
+
**three web searches**. The skill hands over these answers up front — correct by
|
|
174
|
+
construction, no lookups. See `baseline/README.md` for the full control.
|
|
175
|
+
|
|
176
|
+
And the failure isn't a one-off — it's a coin-flip. Running the same no-skill
|
|
177
|
+
Sonnet on the `orders` schema (`UInt8, UUID, Decimal64(2), Enum8`) **5 times in
|
|
178
|
+
isolation**, only **3 of 5** runs decoded correctly; both failures were the same
|
|
179
|
+
UUID byte-order scramble. Even the passing runs varied ~1.9x in generated-code
|
|
180
|
+
throughput. With the skill, every run is correct. So a single A/B undersells the
|
|
181
|
+
gap: from scratch the model is right roughly 60% of the time and silently wrong
|
|
182
|
+
the rest, while the skill makes correctness deterministic.
|
|
183
|
+
|
|
184
|
+
## Examples
|
|
185
|
+
|
|
186
|
+
Six end-to-end examples live in [EXAMPLES.md](EXAMPLES.md). Each ships both an API-combinator
|
|
187
|
+
reader and an optimized, monomorphized one, with a runnable round-trip test and
|
|
188
|
+
a benchmark — so the speedups below are measured, not claimed (Node 24 / V8;
|
|
189
|
+
`npm run bench` for your own numbers):
|
|
190
|
+
|
|
191
|
+
| Example | Columns | Optimized speedup |
|
|
192
|
+
| ----------------- | ---------------------------------------------------- | ------------------- |
|
|
193
|
+
| **orders** | `UUID`, `Decimal64`, `Enum8` | **~3.4x** |
|
|
194
|
+
| **carts** | nested `Array(Tuple(...))`, `Array(Nullable(...))` | **~2.0x** |
|
|
195
|
+
| **telemetry** | `Map`, `Array(Float64)`, `Nullable`, named `Tuple` | **~1.4x** |
|
|
196
|
+
| **observability** | `Variant`, `DateTime64(3)`, `LowCardinality`, nested | **~1.4x** |
|
|
197
|
+
| **profiles** | `Array(String)`, `Nullable(Int32)` | **~1.3x** |
|
|
198
|
+
| **events** | `UInt64`, `String`, `DateTime` scalars | **~1.05x — on par** |
|
|
199
|
+
|
|
200
|
+
Two axes drive the win. **Composite structure** is one: monomorphization pays in
|
|
201
|
+
proportion to how many per-row combinator closures it removes (`carts` /
|
|
202
|
+
`telemetry` / `observability`). **Per-row formatting** is the other, independent
|
|
203
|
+
of composites: `orders` is all-scalar yet the biggest win (~3.4x), almost
|
|
204
|
+
entirely from swapping the BigInt UUID formatter for the lookup-table
|
|
205
|
+
`formatUUIDTable`. The genuinely flat case — a scalar row with no hot formatter
|
|
206
|
+
(`events`) — is on par, so the simpler API reader is the right call there.
|
|
207
|
+
Measure, don't assume.
|
|
208
|
+
|
|
209
|
+
## Scope
|
|
210
|
+
|
|
211
|
+
- **In scope:** `RowBinary`, `RowBinaryWithNames`, and
|
|
212
|
+
`RowBinaryWithNamesAndTypes` decoding for Node.js — full-buffer and streaming
|
|
213
|
+
(chunked) via `advance()`/`NeedMoreData`, `readRows()`, and the async
|
|
214
|
+
`streamRowBatches()` (with a built-in small-chunk warning and the optional
|
|
215
|
+
`coalesceChunks()` debounce filter).
|
|
216
|
+
- **Planned:** RowBinary **writing / encoding** (the inverse of everything above)
|
|
217
|
+
- **Out of scope (for now):** browsers and Edge runtimes, non-RowBinary formats
|
|
218
|
+
(JSON / CSV / TSV / Parquet), and big-endian hosts.
|
|
219
|
+
|
|
220
|
+
## The spirit
|
|
221
|
+
|
|
222
|
+
A RowBinary parser generator is a narrow thing. But it's built as an instance of
|
|
223
|
+
a broader bet about what libraries become once a capable LLM is part of the
|
|
224
|
+
toolchain. Three shifts, each already visible in this repo:
|
|
225
|
+
|
|
226
|
+
- **Self-modifiable software.** The library deliberately ships _several_
|
|
227
|
+
equivalent decoders for the same type — `readUUID` / `readUUIDBigInt` /
|
|
228
|
+
`readUUIDHiLo`, `formatUUID` / `formatUUIDTable`, `new Array(n)` vs `[]`+push,
|
|
229
|
+
streaming vs whole-buffer — because the fastest one depends on the workload,
|
|
230
|
+
not the type. Today the agent picks at generation time from measured
|
|
231
|
+
benchmarks. The next step is to pair the skill with a tracing layer that runs
|
|
232
|
+
variant A against variant B _on the live workload_ and keeps whichever wins for
|
|
233
|
+
this data shape and access pattern — a parser that re-tunes itself as the
|
|
234
|
+
traffic drifts, instead of freezing one author's guess into a release.
|
|
235
|
+
|
|
236
|
+
- **Custom software.** The value here isn't a fixed high-level API; it's the
|
|
237
|
+
benchmarked building blocks plus the judgment to combine them. So the end user
|
|
238
|
+
doesn't bend their code to the authors' generic surface — they have the agent
|
|
239
|
+
assemble the high-level API _they_ actually want, shaped to their queries, row
|
|
240
|
+
shapes, and latency/memory budget. Two teams with different workloads grow two
|
|
241
|
+
different libraries from the same primitives, and neither inherits a design
|
|
242
|
+
decision that was only ever right for the original authors' use case.
|
|
243
|
+
|
|
244
|
+
- **Read-write libraries.** For either of the above to be safe, the source has to
|
|
245
|
+
be legible to an LLM, not merely runnable. So this repo is written _read-write_:
|
|
246
|
+
every tradeoff is commented where it's made — the per-column ClickHouse type
|
|
247
|
+
annotations, the `SAFE TO TOGGLE` markers on the fast variants, each reader's
|
|
248
|
+
doc comment carrying its exact monomorphized form. An LLM can
|
|
249
|
+
read _why_ a decision was made and change it in depth with confidence — not
|
|
250
|
+
just call the public functions, but safely rework the internals.
|
|
251
|
+
|
|
252
|
+
The through-line: the last mile is glue the LLM writes over stable, benchmarked
|
|
253
|
+
blocks, so the authors' job shrinks to exporting good primitives and documenting
|
|
254
|
+
their tradeoffs honestly — rather than trying to bake the right performance
|
|
255
|
+
constants for every possible workload into the library ahead of time.
|