@housekit/orm 0.1.25 → 0.1.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +249 -134
- package/dist/builders/insert.d.ts +25 -22
- package/dist/builders/select.d.ts +3 -1
- package/dist/client.d.ts +15 -0
- package/dist/column.d.ts +2 -2
- package/dist/index.d.ts +45 -6
- package/dist/index.js +445 -150
- package/dist/relational.d.ts +92 -20
- package/dist/schema-builder.d.ts +20 -4
- package/dist/table.d.ts +11 -1
- package/dist/utils/batch-transform.d.ts +0 -2
- package/dist/utils/binary-serializer.d.ts +57 -0
- package/dist/utils/binary-worker-code.d.ts +1 -1
- package/dist/utils/binary-worker-pool.d.ts +2 -2
- package/dist/utils/insert-processing.d.ts +7 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -395,8 +395,8 @@ import { existsSync, readdirSync } from "fs";
|
|
|
395
395
|
// src/client.ts
|
|
396
396
|
import { createClient as createChClient } from "@clickhouse/client";
|
|
397
397
|
import { Readable as Readable2 } from "stream";
|
|
398
|
-
import
|
|
399
|
-
import
|
|
398
|
+
import http2 from "http";
|
|
399
|
+
import https2 from "https";
|
|
400
400
|
|
|
401
401
|
// src/column.ts
|
|
402
402
|
class ClickHouseColumn {
|
|
@@ -1040,27 +1040,41 @@ function chTable(tableName, columns, options = {}) {
|
|
|
1040
1040
|
finalEngineSQL = `ReplacingMergeTree(${finalOptions.versionColumn})`;
|
|
1041
1041
|
}
|
|
1042
1042
|
let sql = `CREATE TABLE IF NOT EXISTS \`${tableName}\`${clusterClause} (${colDefs.join(", ")}) ENGINE = ${finalEngineSQL}`;
|
|
1043
|
+
const mapColumnList = (val) => {
|
|
1044
|
+
if (!val)
|
|
1045
|
+
return "";
|
|
1046
|
+
const list = Array.isArray(val) ? val : val.split(",").map((v) => v.trim());
|
|
1047
|
+
return list.map((id) => {
|
|
1048
|
+
const col = columns[id];
|
|
1049
|
+
if (col)
|
|
1050
|
+
return `\`${col.name}\``;
|
|
1051
|
+
if (/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(id)) {
|
|
1052
|
+
return `\`${id}\``;
|
|
1053
|
+
}
|
|
1054
|
+
return id;
|
|
1055
|
+
}).join(", ");
|
|
1056
|
+
};
|
|
1043
1057
|
if (finalOptions.orderBy) {
|
|
1044
|
-
const orderBy =
|
|
1058
|
+
const orderBy = mapColumnList(finalOptions.orderBy);
|
|
1045
1059
|
sql += ` ORDER BY (${orderBy})`;
|
|
1046
1060
|
} else if (isMergeTreeFamilyEngine) {
|
|
1047
1061
|
if (finalOptions.primaryKey) {
|
|
1048
|
-
const pk =
|
|
1062
|
+
const pk = mapColumnList(finalOptions.primaryKey);
|
|
1049
1063
|
sql += ` ORDER BY (${pk})`;
|
|
1050
1064
|
} else {
|
|
1051
1065
|
sql += ` ORDER BY tuple()`;
|
|
1052
1066
|
}
|
|
1053
1067
|
}
|
|
1054
1068
|
if (finalOptions.primaryKey) {
|
|
1055
|
-
const pk =
|
|
1069
|
+
const pk = mapColumnList(finalOptions.primaryKey);
|
|
1056
1070
|
sql += ` PRIMARY KEY (${pk})`;
|
|
1057
1071
|
}
|
|
1058
1072
|
if (finalOptions.partitionBy) {
|
|
1059
|
-
const partition =
|
|
1073
|
+
const partition = mapColumnList(finalOptions.partitionBy);
|
|
1060
1074
|
sql += ` PARTITION BY (${partition})`;
|
|
1061
1075
|
}
|
|
1062
1076
|
if (finalOptions.sampleBy) {
|
|
1063
|
-
const sample =
|
|
1077
|
+
const sample = mapColumnList(finalOptions.sampleBy);
|
|
1064
1078
|
sql += ` SAMPLE BY (${sample})`;
|
|
1065
1079
|
}
|
|
1066
1080
|
if (finalOptions.ttl) {
|
|
@@ -3498,14 +3512,27 @@ class ClickHouseQueryBuilder {
|
|
|
3498
3512
|
}
|
|
3499
3513
|
|
|
3500
3514
|
// src/utils/insert-processing.ts
|
|
3501
|
-
|
|
3502
|
-
|
|
3515
|
+
var hasNativeUUID = typeof crypto !== "undefined" && typeof crypto.randomUUID === "function";
|
|
3516
|
+
var uuidFns = null;
|
|
3517
|
+
function loadUUIDFns() {
|
|
3518
|
+
if (uuidFns)
|
|
3519
|
+
return uuidFns;
|
|
3520
|
+
const uuid = __require("uuid");
|
|
3521
|
+
uuidFns = {
|
|
3522
|
+
1: uuid.v1,
|
|
3523
|
+
4: hasNativeUUID ? () => crypto.randomUUID() : uuid.v4,
|
|
3524
|
+
6: uuid.v6,
|
|
3525
|
+
7: uuid.v7
|
|
3526
|
+
};
|
|
3527
|
+
return uuidFns;
|
|
3528
|
+
}
|
|
3529
|
+
function buildInsertPlan(table, options) {
|
|
3503
3530
|
const columns = [];
|
|
3504
3531
|
const keyToColumn = new Map;
|
|
3505
3532
|
for (const [propKey, colRaw] of Object.entries(table.$columns)) {
|
|
3506
3533
|
const column = colRaw;
|
|
3507
3534
|
const columnName = column.name;
|
|
3508
|
-
const transform = createTransform(column);
|
|
3535
|
+
const transform = createTransform(column, options?.skipValidation);
|
|
3509
3536
|
const autoUUIDVersion = column.meta?.autoGenerate?.type === "uuid" ? normalizeUUIDVersion(column.meta.autoGenerate.version) : null;
|
|
3510
3537
|
const prepared = {
|
|
3511
3538
|
propKey,
|
|
@@ -3524,7 +3551,7 @@ function buildInsertPlan(table) {
|
|
|
3524
3551
|
}
|
|
3525
3552
|
const columnNames = columns.map((c) => c.columnName);
|
|
3526
3553
|
const useCompact = columns.every((c) => !c.useServerUUID);
|
|
3527
|
-
return { columns, keyToColumn, columnNames, useCompact };
|
|
3554
|
+
return { columns, keyToColumn, columnNames, useCompact, skipValidation: options?.skipValidation };
|
|
3528
3555
|
}
|
|
3529
3556
|
function processRowWithPlan(row, plan, mode = plan.useCompact ? "compact" : "json") {
|
|
3530
3557
|
if (mode === "compact") {
|
|
@@ -3558,10 +3585,10 @@ async function* processRowsStream(rows, plan, mode = plan.useCompact ? "compact"
|
|
|
3558
3585
|
yield processRowWithPlan(row, plan, mode);
|
|
3559
3586
|
}
|
|
3560
3587
|
}
|
|
3561
|
-
function createTransform(col) {
|
|
3588
|
+
function createTransform(col, skipValidation) {
|
|
3562
3589
|
const hasJson = Boolean(col.meta?.isJson);
|
|
3563
3590
|
const enumValues = col.meta?.enumValues;
|
|
3564
|
-
const enumSet = enumValues ? new Set(enumValues) : null;
|
|
3591
|
+
const enumSet = !skipValidation && enumValues ? new Set(enumValues) : null;
|
|
3565
3592
|
const needsJson = hasJson;
|
|
3566
3593
|
const needsEnum = Boolean(enumSet);
|
|
3567
3594
|
if (!needsJson && !needsEnum) {
|
|
@@ -3599,22 +3626,18 @@ function normalizeUUIDVersion(version) {
|
|
|
3599
3626
|
return 4;
|
|
3600
3627
|
}
|
|
3601
3628
|
function generateUUID(version) {
|
|
3602
|
-
|
|
3603
|
-
|
|
3604
|
-
|
|
3605
|
-
|
|
3606
|
-
|
|
3607
|
-
|
|
3608
|
-
|
|
3609
|
-
|
|
3610
|
-
|
|
3611
|
-
|
|
3612
|
-
return v6();
|
|
3613
|
-
case 7:
|
|
3614
|
-
return v7();
|
|
3615
|
-
default:
|
|
3616
|
-
return v4();
|
|
3629
|
+
if (version === 4 && hasNativeUUID) {
|
|
3630
|
+
return crypto.randomUUID();
|
|
3631
|
+
}
|
|
3632
|
+
const fns = loadUUIDFns();
|
|
3633
|
+
const fn2 = fns[version];
|
|
3634
|
+
if (!fn2) {
|
|
3635
|
+
if (version === 3 || version === 5) {
|
|
3636
|
+
throw new Error(`UUID v${version} requires a name and namespace. Use v4, v6, or v7 for auto-generation.`);
|
|
3637
|
+
}
|
|
3638
|
+
return fns[4]();
|
|
3617
3639
|
}
|
|
3640
|
+
return fn2();
|
|
3618
3641
|
}
|
|
3619
3642
|
function processRowCompact(row, plan) {
|
|
3620
3643
|
const out = new Array(plan.columns.length);
|
|
@@ -3655,15 +3678,12 @@ class BatchTransformStream extends Transform {
|
|
|
3655
3678
|
plan;
|
|
3656
3679
|
mode;
|
|
3657
3680
|
batch = [];
|
|
3658
|
-
processingTimer = null;
|
|
3659
3681
|
batchSize;
|
|
3660
|
-
maxProcessingTime;
|
|
3661
3682
|
constructor(plan, mode, options = {}) {
|
|
3662
3683
|
super({ objectMode: true });
|
|
3663
3684
|
this.plan = plan;
|
|
3664
3685
|
this.mode = mode;
|
|
3665
3686
|
this.batchSize = options.batchSize || 100;
|
|
3666
|
-
this.maxProcessingTime = options.maxProcessingTime || 10;
|
|
3667
3687
|
}
|
|
3668
3688
|
_transform(chunk, _encoding, callback) {
|
|
3669
3689
|
this.batch.push(chunk);
|
|
@@ -3681,35 +3701,20 @@ class BatchTransformStream extends Transform {
|
|
|
3681
3701
|
}
|
|
3682
3702
|
}
|
|
3683
3703
|
scheduleProcessing(callback) {
|
|
3684
|
-
|
|
3685
|
-
return;
|
|
3686
|
-
}
|
|
3687
|
-
this.processingTimer = setTimeout(() => {
|
|
3688
|
-
if (this.batch.length > 0) {
|
|
3689
|
-
this.processBatch(callback);
|
|
3690
|
-
} else {
|
|
3691
|
-
callback();
|
|
3692
|
-
}
|
|
3693
|
-
}, this.maxProcessingTime);
|
|
3704
|
+
callback();
|
|
3694
3705
|
}
|
|
3695
3706
|
processBatch(callback) {
|
|
3696
|
-
if (this.processingTimer) {
|
|
3697
|
-
clearTimeout(this.processingTimer);
|
|
3698
|
-
this.processingTimer = null;
|
|
3699
|
-
}
|
|
3700
3707
|
const batchToProcess = this.batch;
|
|
3701
3708
|
this.batch = [];
|
|
3702
|
-
|
|
3703
|
-
|
|
3704
|
-
|
|
3705
|
-
|
|
3706
|
-
this.push(processed);
|
|
3707
|
-
}
|
|
3708
|
-
callback();
|
|
3709
|
-
} catch (error) {
|
|
3710
|
-
callback(error);
|
|
3709
|
+
try {
|
|
3710
|
+
for (const row of batchToProcess) {
|
|
3711
|
+
const processed = processRowWithPlan(row, this.plan, this.mode);
|
|
3712
|
+
this.push(processed);
|
|
3711
3713
|
}
|
|
3712
|
-
|
|
3714
|
+
callback();
|
|
3715
|
+
} catch (error) {
|
|
3716
|
+
callback(error);
|
|
3717
|
+
}
|
|
3713
3718
|
}
|
|
3714
3719
|
}
|
|
3715
3720
|
function createBatchTransformStream(plan, mode, options) {
|
|
@@ -3722,10 +3727,46 @@ import { EventEmitter } from "events";
|
|
|
3722
3727
|
import * as os from "os";
|
|
3723
3728
|
|
|
3724
3729
|
// src/utils/binary-worker-code.ts
|
|
3725
|
-
var binaryWorkerCode = 'import{parentPort as D,workerData as d}from"worker_threads";var X=[];for(let x=0;x<256;++x)X.push((x+256).toString(16).slice(1));function B(x,U=0){return(X[x[U+0]]+X[x[U+1]]+X[x[U+2]]+X[x[U+3]]+"-"+X[x[U+4]]+X[x[U+5]]+"-"+X[x[U+6]]+X[x[U+7]]+"-"+X[x[U+8]]+X[x[U+9]]+"-"+X[x[U+10]]+X[x[U+11]]+X[x[U+12]]+X[x[U+13]]+X[x[U+14]]+X[x[U+15]]).toLowerCase()}import{randomFillSync as o}from"node:crypto";var N=new Uint8Array(256),G=N.length;function P(){if(G>N.length-16)o(N),G=0;return N.slice(G,G+=16)}import{randomUUID as n}from"node:crypto";var S={randomUUID:n};function y(x,U,I){x=x||{};let A=x.random??x.rng?.()??P();if(A.length<16)throw Error("Random bytes length must be >= 16");if(A[6]=A[6]&15|64,A[8]=A[8]&63|128,U){if(I=I||0,I<0||I+16>U.length)throw RangeError(`UUID byte range ${I}:${I+15} is out of buffer bounds`);for(let H=0;H<16;++H)U[I+H]=A[H];return U}return B(A)}function p(x,U,I){if(S.randomUUID&&!U&&!x)return S.randomUUID();return y(x,U,I)}var W=p;class z{buffer;offset=0;constructor(x=4096){this.buffer=Buffer.allocUnsafe(x)}ensureCapacity(x){let U=this.offset+x;if(U>this.buffer.length){let I=Math.max(this.buffer.length*2,U),A=Buffer.allocUnsafe(I);this.buffer.copy(A,0,0,this.offset),this.buffer=A}}reset(){this.offset=0}getBuffer(){return this.buffer.subarray(0,this.offset)}toBuffer(){return Buffer.from(this.buffer.subarray(0,this.offset))}writeInt8(x){this.ensureCapacity(1),this.buffer.writeInt8(x,this.offset),this.offset+=1}writeUInt8(x){this.ensureCapacity(1),this.buffer.writeUInt8(x,this.offset),this.offset+=1}writeInt16(x){this.ensureCapacity(2),this.buffer.writeInt16LE(x,this.offset),this.offset+=2}writeUInt16(x){this.ensureCapacity(2),this.buffer.writeUInt16LE(x,this.offset),this.offset+=2}writeInt32(x){this.ensureCapacity(4),this.buffer.writeInt32LE(x,this.offset),this.offset+=4}writeUInt32(x){this.ensureCapacity(4),this.buffer.writeUInt32LE(x,this.offset),this.offset+=4}writeInt64(x){this.ensureCapacity(8),this.buffer.writeBigInt64LE(BigInt(x),this.offset),this.offset+=8}writeUInt64(x){this.ensureCapacity(8),this.buffer.writeBigUInt64LE(BigInt(x),this.offset),this.offset+=8}writeInt128(x){this.ensureCapacity(16);let U=x&BigInt("0xFFFFFFFFFFFFFFFF"),I=x>>BigInt(64);this.buffer.writeBigUInt64LE(U,this.offset),this.buffer.writeBigInt64LE(I,this.offset+8),this.offset+=16}writeUInt128(x){this.ensureCapacity(16);let U=x&BigInt("0xFFFFFFFFFFFFFFFF"),I=x>>BigInt(64);this.buffer.writeBigUInt64LE(U,this.offset),this.buffer.writeBigUInt64LE(I,this.offset+8),this.offset+=16}writeInt256(x){this.ensureCapacity(32);for(let U=0;U<4;U++){let I=x&BigInt("0xFFFFFFFFFFFFFFFF");this.buffer.writeBigUInt64LE(I,this.offset+U*8),x>>=BigInt(64)}this.offset+=32}writeUInt256(x){this.writeInt256(x)}writeFloat32(x){this.ensureCapacity(4),this.buffer.writeFloatLE(x,this.offset),this.offset+=4}writeFloat64(x){this.ensureCapacity(8),this.buffer.writeDoubleLE(x,this.offset),this.offset+=8}writeVarInt(x){if(x<128){this.ensureCapacity(1),this.buffer[this.offset++]=x;return}if(x<16384){this.ensureCapacity(2),this.buffer[this.offset++]=x&127|128,this.buffer[this.offset++]=x>>7;return}if(x<2097152){this.ensureCapacity(3),this.buffer[this.offset++]=x&127|128,this.buffer[this.offset++]=x>>7&127|128,this.buffer[this.offset++]=x>>14;return}this.ensureCapacity(10);while(x>=128)this.buffer[this.offset++]=x&127|128,x>>>=7;this.buffer[this.offset++]=x}writeString(x){let U=Buffer.from(x,"utf-8");this.writeVarInt(U.length),this.ensureCapacity(U.length),U.copy(this.buffer,this.offset),this.offset+=U.length}writeFixedString(x,U){this.ensureCapacity(U);let I=Buffer.from(x,"utf-8"),A=Math.min(I.length,U);if(I.copy(this.buffer,this.offset,0,A),A<U)this.buffer.fill(0,this.offset+A,this.offset+U);this.offset+=U}writeBytes(x){this.ensureCapacity(x.length),x.copy(this.buffer,this.offset),this.offset+=x.length}writeUUID(x){this.ensureCapacity(16);let U=x.replace(/-/g,"");if(U.length!==32)throw Error(`Invalid UUID: ${x}`);let I=Buffer.from(U,"hex");for(let A=7;A>=0;A--)this.buffer[this.offset++]=I[A];for(let A=15;A>=8;A--)this.buffer[this.offset++]=I[A]}writeDate(x){let U=typeof x==="number"?x:Math.floor(x.getTime()/86400000);this.writeUInt16(U)}writeDate32(x){let U=typeof x==="number"?x:Math.floor(x.getTime()/86400000);this.writeInt32(U)}writeDateTime(x){let U=typeof x==="number"?x:Math.floor(x.getTime()/1000);this.writeUInt32(U)}writeDateTime64(x,U=3){let I=Math.pow(10,U),A=typeof x==="number"?BigInt(Math.round(x*I)):BigInt(Math.round(x.getTime()*I/1000));this.writeInt64(A)}writeNullable(x){this.writeUInt8(x?1:0)}writeArrayLength(x){this.writeUInt64(x)}writeDecimal32(x,U){let I=Math.round(x*Math.pow(10,U));this.writeInt32(I)}writeDecimal64(x,U){let I=BigInt(Math.round(x*Math.pow(10,U)));this.writeInt64(I)}writeDecimal128(x,U){let I;if(typeof x==="bigint")I=x*BigInt(Math.pow(10,U));else I=BigInt(Math.round(x*Math.pow(10,U)));this.writeInt128(I)}writeBool(x){this.writeUInt8(x?1:0)}writeIPv4(x){if(typeof x==="number"){this.writeUInt32(x);return}let U=x.split(".").map(Number);if(U.length!==4)throw Error(`Invalid IPv4: ${x}`);let I=U[0]<<24|U[1]<<16|U[2]<<8|U[3];this.writeUInt32(I>>>0)}writeIPv6(x){if(Buffer.isBuffer(x)){if(x.length!==16)throw Error("IPv6 must be 16 bytes");this.writeBytes(x);return}let I=this.expandIPv6(x).split(":");this.ensureCapacity(16);for(let A of I){let H=parseInt(A,16);this.buffer.writeUInt16BE(H,this.offset),this.offset+=2}}expandIPv6(x){let U=x.split("::");if(U.length===1)return x;let I=U[0]?U[0].split(":"):[],A=U[1]?U[1].split(":"):[],H=8-I.length-A.length,K=Array(H).fill("0000");return[...I,...K,...A].map((L)=>L.padStart(4,"0")).join(":")}writeEnum8(x){this.writeInt8(x)}writeEnum16(x){this.writeInt16(x)}}function m(x){let U=[],I="",A=0;for(let H=0;H<x.length;H++){let K=x[H];if(K===","&&A===0)U.push(I.trim()),I="";else{if(K==="(")A++;if(K===")")A--;I+=K}}if(I.trim())U.push(I.trim());return U}function Q(x,U=!1){let I=x.toLowerCase().trim(),A=I.match(/^nullable\\((.+)\\)$/);if(A){let C=Q(A[1],!1);return($,F)=>{if(F===null||F===void 0)$.writeNullable(!0);else $.writeNullable(!1),C($,F)}}let H=I.match(/^array\\((.+)\\)$/);if(H){let C=Q(H[1],!1);return($,F)=>{let O=Array.isArray(F)?F:[];$.writeArrayLength(O.length);for(let J of O)C($,J)}}let K=I.match(/^map\\((.+)\\)$/);if(K){let[C,$]=m(K[1]);if(!C||!$)throw Error(`Invalid Map type: ${I}`);let F=Q(C),O=Q($);return(J,j)=>{let V;if(j instanceof Map)V=Array.from(j.entries());else if(typeof j==="object"&&j!==null)V=Object.entries(j);else V=[];J.writeUInt64(V.length);for(let[Y,q]of V)F(J,Y),O(J,q)}}let L=I.match(/^tuple\\((.+)\\)$/);if(L){let $=m(L[1]).map((F)=>Q(F));return(F,O)=>{let J=Array.isArray(O)?O:[];for(let j=0;j<$.length;j++)$[j](F,J[j])}}let T=I.match(/^nested\\((.+)\\)$/);if(T){let C=m(T[1]),F=`Tuple(${C.map((J)=>{let j=J.trim().split(/\\s+/);if(j.length<2)return"String";return j.slice(1).join(" ")}).join(", ")})`,O=Q(F);return(J,j)=>{let V=Array.isArray(j)?j:[];J.writeArrayLength(V.length);for(let Y of V){let q=Y;if(!Array.isArray(Y)&&typeof Y==="object")q=C.map((h)=>{let g=h.trim().split(/\\s+/)[0];return Y[g]});O(J,q)}}}let b=I.match(/^lowcardinality\\((.+)\\)$/);if(b)return Q(b[1]);let k=I.match(/^fixedstring\\((\\d+)\\)$/);if(k){let C=parseInt(k[1],10);return($,F)=>{$.writeFixedString(String(F??""),C)}}let E=I.match(/^decimal(?:32|64|128)?\\((\\d+),\\s*(\\d+)\\)$/);if(E){let C=parseInt(E[2],10);if(I.includes("decimal128"))return($,F)=>$.writeDecimal128(F,C);if(I.includes("decimal64"))return($,F)=>$.writeDecimal64(F,C);return($,F)=>$.writeDecimal32(F,C)}let M=I.match(/^datetime64\\((\\d+)/);if(M){let C=parseInt(M[1],10);return($,F)=>$.writeDateTime64(F,C)}if(I.match(/^enum8\\(/))return(C,$)=>C.writeEnum8(Number($));if(I.match(/^enum16\\(/))return(C,$)=>C.writeEnum16(Number($));let R=w(I);if(U)return(C,$)=>{if($===null||$===void 0)C.writeNullable(!0);else C.writeNullable(!1),R(C,$)};if(I==="uuid")return(C,$)=>{if($===void 0||$===null)R(C,W());else R(C,$)};return R}function w(x){if(x==="int8")return(U,I)=>U.writeInt8(Number(I));if(x==="uint8")return(U,I)=>U.writeUInt8(Number(I));if(x==="int16")return(U,I)=>U.writeInt16(Number(I));if(x==="uint16")return(U,I)=>U.writeUInt16(Number(I));if(x==="int32")return(U,I)=>U.writeInt32(Number(I));if(x==="uint32")return(U,I)=>U.writeUInt32(Number(I));if(x==="int64")return(U,I)=>U.writeInt64(I);if(x==="uint64")return(U,I)=>U.writeUInt64(I);if(x==="int128")return(U,I)=>U.writeInt128(BigInt(I));if(x==="uint128")return(U,I)=>U.writeUInt128(BigInt(I));if(x==="int256")return(U,I)=>U.writeInt256(BigInt(I));if(x==="uint256")return(U,I)=>U.writeUInt256(BigInt(I));if(x==="float32")return(U,I)=>U.writeFloat32(Number(I));if(x==="float64")return(U,I)=>U.writeFloat64(Number(I));if(x==="string")return(U,I)=>U.writeString(String(I??""));if(x==="uuid")return(U,I)=>U.writeUUID(String(I));if(x==="date")return(U,I)=>U.writeDate(I);if(x==="date32")return(U,I)=>U.writeDate32(I);if(x==="datetime")return(U,I)=>U.writeDateTime(I);if(x.startsWith("datetime64"))return(U,I)=>U.writeDateTime64(I,3);if(x==="bool"||x==="boolean")return(U,I)=>U.writeBool(Boolean(I));if(x==="ipv4")return(U,I)=>U.writeIPv4(I);if(x==="ipv6")return(U,I)=>U.writeIPv6(I);return(U,I)=>U.writeString(String(I??""))}var Z=null,_=null;function s(x,U){if(!Z||!_)throw Error("Worker not configured");_.reset();for(let I of x)for(let A=0;A<Z.columns.length;A++){let H=Z.columns[A],K=I[H.name];Z.encoders[A](_,K)}return _.toBuffer()}function f(x){try{switch(x.type){case"configure":{let U=x.columns.map((I)=>Q(I.type,I.isNullable));Z={columns:x.columns,keyMapping:new Map,encoders:U},_=new z(1048576),D?.postMessage({type:"ready"});break}case"serialize":{let U=s(x.rows,x.batchId),I=new ArrayBuffer(U.length);new Uint8Array(I).set(U),D?.postMessage({type:"result",batchId:x.batchId,buffer:Buffer.from(I),rowCount:x.rows.length},[I]);break}case"shutdown":process.exit(0)}}catch(U){D?.postMessage({type:"error",batchId:x.batchId,error:U instanceof Error?U.message:String(U)})}}if(D){if(D.on("message",f),d?.columns)f({type:"configure",columns:d.columns})}\n';
|
|
3730
|
+
var binaryWorkerCode = 'var{defineProperty:i,getOwnPropertyNames:g0,getOwnPropertyDescriptor:M0}=Object,h0=Object.prototype.hasOwnProperty;var m0=new WeakMap,b0=(f)=>{var x=m0.get(f),U;if(x)return x;if(x=i({},"__esModule",{value:!0}),f&&typeof f==="object"||typeof f==="function")g0(f).map((m)=>!h0.call(x,m)&&i(x,m,{get:()=>f[m],enumerable:!(U=M0(f,m))||U.enumerable}));return m0.set(f,x),x};var o0=(f,x)=>{for(var U in x)i(f,U,{get:x[U],enumerable:!0,configurable:!0,set:(m)=>x[U]=()=>m})};var L=(f,x)=>()=>(f&&(x=f(f=0)),x);var A0="ffffffff-ffff-ffff-ffff-ffffffffffff";var D0="00000000-0000-0000-0000-000000000000";var S0;var V0=L(()=>{S0=/^(?:[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$/i});function B0(f){return typeof f==="string"&&S0.test(f)}var Z;var C=L(()=>{V0();Z=B0});function p0(f){if(!Z(f))throw TypeError("Invalid UUID");let x;return Uint8Array.of((x=parseInt(f.slice(0,8),16))>>>24,x>>>16&255,x>>>8&255,x&255,(x=parseInt(f.slice(9,13),16))>>>8,x&255,(x=parseInt(f.slice(14,18),16))>>>8,x&255,(x=parseInt(f.slice(19,23),16))>>>8,x&255,(x=parseInt(f.slice(24,36),16))/1099511627776&255,x/4294967296&255,x>>>24&255,x>>>16&255,x>>>8&255,x&255)}var N;var F=L(()=>{C();N=p0});function X(f,x=0){return(_[f[x+0]]+_[f[x+1]]+_[f[x+2]]+_[f[x+3]]+"-"+_[f[x+4]]+_[f[x+5]]+"-"+_[f[x+6]]+_[f[x+7]]+"-"+_[f[x+8]]+_[f[x+9]]+"-"+_[f[x+10]]+_[f[x+11]]+_[f[x+12]]+_[f[x+13]]+_[f[x+14]]+_[f[x+15]]).toLowerCase()}function y0(f,x=0){let U=X(f,x);if(!Z(U))throw TypeError("Stringified UUID is invalid");return U}var _,R0;var Q=L(()=>{C();_=[];for(let f=0;f<256;++f)_.push((f+256).toString(16).slice(1));R0=y0});import{randomFillSync as i0}from"node:crypto";function Y(){if(M>h.length-16)i0(h),M=0;return h.slice(M,M+=16)}var h,M;var b=L(()=>{h=new Uint8Array(256),M=h.length});function c0(f,x,U){let m,A=f?._v6??!1;if(f){let D=Object.keys(f);if(D.length===1&&D[0]==="_v6")f=void 0}if(f)m=$0(f.random??f.rng?.()??Y(),f.msecs,f.nsecs,f.clockseq,f.node,x,U);else{let D=Date.now(),R=Y();w0(K,D,R),m=$0(R,K.msecs,K.nsecs,A?void 0:K.clockseq,A?void 0:K.node,x,U)}return x??X(m)}function w0(f,x,U){if(f.msecs??=-1/0,f.nsecs??=0,x===f.msecs){if(f.nsecs++,f.nsecs>=1e4)f.node=void 0,f.nsecs=0}else if(x>f.msecs)f.nsecs=0;else if(x<f.msecs)f.node=void 0;if(!f.node)f.node=U.slice(10,16),f.node[0]|=1,f.clockseq=(U[8]<<8|U[9])&16383;return f.msecs=x,f}function $0(f,x,U,m,A,D,R=0){if(f.length<16)throw Error("Random bytes length must be >= 16");if(!D)D=new Uint8Array(16),R=0;else if(R<0||R+16>D.length)throw RangeError(`UUID byte range ${R}:${R+15} is out of buffer bounds`);x??=Date.now(),U??=0,m??=(f[8]<<8|f[9])&16383,A??=f.slice(10,16),x+=12219292800000;let q=((x&268435455)*1e4+U)%4294967296;D[R++]=q>>>24&255,D[R++]=q>>>16&255,D[R++]=q>>>8&255,D[R++]=q&255;let H=x/4294967296*1e4&268435455;D[R++]=H>>>8&255,D[R++]=H&255,D[R++]=H>>>24&15|16,D[R++]=H>>>16&255,D[R++]=m>>>8|128,D[R++]=m&255;for(let n=0;n<6;++n)D[R++]=A[n];return D}var K,o;var w=L(()=>{b();Q();K={};o=c0});function z(f){let x=typeof f==="string"?N(f):f,U=r0(x);return typeof f==="string"?X(U):U}function r0(f){return Uint8Array.of((f[6]&15)<<4|f[7]>>4&15,(f[7]&15)<<4|(f[4]&240)>>4,(f[4]&15)<<4|(f[5]&240)>>4,(f[5]&15)<<4|(f[0]&240)>>4,(f[0]&15)<<4|(f[1]&240)>>4,(f[1]&15)<<4|(f[2]&240)>>4,96|f[2]&15,f[3],f[8],f[9],f[10],f[11],f[12],f[13],f[14],f[15])}var r=L(()=>{F();Q()});import{createHash as t0}from"node:crypto";function s0(f){if(Array.isArray(f))f=Buffer.from(f);else if(typeof f==="string")f=Buffer.from(f,"utf8");return t0("md5").update(f).digest()}var L0;var H0=L(()=>{L0=s0});function u0(f){f=unescape(encodeURIComponent(f));let x=new Uint8Array(f.length);for(let U=0;U<f.length;++U)x[U]=f.charCodeAt(U);return x}function P(f,x,U,m,A,D){let R=typeof U==="string"?u0(U):U,q=typeof m==="string"?N(m):m;if(typeof m==="string")m=N(m);if(m?.length!==16)throw TypeError("Namespace must be array-like (16 iterable integer values, 0-255)");let H=new Uint8Array(16+R.length);if(H.set(q),H.set(R,q.length),H=x(H),H[6]=H[6]&15|f,H[8]=H[8]&63|128,A){D=D||0;for(let n=0;n<16;++n)A[D+n]=H[n];return A}return X(H)}var B="6ba7b810-9dad-11d1-80b4-00c04fd430c8",p="6ba7b811-9dad-11d1-80b4-00c04fd430c8";var t=L(()=>{F();Q()});function s(f,x,U,m){return P(48,L0,f,x,U,m)}var _0;var I0=L(()=>{H0();t();s.DNS=B;s.URL=p;_0=s});import{randomUUID as l0}from"node:crypto";var u;var X0=L(()=>{u={randomUUID:l0}});function v0(f,x,U){f=f||{};let m=f.random??f.rng?.()??Y();if(m.length<16)throw Error("Random bytes length must be >= 16");if(m[6]=m[6]&15|64,m[8]=m[8]&63|128,x){if(U=U||0,U<0||U+16>x.length)throw RangeError(`UUID byte range ${U}:${U+15} is out of buffer bounds`);for(let A=0;A<16;++A)x[U+A]=m[A];return x}return X(m)}function a0(f,x,U){if(u.randomUUID&&!x&&!f)return u.randomUUID();return v0(f,x,U)}var j0;var n0=L(()=>{X0();b();Q();j0=a0});import{createHash as e0}from"node:crypto";function f1(f){if(Array.isArray(f))f=Buffer.from(f);else if(typeof f==="string")f=Buffer.from(f,"utf8");return e0("sha1").update(f).digest()}var q0;var J0=L(()=>{q0=f1});function l(f,x,U,m){return P(80,q0,f,x,U,m)}var N0;var Q0=L(()=>{J0();t();l.DNS=B;l.URL=p;N0=l});function x1(f,x,U){f??={},U??=0;let m=o({...f,_v6:!0},new Uint8Array(16));if(m=z(m),x){for(let A=0;A<16;A++)x[U+A]=m[A];return x}return X(m)}var Y0;var T0=L(()=>{Q();w();r();Y0=x1});function v(f){let x=typeof f==="string"?N(f):f,U=U1(x);return typeof f==="string"?X(U):U}function U1(f){return Uint8Array.of((f[3]&15)<<4|f[4]>>4&15,(f[4]&15)<<4|(f[5]&240)>>4,(f[5]&15)<<4|f[6]&15,f[7],(f[1]&15)<<4|(f[2]&240)>>4,(f[2]&15)<<4|(f[3]&240)>>4,16|(f[0]&240)>>4,(f[0]&15)<<4|(f[1]&240)>>4,f[8],f[9],f[10],f[11],f[12],f[13],f[14],f[15])}var G0=L(()=>{F();Q()});function m1(f,x,U){let m;if(f)m=Z0(f.random??f.rng?.()??Y(),f.msecs,f.seq,x,U);else{let A=Date.now(),D=Y();A1(a,A,D),m=Z0(D,a.msecs,a.seq,x,U)}return x??X(m)}function A1(f,x,U){if(f.msecs??=-1/0,f.seq??=0,x>f.msecs)f.seq=U[6]<<23|U[7]<<16|U[8]<<8|U[9],f.msecs=x;else if(f.seq=f.seq+1|0,f.seq===0)f.msecs++;return f}function Z0(f,x,U,m,A=0){if(f.length<16)throw Error("Random bytes length must be >= 16");if(!m)m=new Uint8Array(16),A=0;else if(A<0||A+16>m.length)throw RangeError(`UUID byte range ${A}:${A+15} is out of buffer bounds`);return x??=Date.now(),U??=f[6]*127<<24|f[7]<<16|f[8]<<8|f[9],m[A++]=x/1099511627776&255,m[A++]=x/4294967296&255,m[A++]=x/16777216&255,m[A++]=x/65536&255,m[A++]=x/256&255,m[A++]=x&255,m[A++]=112|U>>>28&15,m[A++]=U>>>20&255,m[A++]=128|U>>>14&63,m[A++]=U>>>6&255,m[A++]=U<<2&255|f[10]&3,m[A++]=f[11],m[A++]=f[12],m[A++]=f[13],m[A++]=f[14],m[A++]=f[15],m}var a,O0;var C0=L(()=>{b();Q();a={};O0=m1});function D1(f){if(!Z(f))throw TypeError("Invalid UUID");return parseInt(f.slice(14,15),16)}var F0;var K0=L(()=>{C();F0=D1});var z0={};o0(z0,{version:()=>F0,validate:()=>Z,v7:()=>O0,v6ToV1:()=>v,v6:()=>Y0,v5:()=>N0,v4:()=>j0,v3:()=>_0,v1ToV6:()=>z,v1:()=>o,stringify:()=>R0,parse:()=>N,NIL:()=>D0,MAX:()=>A0});var P0=L(()=>{F();Q();w();r();I0();n0();Q0();T0();G0();C0();C();K0()});import{parentPort as W,workerData as k0}from"worker_threads";var S1=typeof crypto<"u"&&typeof crypto.randomUUID==="function",y=null;function V1(){if(S1)return crypto.randomUUID();if(y)return y();return y=(P0(),b0(z0)).v4,y()}class f0{buffer;offset=0;constructor(f=4096){this.buffer=Buffer.allocUnsafe(f)}ensureCapacity(f){let x=this.offset+f;if(x>this.buffer.length){let U=Math.max(this.buffer.length*2,x),m=Buffer.allocUnsafe(U);this.buffer.copy(m,0,0,this.offset),this.buffer=m}}reset(){this.offset=0}getBuffer(){return this.buffer.subarray(0,this.offset)}toBuffer(){return Buffer.from(this.buffer.subarray(0,this.offset))}writeInt8(f){this.ensureCapacity(1),this.buffer.writeInt8(f,this.offset),this.offset+=1}writeUInt8(f){this.ensureCapacity(1),this.buffer.writeUInt8(f,this.offset),this.offset+=1}writeInt16(f){this.ensureCapacity(2),this.buffer.writeInt16LE(f,this.offset),this.offset+=2}writeUInt16(f){this.ensureCapacity(2),this.buffer.writeUInt16LE(f,this.offset),this.offset+=2}writeInt32(f){this.ensureCapacity(4),this.buffer.writeInt32LE(f,this.offset),this.offset+=4}writeUInt32(f){this.ensureCapacity(4),this.buffer.writeUInt32LE(f,this.offset),this.offset+=4}writeInt64(f){this.ensureCapacity(8),this.buffer.writeBigInt64LE(BigInt(f),this.offset),this.offset+=8}writeUInt64(f){this.ensureCapacity(8),this.buffer.writeBigUInt64LE(BigInt(f),this.offset),this.offset+=8}writeInt128(f){this.ensureCapacity(16);let x=f&BigInt("0xFFFFFFFFFFFFFFFF"),U=f>>BigInt(64);this.buffer.writeBigUInt64LE(x,this.offset),this.buffer.writeBigInt64LE(U,this.offset+8),this.offset+=16}writeUInt128(f){this.ensureCapacity(16);let x=f&BigInt("0xFFFFFFFFFFFFFFFF"),U=f>>BigInt(64);this.buffer.writeBigUInt64LE(x,this.offset),this.buffer.writeBigUInt64LE(U,this.offset+8),this.offset+=16}writeInt256(f){this.ensureCapacity(32);for(let x=0;x<4;x++){let U=f&BigInt("0xFFFFFFFFFFFFFFFF");this.buffer.writeBigUInt64LE(U,this.offset+x*8),f>>=BigInt(64)}this.offset+=32}writeUInt256(f){this.writeInt256(f)}writeFloat32(f){this.ensureCapacity(4),this.buffer.writeFloatLE(f,this.offset),this.offset+=4}writeFloat64(f){this.ensureCapacity(8),this.buffer.writeDoubleLE(f,this.offset),this.offset+=8}writeVarInt(f){if(f<128){this.ensureCapacity(1),this.buffer[this.offset++]=f;return}if(f<16384){this.ensureCapacity(2),this.buffer[this.offset++]=f&127|128,this.buffer[this.offset++]=f>>7;return}if(f<2097152){this.ensureCapacity(3),this.buffer[this.offset++]=f&127|128,this.buffer[this.offset++]=f>>7&127|128,this.buffer[this.offset++]=f>>14;return}this.ensureCapacity(10);while(f>=128)this.buffer[this.offset++]=f&127|128,f>>>=7;this.buffer[this.offset++]=f}writeString(f){let x=Buffer.from(f,"utf-8");this.writeVarInt(x.length),this.ensureCapacity(x.length),x.copy(this.buffer,this.offset),this.offset+=x.length}writeFixedString(f,x){this.ensureCapacity(x);let U=Buffer.from(f,"utf-8"),m=Math.min(U.length,x);if(U.copy(this.buffer,this.offset,0,m),m<x)this.buffer.fill(0,this.offset+m,this.offset+x);this.offset+=x}writeBytes(f){this.ensureCapacity(f.length),f.copy(this.buffer,this.offset),this.offset+=f.length}writeUUID(f){this.ensureCapacity(16);let x=f.replace(/-/g,"");if(x.length!==32)throw Error(`Invalid UUID: ${f}`);let U=Buffer.from(x,"hex");for(let m=7;m>=0;m--)this.buffer[this.offset++]=U[m];for(let m=15;m>=8;m--)this.buffer[this.offset++]=U[m]}writeDate(f){let x=typeof f==="number"?f:Math.floor(f.getTime()/86400000);this.writeUInt16(x)}writeDate32(f){let x=typeof f==="number"?f:Math.floor(f.getTime()/86400000);this.writeInt32(x)}writeDateTime(f){let x=typeof f==="number"?f:Math.floor(f.getTime()/1000);this.writeUInt32(x)}writeDateTime64(f,x=3){let U=Math.pow(10,x),m=typeof f==="number"?BigInt(Math.round(f*U)):BigInt(Math.round(f.getTime()*U/1000));this.writeInt64(m)}writeNullable(f){this.writeUInt8(f?1:0)}writeArrayLength(f){this.writeUInt64(f)}writeDecimal32(f,x){let U=Math.round(f*Math.pow(10,x));this.writeInt32(U)}writeDecimal64(f,x){let U=BigInt(Math.round(f*Math.pow(10,x)));this.writeInt64(U)}writeDecimal128(f,x){let U;if(typeof f==="bigint")U=f*BigInt(Math.pow(10,x));else U=BigInt(Math.round(f*Math.pow(10,x)));this.writeInt128(U)}writeBool(f){this.writeUInt8(f?1:0)}writeIPv4(f){if(typeof f==="number"){this.writeUInt32(f);return}let x=f.split(".").map(Number);if(x.length!==4)throw Error(`Invalid IPv4: ${f}`);let U=x[0]<<24|x[1]<<16|x[2]<<8|x[3];this.writeUInt32(U>>>0)}writeIPv6(f){if(Buffer.isBuffer(f)){if(f.length!==16)throw Error("IPv6 must be 16 bytes");this.writeBytes(f);return}let U=this.expandIPv6(f).split(":");this.ensureCapacity(16);for(let m of U){let A=parseInt(m,16);this.buffer.writeUInt16BE(A,this.offset),this.offset+=2}}expandIPv6(f){let x=f.split("::");if(x.length===1)return f;let U=x[0]?x[0].split(":"):[],m=x[1]?x[1].split(":"):[],A=8-U.length-m.length,D=Array(A).fill("0000");return[...U,...D,...m].map((R)=>R.padStart(4,"0")).join(":")}writeEnum8(f){this.writeInt8(f)}writeEnum16(f){this.writeInt16(f)}}function e(f){let x=[],U="",m=0;for(let A=0;A<f.length;A++){let D=f[A];if(D===","&&m===0)x.push(U.trim()),U="";else{if(D==="(")m++;if(D===")")m--;U+=D}}if(U.trim())x.push(U.trim());return x}function T(f,x=!1){let U=f.toLowerCase().trim(),m=U.match(/^nullable\\((.+)\\)$/);if(m){let V=T(m[1],!1);return(S,$)=>{if($===null||$===void 0)S.writeNullable(!0);else S.writeNullable(!1),V(S,$)}}let A=U.match(/^array\\((.+)\\)$/);if(A){let V=T(A[1],!1);return(S,$)=>{let J=Array.isArray($)?$:[];S.writeArrayLength(J.length);for(let j of J)V(S,j)}}let D=U.match(/^map\\((.+)\\)$/);if(D){let[V,S]=e(D[1]);if(!V||!S)throw Error(`Invalid Map type: ${U}`);let $=T(V),J=T(S);return(j,I)=>{let G;if(I instanceof Map)G=Array.from(I.entries());else if(typeof I==="object"&&I!==null)G=Object.entries(I);else G=[];j.writeUInt64(G.length);for(let[O,g]of G)$(j,O),J(j,g)}}let R=U.match(/^tuple\\((.+)\\)$/);if(R){let S=e(R[1]).map(($)=>T($));return($,J)=>{let j=Array.isArray(J)?J:[];for(let I=0;I<S.length;I++)S[I]($,j[I])}}let q=U.match(/^nested\\((.+)\\)$/);if(q){let V=e(q[1]),$=`Tuple(${V.map((j)=>{let I=j.trim().split(/\\s+/);if(I.length<2)return"String";return I.slice(1).join(" ")}).join(", ")})`,J=T($);return(j,I)=>{let G=Array.isArray(I)?I:[];j.writeArrayLength(G.length);for(let O of G){let g=O;if(!Array.isArray(O)&&typeof O==="object")g=V.map((W0)=>{let d0=W0.trim().split(/\\s+/)[0];return O[d0]});J(j,g)}}}let H=U.match(/^lowcardinality\\((.+)\\)$/);if(H)return T(H[1]);let n=U.match(/^fixedstring\\((\\d+)\\)$/);if(n){let V=parseInt(n[1],10);return(S,$)=>{S.writeFixedString(String($??""),V)}}let x0=U.match(/^decimal(?:32|64|128)?\\((\\d+),\\s*(\\d+)\\)$/);if(x0){let V=parseInt(x0[2],10);if(U.includes("decimal128"))return(S,$)=>S.writeDecimal128($,V);if(U.includes("decimal64"))return(S,$)=>S.writeDecimal64($,V);return(S,$)=>S.writeDecimal32($,V)}let U0=U.match(/^datetime64\\((\\d+)/);if(U0){let V=parseInt(U0[1],10);return(S,$)=>S.writeDateTime64($,V)}if(U.match(/^enum8\\(/))return(V,S)=>V.writeEnum8(Number(S));if(U.match(/^enum16\\(/))return(V,S)=>V.writeEnum16(Number(S));let d=R1(U);if(x)return(V,S)=>{if(S===null||S===void 0)V.writeNullable(!0);else V.writeNullable(!1),d(V,S)};if(U==="uuid")return(V,S)=>{if(S===void 0||S===null)d(V,V1());else d(V,S)};return d}function R1(f){if(f==="int8")return(x,U)=>x.writeInt8(Number(U));if(f==="uint8")return(x,U)=>x.writeUInt8(Number(U));if(f==="int16")return(x,U)=>x.writeInt16(Number(U));if(f==="uint16")return(x,U)=>x.writeUInt16(Number(U));if(f==="int32")return(x,U)=>x.writeInt32(Number(U));if(f==="uint32")return(x,U)=>x.writeUInt32(Number(U));if(f==="int64")return(x,U)=>x.writeInt64(U);if(f==="uint64")return(x,U)=>x.writeUInt64(U);if(f==="int128")return(x,U)=>x.writeInt128(BigInt(U));if(f==="uint128")return(x,U)=>x.writeUInt128(BigInt(U));if(f==="int256")return(x,U)=>x.writeInt256(BigInt(U));if(f==="uint256")return(x,U)=>x.writeUInt256(BigInt(U));if(f==="float32")return(x,U)=>x.writeFloat32(Number(U));if(f==="float64")return(x,U)=>x.writeFloat64(Number(U));if(f==="string")return(x,U)=>x.writeString(String(U??""));if(f==="uuid")return(x,U)=>x.writeUUID(String(U));if(f==="date")return(x,U)=>x.writeDate(U);if(f==="date32")return(x,U)=>x.writeDate32(U);if(f==="datetime")return(x,U)=>x.writeDateTime(U);if(f.startsWith("datetime64"))return(x,U)=>x.writeDateTime64(U,3);if(f==="bool"||f==="boolean")return(x,U)=>x.writeBool(Boolean(U));if(f==="ipv4")return(x,U)=>x.writeIPv4(U);if(f==="ipv6")return(x,U)=>x.writeIPv6(U);return(x,U)=>x.writeString(String(U??""))}var k=null,E=null;function $1(f,x){if(!k||!E)throw Error("Worker not configured");E.reset();for(let U of f)for(let m=0;m<k.columns.length;m++){let A=k.columns[m],D=U[A.name];k.encoders[m](E,D)}return E.toBuffer()}function E0(f){try{switch(f.type){case"configure":{let x=f.columns.map((U)=>T(U.type,U.isNullable));k={columns:f.columns,keyMapping:new Map,encoders:x},E=new f0(1048576),W?.postMessage({type:"ready"});break}case"serialize":{let x=$1(f.rows,f.batchId),U=new ArrayBuffer(x.length);new Uint8Array(U).set(x),W?.postMessage({type:"result",batchId:f.batchId,buffer:Buffer.from(U),rowCount:f.rows.length},[U]);break}case"shutdown":process.exit(0)}}catch(x){W?.postMessage({type:"error",batchId:f.batchId,error:x instanceof Error?x.message:String(x)})}}if(W){if(W.on("message",E0),k0?.columns)E0({type:"configure",columns:k0.columns})}\n';
|
|
3726
3731
|
|
|
3727
3732
|
// src/utils/binary-serializer.ts
|
|
3728
|
-
|
|
3733
|
+
var hasNativeUUID2 = typeof crypto !== "undefined" && typeof crypto.randomUUID === "function";
|
|
3734
|
+
var uuidv4Fn = null;
|
|
3735
|
+
function generateUUIDv4() {
|
|
3736
|
+
if (hasNativeUUID2)
|
|
3737
|
+
return crypto.randomUUID();
|
|
3738
|
+
if (uuidv4Fn)
|
|
3739
|
+
return uuidv4Fn();
|
|
3740
|
+
const uuid = __require("uuid");
|
|
3741
|
+
uuidv4Fn = uuid.v4;
|
|
3742
|
+
return uuidv4Fn();
|
|
3743
|
+
}
|
|
3744
|
+
var WRITER_POOL_SIZE = 8;
|
|
3745
|
+
var writerPool = [];
|
|
3746
|
+
var poolInitialized = false;
|
|
3747
|
+
function initWriterPool() {
|
|
3748
|
+
if (poolInitialized)
|
|
3749
|
+
return;
|
|
3750
|
+
for (let i = 0;i < WRITER_POOL_SIZE; i++) {
|
|
3751
|
+
writerPool.push(new BinaryWriter(64 * 1024));
|
|
3752
|
+
}
|
|
3753
|
+
poolInitialized = true;
|
|
3754
|
+
}
|
|
3755
|
+
function acquireWriter() {
|
|
3756
|
+
initWriterPool();
|
|
3757
|
+
const writer = writerPool.pop();
|
|
3758
|
+
if (writer) {
|
|
3759
|
+
writer.reset();
|
|
3760
|
+
return writer;
|
|
3761
|
+
}
|
|
3762
|
+
return new BinaryWriter(64 * 1024);
|
|
3763
|
+
}
|
|
3764
|
+
function releaseWriter(writer) {
|
|
3765
|
+
if (writerPool.length < WRITER_POOL_SIZE) {
|
|
3766
|
+
writer.reset();
|
|
3767
|
+
writerPool.push(writer);
|
|
3768
|
+
}
|
|
3769
|
+
}
|
|
3729
3770
|
|
|
3730
3771
|
class BinaryWriter {
|
|
3731
3772
|
buffer;
|
|
@@ -4137,7 +4178,7 @@ function createBinaryEncoder(clickhouseType, isNullable = false) {
|
|
|
4137
4178
|
if (type === "uuid") {
|
|
4138
4179
|
return (writer, value) => {
|
|
4139
4180
|
if (value === undefined || value === null) {
|
|
4140
|
-
baseEncoder(writer,
|
|
4181
|
+
baseEncoder(writer, generateUUIDv4());
|
|
4141
4182
|
} else {
|
|
4142
4183
|
baseEncoder(writer, value);
|
|
4143
4184
|
}
|
|
@@ -4231,6 +4272,95 @@ function serializeRowsBinary(rows, config) {
|
|
|
4231
4272
|
}
|
|
4232
4273
|
return writer.toBuffer();
|
|
4233
4274
|
}
|
|
4275
|
+
function createAccessor(propKey, columnName) {
|
|
4276
|
+
if (propKey === columnName) {
|
|
4277
|
+
return (row) => row[propKey];
|
|
4278
|
+
}
|
|
4279
|
+
return (row) => {
|
|
4280
|
+
const v = row[propKey];
|
|
4281
|
+
return v !== undefined ? v : row[columnName];
|
|
4282
|
+
};
|
|
4283
|
+
}
|
|
4284
|
+
function buildOptimizedBinaryConfig(columns, options) {
|
|
4285
|
+
const encoders = [];
|
|
4286
|
+
const accessors = [];
|
|
4287
|
+
const columnInfos = [];
|
|
4288
|
+
for (const col of columns) {
|
|
4289
|
+
encoders.push(createBinaryEncoder(col.type, col.isNull));
|
|
4290
|
+
accessors.push(createAccessor(col.propKey, col.name));
|
|
4291
|
+
columnInfos.push({
|
|
4292
|
+
name: col.name,
|
|
4293
|
+
type: col.type,
|
|
4294
|
+
isNullable: col.isNull
|
|
4295
|
+
});
|
|
4296
|
+
}
|
|
4297
|
+
return {
|
|
4298
|
+
columns: columnInfos,
|
|
4299
|
+
encoders,
|
|
4300
|
+
accessors,
|
|
4301
|
+
skipValidation: options?.skipValidation
|
|
4302
|
+
};
|
|
4303
|
+
}
|
|
4304
|
+
function serializeRowsOptimized(rows, config) {
|
|
4305
|
+
const writer = acquireWriter();
|
|
4306
|
+
try {
|
|
4307
|
+
const { encoders, accessors } = config;
|
|
4308
|
+
const colCount = encoders.length;
|
|
4309
|
+
for (let r = 0;r < rows.length; r++) {
|
|
4310
|
+
const row = rows[r];
|
|
4311
|
+
for (let c = 0;c < colCount; c++) {
|
|
4312
|
+
encoders[c](writer, accessors[c](row));
|
|
4313
|
+
}
|
|
4314
|
+
}
|
|
4315
|
+
return writer.toBuffer();
|
|
4316
|
+
} finally {
|
|
4317
|
+
releaseWriter(writer);
|
|
4318
|
+
}
|
|
4319
|
+
}
|
|
4320
|
+
function isNumericHeavySchema(columns) {
|
|
4321
|
+
const numericTypes = [
|
|
4322
|
+
"int8",
|
|
4323
|
+
"uint8",
|
|
4324
|
+
"int16",
|
|
4325
|
+
"uint16",
|
|
4326
|
+
"int32",
|
|
4327
|
+
"uint32",
|
|
4328
|
+
"int64",
|
|
4329
|
+
"uint64",
|
|
4330
|
+
"float32",
|
|
4331
|
+
"float64"
|
|
4332
|
+
];
|
|
4333
|
+
let numericCount = 0;
|
|
4334
|
+
for (const col of columns) {
|
|
4335
|
+
const type = col.type.toLowerCase();
|
|
4336
|
+
if (numericTypes.some((t) => type === t || type.startsWith(t))) {
|
|
4337
|
+
numericCount++;
|
|
4338
|
+
}
|
|
4339
|
+
}
|
|
4340
|
+
return numericCount > columns.length / 2;
|
|
4341
|
+
}
|
|
4342
|
+
function serializeNumericBatch(rows, config, numericIndices) {
|
|
4343
|
+
const rowCount = rows.length;
|
|
4344
|
+
const numericCount = numericIndices.length;
|
|
4345
|
+
const numericBuffer = new Float64Array(rowCount * numericCount);
|
|
4346
|
+
const otherData = [];
|
|
4347
|
+
for (let r = 0;r < rowCount; r++) {
|
|
4348
|
+
const row = rows[r];
|
|
4349
|
+
const otherRow = [];
|
|
4350
|
+
let numIdx = 0;
|
|
4351
|
+
for (let c = 0;c < config.accessors.length; c++) {
|
|
4352
|
+
const value = config.accessors[c](row);
|
|
4353
|
+
if (numericIndices.includes(c)) {
|
|
4354
|
+
numericBuffer[r * numericCount + numIdx] = Number(value) || 0;
|
|
4355
|
+
numIdx++;
|
|
4356
|
+
} else {
|
|
4357
|
+
otherRow.push(value);
|
|
4358
|
+
}
|
|
4359
|
+
}
|
|
4360
|
+
otherData.push(otherRow);
|
|
4361
|
+
}
|
|
4362
|
+
return { numericBuffer: numericBuffer.buffer, otherData };
|
|
4363
|
+
}
|
|
4234
4364
|
|
|
4235
4365
|
// src/utils/binary-worker-pool.ts
|
|
4236
4366
|
class BinaryWorkerPool extends EventEmitter {
|
|
@@ -4384,22 +4514,24 @@ class BinaryWorkerPool extends EventEmitter {
|
|
|
4384
4514
|
class SyncBinarySerializer {
|
|
4385
4515
|
encoders = [];
|
|
4386
4516
|
columns = [];
|
|
4387
|
-
writer;
|
|
4388
4517
|
constructor(columns) {
|
|
4389
4518
|
this.columns = columns;
|
|
4390
4519
|
this.encoders = columns.map((col) => createBinaryEncoder(col.type, col.isNullable));
|
|
4391
|
-
this.writer = new BinaryWriter(1024 * 1024);
|
|
4392
4520
|
}
|
|
4393
4521
|
serialize(rows) {
|
|
4394
|
-
|
|
4395
|
-
|
|
4396
|
-
for (
|
|
4397
|
-
|
|
4398
|
-
|
|
4399
|
-
|
|
4522
|
+
const writer = acquireWriter();
|
|
4523
|
+
try {
|
|
4524
|
+
for (const row of rows) {
|
|
4525
|
+
for (let i = 0;i < this.columns.length; i++) {
|
|
4526
|
+
const col = this.columns[i];
|
|
4527
|
+
const value = row[col.name];
|
|
4528
|
+
this.encoders[i](writer, value);
|
|
4529
|
+
}
|
|
4400
4530
|
}
|
|
4531
|
+
return writer.toBuffer();
|
|
4532
|
+
} finally {
|
|
4533
|
+
releaseWriter(writer);
|
|
4401
4534
|
}
|
|
4402
|
-
return this.writer.toBuffer();
|
|
4403
4535
|
}
|
|
4404
4536
|
async* serializeStream(rows, batchSize = 1e4) {
|
|
4405
4537
|
let batch = [];
|
|
@@ -4417,7 +4549,7 @@ class SyncBinarySerializer {
|
|
|
4417
4549
|
}
|
|
4418
4550
|
|
|
4419
4551
|
// src/builders/insert.ts
|
|
4420
|
-
import { Readable
|
|
4552
|
+
import { Readable } from "stream";
|
|
4421
4553
|
|
|
4422
4554
|
// src/utils/background-batcher.ts
|
|
4423
4555
|
class BackgroundBatcher {
|
|
@@ -4509,75 +4641,76 @@ var globalBatcher = (client) => {
|
|
|
4509
4641
|
};
|
|
4510
4642
|
|
|
4511
4643
|
// src/builders/insert.ts
|
|
4512
|
-
import
|
|
4513
|
-
|
|
4514
|
-
|
|
4515
|
-
|
|
4516
|
-
|
|
4517
|
-
|
|
4518
|
-
|
|
4519
|
-
|
|
4520
|
-
|
|
4521
|
-
|
|
4522
|
-
}
|
|
4523
|
-
|
|
4524
|
-
|
|
4525
|
-
|
|
4526
|
-
|
|
4527
|
-
|
|
4528
|
-
|
|
4529
|
-
|
|
4530
|
-
|
|
4531
|
-
|
|
4532
|
-
|
|
4533
|
-
|
|
4534
|
-
|
|
4535
|
-
callback();
|
|
4536
|
-
}
|
|
4537
|
-
_flush(callback) {
|
|
4538
|
-
if (this.batch.length > 0) {
|
|
4539
|
-
try {
|
|
4540
|
-
const buffer = this.serializer.serialize(this.batch);
|
|
4541
|
-
this.push(buffer);
|
|
4542
|
-
} catch (error) {
|
|
4543
|
-
callback(error);
|
|
4544
|
-
return;
|
|
4545
|
-
}
|
|
4546
|
-
}
|
|
4547
|
-
callback();
|
|
4548
|
-
}
|
|
4644
|
+
import http from "http";
|
|
4645
|
+
import https from "https";
|
|
4646
|
+
var uuidv4Fn2 = null;
|
|
4647
|
+
var uuidv7Fn = null;
|
|
4648
|
+
var uuidv1 = null;
|
|
4649
|
+
var uuidv6 = null;
|
|
4650
|
+
var hasNativeUUID3 = typeof crypto !== "undefined" && typeof crypto.randomUUID === "function";
|
|
4651
|
+
function getUUIDv4() {
|
|
4652
|
+
if (hasNativeUUID3) {
|
|
4653
|
+
return crypto.randomUUID();
|
|
4654
|
+
}
|
|
4655
|
+
if (uuidv4Fn2)
|
|
4656
|
+
return uuidv4Fn2();
|
|
4657
|
+
const uuid = __require("uuid");
|
|
4658
|
+
uuidv4Fn2 = uuid.v4;
|
|
4659
|
+
return uuidv4Fn2();
|
|
4660
|
+
}
|
|
4661
|
+
function getUUIDv7() {
|
|
4662
|
+
if (uuidv7Fn)
|
|
4663
|
+
return uuidv7Fn();
|
|
4664
|
+
const uuid = __require("uuid");
|
|
4665
|
+
uuidv7Fn = uuid.v7;
|
|
4666
|
+
return uuidv7Fn();
|
|
4549
4667
|
}
|
|
4550
4668
|
|
|
4551
4669
|
class ClickHouseInsertBuilder {
|
|
4552
4670
|
client;
|
|
4553
4671
|
table;
|
|
4672
|
+
connectionConfig;
|
|
4554
4673
|
_values = null;
|
|
4555
4674
|
_async = true;
|
|
4556
4675
|
_waitForAsync = true;
|
|
4557
4676
|
_batchOptions = {};
|
|
4558
4677
|
_format = "auto";
|
|
4559
|
-
_batchSize = 1000;
|
|
4560
4678
|
_batchConfig = null;
|
|
4561
4679
|
_forceJson = false;
|
|
4562
4680
|
_returning = false;
|
|
4563
|
-
|
|
4681
|
+
_isSingle = false;
|
|
4682
|
+
_skipValidation = false;
|
|
4683
|
+
constructor(client, table, connectionConfig) {
|
|
4564
4684
|
this.client = client;
|
|
4565
4685
|
this.table = table;
|
|
4686
|
+
this.connectionConfig = connectionConfig;
|
|
4566
4687
|
if (table.$options?.asyncInsert !== undefined) {
|
|
4567
4688
|
this._async = table.$options.asyncInsert;
|
|
4568
4689
|
}
|
|
4690
|
+
if (connectionConfig?.skipValidation) {
|
|
4691
|
+
this._skipValidation = true;
|
|
4692
|
+
}
|
|
4693
|
+
}
|
|
4694
|
+
skipValidation() {
|
|
4695
|
+
this._skipValidation = true;
|
|
4696
|
+
return this;
|
|
4569
4697
|
}
|
|
4570
4698
|
values(value) {
|
|
4571
4699
|
this._values = value;
|
|
4700
|
+
this._isSingle = !Array.isArray(value) && !isIterable(value) && !isAsyncIterable(value) && !(value instanceof Readable);
|
|
4572
4701
|
return this;
|
|
4573
4702
|
}
|
|
4574
4703
|
async insert(data) {
|
|
4575
|
-
return this.values(data);
|
|
4704
|
+
return this.values(data).execute();
|
|
4576
4705
|
}
|
|
4577
4706
|
returning() {
|
|
4578
4707
|
this._returning = true;
|
|
4579
4708
|
return this;
|
|
4580
4709
|
}
|
|
4710
|
+
noReturning() {
|
|
4711
|
+
this._returning = false;
|
|
4712
|
+
return this;
|
|
4713
|
+
}
|
|
4581
4714
|
syncInsert() {
|
|
4582
4715
|
this._async = false;
|
|
4583
4716
|
return this;
|
|
@@ -4599,14 +4732,14 @@ class ClickHouseInsertBuilder {
|
|
|
4599
4732
|
return this;
|
|
4600
4733
|
}
|
|
4601
4734
|
batchSize(size) {
|
|
4602
|
-
this.
|
|
4735
|
+
this._batchOptions.batchSize = size;
|
|
4603
4736
|
return this;
|
|
4604
4737
|
}
|
|
4605
4738
|
async append(row) {
|
|
4606
4739
|
if (!this._batchConfig) {
|
|
4607
4740
|
this.batch();
|
|
4608
4741
|
}
|
|
4609
|
-
const plan = buildInsertPlan(this.table);
|
|
4742
|
+
const plan = buildInsertPlan(this.table, { skipValidation: this._skipValidation });
|
|
4610
4743
|
const batcher = globalBatcher(this.client);
|
|
4611
4744
|
const oldValues = this._values;
|
|
4612
4745
|
this._values = [row];
|
|
@@ -4642,7 +4775,7 @@ class ClickHouseInsertBuilder {
|
|
|
4642
4775
|
if (Array.isArray(this._values) && this._values.length === 0) {
|
|
4643
4776
|
throw new Error("❌ No values to insert");
|
|
4644
4777
|
}
|
|
4645
|
-
const plan = buildInsertPlan(this.table);
|
|
4778
|
+
const plan = buildInsertPlan(this.table, { skipValidation: this._skipValidation });
|
|
4646
4779
|
if (this._returning) {
|
|
4647
4780
|
if (this._batchConfig) {
|
|
4648
4781
|
throw new Error("❌ returning() cannot be used with background batching");
|
|
@@ -4658,6 +4791,9 @@ class ClickHouseInsertBuilder {
|
|
|
4658
4791
|
wait_for_async_insert: this._waitForAsync ? 1 : 0
|
|
4659
4792
|
}
|
|
4660
4793
|
});
|
|
4794
|
+
if (this._isSingle && resultRows.length > 0) {
|
|
4795
|
+
return resultRows[0];
|
|
4796
|
+
}
|
|
4661
4797
|
return resultRows;
|
|
4662
4798
|
}
|
|
4663
4799
|
if (this._batchConfig && !this._forceJson) {
|
|
@@ -4719,32 +4855,70 @@ class ClickHouseInsertBuilder {
|
|
|
4719
4855
|
});
|
|
4720
4856
|
}
|
|
4721
4857
|
async executeBinaryInsert(plan, tableName) {
|
|
4858
|
+
if (!this.connectionConfig) {
|
|
4859
|
+
throw new Error("❌ Binary format requires connection configuration. This is an internal error - please report it.");
|
|
4860
|
+
}
|
|
4722
4861
|
const columns = plan.columns.map((col) => ({
|
|
4723
4862
|
name: col.columnName,
|
|
4724
4863
|
type: col.column.type,
|
|
4725
4864
|
isNullable: col.column.isNull,
|
|
4726
4865
|
propKey: col.propKey
|
|
4727
4866
|
}));
|
|
4728
|
-
const
|
|
4729
|
-
|
|
4867
|
+
const serializer = new SyncBinarySerializer(columns);
|
|
4868
|
+
const allRows = [];
|
|
4730
4869
|
if (this._values instanceof Readable) {
|
|
4731
|
-
|
|
4870
|
+
for await (const chunk of this._values) {
|
|
4871
|
+
allRows.push(chunk);
|
|
4872
|
+
}
|
|
4732
4873
|
} else {
|
|
4733
|
-
const
|
|
4734
|
-
|
|
4735
|
-
}
|
|
4736
|
-
const binaryStream = sourceStream.pipe(binaryTransform);
|
|
4737
|
-
await this.client.insert({
|
|
4738
|
-
table: tableName,
|
|
4739
|
-
values: binaryStream,
|
|
4740
|
-
format: "RowBinary",
|
|
4741
|
-
clickhouse_settings: {
|
|
4742
|
-
async_insert: this._async ? 1 : 0,
|
|
4743
|
-
wait_for_async_insert: this._waitForAsync ? 1 : 0
|
|
4874
|
+
for await (const row of this.processRows(plan, true)) {
|
|
4875
|
+
allRows.push(row);
|
|
4744
4876
|
}
|
|
4877
|
+
}
|
|
4878
|
+
if (allRows.length === 0) {
|
|
4879
|
+
return;
|
|
4880
|
+
}
|
|
4881
|
+
const binaryData = serializer.serialize(allRows);
|
|
4882
|
+
const { url: baseUrl, username, password, database } = this.connectionConfig;
|
|
4883
|
+
const url2 = new URL(baseUrl);
|
|
4884
|
+
const queryParams = new URLSearchParams({
|
|
4885
|
+
query: `INSERT INTO ${tableName} FORMAT RowBinary`,
|
|
4886
|
+
database
|
|
4887
|
+
});
|
|
4888
|
+
if (this._async) {
|
|
4889
|
+
queryParams.set("async_insert", "1");
|
|
4890
|
+
queryParams.set("wait_for_async_insert", this._waitForAsync ? "1" : "0");
|
|
4891
|
+
}
|
|
4892
|
+
url2.search = queryParams.toString();
|
|
4893
|
+
const isHttps = url2.protocol === "https:";
|
|
4894
|
+
const httpModule = isHttps ? https : http;
|
|
4895
|
+
const authHeader = Buffer.from(`${username}:${password}`).toString("base64");
|
|
4896
|
+
return new Promise((resolve, reject) => {
|
|
4897
|
+
const req = httpModule.request(url2, {
|
|
4898
|
+
method: "POST",
|
|
4899
|
+
headers: {
|
|
4900
|
+
"Content-Type": "application/octet-stream",
|
|
4901
|
+
Authorization: `Basic ${authHeader}`
|
|
4902
|
+
}
|
|
4903
|
+
}, (res) => {
|
|
4904
|
+
let body = "";
|
|
4905
|
+
res.on("data", (chunk) => {
|
|
4906
|
+
body += chunk;
|
|
4907
|
+
});
|
|
4908
|
+
res.on("end", () => {
|
|
4909
|
+
if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) {
|
|
4910
|
+
resolve();
|
|
4911
|
+
} else {
|
|
4912
|
+
reject(new Error(`ClickHouse error (${res.statusCode}): ${body}`));
|
|
4913
|
+
}
|
|
4914
|
+
});
|
|
4915
|
+
});
|
|
4916
|
+
req.on("error", reject);
|
|
4917
|
+
req.write(binaryData);
|
|
4918
|
+
req.end();
|
|
4745
4919
|
});
|
|
4746
4920
|
}
|
|
4747
|
-
async* processRows(plan) {
|
|
4921
|
+
async* processRows(plan, skipDateTransform = false) {
|
|
4748
4922
|
const values2 = this._values;
|
|
4749
4923
|
const iterable = Array.isArray(values2) ? values2 : isIterable(values2) ? values2 : isAsyncIterable(values2) ? values2 : [values2];
|
|
4750
4924
|
for await (const row of iterable) {
|
|
@@ -4755,12 +4929,12 @@ class ClickHouseInsertBuilder {
|
|
|
4755
4929
|
if (col.defaultFn) {
|
|
4756
4930
|
value = col.defaultFn(row);
|
|
4757
4931
|
} else if (col.autoUUIDVersion !== null && !col.useServerUUID) {
|
|
4758
|
-
value = col.autoUUIDVersion === 7 ?
|
|
4932
|
+
value = col.autoUUIDVersion === 7 ? getUUIDv7() : getUUIDv4();
|
|
4759
4933
|
} else if (col.hasDefault) {
|
|
4760
4934
|
value = col.defaultValue;
|
|
4761
4935
|
}
|
|
4762
4936
|
}
|
|
4763
|
-
if (value !== undefined) {
|
|
4937
|
+
if (value !== undefined && !skipDateTransform) {
|
|
4764
4938
|
value = col.transform(value);
|
|
4765
4939
|
}
|
|
4766
4940
|
processedRow[col.columnName] = value;
|
|
@@ -4819,13 +4993,13 @@ class ClickHouseInsertBuilder {
|
|
|
4819
4993
|
return new Date;
|
|
4820
4994
|
}
|
|
4821
4995
|
if (normalized === "generateuuidv4()")
|
|
4822
|
-
return
|
|
4996
|
+
return getUUIDv4();
|
|
4823
4997
|
if (normalized === "generateuuidv7()")
|
|
4824
|
-
return
|
|
4998
|
+
return getUUIDv7();
|
|
4825
4999
|
if (normalized === "generateuuidv1()")
|
|
4826
|
-
return uuidv1();
|
|
5000
|
+
return uuidv1?.() ?? getUUIDv4();
|
|
4827
5001
|
if (normalized === "generateuuidv6()")
|
|
4828
|
-
return uuidv6();
|
|
5002
|
+
return uuidv6?.() ?? getUUIDv7();
|
|
4829
5003
|
return;
|
|
4830
5004
|
}
|
|
4831
5005
|
async then(onfulfilled, onrejected) {
|
|
@@ -5134,6 +5308,29 @@ function wrapClientWithLogger(client, logger) {
|
|
|
5134
5308
|
}
|
|
5135
5309
|
|
|
5136
5310
|
// src/relational.ts
|
|
5311
|
+
var operators = {
|
|
5312
|
+
eq,
|
|
5313
|
+
ne,
|
|
5314
|
+
gt,
|
|
5315
|
+
gte,
|
|
5316
|
+
lt,
|
|
5317
|
+
lte,
|
|
5318
|
+
inArray,
|
|
5319
|
+
notInArray,
|
|
5320
|
+
between,
|
|
5321
|
+
notBetween,
|
|
5322
|
+
has,
|
|
5323
|
+
hasAll,
|
|
5324
|
+
hasAny,
|
|
5325
|
+
sql,
|
|
5326
|
+
and,
|
|
5327
|
+
or,
|
|
5328
|
+
not,
|
|
5329
|
+
isNull,
|
|
5330
|
+
isNotNull,
|
|
5331
|
+
asc,
|
|
5332
|
+
desc
|
|
5333
|
+
};
|
|
5137
5334
|
function buildJoinCondition(fields, references) {
|
|
5138
5335
|
if (!fields || !references || fields.length === 0 || references.length === 0)
|
|
5139
5336
|
return null;
|
|
@@ -5157,7 +5354,8 @@ function buildRelationalAPI(client, schema) {
|
|
|
5157
5354
|
api[tableKey] = {
|
|
5158
5355
|
findMany: async (opts) => {
|
|
5159
5356
|
let builder = new ClickHouseQueryBuilder(client).from(tableDef);
|
|
5160
|
-
const
|
|
5357
|
+
const selectedColumns = opts?.columns;
|
|
5358
|
+
const baseColumns = selectedColumns ? Object.entries(tableDef.$columns).filter(([key]) => selectedColumns[key] === true) : Object.entries(tableDef.$columns);
|
|
5161
5359
|
const relations = tableDef.$relations;
|
|
5162
5360
|
const requestedTopLevel = opts?.with ? Object.entries(opts.with).filter(([, v]) => v) : [];
|
|
5163
5361
|
const requestedRelations = requestedTopLevel.map(([relName, val]) => {
|
|
@@ -5165,16 +5363,19 @@ function buildRelationalAPI(client, schema) {
|
|
|
5165
5363
|
return { relName, rel, options: typeof val === "object" ? val : {} };
|
|
5166
5364
|
}).filter((r) => Boolean(r.rel));
|
|
5167
5365
|
const needsGrouping = requestedRelations.length > 0;
|
|
5168
|
-
const
|
|
5366
|
+
const allColumns = Object.entries(tableDef.$columns);
|
|
5367
|
+
const groupByColumns = needsGrouping ? allColumns.map(([, col]) => col) : [];
|
|
5169
5368
|
const joinStrategy = opts?.joinStrategy || "auto";
|
|
5170
5369
|
const isDistributed = isDistributedTable(tableDef);
|
|
5171
5370
|
const useGlobal = joinStrategy === "global" || joinStrategy === "global_any" || joinStrategy === "auto" && isDistributed;
|
|
5172
5371
|
const useAny = joinStrategy === "any" || joinStrategy === "global_any";
|
|
5173
|
-
function buildNestedSelection(currentTableDef, currentWith, prefix = "", outerJoinStrategy, outerUseGlobal, outerUseAny) {
|
|
5372
|
+
function buildNestedSelection(currentTableDef, currentWith, prefix = "", outerJoinStrategy, outerUseGlobal, outerUseAny, columnsFilter) {
|
|
5174
5373
|
let currentSelection = {};
|
|
5175
5374
|
let currentJoins = [];
|
|
5176
5375
|
Object.entries(currentTableDef.$columns).forEach(([key, col]) => {
|
|
5177
|
-
|
|
5376
|
+
if (!columnsFilter || columnsFilter[key] === true) {
|
|
5377
|
+
currentSelection[`${prefix}${key}`] = col;
|
|
5378
|
+
}
|
|
5178
5379
|
});
|
|
5179
5380
|
if (!currentWith)
|
|
5180
5381
|
return { selection: currentSelection, joins: currentJoins };
|
|
@@ -5185,7 +5386,22 @@ function buildRelationalAPI(client, schema) {
|
|
|
5185
5386
|
}).filter((r) => Boolean(r.rel));
|
|
5186
5387
|
for (const { relName, rel, options } of requestedNested) {
|
|
5187
5388
|
const newPrefix = prefix ? `${prefix}_${relName}_` : `${relName}_`;
|
|
5188
|
-
|
|
5389
|
+
let relWhere = null;
|
|
5390
|
+
if (options.where) {
|
|
5391
|
+
if (typeof options.where === "function") {
|
|
5392
|
+
relWhere = options.where(rel.table.$columns, operators);
|
|
5393
|
+
} else if (typeof options.where === "object" && !("toSQL" in options.where)) {
|
|
5394
|
+
const conditions = Object.entries(options.where).map(([key, value]) => {
|
|
5395
|
+
const col = rel.table.$columns[key];
|
|
5396
|
+
if (!col)
|
|
5397
|
+
throw new Error(`Column '${key}' not found in relation '${relName}'`);
|
|
5398
|
+
return eq(col, value);
|
|
5399
|
+
});
|
|
5400
|
+
relWhere = conditions.length === 1 ? conditions[0] : and(...conditions);
|
|
5401
|
+
} else {
|
|
5402
|
+
relWhere = options.where;
|
|
5403
|
+
}
|
|
5404
|
+
}
|
|
5189
5405
|
let joinCondition = buildJoinCondition(rel.fields, rel.references);
|
|
5190
5406
|
if (joinCondition) {
|
|
5191
5407
|
const joinType = (() => {
|
|
@@ -5212,7 +5428,7 @@ function buildRelationalAPI(client, schema) {
|
|
|
5212
5428
|
}
|
|
5213
5429
|
currentJoins.push({ type: joinType, table: rel.table, on: joinCondition });
|
|
5214
5430
|
}
|
|
5215
|
-
const nestedResult = buildNestedSelection(rel.table, options.with, newPrefix, outerJoinStrategy, outerUseGlobal, outerUseAny);
|
|
5431
|
+
const nestedResult = buildNestedSelection(rel.table, options.with, newPrefix, outerJoinStrategy, outerUseGlobal, outerUseAny, options.columns);
|
|
5216
5432
|
if (!(rel.relation === "many" && !prefix)) {
|
|
5217
5433
|
Object.assign(currentSelection, nestedResult.selection);
|
|
5218
5434
|
}
|
|
@@ -5220,7 +5436,7 @@ function buildRelationalAPI(client, schema) {
|
|
|
5220
5436
|
}
|
|
5221
5437
|
return { selection: currentSelection, joins: currentJoins };
|
|
5222
5438
|
}
|
|
5223
|
-
const { selection: flatSelection, joins: allJoins } = buildNestedSelection(tableDef, opts?.with, "", joinStrategy, useGlobal, useAny);
|
|
5439
|
+
const { selection: flatSelection, joins: allJoins } = buildNestedSelection(tableDef, opts?.with, "", joinStrategy, useGlobal, useAny, opts?.columns);
|
|
5224
5440
|
for (const joinDef of allJoins) {
|
|
5225
5441
|
builder = joinDef.type.call(builder, joinDef.table, joinDef.on);
|
|
5226
5442
|
}
|
|
@@ -5229,7 +5445,29 @@ function buildRelationalAPI(client, schema) {
|
|
|
5229
5445
|
builder = builder.groupBy(...groupByColumns);
|
|
5230
5446
|
}
|
|
5231
5447
|
if (opts?.where) {
|
|
5232
|
-
|
|
5448
|
+
let whereValue;
|
|
5449
|
+
if (typeof opts.where === "function") {
|
|
5450
|
+
whereValue = opts.where(tableDef.$columns, operators);
|
|
5451
|
+
} else if (opts.where && typeof opts.where === "object" && !("toSQL" in opts.where)) {
|
|
5452
|
+
const conditions = Object.entries(opts.where).map(([key, value]) => {
|
|
5453
|
+
const col = tableDef.$columns[key];
|
|
5454
|
+
if (!col)
|
|
5455
|
+
throw new Error(`Column '${key}' not found in table`);
|
|
5456
|
+
return eq(col, value);
|
|
5457
|
+
});
|
|
5458
|
+
whereValue = conditions.length === 1 ? conditions[0] : and(...conditions);
|
|
5459
|
+
} else {
|
|
5460
|
+
whereValue = opts.where;
|
|
5461
|
+
}
|
|
5462
|
+
builder = builder.where(whereValue);
|
|
5463
|
+
}
|
|
5464
|
+
if (opts?.orderBy) {
|
|
5465
|
+
const orderByFns = { asc, desc };
|
|
5466
|
+
const orderByValue = typeof opts.orderBy === "function" ? opts.orderBy(tableDef.$columns, orderByFns) : opts.orderBy;
|
|
5467
|
+
const orderByArray = Array.isArray(orderByValue) ? orderByValue : [orderByValue];
|
|
5468
|
+
for (const ob of orderByArray) {
|
|
5469
|
+
builder = builder.orderBy(ob.col, ob.dir);
|
|
5470
|
+
}
|
|
5233
5471
|
}
|
|
5234
5472
|
if (opts?.limit)
|
|
5235
5473
|
builder = builder.limit(opts.limit);
|
|
@@ -5316,6 +5554,21 @@ function buildRelationalAPI(client, schema) {
|
|
|
5316
5554
|
findFirst: async (opts) => {
|
|
5317
5555
|
const rows = await api[tableKey].findMany({ ...opts, limit: 1 });
|
|
5318
5556
|
return rows[0] ?? null;
|
|
5557
|
+
},
|
|
5558
|
+
findById: async (id, opts) => {
|
|
5559
|
+
const pkOption = tableDef.$options?.primaryKey;
|
|
5560
|
+
const pkCols = pkOption ? Array.isArray(pkOption) ? pkOption : [pkOption] : [Object.keys(tableDef.$columns)[0]];
|
|
5561
|
+
const pkColName = pkCols[0];
|
|
5562
|
+
const pkColumn = tableDef.$columns[pkColName];
|
|
5563
|
+
if (!pkColumn) {
|
|
5564
|
+
throw new Error(`Primary key column '${pkColName}' not found in table '${tableKey}'`);
|
|
5565
|
+
}
|
|
5566
|
+
const rows = await api[tableKey].findMany({
|
|
5567
|
+
...opts,
|
|
5568
|
+
where: eq(pkColumn, id),
|
|
5569
|
+
limit: 1
|
|
5570
|
+
});
|
|
5571
|
+
return rows[0] ?? null;
|
|
5319
5572
|
}
|
|
5320
5573
|
};
|
|
5321
5574
|
}
|
|
@@ -5323,8 +5576,26 @@ function buildRelationalAPI(client, schema) {
|
|
|
5323
5576
|
}
|
|
5324
5577
|
|
|
5325
5578
|
// src/client.ts
|
|
5579
|
+
var agentPool = new Map;
|
|
5580
|
+
function getOrCreateAgent(url2, config = {}) {
|
|
5581
|
+
const isHttps = url2.startsWith("https");
|
|
5582
|
+
const key = `${url2}-${config.maxSockets ?? 100}`;
|
|
5583
|
+
let agent = agentPool.get(key);
|
|
5584
|
+
if (agent)
|
|
5585
|
+
return agent;
|
|
5586
|
+
const agentConfig = {
|
|
5587
|
+
keepAlive: config.keepAlive ?? true,
|
|
5588
|
+
keepAliveMsecs: config.keepAliveInitialDelay ?? 1000,
|
|
5589
|
+
maxSockets: config.maxSockets ?? 100,
|
|
5590
|
+
maxFreeSockets: Math.floor((config.maxSockets ?? 100) / 2),
|
|
5591
|
+
timeout: config.timeout ?? 30000
|
|
5592
|
+
};
|
|
5593
|
+
agent = isHttps ? new https2.Agent(agentConfig) : new http2.Agent(agentConfig);
|
|
5594
|
+
agentPool.set(key, agent);
|
|
5595
|
+
return agent;
|
|
5596
|
+
}
|
|
5326
5597
|
function createHousekitClient(config) {
|
|
5327
|
-
const { schema, logger, ...configRest } = config;
|
|
5598
|
+
const { schema, logger, pool, skipValidation, ...configRest } = config;
|
|
5328
5599
|
const normalizedConfig = { ...configRest };
|
|
5329
5600
|
if ("host" in normalizedConfig && !("url" in normalizedConfig)) {
|
|
5330
5601
|
const host = normalizedConfig.host;
|
|
@@ -5332,8 +5603,7 @@ function createHousekitClient(config) {
|
|
|
5332
5603
|
delete normalizedConfig.host;
|
|
5333
5604
|
}
|
|
5334
5605
|
const urlStr = normalizedConfig.url?.toString() ?? "";
|
|
5335
|
-
const
|
|
5336
|
-
const keepAliveAgent = isHttps ? new https.Agent({ keepAlive: true, maxSockets: 100 }) : new http.Agent({ keepAlive: true, maxSockets: 100 });
|
|
5606
|
+
const keepAliveAgent = getOrCreateAgent(urlStr, pool);
|
|
5337
5607
|
const clientConfig = {
|
|
5338
5608
|
...normalizedConfig,
|
|
5339
5609
|
clickhouse_settings: {
|
|
@@ -5363,6 +5633,13 @@ function createHousekitClient(config) {
|
|
|
5363
5633
|
const clause = options?.ifExists === false ? "" : " IF EXISTS";
|
|
5364
5634
|
await client.query({ query: `DROP TABLE${clause} \`${tableName}\`` });
|
|
5365
5635
|
};
|
|
5636
|
+
const binaryInsertConfig = {
|
|
5637
|
+
url: normalizedConfig.url?.toString() || "http://localhost:8123",
|
|
5638
|
+
username: normalizedConfig.username || "default",
|
|
5639
|
+
password: normalizedConfig.password || "",
|
|
5640
|
+
database: normalizedConfig.database || "default",
|
|
5641
|
+
skipValidation
|
|
5642
|
+
};
|
|
5366
5643
|
const baseClient = {
|
|
5367
5644
|
rawClient: client,
|
|
5368
5645
|
select: (fieldsOrTable) => {
|
|
@@ -5378,7 +5655,7 @@ function createHousekitClient(config) {
|
|
|
5378
5655
|
const builder = new ClickHouseQueryBuilder(client);
|
|
5379
5656
|
return builder.with(name, query);
|
|
5380
5657
|
},
|
|
5381
|
-
insert: (table) => new ClickHouseInsertBuilder(client, table),
|
|
5658
|
+
insert: (table) => new ClickHouseInsertBuilder(client, table, binaryInsertConfig),
|
|
5382
5659
|
insertMany: async (table, values2, opts) => {
|
|
5383
5660
|
const blockSize = Math.max(opts?.maxBlockSize || 1e4, 1);
|
|
5384
5661
|
const plan = buildInsertPlan(table);
|
|
@@ -6465,6 +6742,20 @@ function lead(col, offset = 1, defaultValue, spec) {
|
|
|
6465
6742
|
}
|
|
6466
6743
|
var windowFns = { over, rowNumber, rank, denseRank, lag, lead };
|
|
6467
6744
|
// src/schema-builder.ts
|
|
6745
|
+
function primaryUuid(name) {
|
|
6746
|
+
const colName = name ?? "id";
|
|
6747
|
+
const column = new ClickHouseColumn(colName, "UUID").autoGenerate().primaryKey().default("generateUUIDv4()");
|
|
6748
|
+
return {
|
|
6749
|
+
[colName]: column
|
|
6750
|
+
};
|
|
6751
|
+
}
|
|
6752
|
+
function primaryUuidV7(name) {
|
|
6753
|
+
const colName = name ?? "id";
|
|
6754
|
+
const column = new ClickHouseColumn(colName, "UUID").autoGenerate({ version: 7 }).primaryKey().default("generateUUIDv7()");
|
|
6755
|
+
return {
|
|
6756
|
+
[colName]: column
|
|
6757
|
+
};
|
|
6758
|
+
}
|
|
6468
6759
|
var t = {
|
|
6469
6760
|
int8: (name) => new ClickHouseColumn(name, "Int8"),
|
|
6470
6761
|
int16: (name) => new ClickHouseColumn(name, "Int16"),
|
|
@@ -6532,12 +6823,8 @@ var t = {
|
|
|
6532
6823
|
created_at: new ClickHouseColumn("created_at", "DateTime").default("now()"),
|
|
6533
6824
|
updated_at: new ClickHouseColumn("updated_at", "DateTime").default("now()")
|
|
6534
6825
|
}),
|
|
6535
|
-
primaryUuid
|
|
6536
|
-
|
|
6537
|
-
return {
|
|
6538
|
-
[colName]: new ClickHouseColumn(colName, "UUID").autoGenerate().primaryKey().default("generateUUIDv4()")
|
|
6539
|
-
};
|
|
6540
|
-
},
|
|
6826
|
+
primaryUuid,
|
|
6827
|
+
primaryUuidV7,
|
|
6541
6828
|
softDeletes: () => ({
|
|
6542
6829
|
is_deleted: new ClickHouseColumn("is_deleted", "Bool").default(false),
|
|
6543
6830
|
deleted_at: new ClickHouseColumn("deleted_at", "DateTime").nullable()
|
|
@@ -6789,8 +7076,10 @@ export {
|
|
|
6789
7076
|
sipHash64,
|
|
6790
7077
|
sha256,
|
|
6791
7078
|
sha1,
|
|
7079
|
+
serializeRowsOptimized,
|
|
6792
7080
|
serializeRowsBinary,
|
|
6793
7081
|
serializeRowBinary,
|
|
7082
|
+
serializeNumericBatch,
|
|
6794
7083
|
s3Cluster,
|
|
6795
7084
|
s3,
|
|
6796
7085
|
rowNumber,
|
|
@@ -6802,6 +7091,8 @@ export {
|
|
|
6802
7091
|
replace,
|
|
6803
7092
|
repeat,
|
|
6804
7093
|
renderSchema,
|
|
7094
|
+
renderEngineSQL,
|
|
7095
|
+
releaseWriter,
|
|
6805
7096
|
relations,
|
|
6806
7097
|
rate,
|
|
6807
7098
|
rank,
|
|
@@ -6901,6 +7192,7 @@ export {
|
|
|
6901
7192
|
jsonArrayLength,
|
|
6902
7193
|
jsonArrayExtract,
|
|
6903
7194
|
isValidGeometry,
|
|
7195
|
+
isNumericHeavySchema,
|
|
6904
7196
|
isNull,
|
|
6905
7197
|
isNotNull,
|
|
6906
7198
|
int64PrimaryKey,
|
|
@@ -6992,6 +7284,7 @@ export {
|
|
|
6992
7284
|
createClientFromConfig,
|
|
6993
7285
|
createClient,
|
|
6994
7286
|
createBinaryEncoder,
|
|
7287
|
+
createAccessor,
|
|
6995
7288
|
covarSamp,
|
|
6996
7289
|
covarPop,
|
|
6997
7290
|
countryCode,
|
|
@@ -7012,6 +7305,7 @@ export {
|
|
|
7012
7305
|
ceil,
|
|
7013
7306
|
caseWhen,
|
|
7014
7307
|
buildRelationalAPI,
|
|
7308
|
+
buildOptimizedBinaryConfig,
|
|
7015
7309
|
buildHousekitMetadata,
|
|
7016
7310
|
buildBinaryConfig,
|
|
7017
7311
|
boundingBox,
|
|
@@ -7049,6 +7343,7 @@ export {
|
|
|
7049
7343
|
anyLast,
|
|
7050
7344
|
any,
|
|
7051
7345
|
and,
|
|
7346
|
+
acquireWriter,
|
|
7052
7347
|
abs,
|
|
7053
7348
|
SyncBinarySerializer,
|
|
7054
7349
|
SQL,
|