@chkit/plugin-backfill 0.1.0-beta.20 → 0.1.0-beta.21

package/dist/chunking/types.d.ts

@@ -1,29 +1,139 @@
-export interface PartitionInfo {
+export type RowProbeStrategy = 'explain-estimate' | 'count';
+export type SortKeyCategory = 'numeric' | 'datetime' | 'string';
+export type SortKeyBoundaryEncoding = 'literal' | 'hex-latin1';
+export type EstimateConfidence = 'high' | 'low' | 'exact';
+export type EstimateReason = 'partition-metadata' | 'quantile-estimate' | 'string-prefix-distribution' | 'group-by-key-distribution' | 'temporal-distribution' | 'equal-width-distribution' | 'exact-count';
+export interface SortKey {
+    name: string;
+    type: string;
+    category: SortKeyCategory;
+    boundaryEncoding: SortKeyBoundaryEncoding;
+}
+export interface ChunkRange {
+    dimensionIndex: number;
+    from?: string;
+    to?: string;
+}
+export interface ChunkDerivationStep {
+    strategyId: string;
+    dimensionIndex?: number;
+    reason: string;
+}
+export interface ChunkEstimate {
+    rows: number;
+    bytesCompressed: number;
+    bytesUncompressed: number;
+    confidence: EstimateConfidence;
+    reason: EstimateReason;
+}
+export interface FocusedValue {
+    dimensionIndex: number;
+    value: string;
+}
+export interface ChunkAnalysis {
+    focusedValue?: FocusedValue;
+    lineage: ChunkDerivationStep[];
+}
+export interface Chunk {
+    id: string;
+    partitionId: string;
+    ranges: ChunkRange[];
+    estimate: ChunkEstimate;
+    analysis: ChunkAnalysis;
+}
+export interface PartitionDiagnostics {
+    estimatedRowSum: number;
+    exactPartitionRows: number;
+    estimateToExactRatio: number;
+    suspiciousEstimate: boolean;
+    lowConfidenceChunkCount: number;
+    usedDistributionFallback: boolean;
+    usedLowConfidenceChunkRefinement: boolean;
+    usedExactCountFallback: boolean;
+}
+export interface Partition {
     partitionId: string;
     rows: number;
-    bytesOnDisk: number;
+    bytesCompressed: number;
+    bytesUncompressed: number;
     minTime: string;
     maxTime: string;
+    diagnostics?: PartitionDiagnostics;
 }
-export interface SortKeyInfo {
-    column: string;
-    type: string;
-    category: 'numeric' | 'datetime' | 'string';
+export interface TableProfile {
+    database: string;
+    table: string;
+    sortKeys: SortKey[];
+}
+export interface ChunkPlanStats {
+    totalPartitions: number;
+    oversizedPartitions: number;
+    focusedChunks: number;
+    totalChunks: number;
+    avgChunkBytes: number;
+    maxChunkBytes: number;
+    minChunkBytes: number;
 }
-export interface ChunkBoundary {
+export interface ChunkPlan {
+    planId: string;
+    generatedAt: string;
+    rowProbeStrategy: RowProbeStrategy;
+    targetChunkBytes: number;
+    table: TableProfile;
+    partitions: Partition[];
+    chunks: Chunk[];
+    totalRows: number;
+    totalBytesCompressed: number;
+    totalBytesUncompressed: number;
+    stats: ChunkPlanStats;
+}
+export type PlannerQuery = <T>(sql: string, settings?: Record<string, string | number | boolean | undefined>) => Promise<T[]>;
+export interface PlannerContext {
+    database: string;
+    table: string;
+    from?: string;
+    to?: string;
+    targetChunkBytes: number;
+    query: PlannerQuery;
+    querySettings?: Record<string, string | number | boolean | undefined>;
+    rowProbeStrategy: RowProbeStrategy;
+}
+export interface EstimateFilter {
     partitionId: string;
-    sortKeyFrom?: string;
-    sortKeyTo?: string;
-    estimatedBytes: number;
+    ranges: ChunkRange[];
+    exactDimensionIndex?: number;
+    exactValue?: string;
 }
-export interface PlannedChunk {
-    id: string;
+export interface StringPrefixBucket {
+    value: string;
+    rowCount: number;
+    isExactValue: boolean;
+}
+export interface TemporalBucket {
+    start: string;
+    rowCount: number;
+}
+export interface PartitionSlice {
     partitionId: string;
-    sortKeyFrom?: string;
-    sortKeyTo?: string;
-    estimatedBytes: number;
-    idempotencyToken: string;
-    from: string;
-    to: string;
+    ranges: ChunkRange[];
+    estimate: ChunkEstimate;
+    analysis: ChunkAnalysis;
+}
+export interface PartitionBuildResult {
+    slices: PartitionSlice[];
+    diagnostics: PartitionDiagnostics;
+}
+export interface PlanChunkOptions {
+    requireIdempotencyToken: boolean;
+}
+export interface GenerateChunkPlanInput {
+    database: string;
+    table: string;
+    from?: string;
+    to?: string;
+    targetChunkBytes: number;
+    query: PlannerQuery;
+    querySettings?: Record<string, string | number | boolean | undefined>;
+    rowProbeStrategy?: RowProbeStrategy;
 }
 //# sourceMappingURL=types.d.ts.map

package/dist/chunking/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/chunking/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,MAAM,CAAA;IACnB,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,SAAS,GAAG,UAAU,GAAG,QAAQ,CAAA;CAC5C;AAED,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,cAAc,EAAE,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAA;IACV,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,cAAc,EAAE,MAAM,CAAA;IACtB,gBAAgB,EAAE,MAAM,CAAA;IACxB,IAAI,EAAE,MAAM,CAAA;IACZ,EAAE,EAAE,MAAM,CAAA;CACX"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/chunking/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,gBAAgB,GAAG,kBAAkB,GAAG,OAAO,CAAA;AAE3D,MAAM,MAAM,eAAe,GAAG,SAAS,GAAG,UAAU,GAAG,QAAQ,CAAA;AAE/D,MAAM,MAAM,uBAAuB,GAAG,SAAS,GAAG,YAAY,CAAA;AAE9D,MAAM,MAAM,kBAAkB,GAAG,MAAM,GAAG,KAAK,GAAG,OAAO,CAAA;AAEzD,MAAM,MAAM,cAAc,GACtB,oBAAoB,GACpB,mBAAmB,GACnB,4BAA4B,GAC5B,2BAA2B,GAC3B,uBAAuB,GACvB,0BAA0B,GAC1B,aAAa,CAAA;AAEjB,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,eAAe,CAAA;IACzB,gBAAgB,EAAE,uBAAuB,CAAA;CAC1C;AAED,MAAM,WAAW,UAAU;IACzB,cAAc,EAAE,MAAM,CAAA;IACtB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,EAAE,CAAC,EAAE,MAAM,CAAA;CACZ;AAED,MAAM,WAAW,mBAAmB;IAClC,UAAU,EAAE,MAAM,CAAA;IAClB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAA;IACZ,eAAe,EAAE,MAAM,CAAA;IACvB,iBAAiB,EAAE,MAAM,CAAA;IACzB,UAAU,EAAE,kBAAkB,CAAA;IAC9B,MAAM,EAAE,cAAc,CAAA;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,cAAc,EAAE,MAAM,CAAA;IACtB,KAAK,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,aAAa;IAC5B,YAAY,CAAC,EAAE,YAAY,CAAA;IAC3B,OAAO,EAAE,mBAAmB,EAAE,CAAA;CAC/B;AAED,MAAM,WAAW,KAAK;IACpB,EAAE,EAAE,MAAM,CAAA;IACV,WAAW,EAAE,MAAM,CAAA;IACnB,MAAM,EAAE,UAAU,EAAE,CAAA;IACpB,QAAQ,EAAE,aAAa,CAAA;IACvB,QAAQ,EAAE,aAAa,CAAA;CACxB;AAED,MAAM,WAAW,oBAAoB;IACnC,eAAe,EAAE,MAAM,CAAA;IACvB,kBAAkB,EAAE,MAAM,CAAA;IAC1B,oBAAoB,EAAE,MAAM,CAAA;IAC5B,kBAAkB,EAAE,OAAO,CAAA;IAC3B,uBAAuB,EAAE,MAAM,CAAA;IAC/B,wBAAwB,EAAE,OAAO,CAAA;IACjC,gCAAgC,EAAE,OAAO,CAAA;IACzC,sBAAsB,EAAE,OAAO,CAAA;CAChC;AAED,MAAM,WAAW,SAAS;IACxB,WAAW,EAAE,MAAM,CAAA;IACnB,IAAI,EAAE,MAAM,CAAA;IACZ,eAAe,EAAE,MAAM,CAAA;IACvB,iBAAiB,EAAE,MAAM,CAAA;IACzB,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,CAAC,EAAE,oBAAoB,CAAA;CACnC;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,OAAO,EAAE,CAAA;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,eAAe,EAAE,MAAM,CAAA;IACvB,mBAAmB,EAAE,MAAM,CAAA;IAC3B,aAAa,EAAE,MAAM,CAAA;IACrB,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa,EAAE,MAAM,CAAA;IACrB,aAAa,EAAE,MAAM,CAAA;IACrB,aAAa,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,MAAM,CAAA;IACnB,gBAAgB,EAAE,gBAAgB,CAAA;IAClC,gBAAgB,EAAE,MAAM,CAAA;IACxB,KAAK,EAAE,YAAY,CAAA;IACnB,UAAU,EAAE,SAAS,EAAE,CAAA;IACvB,MAAM,EAAE,KAAK,EAAE,CAAA;IACf,SAAS,EAAE,MAAM,CAAA;IACjB,oBAAoB,EAAE,MAAM,CAAA;IAC5B,sBAAsB,EAAE,MAAM,CAAA;IAC9B,KAAK,EAAE,cAAc,CAAA;CACtB;AAED,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC,KAAK,OAAO,CAAC,CAAC,EAAE,CAAC,CAAA;AAE7H,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,gBAAgB,EAAE,MAAM,CAAA;IACxB,KAAK,EAAE,YAAY,CAAA;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC,CAAA;IACrE,gBAAgB,EAAE,gBAAgB,CAAA;CACnC;AAED,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAA;IACnB,MAAM,EAAE,UAAU,EAAE,CAAA;IACpB,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;IAChB,YAAY,EAAE,OAAO,CAAA;CACtB;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAA;IACnB,MAAM,EAAE,UAAU,EAAE,CAAA;IACpB,QAAQ,EAAE,aAAa,CAAA;IACvB,QAAQ,EAAE,aAAa,CAAA;CACxB;AAED,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,cAAc,EAAE,CAAA;IACxB,WAAW,EAAE,oBAAoB,CAAA;CAClC;AAED,MAAM,WAAW,gBAAgB;IAC/B,uBAAuB,EAAE,OAAO,CAAA;CACjC;AAED,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,gBAAgB,EAAE,MAAM,CAAA;IACxB,KAAK,EAAE,YAAY,CAAA;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC,CAAA;IACrE,gBAAgB,CAAC,EAAE,gBAAgB,CAAA;CACpC"}

package/dist/chunking/utils/binary-string.d.ts

@@ -0,0 +1,8 @@
+export declare function compareBinaryStrings(left: string, right: string): number;
+export declare function minBinaryString(left: string, right: string): string;
+export declare function maxBinaryString(left: string, right: string): string;
+export declare function nextPrefixValue(prefix: string): string | undefined;
+export declare function buildObservedStringUpperBound(maxValue: string): string;
+export declare function strToBigInt(value: string, padTo: number): bigint;
+export declare function bigIntToStr(value: bigint, length: number, minLength?: number): string;
+//# sourceMappingURL=binary-string.d.ts.map

package/dist/chunking/utils/binary-string.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"binary-string.d.ts","sourceRoot":"","sources":["../../../src/chunking/utils/binary-string.ts"],"names":[],"mappings":"AAAA,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAExE;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAEnE;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAEnE;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAclE;AAED,wBAAgB,6BAA6B,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAEtE;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAUhE;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,SAAI,GAAG,MAAM,CAiBhF"}

package/dist/chunking/utils/binary-string.js

@@ -0,0 +1,52 @@
+export function compareBinaryStrings(left, right) {
+    return Buffer.from(left, 'latin1').compare(Buffer.from(right, 'latin1'));
+}
+export function minBinaryString(left, right) {
+    return compareBinaryStrings(left, right) <= 0 ? left : right;
+}
+export function maxBinaryString(left, right) {
+    return compareBinaryStrings(left, right) >= 0 ? left : right;
+}
+export function nextPrefixValue(prefix) {
+    if (prefix.length === 0)
+        return undefined;
+    const buffer = Buffer.from(prefix, 'latin1');
+    for (let index = buffer.length - 1; index >= 0; index--) {
+        const byte = buffer[index];
+        if (byte === undefined || byte === 0xff)
+            continue;
+        const next = Buffer.from(buffer.subarray(0, index + 1));
+        next[index] = byte + 1;
+        return next.toString('latin1');
+    }
+    return undefined;
+}
+export function buildObservedStringUpperBound(maxValue) {
+    return `${maxValue}\0`;
+}
+export function strToBigInt(value, padTo) {
+    const buffer = Buffer.from(value, 'latin1');
+    let result = 0n;
+    for (let index = 0; index < padTo; index++) {
+        const byte = index < buffer.length ? (buffer[index] ?? 0) : 0;
+        result = (result << 8n) | BigInt(byte);
+    }
+    return result;
+}
+export function bigIntToStr(value, length, minLength = 0) {
+    const buffer = Buffer.alloc(length);
+    let remaining = value;
+    for (let index = length - 1; index >= 0; index--) {
+        buffer[index] = Number(remaining & 0xffn);
+        remaining >>= 8n;
+    }
+    // Strip trailing null bytes so boundaries match real string values
+    // in ClickHouse comparisons (where "abc" < "abc\0"), but preserve
+    // at least minLength bytes to avoid losing meaningful trailing nulls
+    // (e.g. from buildObservedStringUpperBound which appends "\0").
+    let end = length;
+    while (end > minLength && buffer[end - 1] === 0)
+        end--;
+    return buffer.subarray(0, end).toString('latin1');
+}
+//# sourceMappingURL=binary-string.js.map

package/dist/chunking/utils/binary-string.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"binary-string.js","sourceRoot":"","sources":["../../../src/chunking/utils/binary-string.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,oBAAoB,CAAC,IAAY,EAAE,KAAa;IAC9D,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAA;AAC1E,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,KAAa;IACzD,OAAO,oBAAoB,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAA;AAC9D,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,KAAa;IACzD,OAAO,oBAAoB,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAA;AAC9D,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,MAAc;IAC5C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAA;IAEzC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;IAC5C,KAAK,IAAI,KAAK,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;QACxD,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA;QAC1B,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,IAAI;YAAE,SAAQ;QAEjD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAA;QACvD,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,CAAA;QACtB,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;IAChC,CAAC;IAED,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,QAAgB;IAC5D,OAAO,GAAG,QAAQ,IAAI,CAAA;AACxB,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAa,EAAE,KAAa;IACtD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;IAC3C,IAAI,MAAM,GAAG,EAAE,CAAA;IAEf,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAC7D,MAAM,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA;IACxC,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAa,EAAE,MAAc,EAAE,SAAS,GAAG,CAAC;IACtE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IACnC,IAAI,SAAS,GAAG,KAAK,CAAA;IAErB,KAAK,IAAI,KAAK,GAAG,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;QACjD,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,SAAS,GAAG,KAAK,CAAC,CAAA;QACzC,SAAS,KAAK,EAAE,CAAA;IAClB,CAAC;IAED,mEAAmE;IACnE,kEAAkE;IAClE,qEAAqE;IACrE,gEAAgE;IAChE,IAAI,GAAG,GAAG,MAAM,CAAA;IAChB,OAAO,GAAG,GAAG,SAAS,IAAI,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC;QAAE,GAAG,EAAE,CAAA;IAEtD,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;AACnD,CAAC"}

package/dist/chunking/utils/ids.d.ts

@@ -0,0 +1,4 @@
+export declare function generatePlanId(): string;
+export declare function generateChunkId(planId: string, partitionId: string, index: number): string;
+export declare function generateIdempotencyToken(planId: string, chunkId: string): string;
+//# sourceMappingURL=ids.d.ts.map

package/dist/chunking/utils/ids.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ids.d.ts","sourceRoot":"","sources":["../../../src/chunking/utils/ids.ts"],"names":[],"mappings":"AAEA,wBAAgB,cAAc,IAAI,MAAM,CAEvC;AAED,wBAAgB,eAAe,CAC7B,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,MAAM,GACZ,MAAM,CAER;AAED,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAEhF"}

package/dist/chunking/utils/ids.js

@@ -0,0 +1,11 @@
+import { hashId, randomPlanId } from '../../state.js';
+export function generatePlanId() {
+    return randomPlanId();
+}
+export function generateChunkId(planId, partitionId, index) {
+    return hashId(`chunk:${planId}:${partitionId}:${index}`).slice(0, 16);
+}
+export function generateIdempotencyToken(planId, chunkId) {
+    return hashId(`token:${planId}:${chunkId}`);
+}
+//# sourceMappingURL=ids.js.map

package/dist/chunking/utils/ids.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ids.js","sourceRoot":"","sources":["../../../src/chunking/utils/ids.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAA;AAErD,MAAM,UAAU,cAAc;IAC5B,OAAO,YAAY,EAAE,CAAA;AACvB,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,MAAc,EACd,WAAmB,EACnB,KAAa;IAEb,OAAO,MAAM,CAAC,SAAS,MAAM,IAAI,WAAW,IAAI,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;AACvE,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,MAAc,EAAE,OAAe;IACtE,OAAO,MAAM,CAAC,SAAS,MAAM,IAAI,OAAO,EAAE,CAAC,CAAA;AAC7C,CAAC"}

package/dist/chunking/utils/ranges.d.ts

@@ -0,0 +1,5 @@
+import type { ChunkRange, PartitionSlice } from '../types.js';
+export declare function getChunkRange(slice: Pick<PartitionSlice, 'ranges'>, dimensionIndex: number): ChunkRange;
+export declare function replaceChunkRange(slice: Pick<PartitionSlice, 'ranges'>, dimensionIndex: number, from: string | undefined, to: string | undefined): ChunkRange[];
+export declare function isExactChunkRange(range: Pick<ChunkRange, 'from' | 'to'>): boolean;
+//# sourceMappingURL=ranges.d.ts.map

package/dist/chunking/utils/ranges.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ranges.d.ts","sourceRoot":"","sources":["../../../src/chunking/utils/ranges.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE7D,wBAAgB,aAAa,CAC3B,KAAK,EAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,EACrC,cAAc,EAAE,MAAM,GACrB,UAAU,CAQZ;AAED,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,EACrC,cAAc,EAAE,MAAM,EACtB,IAAI,EAAE,MAAM,GAAG,SAAS,EACxB,EAAE,EAAE,MAAM,GAAG,SAAS,GACrB,UAAU,EAAE,CAKd;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAGjF"}

package/dist/chunking/utils/ranges.js

@@ -0,0 +1,19 @@
+export function getChunkRange(slice, dimensionIndex) {
+    return (slice.ranges.find((range) => range.dimensionIndex === dimensionIndex) ?? {
+        dimensionIndex,
+        from: undefined,
+        to: undefined,
+    });
+}
+export function replaceChunkRange(slice, dimensionIndex, from, to) {
+    return slice.ranges
+        .filter((range) => range.dimensionIndex !== dimensionIndex)
+        .concat([{ dimensionIndex, from, to }])
+        .sort((left, right) => left.dimensionIndex - right.dimensionIndex);
+}
+export function isExactChunkRange(range) {
+    if (range.from === undefined || range.to === undefined)
+        return false;
+    return range.to === `${range.from}\0`;
+}
+//# sourceMappingURL=ranges.js.map

package/dist/chunking/utils/ranges.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ranges.js","sourceRoot":"","sources":["../../../src/chunking/utils/ranges.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,aAAa,CAC3B,KAAqC,EACrC,cAAsB;IAEtB,OAAO,CACL,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,cAAc,KAAK,cAAc,CAAC,IAAI;QACvE,cAAc;QACd,IAAI,EAAE,SAAS;QACf,EAAE,EAAE,SAAS;KACd,CACF,CAAA;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,KAAqC,EACrC,cAAsB,EACtB,IAAwB,EACxB,EAAsB;IAEtB,OAAO,KAAK,CAAC,MAAM;SAChB,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,cAAc,KAAK,cAAc,CAAC;SAC1D,MAAM,CAAC,CAAC,EAAE,cAAc,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;SACtC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC,cAAc,CAAC,CAAA;AACtE,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAsC;IACtE,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,EAAE,KAAK,SAAS;QAAE,OAAO,KAAK,CAAA;IACpE,OAAO,KAAK,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,IAAI,IAAI,CAAA;AACvC,CAAC"}

package/dist/index.d.ts

@@ -1,11 +1,6 @@
 import './table-config.js';
 export { backfill, createBackfillPlugin } from './plugin.js';
-export { executeBackfill, syncProgress } from './async-backfill.js';
-export { analyzeAndChunk } from './chunking/analyze.js';
-export type { BackfillOptions, BackfillChunkState, BackfillProgress, BackfillResult, } from './async-backfill.js';
 export type { BackfillPlugin, BackfillPluginOptions, BackfillPluginRegistration } from './types.js';
 export type { PluginConfig } from './options.js';
 export type { BackfillTableConfig } from './table-config.js';
-export type { AnalyzeAndChunkInput, AnalyzeAndChunkResult } from './chunking/analyze.js';
-export type { PlannedChunk, PartitionInfo, SortKeyInfo } from './chunking/types.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,mBAAmB,CAAA;AAE1B,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAA;AAC5D,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAA;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AACvD,YAAY,EACV,eAAe,EACf,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,GACf,MAAM,qBAAqB,CAAA;AAC5B,YAAY,EAAE,cAAc,EAAE,qBAAqB,EAAE,0BAA0B,EAAE,MAAM,YAAY,CAAA;AACnG,YAAY,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAChD,YAAY,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAA;AAC5D,YAAY,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AACxF,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,mBAAmB,CAAA;AAE1B,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAA;AAC5D,YAAY,EAAE,cAAc,EAAE,qBAAqB,EAAE,0BAA0B,EAAE,MAAM,YAAY,CAAA;AACnG,YAAY,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAChD,YAAY,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAA"}

package/dist/index.js

@@ -1,5 +1,3 @@
 import './table-config.js';
 export { backfill, createBackfillPlugin } from './plugin.js';
-export { executeBackfill, syncProgress } from './async-backfill.js';
-export { analyzeAndChunk } from './chunking/analyze.js';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,mBAAmB,CAAA;AAE1B,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAA;AAC5D,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAA;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,mBAAmB,CAAA;AAE1B,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAA"}

package/dist/logging.d.ts

@@ -0,0 +1,12 @@
+import { type Logger } from '@logtape/logtape';
+export declare const CHKIT_LOGGER_CATEGORY: readonly ["chkit"];
+export declare const CHKIT_BACKFILL_LOGGER_CATEGORY: readonly ["chkit", "backfill"];
+export declare const SLOW_CLICKHOUSE_QUERY_MS = 5000;
+export declare const SLOW_CLICKHOUSE_QUERY_REPEAT_INITIAL_MS = 5000;
+export declare const SLOW_CLICKHOUSE_QUERY_REPEAT_MAX_MS = 30000;
+export declare function getBackfillLogger(...segments: string[]): Logger;
+export declare function formatBytes(bytes: number): string;
+export declare function summarizeSql(sql: string, maxLength?: number): string;
+export declare function describeSqlOperation(sql: string): string;
+export declare function describeSqlContext(sql: string): string | undefined;
+//# sourceMappingURL=logging.d.ts.map

package/dist/logging.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"logging.d.ts","sourceRoot":"","sources":["../src/logging.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,KAAK,MAAM,EAAE,MAAM,kBAAkB,CAAA;AAEzD,eAAO,MAAM,qBAAqB,oBAAqB,CAAA;AACvD,eAAO,MAAM,8BAA8B,gCAAkD,CAAA;AAC7F,eAAO,MAAM,wBAAwB,OAAO,CAAA;AAC5C,eAAO,MAAM,uCAAuC,OAAO,CAAA;AAC3D,eAAO,MAAM,mCAAmC,QAAQ,CAAA;AAExD,wBAAgB,iBAAiB,CAAC,GAAG,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAE/D;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAMjD;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,SAAM,GAAG,MAAM,CAIjE;AAED,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAqBxD;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAMlE"}

package/dist/logging.js

@@ -0,0 +1,61 @@
+import { getLogger } from '@logtape/logtape';
+export const CHKIT_LOGGER_CATEGORY = ['chkit'];
+export const CHKIT_BACKFILL_LOGGER_CATEGORY = [...CHKIT_LOGGER_CATEGORY, 'backfill'];
+export const SLOW_CLICKHOUSE_QUERY_MS = 5000;
+export const SLOW_CLICKHOUSE_QUERY_REPEAT_INITIAL_MS = 5000;
+export const SLOW_CLICKHOUSE_QUERY_REPEAT_MAX_MS = 30000;
+export function getBackfillLogger(...segments) {
+    return getLogger([...CHKIT_BACKFILL_LOGGER_CATEGORY, ...segments]);
+}
+export function formatBytes(bytes) {
+    if (bytes >= 1024 ** 4)
+        return `${(bytes / 1024 ** 4).toFixed(1)} TiB`;
+    if (bytes >= 1024 ** 3)
+        return `${(bytes / 1024 ** 3).toFixed(1)} GiB`;
+    if (bytes >= 1024 ** 2)
+        return `${(bytes / 1024 ** 2).toFixed(1)} MiB`;
+    if (bytes >= 1024)
+        return `${(bytes / 1024).toFixed(1)} KiB`;
+    return `${bytes} B`;
+}
+export function summarizeSql(sql, maxLength = 240) {
+    const normalized = normalizeSql(sql);
+    if (normalized.length <= maxLength)
+        return normalized;
+    return `${normalized.slice(0, maxLength - 3)}...`;
+}
+export function describeSqlOperation(sql) {
+    const normalized = normalizeSql(sql);
+    const prefixDistribution = normalized.match(/^SELECT substring\((\w+), 1, \d+\) AS prefix, count\(\) AS cnt /);
+    if (prefixDistribution?.[1])
+        return `prefix distribution on ${prefixDistribution[1]}`;
+    const temporalDistribution = normalized.match(/^SELECT formatDateTime\(toStartOf(Day|Hour)\((\w+)\)/);
+    if (temporalDistribution?.[1] && temporalDistribution[2]) {
+        return `${temporalDistribution[1].toLowerCase()} distribution on ${temporalDistribution[2]}`;
+    }
+    const minMaxProbe = normalized.match(/^SELECT toString\(min\((\w+)\)\) AS minVal, toString\(max\(\1\)\) AS maxVal /);
+    if (minMaxProbe?.[1])
+        return `range probe on ${minMaxProbe[1]}`;
+    if (normalized.startsWith('SELECT count() AS cnt FROM '))
+        return 'row count probe';
+    if (normalized.startsWith('SELECT sorting_key FROM system.tables'))
+        return 'sort key introspection';
+    if (normalized.startsWith('SELECT name, type FROM system.columns'))
+        return 'column introspection';
+    if (normalized.startsWith('SELECT partition_id,'))
+        return 'partition introspection';
+    if (normalized.startsWith('SELECT 1 FROM '))
+        return 'table existence probe';
+    return summarizeSql(normalized, 100);
+}
+export function describeSqlContext(sql) {
+    const normalized = normalizeSql(sql);
+    const partitionId = normalized.match(/_partition_id = '([^']+)'/)?.[1];
+    if (partitionId)
+        return `partition ${partitionId}`;
+    return undefined;
+}
+function normalizeSql(sql) {
+    return sql.replace(/\s+/g, ' ').trim();
+}
+//# sourceMappingURL=logging.js.map

package/dist/logging.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"logging.js","sourceRoot":"","sources":["../src/logging.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAe,MAAM,kBAAkB,CAAA;AAEzD,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,OAAO,CAAU,CAAA;AACvD,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC,GAAG,qBAAqB,EAAE,UAAU,CAAU,CAAA;AAC7F,MAAM,CAAC,MAAM,wBAAwB,GAAG,IAAI,CAAA;AAC5C,MAAM,CAAC,MAAM,uCAAuC,GAAG,IAAI,CAAA;AAC3D,MAAM,CAAC,MAAM,mCAAmC,GAAG,KAAK,CAAA;AAExD,MAAM,UAAU,iBAAiB,CAAC,GAAG,QAAkB;IACrD,OAAO,SAAS,CAAC,CAAC,GAAG,8BAA8B,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAA;AACpE,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC;QAAE,OAAO,GAAG,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAA;IACtE,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC;QAAE,OAAO,GAAG,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAA;IACtE,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC;QAAE,OAAO,GAAG,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAA;IACtE,IAAI,KAAK,IAAI,IAAI;QAAE,OAAO,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAA;IAC5D,OAAO,GAAG,KAAK,IAAI,CAAA;AACrB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAW,EAAE,SAAS,GAAG,GAAG;IACvD,MAAM,UAAU,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;IACpC,IAAI,UAAU,CAAC,MAAM,IAAI,SAAS;QAAE,OAAO,UAAU,CAAA;IACrD,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,GAAG,CAAC,CAAC,KAAK,CAAA;AACnD,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,GAAW;IAC9C,MAAM,UAAU,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;IAEpC,MAAM,kBAAkB,GAAG,UAAU,CAAC,KAAK,CAAC,iEAAiE,CAAC,CAAA;IAC9G,IAAI,kBAAkB,EAAE,CAAC,CAAC,CAAC;QAAE,OAAO,0BAA0B,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAA;IAErF,MAAM,oBAAoB,GAAG,UAAU,CAAC,KAAK,CAAC,sDAAsD,CAAC,CAAA;IACrG,IAAI,oBAAoB,EAAE,CAAC,CAAC,CAAC,IAAI,oBAAoB,CAAC,CAAC,CAAC,EAAE,CAAC;QACzD,OAAO,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,oBAAoB,oBAAoB,CAAC,CAAC,CAAC,EAAE,CAAA;IAC9F,CAAC;IAED,MAAM,WAAW,GAAG,UAAU,CAAC,KAAK,CAAC,8EAA8E,CAAC,CAAA;IACpH,IAAI,WAAW,EAAE,CAAC,CAAC,CAAC;QAAE,OAAO,kBAAkB,WAAW,CAAC,CAAC,CAAC,EAAE,CAAA;IAE/D,IAAI,UAAU,CAAC,UAAU,CAAC,6BAA6B,CAAC;QAAE,OAAO,iBAAiB,CAAA;IAClF,IAAI,UAAU,CAAC,UAAU,CAAC,uCAAuC,CAAC;QAAE,OAAO,wBAAwB,CAAA;IACnG,IAAI,UAAU,CAAC,UAAU,CAAC,uCAAuC,CAAC;QAAE,OAAO,sBAAsB,CAAA;IACjG,IAAI,UAAU,CAAC,UAAU,CAAC,sBAAsB,CAAC;QAAE,OAAO,yBAAyB,CAAA;IACnF,IAAI,UAAU,CAAC,UAAU,CAAC,gBAAgB,CAAC;QAAE,OAAO,uBAAuB,CAAA;IAE3E,OAAO,YAAY,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;AACtC,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC5C,MAAM,UAAU,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;IACpC,MAAM,WAAW,GAAG,UAAU,CAAC,KAAK,CAAC,2BAA2B,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAEtE,IAAI,WAAW;QAAE,OAAO,aAAa,WAAW,EAAE,CAAA;IAClD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,SAAS,YAAY,CAAC,GAAW;IAC/B,OAAO,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;AACxC,CAAC"}

package/dist/payload.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"payload.d.ts","sourceRoot":"","sources":["../src/payload.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,oBAAoB,EACpB,kBAAkB,EAClB,qBAAqB,EACrB,uBAAuB,EACxB,MAAM,YAAY,CAAA;AAEnB,wBAAgB,WAAW,CAAC,MAAM,EAAE,uBAAuB,GAAG;IAC5D,EAAE,EAAE,IAAI,CAAA;IACR,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;IACZ,EAAE,EAAE,MAAM,CAAA;IACV,UAAU,EAAE,MAAM,CAAA;IAClB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB,CAkBA;AAED,wBAAgB,aAAa,CAAC,OAAO,EAAE,qBAAqB,GAAG;IAC7D,EAAE,EAAE,OAAO,CAAA;IACX,OAAO,EAAE,QAAQ,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,kBAAkB,CAAA;IAC1B,WAAW,EAAE,qBAAqB,CAAC,QAAQ,CAAC,CAAA;IAC5C,WAAW,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,MAAM,CAAA;IACf,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB,CAYA;AAED,wBAAgB,aAAa,CAAC,OAAO,EAAE,qBAAqB,GAAG;IAC7D,EAAE,EAAE,OAAO,CAAA;IACX,OAAO,EAAE,QAAQ,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,kBAAkB,CAAA;IAC1B,WAAW,EAAE,qBAAqB,CAAC,QAAQ,CAAC,CAAA;IAC5C,OAAO,EAAE,MAAM,CAAA;CAChB,CASA;AAED,wBAAgB,aAAa,CAAC,MAAM,EAAE,oBAAoB,GAAG;IAC3D,EAAE,EAAE,OAAO,CAAA;IACX,OAAO,EAAE,QAAQ,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,kBAAkB,CAAA;IAC1B,UAAU,EAAE,MAAM,EAAE,CAAA;IACpB,eAAe,EAAE,MAAM,EAAE,CAAA;IACzB,cAAc,EAAE,MAAM,EAAE,CAAA;CACzB,CAUA"}
+{"version":3,"file":"payload.d.ts","sourceRoot":"","sources":["../src/payload.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,oBAAoB,EACpB,kBAAkB,EAClB,qBAAqB,EACrB,uBAAuB,EACxB,MAAM,YAAY,CAAA;AAEnB,wBAAgB,WAAW,CAAC,MAAM,EAAE,uBAAuB,GAAG;IAC5D,EAAE,EAAE,IAAI,CAAA;IACR,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;IACZ,EAAE,EAAE,MAAM,CAAA;IACV,UAAU,EAAE,MAAM,CAAA;IAClB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB,CAgBA;AAED,wBAAgB,aAAa,CAAC,OAAO,EAAE,qBAAqB,GAAG;IAC7D,EAAE,EAAE,OAAO,CAAA;IACX,OAAO,EAAE,QAAQ,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,kBAAkB,CAAA;IAC1B,WAAW,EAAE,qBAAqB,CAAC,QAAQ,CAAC,CAAA;IAC5C,WAAW,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,MAAM,CAAA;IACf,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB,CAYA;AAED,wBAAgB,aAAa,CAAC,OAAO,EAAE,qBAAqB,GAAG;IAC7D,EAAE,EAAE,OAAO,CAAA;IACX,OAAO,EAAE,QAAQ,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,kBAAkB,CAAA;IAC1B,WAAW,EAAE,qBAAqB,CAAC,QAAQ,CAAC,CAAA;IAC5C,OAAO,EAAE,MAAM,CAAA;CAChB,CASA;AAED,wBAAgB,aAAa,CAAC,MAAM,EAAE,oBAAoB,GAAG;IAC3D,EAAE,EAAE,OAAO,CAAA;IACX,OAAO,EAAE,QAAQ,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,kBAAkB,CAAA;IAC1B,UAAU,EAAE,MAAM,EAAE,CAAA;IACpB,eAAe,EAAE,MAAM,EAAE,CAAA;IACzB,cAAc,EAAE,MAAM,EAAE,CAAA;CACzB,CAUA"}

package/dist/payload.js

@@ -6,15 +6,13 @@ export function planPayload(output) {
         target: output.plan.target,
         from: output.plan.from,
         to: output.plan.to,
-        chunkCount: output.plan.chunks.length,
+        chunkCount: output.plan.chunkPlan.chunks.length,
         maxChunkBytes: output.plan.options.maxChunkBytes,
         sortKeyColumn: output.plan.options.sortKeyColumn,
         planPath: output.planPath,
-        strategy: output.plan.strategy,
-        partitionCount: output.plan.partitions?.length,
-        totalBytes: output.plan.partitions
-            ? output.plan.partitions.reduce((sum, p) => sum + p.bytesOnDisk, 0)
-            : undefined,
+        strategy: output.plan.execution.mode,
+        partitionCount: output.plan.chunkPlan.partitions.length,
+        totalBytes: output.plan.chunkPlan.totalBytesCompressed,
     };
 }
 export function statusPayload(summary) {

package/dist/payload.js.map

@@ -1 +1 @@
-{"version":3,"file":"payload.js","sourceRoot":"","sources":["../src/payload.ts"],"names":[],"mappings":"AAOA,MAAM,UAAU,WAAW,CAAC,MAA+B;IAezD,OAAO;QACL,EAAE,EAAE,IAAI;QACR,OAAO,EAAE,MAAM;QACf,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM;QAC1B,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM;QAC1B,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI;QACtB,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;QAClB,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM;QACrC,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa;QAChD,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa;QAChD,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;QAC9B,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM;QAC9C,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU;YAChC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;YACnE,CAAC,CAAC,SAAS;KACd,CAAA;AACH,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAA8B;IAW1D,OAAO;QACL,EAAE,EAAE,OAAO,CAAC,MAAM,KAAK,QAAQ;QAC/B,OAAO,EAAE,QAAQ;QACjB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,WAAW,EAAE,OAAO,CAAC,MAAM;QAC3B,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAA;AACH,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAA8B;IAQ1D,OAAO;QACL,EAAE,EAAE,OAAO,CAAC,MAAM,KAAK,WAAW;QAClC,OAAO,EAAE,QAAQ;QACjB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,WAAW,EAAE,OAAO,CAAC,MAAM;QAC3B,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAA;AACH,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAA4B;IASxD,OAAO;QACL,EAAE,EAAE,MAAM,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC;QAClC,OAAO,EAAE,QAAQ;QACjB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,eAAe,EAAE,MAAM,CAAC,eAAe;QACvC,cAAc,EAAE,MAAM,CAAC,cAAc;KACtC,CAAA;AACH,CAAC"}
+{"version":3,"file":"payload.js","sourceRoot":"","sources":["../src/payload.ts"],"names":[],"mappings":"AAOA,MAAM,UAAU,WAAW,CAAC,MAA+B;IAezD,OAAO;QACL,EAAE,EAAE,IAAI;QACR,OAAO,EAAE,MAAM;QACf,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM;QAC1B,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM;QAC1B,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI;QACtB,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;QAClB,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM;QAC/C,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa;QAChD,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa;QAChD,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI;QACpC,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM;QACvD,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,oBAAoB;KACvD,CAAA;AACH,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAA8B;IAW1D,OAAO;QACL,EAAE,EAAE,OAAO,CAAC,MAAM,KAAK,QAAQ;QAC/B,OAAO,EAAE,QAAQ;QACjB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,WAAW,EAAE,OAAO,CAAC,MAAM;QAC3B,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAA;AACH,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAA8B;IAQ1D,OAAO;QACL,EAAE,EAAE,OAAO,CAAC,MAAM,KAAK,WAAW;QAClC,OAAO,EAAE,QAAQ;QACjB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,WAAW,EAAE,OAAO,CAAC,MAAM;QAC3B,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAA;AACH,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAA4B;IASxD,OAAO;QACL,EAAE,EAAE,MAAM,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC;QAClC,OAAO,EAAE,QAAQ;QACjB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,eAAe,EAAE,MAAM,CAAC,eAAe;QACvC,cAAc,EAAE,MAAM,CAAC,cAAc;KACtC,CAAA;AACH,CAAC"}

package/dist/planner.d.ts

@@ -9,6 +9,7 @@ export declare function buildBackfillPlan(input: {
         url: string;
         database: string;
     };
-    clickhouseQuery: <T>(sql: string) => Promise<T[]>;
+    clickhouseQuery: <T>(sql: string, settings?: Record<string, string | number | boolean | undefined>) => Promise<T[]>;
+    querySettings?: Record<string, string | number | boolean | undefined>;
 }): Promise<BuildBackfillPlanOutput>;
 //# sourceMappingURL=planner.d.ts.map

package/dist/planner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"planner.d.ts","sourceRoot":"","sources":["../src/planner.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAMpD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAA;AAO/C,OAAO,KAAK,EAEV,uBAAuB,EAExB,MAAM,YAAY,CAAA;AAEnB,wBAAsB,iBAAiB,CAAC,KAAK,EAAE;IAC7C,IAAI,EAAE,WAAW,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,IAAI,CAAC,iBAAiB,EAAE,SAAS,GAAG,QAAQ,CAAC,CAAA;IACrD,UAAU,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAA;IAC9C,eAAe,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,EAAE,CAAC,CAAA;CAClD,GAAG,OAAO,CAAC,uBAAuB,CAAC,CAwHnC"}
+{"version":3,"file":"planner.d.ts","sourceRoot":"","sources":["../src/planner.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAOpD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAA;AAQ/C,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,YAAY,CAAA;AAEzD,wBAAsB,iBAAiB,CAAC,KAAK,EAAE;IAC7C,IAAI,EAAE,WAAW,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,IAAI,CAAC,iBAAiB,EAAE,SAAS,GAAG,QAAQ,CAAC,CAAA;IACrD,UAAU,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAA;IAC9C,eAAe,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC,KAAK,OAAO,CAAC,CAAC,EAAE,CAAC,CAAA;IACnH,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC,CAAA;CACtE,GAAG,OAAO,CAAC,uBAAuB,CAAC,CAqGnC"}

package/dist/planner.js

@@ -1,36 +1,34 @@
 import { dirname } from 'node:path';
 import { loadSchemaDefinitions } from '@chkit/core/schema-loader';
-import { analyzeAndChunk } from './chunking/analyze.js';
-import { buildChunkSql } from './chunking/sql.js';
+import { encodeChunkPlanForPersistence } from './chunking/boundary-codec.js';
+import { generateChunkPlan } from './chunking/planner.js';
 import { findMvForTarget } from './detect.js';
 import { BackfillConfigError } from './errors.js';
-import { backfillPaths, computeBackfillStateDir, computeEnvironmentFingerprint, writeJson, } from './state.js';
+import { backfillPaths, computeBackfillStateDir, computeEnvironmentFingerprint, nowIso, writeJson, } from './state.js';
 export async function buildBackfillPlan(input) {
     const { opts } = input;
     const [database, table] = opts.target.split('.');
     if (!database || !table) {
         throw new BackfillConfigError('Invalid target format. Expected <database.table>.');
     }
-    const env = computeEnvironmentFingerprint(input.clickhouse);
-    // 1. Analyze table and build planned chunks
-    const { planId, partitions, sortKey, chunks: plannedChunks } = await analyzeAndChunk({
+    const chunkPlan = await generateChunkPlan({
         database,
         table,
         from: opts.from,
         to: opts.to,
-        maxChunkBytes: opts.maxChunkBytes,
-        requireIdempotencyToken: opts.requireIdempotencyToken,
+        targetChunkBytes: opts.maxChunkBytes,
         query: input.clickhouseQuery,
+        querySettings: input.querySettings,
     });
-    if (partitions.length === 0) {
+    const firstPartition = chunkPlan.partitions[0];
+    if (!firstPartition) {
         throw new BackfillConfigError(`No partitions found for ${opts.target}${opts.from || opts.to ? ' within the specified time range' : ''}. The table may be empty.`);
     }
-    const firstPartition = partitions[0];
-    const derivedFrom = opts.from ?? partitions.reduce((min, p) => (p.minTime < min ? p.minTime : min), firstPartition.minTime);
-    const derivedTo = opts.to ?? partitions.reduce((max, p) => (p.maxTime > max ? p.maxTime : max), firstPartition.maxTime);
+    const env = computeEnvironmentFingerprint(input.clickhouse);
+    const derivedFrom = opts.from ?? chunkPlan.partitions.reduce((min, partition) => (partition.minTime < min ? partition.minTime : min), firstPartition.minTime);
+    const derivedTo = opts.to ?? chunkPlan.partitions.reduce((max, partition) => (partition.maxTime > max ? partition.maxTime : max), firstPartition.maxTime);
     const stateDir = computeBackfillStateDir(input.config, input.configPath, opts.stateDir);
-    const paths = backfillPaths(stateDir, planId);
-    // 2. Detect MV for replay strategy
+    const paths = backfillPaths(stateDir, chunkPlan.planId);
     let mvAsQuery;
     let targetColumns;
     try {
@@ -40,58 +38,36 @@ export async function buildBackfillPlan(input) {
         const mv = findMvForTarget(definitions, database, table);
         if (mv) {
             mvAsQuery = mv.as;
-            const tableDef = definitions.find((d) => d.kind === 'table' && d.database === database && d.name === table);
-            if (tableDef && tableDef.kind === 'table') {
-                targetColumns = tableDef.columns.map((c) => c.name);
+            const tableDef = definitions.find((definition) => definition.kind === 'table' && definition.database === database && definition.name === table);
+            if (tableDef?.kind === 'table') {
+                targetColumns = tableDef.columns.map((column) => column.name);
             }
         }
     }
     catch {
-        // Schema load failed — fall back to direct copy
+        // Schema load failed, fall back to direct copy.
     }
-    // 3. Stamp SQL on each planned chunk to produce BackfillChunk[]
-    const chunks = plannedChunks.map(planned => {
-        const sqlTemplate = buildChunkSql({
-            planId,
-            chunk: planned,
-            target: opts.target,
-            sortKey,
-            mvAsQuery,
-            targetColumns,
-        });
-        return {
-            id: planned.id,
-            from: planned.from,
-            to: planned.to,
-            status: 'pending',
-            attempts: 0,
-            idempotencyToken: planned.idempotencyToken,
-            sqlTemplate,
-            partitionId: planned.partitionId,
-            estimatedBytes: planned.estimatedBytes,
-            ...(planned.sortKeyFrom !== undefined ? { sortKeyFrom: planned.sortKeyFrom } : {}),
-            ...(planned.sortKeyTo !== undefined ? { sortKeyTo: planned.sortKeyTo } : {}),
-        };
-    });
-    const strategy = mvAsQuery ? 'mv_replay' : 'partition';
     const plan = {
-        planId,
+        planId: chunkPlan.planId,
         target: opts.target,
-        createdAt: '1970-01-01T00:00:00.000Z',
-        status: 'planned',
-        strategy: strategy,
+        createdAt: nowIso(),
         ...(env ? { environment: env } : {}),
         from: derivedFrom,
         to: derivedTo,
-        chunks,
-        partitions,
-        sortKey,
+        chunkPlan,
+        execution: {
+            mode: mvAsQuery ? 'mv_replay' : 'copy',
+            sourceTarget: opts.target,
+            ...(mvAsQuery ? { mvAsQuery } : {}),
+            ...(targetColumns ? { targetColumns } : {}),
+            requireIdempotencyToken: opts.requireIdempotencyToken,
+        },
         options: {
             maxChunkBytes: opts.maxChunkBytes,
             maxParallelChunks: opts.maxParallelChunks,
             maxRetriesPerChunk: opts.maxRetriesPerChunk,
             requireIdempotencyToken: opts.requireIdempotencyToken,
-            sortKeyColumn: sortKey?.column,
+            sortKeyColumn: chunkPlan.table.sortKeys[0]?.name,
         },
         policy: {
             requireDryRunBeforeRun: opts.requireDryRunBeforeRun,
@@ -104,7 +80,10 @@ export async function buildBackfillPlan(input) {
             minChunkMinutes: opts.minChunkMinutes,
         },
     };
-    await writeJson(paths.planPath, plan);
+    await writeJson(paths.planPath, {
+        ...plan,
+        chunkPlan: encodeChunkPlanForPersistence(plan.chunkPlan),
+    });
     return {
         plan,
         planPath: paths.planPath,