@chkit/plugin-backfill 0.1.0-beta.9 → 0.1.0

package/dist/chunking/introspect.js

@@ -0,0 +1,187 @@
+const NUMERIC_TYPES = new Set([
+    'Int8', 'Int16', 'Int32', 'Int64', 'Int128', 'Int256',
+    'UInt8', 'UInt16', 'UInt32', 'UInt64', 'UInt128', 'UInt256',
+    'Float32', 'Float64',
+]);
+const DATETIME_TYPES = new Set(['Date', 'Date32', 'DateTime', 'DateTime64']);
+function classifySortKeyType(type) {
+    if (NUMERIC_TYPES.has(type))
+        return 'numeric';
+    if (DATETIME_TYPES.has(type))
+        return 'datetime';
+    if (type.startsWith('DateTime64('))
+        return 'datetime';
+    if (type.startsWith("DateTime('"))
+        return 'datetime';
+    return 'string';
+}
+export async function queryPartitionInfo(input) {
+    // Force replica sync on the target table before reading system.parts.
+    // select_sequential_consistency is only effective on user tables, not system
+    // tables, so this preliminary query ensures the replica has caught up with
+    // all pending writes before we inspect part metadata.
+    await input.query(`SELECT 1 FROM ${input.database}.${input.table} LIMIT 1 SETTINGS select_sequential_consistency = 1`);
+    const rows = await input.query(`SELECT
+  partition_id,
+  toString(sum(rows)) AS total_rows,
+  toString(sum(bytes_on_disk)) AS total_bytes,
+  toString(sum(data_uncompressed_bytes)) AS total_uncompressed_bytes,
+  toString(min(min_time)) AS min_time,
+  toString(max(max_time)) AS max_time
+FROM system.parts
+WHERE database = '${input.database}'
+  AND table = '${input.table}'
+  AND active = 1
+GROUP BY partition_id
+ORDER BY partition_id
+SETTINGS select_sequential_consistency = 1`);
+    const partitions = rows.map((row) => ({
+        partitionId: row.partition_id,
+        rows: Number(row.total_rows),
+        bytesOnDisk: Number(row.total_bytes),
+        bytesUncompressed: Number(row.total_uncompressed_bytes ?? row.total_bytes),
+        minTime: new Date(row.min_time).toISOString(),
+        maxTime: new Date(row.max_time).toISOString(),
+    }));
+    return partitions.filter((p) => {
+        if (input.from && p.maxTime < input.from)
+            return false;
+        if (input.to && p.minTime >= input.to)
+            return false;
+        return true;
+    });
+}
+function extractSortKeyColumns(sortingKey) {
+    return splitTopLevelCsv(sortingKey)
+        .map((part) => part.trim())
+        .filter((part) => part.length > 0);
+}
+function splitTopLevelCsv(input) {
+    const parts = [];
+    let current = '';
+    let depth = 0;
+    let quote;
+    for (let index = 0; index < input.length; index++) {
+        const char = input[index];
+        if (char === undefined)
+            continue;
+        if (quote) {
+            current += char;
+            if (char === quote && input[index - 1] !== '\\') {
+                quote = undefined;
+            }
+            continue;
+        }
+        if (char === '\'' || char === '"') {
+            quote = char;
+            current += char;
+            continue;
+        }
+        if (char === '(') {
+            depth += 1;
+            current += char;
+            continue;
+        }
+        if (char === ')') {
+            depth = Math.max(0, depth - 1);
+            current += char;
+            continue;
+        }
+        if (char === ',' && depth === 0) {
+            parts.push(current.trim());
+            current = '';
+            continue;
+        }
+        current += char;
+    }
+    if (current.trim().length > 0) {
+        parts.push(current.trim());
+    }
+    return parts;
+}
+function resolveSortKeyColumn(expression, knownColumns) {
+    const trimmed = expression.trim();
+    if (knownColumns.has(trimmed)) {
+        return trimmed;
+    }
+    const identifiers = Array.from(trimmed.matchAll(/\b[A-Za-z_][A-Za-z0-9_]*\b/g))
+        .map((match) => match[0])
+        .filter((identifier) => Boolean(identifier));
+    const matches = Array.from(new Set(identifiers.filter((identifier) => knownColumns.has(identifier))));
+    if (matches.length === 1) {
+        return matches[0];
+    }
+    return undefined;
+}
+function resolveSortKeyColumnWithoutSchema(expression) {
+    const trimmed = expression.trim();
+    if (/^[A-Za-z_][A-Za-z0-9_]*$/.test(trimmed)) {
+        return trimmed;
+    }
+    const identifiers = Array.from(trimmed.matchAll(/\b[A-Za-z_][A-Za-z0-9_]*\b/g))
+        .map((match) => match[0])
+        .filter((identifier) => Boolean(identifier));
+    return identifiers.length > 0 ? identifiers[identifiers.length - 1] : undefined;
+}
+export async function querySortKeys(input) {
+    const tableRows = await input.query(`SELECT sorting_key FROM system.tables WHERE database = '${input.database}' AND name = '${input.table}'`);
+    const sortingKey = tableRows[0]?.sorting_key;
+    if (!sortingKey)
+        return [];
+    const expressions = extractSortKeyColumns(sortingKey);
+    if (expressions.length === 0)
+        return [];
+    const columnRows = await input.query(`SELECT name, type FROM system.columns WHERE database = '${input.database}' AND table = '${input.table}'`);
+    const typeByName = new Map(columnRows
+        .filter((row) => Boolean(row.name))
+        .map((row) => [row.name, row.type]));
+    const knownColumns = new Set(typeByName.keys());
+    if (knownColumns.size === 0) {
+        return expressions.flatMap((expression, index) => {
+            const column = resolveSortKeyColumnWithoutSchema(expression);
+            const type = columnRows[index]?.type ?? columnRows[0]?.type;
+            if (!column || !type)
+                return [];
+            return [{
+                    column,
+                    type,
+                    category: classifySortKeyType(type),
+                }];
+        });
+    }
+    return expressions.flatMap((expression) => {
+        const column = resolveSortKeyColumn(expression, knownColumns);
+        if (!column)
+            return [];
+        const type = typeByName.get(column) ?? 'String';
+        return [{
+                column,
+                type,
+                category: classifySortKeyType(type),
+            }];
+    });
+}
+export async function querySortKeyInfo(input) {
+    return (await querySortKeys(input))[0];
+}
+export async function querySortKeyRanges(input) {
+    if (input.partitionIds.length === 0)
+        return new Map();
+    const inList = input.partitionIds.map((id) => `'${id}'`).join(', ');
+    const rows = await input.query(`SELECT _partition_id AS partition_id, toString(min(${input.sortKeyColumn})) AS min_val, toString(max(${input.sortKeyColumn})) AS max_val FROM ${input.database}.${input.table} WHERE _partition_id IN (${inList}) GROUP BY _partition_id SETTINGS select_sequential_consistency = 1`);
+    const result = new Map();
+    for (const row of rows) {
+        result.set(row.partition_id, { min: row.min_val, max: row.max_val });
+    }
+    return result;
+}
+export async function introspectTable(input) {
+    const partitions = await queryPartitionInfo(input);
+    const sortKeys = await querySortKeys({
+        database: input.database,
+        table: input.table,
+        query: input.query,
+    });
+    return { partitions, sortKey: sortKeys[0], sortKeys };
+}
+//# sourceMappingURL=introspect.js.map

package/dist/chunking/introspect.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"introspect.js","sourceRoot":"","sources":["../../src/chunking/introspect.ts"],"names":[],"mappings":"AAEA,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC;IAC5B,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ;IACrD,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS;IAC3D,SAAS,EAAE,SAAS;CACrB,CAAC,CAAA;AAEF,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC,CAAA;AAE5E,SAAS,mBAAmB,CAAC,IAAY;IACvC,IAAI,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAA;IAC7C,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IAC/C,IAAI,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC;QAAE,OAAO,UAAU,CAAA;IACrD,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO,UAAU,CAAA;IACpD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,KAMxC;IACC,sEAAsE;IACtE,6EAA6E;IAC7E,2EAA2E;IAC3E,sDAAsD;IACtD,MAAM,KAAK,CAAC,KAAK,CACf,iBAAiB,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,KAAK,qDAAqD,CACpG,CAAA;IAED,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,CAQ5B;;;;;;;;oBAQgB,KAAK,CAAC,QAAQ;iBACjB,KAAK,CAAC,KAAK;;;;2CAIe,CACxC,CAAA;IAED,MAAM,UAAU,GAAoB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACrD,WAAW,EAAE,GAAG,CAAC,YAAY;QAC7B,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC;QAC5B,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;QACpC,iBAAiB,EAAE,MAAM,CAAC,GAAG,CAAC,wBAAwB,IAAI,GAAG,CAAC,WAAW,CAAC;QAC1E,OAAO,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE;QAC7C,OAAO,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE;KAC9C,CAAC,CAAC,CAAA;IAEH,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QAC7B,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,IAAI;YAAE,OAAO,KAAK,CAAA;QACtD,IAAI,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC,EAAE;YAAE,OAAO,KAAK,CAAA;QACnD,OAAO,IAAI,CAAA;IACb,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,UAAkB;IAC/C,OAAO,gBAAgB,CAAC,UAAU,CAAC;SAChC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;AACtD,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAa;IACrC,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,IAAI,OAAO,GAAG,EAAE,CAAA;IAChB,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,KAA4B,CAAA;IAEhC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAClD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAA;QACzB,IAAI,IAAI,KAAK,SAAS;YAAE,SAAQ;QAEhC,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,IAAI,IAAI,CAAA;YACf,IAAI,IAAI,KAAK,KAAK,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBAChD,KAAK,GAAG,SAAS,CAAA;YACnB,CAAC;YACD,SAAQ;QACV,CAAC;QAED,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YAClC,KAAK,GAAG,IAAI,CAAA;YACZ,OAAO,IAAI,IAAI,CAAA;YACf,SAAQ;QACV,CAAC;QAED,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACjB,KAAK,IAAI,CAAC,CAAA;YACV,OAAO,IAAI,IAAI,CAAA;YACf,SAAQ;QACV,CAAC;QAED,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACjB,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;YAC9B,OAAO,IAAI,IAAI,CAAA;YACf,SAAQ;QACV,CAAC;QAED,IAAI,IAAI,KAAK,GAAG,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAA;YAC1B,OAAO,GAAG,EAAE,CAAA;YACZ,SAAQ;QACV,CAAC;QAED,OAAO,IAAI,IAAI,CAAA;IACjB,CAAC;IAED,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAA;IAC5B,CAAC;IAED,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,oBAAoB,CAAC,UAAkB,EAAE,YAAyB;IACzE,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,EAAE,CAAA;IACjC,IAAI,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QAC9B,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,6BAA6B,CAAC,CAAC;SAC5E,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SACxB,MAAM,CAAC,CAAC,UAAU,EAAwB,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAA;IAEpE,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAA;IACrG,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,OAAO,CAAC,CAAC,CAAC,CAAA;IACnB,CAAC;IAED,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,SAAS,iCAAiC,CAAC,UAAkB;IAC3D,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,EAAE,CAAA;IACjC,IAAI,0BAA0B,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC7C,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,6BAA6B,CAAC,CAAC;SAC5E,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SACxB,MAAM,CAAC,CAAC,UAAU,EAAwB,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAA;IAEpE,OAAO,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;AACjF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,KAInC;IACC,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,KAAK,CACjC,2DAA2D,KAAK,CAAC,QAAQ,iBAAiB,KAAK,CAAC,KAAK,GAAG,CACzG,CAAA;IAED,MAAM,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,CAAA;IAC5C,IAAI,CAAC,UAAU;QAAE,OAAO,EAAE,CAAA;IAE1B,MAAM,WAAW,GAAG,qBAAqB,CAAC,UAAU,CAAC,CAAA;IACrD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAA;IAEvC,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,KAAK,CAClC,2DAA2D,KAAK,CAAC,QAAQ,kBAAkB,KAAK,CAAC,KAAK,GAAG,CAC1G,CAAA;IACD,MAAM,UAAU,GAAG,IAAI,GAAG,CACxB,UAAU;SACP,MAAM,CAAC,CAAC,GAAG,EAAyC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;SACzE,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CACtC,CAAA;IACD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAA;IAE/C,IAAI,YAAY,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,WAAW,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE;YAC/C,MAAM,MAAM,GAAG,iCAAiC,CAAC,UAAU,CAAC,CAAA;YAC5D,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,IAAI,IAAI,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,CAAA;YAC3D,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI;gBAAE,OAAO,EAAE,CAAA;YAE/B,OAAO,CAAC;oBACN,MAAM;oBACN,IAAI;oBACJ,QAAQ,EAAE,mBAAmB,CAAC,IAAI,CAAC;iBACpC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,OAAO,WAAW,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE;QACxC,MAAM,MAAM,GAAG,oBAAoB,CAAC,UAAU,EAAE,YAAY,CAAC,CAAA;QAC7D,IAAI,CAAC,MAAM;YAAE,OAAO,EAAE,CAAA;QACtB,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAA;QAC/C,OAAO,CAAC;gBACN,MAAM;gBACN,IAAI;gBACJ,QAAQ,EAAE,mBAAmB,CAAC,IAAI,CAAC;aACpC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,KAItC;IACC,OAAO,CAAC,MAAM,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;AACxC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,KAMxC;IACC,IAAI,KAAK,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,GAAG,EAAE,CAAA;IAErD,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACnE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,CAK5B,sDAAsD,KAAK,CAAC,aAAa,+BAA+B,KAAK,CAAC,aAAa,sBAAsB,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,KAAK,4BAA4B,MAAM,qEAAqE,CACtR,CAAA;IAED,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwC,CAAA;IAC9D,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,YAAY,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAA;IACtE,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,KAMrC;IACC,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAA;IAClD,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC;QACnC,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,KAAK,EAAE,KAAK,CAAC,KAAK;KACnB,CAAC,CAAA;IAEF,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAA;AACvD,CAAC"}

package/dist/chunking/partition-slices.d.ts

@@ -0,0 +1,14 @@
+import type { ChunkEstimate, EstimateConfidence, EstimateReason, Partition, PartitionSlice, ChunkDerivationStep, ChunkRange } from './types.js';
+export declare function buildRootSlice(partition: Partition): PartitionSlice;
+export declare function buildSliceEstimate(partition: Partition, rows: number, confidence: EstimateConfidence, reason: EstimateReason): ChunkEstimate;
+export declare function buildSliceFromRows(partition: Partition, input: {
+    ranges: ChunkRange[];
+    rows: number;
+    focusedValue?: PartitionSlice['analysis']['focusedValue'];
+    confidence: EstimateConfidence;
+    reason: EstimateReason;
+    lineage: ChunkDerivationStep[];
+}): PartitionSlice;
+export declare function getTargetChunkRows(partition: Partition, targetChunkBytes: number): number;
+export declare function mergeAdjacentSlices(slices: PartitionSlice[], targetChunkBytes: number): PartitionSlice[];
+//# sourceMappingURL=partition-slices.d.ts.map

package/dist/chunking/partition-slices.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"partition-slices.d.ts","sourceRoot":"","sources":["../../src/chunking/partition-slices.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,aAAa,EACb,kBAAkB,EAClB,cAAc,EACd,SAAS,EACT,cAAc,EACd,mBAAmB,EACnB,UAAU,EACX,MAAM,YAAY,CAAA;AAEnB,wBAAgB,cAAc,CAAC,SAAS,EAAE,SAAS,GAAG,cAAc,CAenE;AAED,wBAAgB,kBAAkB,CAChC,SAAS,EAAE,SAAS,EACpB,IAAI,EAAE,MAAM,EACZ,UAAU,EAAE,kBAAkB,EAC9B,MAAM,EAAE,cAAc,GACrB,aAAa,CAef;AAED,wBAAgB,kBAAkB,CAChC,SAAS,EAAE,SAAS,EACpB,KAAK,EAAE;IACL,MAAM,EAAE,UAAU,EAAE,CAAA;IACpB,IAAI,EAAE,MAAM,CAAA;IACZ,YAAY,CAAC,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC,cAAc,CAAC,CAAA;IACzD,UAAU,EAAE,kBAAkB,CAAA;IAC9B,MAAM,EAAE,cAAc,CAAA;IACtB,OAAO,EAAE,mBAAmB,EAAE,CAAA;CAC/B,GACA,cAAc,CAUhB;AAED,wBAAgB,kBAAkB,CAChC,SAAS,EAAE,SAAS,EACpB,gBAAgB,EAAE,MAAM,GACvB,MAAM,CAGR;AAED,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,cAAc,EAAE,EACxB,gBAAgB,EAAE,MAAM,GACvB,cAAc,EAAE,CAuClB"}

package/dist/chunking/partition-slices.js

@@ -0,0 +1,111 @@
+export function buildRootSlice(partition) {
+    return {
+        partitionId: partition.partitionId,
+        ranges: [],
+        estimate: {
+            rows: partition.rows,
+            bytesCompressed: partition.bytesCompressed,
+            bytesUncompressed: partition.bytesUncompressed,
+            confidence: 'high',
+            reason: 'partition-metadata',
+        },
+        analysis: {
+            lineage: [],
+        },
+    };
+}
+export function buildSliceEstimate(partition, rows, confidence, reason) {
+    const bytesCompressed = partition.rows > 0
+        ? Math.round((rows / partition.rows) * partition.bytesCompressed)
+        : 0;
+    const bytesUncompressed = partition.rows > 0
+        ? Math.round((rows / partition.rows) * partition.bytesUncompressed)
+        : 0;
+    return {
+        rows,
+        bytesCompressed,
+        bytesUncompressed,
+        confidence,
+        reason,
+    };
+}
+export function buildSliceFromRows(partition, input) {
+    return {
+        partitionId: partition.partitionId,
+        ranges: input.ranges,
+        estimate: buildSliceEstimate(partition, input.rows, input.confidence, input.reason),
+        analysis: {
+            focusedValue: input.focusedValue,
+            lineage: input.lineage,
+        },
+    };
+}
+export function getTargetChunkRows(partition, targetChunkBytes) {
+    if (partition.bytesUncompressed <= 0)
+        return partition.rows;
+    return (targetChunkBytes * partition.rows) / partition.bytesUncompressed;
+}
+export function mergeAdjacentSlices(slices, targetChunkBytes) {
+    if (slices.length <= 1)
+        return slices;
+    const merged = [];
+    let current;
+    for (const slice of slices) {
+        if (!current) {
+            current = slice;
+            continue;
+        }
+        const canMerge = !current.analysis.focusedValue &&
+            !slice.analysis.focusedValue &&
+            haveSameTrailingRanges(current.ranges, slice.ranges) &&
+            current.estimate.bytesUncompressed + slice.estimate.bytesUncompressed <= targetChunkBytes * 1.1;
+        if (!canMerge) {
+            merged.push(current);
+            current = slice;
+            continue;
+        }
+        current = {
+            ...current,
+            ranges: mergeRanges(current.ranges, slice.ranges),
+            estimate: {
+                ...current.estimate,
+                rows: current.estimate.rows + slice.estimate.rows,
+                bytesCompressed: current.estimate.bytesCompressed + slice.estimate.bytesCompressed,
+                bytesUncompressed: current.estimate.bytesUncompressed + slice.estimate.bytesUncompressed,
+            },
+        };
+    }
+    if (current)
+        merged.push(current);
+    return merged;
+}
+function mergeRanges(left, right) {
+    return left.map((leftRange) => {
+        const rightRange = right.find((candidate) => candidate.dimensionIndex === leftRange.dimensionIndex);
+        return rightRange === undefined
+            ? leftRange
+            : {
+                dimensionIndex: leftRange.dimensionIndex,
+                from: leftRange.from,
+                to: rightRange.to,
+            };
+    });
+}
+function haveSameTrailingRanges(left, right) {
+    if (left.length !== right.length)
+        return false;
+    let differingDimensions = 0;
+    for (const leftRange of left) {
+        const rightRange = right.find((candidate) => candidate.dimensionIndex === leftRange.dimensionIndex);
+        if (!rightRange)
+            return false;
+        const same = leftRange.from === rightRange.from && leftRange.to === rightRange.to;
+        if (!same) {
+            differingDimensions += 1;
+            if (leftRange.to !== rightRange.from)
+                return false;
+        }
+    }
+    return differingDimensions <= 1;
+}
+//# sourceMappingURL=partition-slices.js.map

package/dist/chunking/partition-slices.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"partition-slices.js","sourceRoot":"","sources":["../../src/chunking/partition-slices.ts"],"names":[],"mappings":"AAUA,MAAM,UAAU,cAAc,CAAC,SAAoB;IACjD,OAAO;QACL,WAAW,EAAE,SAAS,CAAC,WAAW;QAClC,MAAM,EAAE,EAAE;QACV,QAAQ,EAAE;YACR,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,eAAe,EAAE,SAAS,CAAC,eAAe;YAC1C,iBAAiB,EAAE,SAAS,CAAC,iBAAiB;YAC9C,UAAU,EAAE,MAAM;YAClB,MAAM,EAAE,oBAAoB;SAC7B;QACD,QAAQ,EAAE;YACR,OAAO,EAAE,EAAE;SACZ;KACF,CAAA;AACH,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,SAAoB,EACpB,IAAY,EACZ,UAA8B,EAC9B,MAAsB;IAEtB,MAAM,eAAe,GAAG,SAAS,CAAC,IAAI,GAAG,CAAC;QACxC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,eAAe,CAAC;QACjE,CAAC,CAAC,CAAC,CAAA;IACL,MAAM,iBAAiB,GAAG,SAAS,CAAC,IAAI,GAAG,CAAC;QAC1C,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,iBAAiB,CAAC;QACnE,CAAC,CAAC,CAAC,CAAA;IAEL,OAAO;QACL,IAAI;QACJ,eAAe;QACf,iBAAiB;QACjB,UAAU;QACV,MAAM;KACP,CAAA;AACH,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,SAAoB,EACpB,KAOC;IAED,OAAO;QACL,WAAW,EAAE,SAAS,CAAC,WAAW;QAClC,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,QAAQ,EAAE,kBAAkB,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC;QACnF,QAAQ,EAAE;YACR,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,OAAO,EAAE,KAAK,CAAC,OAAO;SACvB;KACF,CAAA;AACH,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,SAAoB,EACpB,gBAAwB;IAExB,IAAI,SAAS,CAAC,iBAAiB,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC,IAAI,CAAA;IAC3D,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,iBAAiB,CAAA;AAC1E,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,MAAwB,EACxB,gBAAwB;IAExB,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,MAAM,CAAA;IAErC,MAAM,MAAM,GAAqB,EAAE,CAAA;IACnC,IAAI,OAAmC,CAAA;IAEvC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,GAAG,KAAK,CAAA;YACf,SAAQ;QACV,CAAC;QAED,MAAM,QAAQ,GACZ,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY;YAC9B,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY;YAC5B,sBAAsB,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC;YACpD,OAAO,CAAC,QAAQ,CAAC,iBAAiB,GAAG,KAAK,CAAC,QAAQ,CAAC,iBAAiB,IAAI,gBAAgB,GAAG,GAAG,CAAA;QAEjG,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACpB,OAAO,GAAG,KAAK,CAAA;YACf,SAAQ;QACV,CAAC;QAED,OAAO,GAAG;YACR,GAAG,OAAO;YACV,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC;YACjD,QAAQ,EAAE;gBACR,GAAG,OAAO,CAAC,QAAQ;gBACnB,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAC,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,IAAI;gBACjD,eAAe,EAAE,OAAO,CAAC,QAAQ,CAAC,eAAe,GAAG,KAAK,CAAC,QAAQ,CAAC,eAAe;gBAClF,iBAAiB,EAAE,OAAO,CAAC,QAAQ,CAAC,iBAAiB,GAAG,KAAK,CAAC,QAAQ,CAAC,iBAAiB;aAEzF;SACF,CAAA;IACH,CAAC;IAED,IAAI,OAAO;QAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACjC,OAAO,MAAM,CAAA;AACf,CAAC;AAED,SAAS,WAAW,CAAC,IAAkB,EAAE,KAAmB;IAC1D,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE;QAC5B,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,cAAc,KAAK,SAAS,CAAC,cAAc,CAAC,CAAA;QACnG,OAAO,UAAU,KAAK,SAAS;YAC7B,CAAC,CAAC,SAAS;YACX,CAAC,CAAC;gBACA,cAAc,EAAE,SAAS,CAAC,cAAc;gBACxC,IAAI,EAAE,SAAS,CAAC,IAAI;gBACpB,EAAE,EAAE,UAAU,CAAC,EAAE;aAClB,CAAA;IACL,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,SAAS,sBAAsB,CAAC,IAAkB,EAAE,KAAmB;IACrE,IAAI,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;QAAE,OAAO,KAAK,CAAA;IAE9C,IAAI,mBAAmB,GAAG,CAAC,CAAA;IAE3B,KAAK,MAAM,SAAS,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,cAAc,KAAK,SAAS,CAAC,cAAc,CAAC,CAAA;QACnG,IAAI,CAAC,UAAU;YAAE,OAAO,KAAK,CAAA;QAE7B,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,KAAK,UAAU,CAAC,IAAI,IAAI,SAAS,CAAC,EAAE,KAAK,UAAU,CAAC,EAAE,CAAA;QACjF,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,mBAAmB,IAAI,CAAC,CAAA;YACxB,IAAI,SAAS,CAAC,EAAE,KAAK,UAAU,CAAC,IAAI;gBAAE,OAAO,KAAK,CAAA;QACpD,CAAC;IACH,CAAC;IAED,OAAO,mBAAmB,IAAI,CAAC,CAAA;AACjC,CAAC"}

package/dist/chunking/planner.d.ts

@@ -0,0 +1,3 @@
+import type { ChunkPlan, GenerateChunkPlanInput } from './types.js';
+export declare function generateChunkPlan(input: GenerateChunkPlanInput): Promise<ChunkPlan>;
+//# sourceMappingURL=planner.d.ts.map

package/dist/chunking/planner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"planner.d.ts","sourceRoot":"","sources":["../../src/chunking/planner.ts"],"names":[],"mappings":"AAyBA,OAAO,KAAK,EAEV,SAAS,EACT,sBAAsB,EAQvB,MAAM,YAAY,CAAA;AASnB,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,SAAS,CAAC,CAyEzF"}