@nshiab/simple-data-analysis-core 1.0.1 → 2.0.0

package/esm/methods/cache.js

@@ -1,10 +1,22 @@
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync, } from "node:fs";
 import crypto from "node:crypto";
-import formatDate from "../helpers/formatDate.js";
+import flushAllTables from "../helpers/flushAllTables.js";
 import prettyDuration from "../helpers/prettyDuration.js";
-export default async function cache(table, run, options = {}) {
+import { cacheEntryExpired, cacheLoadMessage, cacheTtlMessage, } from "../helpers/cacheDiagnostics.js";
+import serializeCacheInputs, { getCacheInputLabel, } from "../helpers/serializeCacheInputs.js";
+import { createTableGenerationId, peekTableGeneration, restoreTableGeneration, } from "../helpers/tableGeneration.js";
+import { loadCacheDatabase, writeCacheDatabase, } from "../helpers/cacheDatabase.js";
+import { cacheTableDependenciesMatch, captureCacheTableDependencies, changedCacheTableDependenciesMessage, recordCacheTableAccess, unchangedCacheTableDependenciesMessage, } from "../helpers/cacheTableDependencies.js";
+const CACHE_FORMAT_VERSION = "duckdb-v4";
+export default async function cache(table, compute, options = {}) {
+    // If cache() is itself called from another cache computation, the outer
+    // computation depends on the table produced here.
+    recordCacheTableAccess(table);
     options.verbose &&
         console.log(`\ncache() for ${table.name}`);
+    // cache() is an observation point even when a cache hit has no artifact to
+    // load, so all earlier database-wide work must complete before it returns.
+    await flushAllTables(table.sdb);
     const cachePath = "./.sda-cache";
     if (!existsSync(cachePath)) {
         mkdirSync(cachePath);
@@ -14,70 +26,107 @@ export default async function cache(table, run, options = {}) {
     if (existsSync(cacheSourcesPath)) {
         cacheSources = JSON.parse(readFileSync(cacheSourcesPath, "utf-8"));
     }
-    const functionBody = run.toString();
-    const hash = crypto
-        .createHash("sha256")
-        .update(table.name + options.toString() + functionBody)
-        .digest("hex");
-    const id = `${table.name}.${hash}`;
-    const cache = cacheSources[id];
+    const functionBody = compute.toString();
+    const entryGeneration = peekTableGeneration(table) ?? null;
+    const inputs = options.inputs?.filter((input) => input !== table) ?? [];
+    const hasInputs = inputs.length > 0;
+    const serializedInputs = hasInputs ? serializeCacheInputs(inputs) : null;
+    const codeHash = createHash(functionBody);
+    const inputHashes = hasInputs
+        ? inputs.map((input) => createHash(serializeCacheInputs([input])))
+        : [];
+    const hash = createHash(JSON.stringify([
+        CACHE_FORMAT_VERSION,
+        table.name,
+        entryGeneration,
+        functionBody,
+        serializedInputs,
+    ]));
+    const requestedId = `${table.name}.${hash}`;
+    let id = requestedId;
+    let cache = cacheSources[id];
+    if (cache === undefined) {
+        const alreadyApplied = findAlreadyAppliedCache(table, cacheSources, entryGeneration, codeHash, inputHashes);
+        if (alreadyApplied !== undefined) {
+            [id, cache] = alreadyApplied;
+        }
+    }
+    if (cache !== undefined &&
+        !cacheTableDependenciesMatch(table, cache.tableDependencies)) {
+        cache = undefined;
+    }
     const now = Date.now();
+    if (cache !== undefined) {
+        ensureCacheDiagnostics(cache, cacheSources, cacheSourcesPath, codeHash, inputHashes);
+    }
     if (cache === undefined) {
-        options.verbose &&
-            console.log(`Nothing in cache. Running and storing in cache.`);
-        await runAndWrite(table, run, cacheSources, cacheSourcesPath, cachePath, id);
+        if (options.verbose) {
+            logCacheMiss(table, cacheSources, codeHash, inputHashes, inputs, entryGeneration);
+        }
+        await runAndWrite(table, compute, cacheSources, cacheSourcesPath, cachePath, id, codeHash, inputHashes, entryGeneration);
     }
-    else if (cache &&
-        typeof options.ttl === "number" &&
-        now - cache.creation > options.ttl * 1000) {
+    else if (options.ttl !== undefined &&
+        cacheEntryExpired(cache.creation, options.ttl, now)) {
+        const refreshId = cache.entryGeneration === entryGeneration
+            ? id
+            : requestedId;
         options.verbose &&
-            console.log(`Found in cache.\nttl of ${prettyDuration(0, { end: options.ttl * 1000 })} has expired.\nThe creation date is ${formatDate(new Date(cache.creation), "Month DD, YYYY, at HH:MM period")}.\nIt's is ${prettyDuration(cache.creation, { end: now })} ago.\nRunning and storing in cache.`);
-        await runAndWrite(table, run, cacheSources, cacheSourcesPath, cachePath, id);
+            console.log(`Cache entry is stale.\n${identityMatchMessage(inputHashes, cache.tableDependencies ?? [])}\n${cacheTtlMessage(cache.creation, options.ttl, now, true)}\nRunning computations and refreshing the cache entry.`);
+        await runAndWrite(table, compute, cacheSources, cacheSourcesPath, cachePath, refreshId, codeHash, inputHashes, entryGeneration);
     }
     else {
         options.verbose &&
-            console.log(`Found in cache.`);
+            console.log(`Cache hit.\n${identityMatchMessage(inputHashes, cache.tableDependencies ?? [])}`);
         if (typeof options.ttl === "number") {
-            const ttlLimit = new Date(cache.creation + options.ttl * 1000);
             (options.verbose) &&
-                console.log(`ttl of ${prettyDuration(0, { end: options.ttl * 1000 })} has not expired.\nThe creation date is ${formatDate(new Date(cache.creation), "Month DD, YYYY, at HH:MM period")}.\nThere are ${prettyDuration(now, { end: ttlLimit })} left.`);
+                console.log(cacheTtlMessage(cache.creation, options.ttl, now, false));
         }
         if (cache.file === null) {
             console.log("No data in cache. Nothing to load.");
         }
-        else if (cache.geo) {
+        else {
             const start = Date.now();
-            await table.loadGeoData(cache.file);
-            if (table.sdb.cacheSourcesUsed.indexOf(id) < 0) {
-                table.sdb.cacheSourcesUsed.push(id);
+            try {
+                await loadCacheDatabase(table, cache.file, cache.indexes ?? [], cache.geo);
             }
-            const end = Date.now();
-            const duration = end - start;
-            if (options.verbose) {
-                console.log(`Data loaded in ${prettyDuration(start, { end })}.\nRunning computations previously took ${prettyDuration(0, { end: cache.duration })}.\nYou saved ${prettyDuration(duration, { end: cache.duration })}.\n`);
-                table.sdb.cacheTimeSaved += cache.duration - duration;
+            catch {
+                options.verbose &&
+                    console.log("Could not load cached data. Recomputing...");
+                if (existsSync(cache.file)) {
+                    rmSync(cache.file);
+                }
+                delete cacheSources[id];
+                await runAndWrite(table, compute, cacheSources, cacheSourcesPath, cachePath, requestedId, codeHash, inputHashes, entryGeneration);
+                return;
             }
-        }
-        else {
-            const start = Date.now();
-            await table.loadData(cache.file);
             if (table.sdb.cacheSourcesUsed.indexOf(id) < 0) {
                 table.sdb.cacheSourcesUsed.push(id);
             }
             const end = Date.now();
             const duration = end - start;
             if (options.verbose) {
-                console.log(`Data loaded in ${prettyDuration(start, { end })}.\nRunning computations previously took ${prettyDuration(0, { end: cache.duration })}.\nYou saved ${prettyDuration(duration, { end: cache.duration })}.\n`);
+                console.log(`${cacheLoadMessage(start, end, cache.duration)}\n`);
                 table.sdb.cacheTimeSaved += cache.duration - duration;
             }
         }
+        restoreCachedGeneration(table, cache, cacheSources, cacheSourcesPath);
     }
 }
-async function runAndWrite(table, run, cacheSources, cacheSourcesPath, cachePath, id) {
+async function runAndWrite(table, compute, cacheSources, cacheSourcesPath, cachePath, id, codeHash, inputHashes, entryGeneration) {
+    // Complete any work queued before cache() before dependency observation
+    // begins. Operations queued by compute are flushed inside the observation.
+    await flushAllTables(table.sdb);
     const start = Date.now();
-    await run();
+    const tableDependencies = await captureCacheTableDependencies(table, async () => {
+        await compute(table);
+        // run() only queues the sync builders in the user's callback; the actual
+        // computation happens at the flush, so it must be included in the timing
+        // that decides how much the cache saves on later hits.
+        await flushAllTables(table.sdb);
+    });
     const end = Date.now();
     const duration = end - start;
+    const generationId = createTableGenerationId();
     table.sdb.cacheVerbose &&
         console.log(`Computations done in ${prettyDuration(start, { end })}.`);
     if (!(await table.sdb.hasTable(table.name))) {
@@ -87,47 +136,155 @@ async function runAndWrite(table, run, cacheSources, cacheSourcesPath, cachePath
             duration,
             file: null,
             geo: false,
+            entryGeneration,
+            generationId,
+            indexes: [],
+            formatVersion: CACHE_FORMAT_VERSION,
+            codeHash,
+            inputHashes,
+            tableDependencies,
         };
     }
     else {
         const types = await table.getTypes();
         const geometriesColumns = Object.values(types).filter((d) => d.toLowerCase().includes("geometry")).length;
-        if (geometriesColumns > 0) {
-            const file = `${cachePath}/${id}.geoparquet`;
-            const writeStart = Date.now();
-            await table.writeGeoData(file);
-            cacheSources[id] = {
-                creation: Date.now(),
-                duration,
-                file,
-                geo: true,
-            };
-            const writeEnd = Date.now();
-            table.sdb.cacheVerbose &&
-                console.log(`Wrote in cache in ${prettyDuration(writeStart, { end: writeEnd })}.\n`);
-            table.sdb.cacheTimeWriting += writeEnd - writeStart;
-            if (table.sdb.cacheSourcesUsed.indexOf(id) < 0) {
-                table.sdb.cacheSourcesUsed.push(id);
-            }
+        const file = `${cachePath}/${id}.db`;
+        const indexes = structuredClone(table.indexes);
+        const writeStart = Date.now();
+        await writeCacheDatabase(table, file, indexes);
+        const writeEnd = Date.now();
+        table.sdb.cacheVerbose &&
+            console.log(`Wrote in cache in ${prettyDuration(writeStart, { end: writeEnd })}.\n`);
+        table.sdb.cacheTimeWriting += writeEnd - writeStart;
+        cacheSources[id] = {
+            creation: Date.now(),
+            duration,
+            file,
+            geo: geometriesColumns > 0,
+            entryGeneration,
+            generationId,
+            indexes,
+            formatVersion: CACHE_FORMAT_VERSION,
+            codeHash,
+            inputHashes,
+            tableDependencies,
+        };
+        if (table.sdb.cacheSourcesUsed.indexOf(id) < 0) {
+            table.sdb.cacheSourcesUsed.push(id);
+        }
+    }
+    writeFileSync(cacheSourcesPath, JSON.stringify(cacheSources));
+    restoreTableGeneration(table, generationId);
+}
+function restoreCachedGeneration(table, cache, cacheSources, cacheSourcesPath) {
+    if (cache.generationId === undefined) {
+        cache.generationId = createTableGenerationId();
+        writeFileSync(cacheSourcesPath, JSON.stringify(cacheSources));
+    }
+    restoreTableGeneration(table, cache.generationId);
+}
+function ensureCacheDiagnostics(cache, cacheSources, cacheSourcesPath, codeHash, inputHashes) {
+    if (cache.formatVersion === undefined || cache.codeHash === undefined ||
+        cache.inputHashes === undefined) {
+        cache.formatVersion = CACHE_FORMAT_VERSION;
+        cache.codeHash = codeHash;
+        cache.inputHashes = inputHashes;
+        writeFileSync(cacheSourcesPath, JSON.stringify(cacheSources));
+    }
+}
+function createHash(value) {
+    return crypto.createHash("sha256").update(value).digest("hex");
+}
+function identityMatchMessage(inputHashes, tableDependencies) {
+    const inputsMessage = inputHashes.length === 0
+        ? "Compute function unchanged."
+        : `Compute function unchanged.\nInputs unchanged (${inputHashes.length} checked).`;
+    const dependenciesMessage = tableDependencies.length === 0
+        ? ""
+        : `\n${unchangedCacheTableDependenciesMessage(tableDependencies)}`;
+    return `${inputsMessage}${dependenciesMessage}`;
+}
+function findAlreadyAppliedCache(table, cacheSources, entryGeneration, codeHash, inputHashes) {
+    if (entryGeneration === null) {
+        return undefined;
+    }
+    return Object.entries(cacheSources)
+        .filter(([id, source]) => id.startsWith(`${table.name}.`) &&
+        source.formatVersion === CACHE_FORMAT_VERSION &&
+        source.generationId === entryGeneration &&
+        source.codeHash === codeHash &&
+        arraysEqual(source.inputHashes, inputHashes) &&
+        cacheTableDependenciesMatch(table, source.tableDependencies))
+        .reduce((latest, candidate) => latest === undefined || candidate[1].creation > latest[1].creation
+        ? candidate
+        : latest, undefined);
+}
+function arraysEqual(left, right) {
+    return left !== undefined && left.length === right.length &&
+        left.every((value, index) => value === right[index]);
+}
+function logCacheMiss(table, cacheSources, codeHash, inputHashes, inputs, entryGeneration) {
+    const candidates = Object.entries(cacheSources)
+        .filter(([id]) => id.startsWith(`${table.name}.`))
+        .map(([, source]) => source)
+        .filter(isCacheDiagnosticSource);
+    let reason;
+    if (candidates.length === 0) {
+        const hasLegacyCandidate = Object.keys(cacheSources).some((id) => id.startsWith(`${table.name}.`));
+        reason = hasLegacyCandidate
+            ? "A previous entry exists, but it predates detailed cache diagnostics. Its code, inputs, or cache format changed."
+            : "No matching cache entry exists for this computation.";
+    }
+    else {
+        const sameFormat = candidates.filter((source) => source.formatVersion === CACHE_FORMAT_VERSION);
+        const sameCode = latestCacheSource(sameFormat.filter((source) => source.codeHash === codeHash));
+        if (sameCode !== undefined) {
+            const inputsMessage = changedInputsMessage(sameCode.inputHashes, inputHashes, inputs);
+            const tableMessage = sameCode.entryGeneration === entryGeneration
+                ? ""
+                : "The current table has changed.";
+            const dependenciesMessage = changedCacheTableDependenciesMessage(table, sameCode.tableDependencies);
+            const changes = [tableMessage, inputsMessage, dependenciesMessage]
+                .filter((message) => message.length > 0);
+            reason = changes.length === 0
+                ? "Compute function unchanged."
+                : `Compute function unchanged.\n${changes.join("\n")}`;
+        }
+        else if (sameFormat.length === 0) {
+            reason = "The cache format changed.";
         }
         else {
-            const file = `${cachePath}/${id}.parquet`;
-            const writeStart = Date.now();
-            await table.writeData(file);
-            const writeEnd = Date.now();
-            table.sdb.cacheVerbose &&
-                console.log(`Wrote in cache in ${prettyDuration(writeStart, { end: writeEnd })}.\n`);
-            table.sdb.cacheTimeWriting += writeEnd - writeStart;
-            cacheSources[id] = {
-                creation: Date.now(),
-                duration,
-                file,
-                geo: false,
-            };
-            if (table.sdb.cacheSourcesUsed.indexOf(id) < 0) {
-                table.sdb.cacheSourcesUsed.push(id);
-            }
+            reason = "No matching cache entry exists for this computation.";
         }
     }
-    writeFileSync(cacheSourcesPath, JSON.stringify(cacheSources));
+    console.log(`Cache miss.\n${reason}\nRunning computations and storing a new cache entry.`);
+}
+function latestCacheSource(cacheSources) {
+    return cacheSources.reduce((latest, source) => latest === undefined || source.creation > latest.creation
+        ? source
+        : latest, undefined);
+}
+function isCacheDiagnosticSource(source) {
+    return source.formatVersion !== undefined && source.codeHash !== undefined &&
+        source.inputHashes !== undefined && source.tableDependencies !== undefined;
+}
+function changedInputsMessage(previousHashes, inputHashes, inputs) {
+    const changes = [];
+    const sharedLength = Math.min(previousHashes.length, inputHashes.length);
+    for (let i = 0; i < sharedLength; i++) {
+        if (previousHashes[i] !== inputHashes[i]) {
+            changes.push(`${getCacheInputLabel(inputs[i]) ?? `inputs[${i}]`} changed`);
+        }
+    }
+    for (let i = sharedLength; i < inputHashes.length; i++) {
+        changes.push(`${getCacheInputLabel(inputs[i]) ?? `inputs[${i}]`} was added`);
+    }
+    for (let i = sharedLength; i < previousHashes.length; i++) {
+        changes.push(`inputs[${i}] was removed`);
+    }
+    return changes.length === 0
+        ? inputHashes.length === 0
+            ? ""
+            : `Inputs unchanged (${inputHashes.length} checked).`
+        : `Inputs changed: ${changes.join(", ")}.`;
 }

package/esm/methods/capitalize.d.ts

@@ -0,0 +1,3 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function capitalize(simpleTable: SimpleTable, columns: string | string[]): void;
+//# sourceMappingURL=capitalize.d.ts.map

package/esm/methods/capitalize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"capitalize.d.ts","sourceRoot":"","sources":["../../src/methods/capitalize.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,UAAU,CAChC,WAAW,EAAE,WAAW,EACxB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,QAmB3B"}

package/esm/methods/capitalize.js

@@ -0,0 +1,15 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.js";
+import stringToArray from "../helpers/stringToArray.js";
+import queueOp from "../helpers/queueOp.js";
+export default function capitalize(simpleTable, columns) {
+    const cols = stringToArray(columns);
+    queueOp(simpleTable, {
+        kind: "fusable",
+        method: "capitalize()",
+        parameters: { columns },
+        needsSchema: false,
+        buildSelect: (input) => `SELECT * REPLACE (${cols
+            .map((c) => `CONCAT(UPPER(LEFT(${quoteIdentifier(c)}, 1)), LOWER(RIGHT(${quoteIdentifier(c)}, LENGTH(${quoteIdentifier(c)})-1))) AS ${quoteIdentifier(c)}`)
+            .join(", ")}) FROM ${input}`,
+    });
+}

package/esm/methods/centroid.d.ts

@@ -0,0 +1,5 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function centroid(simpleTable: SimpleTable, newColumn: string, options?: {
+    column?: string;
+}): void;
+//# sourceMappingURL=centroid.d.ts.map

package/esm/methods/centroid.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"centroid.d.ts","sourceRoot":"","sources":["../../src/methods/centroid.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,QAAQ,CAC9B,WAAW,EAAE,WAAW,EACxB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,QA2BlC"}

package/esm/methods/centroid.js

@@ -0,0 +1,24 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.js";
+import findGeoColumnFromSchema from "../helpers/findGeoColumnFromSchema.js";
+import queueOp from "../helpers/queueOp.js";
+export default function centroid(simpleTable, newColumn, options = {}) {
+    options = structuredClone(options);
+    queueOp(simpleTable, {
+        kind: "fusable",
+        method: "centroid()",
+        parameters: { newColumn, options },
+        needsSchema: true,
+        needsSpatial: true,
+        buildSelect: (input, types) => {
+            const column = typeof options.column === "string"
+                ? options.column
+                : findGeoColumnFromSchema(types, "centroid()", simpleTable.name);
+            // The schema type carries the projection (e.g. GEOMETRY('EPSG:4326')),
+            // so the cast keeps it on the new geometries.
+            const expression = `ST_Centroid(${quoteIdentifier(column)})::${types[column]}`;
+            return Object.keys(types).includes(newColumn)
+                ? `SELECT * REPLACE (${expression} AS ${quoteIdentifier(newColumn)}) FROM ${input}`
+                : `SELECT *, ${expression} AS ${quoteIdentifier(newColumn)} FROM ${input}`;
+        },
+    });
+}

package/esm/methods/clone.d.ts

@@ -0,0 +1,9 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function clone(simpleTable: SimpleTable, nameOrOptions?: string | {
+    name?: string;
+    conditions?: string;
+    columns?: string | string[];
+    limit?: number;
+    offset?: number;
+}): SimpleTable;
+//# sourceMappingURL=clone.d.ts.map

package/esm/methods/clone.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"clone.d.ts","sourceRoot":"","sources":["../../src/methods/clone.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,KAAK,CAC3B,WAAW,EAAE,WAAW,EACxB,aAAa,GAAE,MAAM,GAAG;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACZ,GACL,WAAW,CAuCb"}

package/esm/methods/clone.js

@@ -0,0 +1,35 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.js";
+import queueOp from "../helpers/queueOp.js";
+import stringToArray from "../helpers/stringToArray.js";
+export default function clone(simpleTable, nameOrOptions = {}) {
+    const columns = typeof nameOrOptions === "object" && nameOrOptions.columns
+        ? stringToArray(nameOrOptions.columns)
+        : [];
+    // The cloned table instance is created at call time so it can be returned
+    // synchronously and chained on right away. Delegating to sdb.newTable()
+    // makes subclasses using tableClass work correctly.
+    let clonedTable;
+    const options = typeof nameOrOptions === "string"
+        ? { name: nameOrOptions }
+        : { ...nameOrOptions };
+    if (typeof options.name === "string") {
+        clonedTable = simpleTable.sdb.newTable(options.name);
+    }
+    else {
+        clonedTable = simpleTable.sdb.newTable(undefined);
+    }
+    const selectClause = columns.length > 0
+        ? columns.map((col) => `${quoteIdentifier(col)}`).join(", ")
+        : "*";
+    queueOp(clonedTable, {
+        kind: "source",
+        method: "clone()",
+        parameters: { options },
+        // The clone reads simpleTable by name rather than through clonedTable's
+        // own (nonexistent) chain, so simpleTable's pending work must close and
+        // execute before this SELECT runs, as it would at this call position.
+        rawSQL: [`${quoteIdentifier(simpleTable.name)}`],
+        buildSelect: () => `SELECT ${selectClause} FROM ${quoteIdentifier(simpleTable.name)}${options.conditions ? ` WHERE ${options.conditions}` : ""}${typeof options.limit === "number" ? ` LIMIT ${options.limit}` : ""}${typeof options.offset === "number" ? ` OFFSET ${options.offset}` : ""}`,
+    });
+    return clonedTable;
+}

package/esm/methods/cloneColumn.d.ts

@@ -1,4 +1,3 @@
-export default function cloneColumnQuery(table: string, originalColumn: string, newColumn: string, types: {
-    [key: string]: string;
-}): string;
+import type SimpleTable from "../class/SimpleTable.js";
+export default function cloneColumn(simpleTable: SimpleTable, column: string, newColumn: string): void;
 //# sourceMappingURL=cloneColumn.d.ts.map

package/esm/methods/cloneColumn.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cloneColumn.d.ts","sourceRoot":"","sources":["../../src/methods/cloneColumn.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,gBAAgB,CACtC,KAAK,EAAE,MAAM,EACb,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAE,UAejC"}
+{"version":3,"file":"cloneColumn.d.ts","sourceRoot":"","sources":["../../src/methods/cloneColumn.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,WAAW,CACjC,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,QAiBlB"}

package/esm/methods/cloneColumn.js

@@ -1,13 +1,18 @@
-export default function cloneColumnQuery(table, originalColumn, newColumn, types) {
-    let query = "";
-    const originalColumnType = types[originalColumn];
-    if (originalColumnType) {
-        query +=
-            `ALTER TABLE "${table}" ADD COLUMN "${newColumn}" ${originalColumnType};
-        UPDATE "${table}" SET "${newColumn}" = "${originalColumn}"`;
-    }
-    else {
-        throw new Error(`Can't find type of ${originalColumn}`);
-    }
-    return query;
+import quoteIdentifier from "../helpers/quoteIdentifier.js";
+import assertNewColumns from "../helpers/assertNewColumns.js";
+import queueOp from "../helpers/queueOp.js";
+export default function cloneColumn(simpleTable, column, newColumn) {
+    queueOp(simpleTable, {
+        kind: "fusable",
+        method: "cloneColumn()",
+        parameters: { column, newColumn },
+        needsSchema: true,
+        buildSelect: (input, types) => {
+            if (!types[column]) {
+                throw new Error(`Can't find type of ${column}`);
+            }
+            assertNewColumns(types, [newColumn], "cloneColumn()");
+            return `SELECT *, ${quoteIdentifier(column)} AS ${quoteIdentifier(newColumn)} FROM ${input}`;
+        },
+    });
 }

package/esm/methods/cloneColumnWithOffset.d.ts

@@ -0,0 +1,6 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function cloneColumnWithOffset(simpleTable: SimpleTable, column: string, newColumn: string, options?: {
+    offset?: number;
+    by?: string | string[];
+}): void;
+//# sourceMappingURL=cloneColumnWithOffset.d.ts.map

package/esm/methods/cloneColumnWithOffset.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cloneColumnWithOffset.d.ts","sourceRoot":"","sources":["../../src/methods/cloneColumnWithOffset.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,qBAAqB,CAC3C,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CACnB,QAiDP"}

package/esm/methods/cloneColumnWithOffset.js

@@ -0,0 +1,35 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.js";
+import assertNewColumns from "../helpers/assertNewColumns.js";
+import mergeOptions from "../helpers/mergeOptions.js";
+import queryDB from "../helpers/queryDB.js";
+import queueOp from "../helpers/queueOp.js";
+import stringToArray from "../helpers/stringToArray.js";
+export default function cloneColumnWithOffset(simpleTable, column, newColumn, options = {}) {
+    // The offset is based on rowid, which only exists on the materialized
+    // table, not on the output of a fused step: it executes as a barrier.
+    options = structuredClone(options);
+    queueOp(simpleTable, {
+        kind: "barrier",
+        method: "cloneColumnWithOffset()",
+        parameters: { column, newColumn, options },
+        execute: async () => {
+            // A SELECT *, expr AS col colliding with an existing column would be
+            // silently renamed by DuckDB (col -> col_1) instead of erroring.
+            assertNewColumns(await simpleTable.getTypes(), [newColumn], "cloneColumnWithOffset()");
+            const offset = options.offset ?? 1;
+            const by = options.by ? stringToArray(options.by) : [];
+            const partition = by.length > 0
+                ? `PARTITION BY ${by.map((d) => `${quoteIdentifier(d)}`).join(", ")}`
+                : "";
+            // When by are specified, also sort the final result by
+            // by.
+            await queryDB(simpleTable, `CREATE OR REPLACE TABLE ${quoteIdentifier(simpleTable.name)} AS SELECT *, LEAD(${quoteIdentifier(column)}, ${offset}) OVER(${partition} ORDER BY rowid) AS ${quoteIdentifier(newColumn)} FROM ${quoteIdentifier(simpleTable.name)}${by.length > 0
+                ? ` ORDER BY ${by.map((d) => `${quoteIdentifier(d)}`).join(", ")}, rowid`
+                : ` ORDER BY rowid`};`, mergeOptions(simpleTable, {
+                table: simpleTable.name,
+                method: "cloneColumnWithOffset()",
+                parameters: { column, newColumn },
+            }));
+        },
+    });
+}

package/esm/methods/columnProportions.d.ts

@@ -0,0 +1,6 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function columnProportions(simpleTable: SimpleTable, column: string, newColumn: string, options?: {
+    by?: string | string[];
+    decimals?: number;
+}): void;
+//# sourceMappingURL=columnProportions.d.ts.map

package/esm/methods/columnProportions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"columnProportions.d.ts","sourceRoot":"","sources":["../../src/methods/columnProportions.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,iBAAiB,CACvC,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACd,QA4BP"}

package/esm/methods/columnProportions.js

@@ -0,0 +1,23 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.js";
+import assertNewColumns from "../helpers/assertNewColumns.js";
+import queueOp from "../helpers/queueOp.js";
+import stringToArray from "../helpers/stringToArray.js";
+export default function columnProportions(simpleTable, column, newColumn, options = {}) {
+    options = structuredClone(options);
+    queueOp(simpleTable, {
+        kind: "fusable",
+        method: "columnProportions()",
+        parameters: { column, newColumn, options },
+        needsSchema: true,
+        buildSelect: (input, schema) => {
+            assertNewColumns(schema, [newColumn], "columnProportions()");
+            const by = options.by ? stringToArray(options.by) : [];
+            const partition = by.length === 0
+                ? ""
+                : `PARTITION BY ${by.map((d) => `${quoteIdentifier(d)}`).join(",")}`;
+            return typeof options.decimals === "number"
+                ? `SELECT *, ROUND(${quoteIdentifier(column)} / sum(${quoteIdentifier(column)}) OVER(${partition}), ${options.decimals}) AS ${quoteIdentifier(newColumn)} FROM ${input}`
+                : `SELECT *, ${quoteIdentifier(column)} / sum(${quoteIdentifier(column)}) OVER(${partition}) AS ${quoteIdentifier(newColumn)} FROM ${input}`;
+        },
+    });
+}

package/esm/methods/concatenate.d.ts

@@ -0,0 +1,5 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function concatenate(simpleTable: SimpleTable, columns: string[], newColumn: string, options?: {
+    separator?: string;
+}): void;
+//# sourceMappingURL=concatenate.d.ts.map

package/esm/methods/concatenate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"concatenate.d.ts","sourceRoot":"","sources":["../../src/methods/concatenate.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,WAAW,CACjC,WAAW,EAAE,WAAW,EACxB,OAAO,EAAE,MAAM,EAAE,EACjB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,SAAS,CAAC,EAAE,MAAM,CAAC;CACf,QA0BP"}

package/esm/methods/concatenate.js

@@ -0,0 +1,25 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.js";
+import assertNewColumns from "../helpers/assertNewColumns.js";
+import queueOp from "../helpers/queueOp.js";
+export default function concatenate(simpleTable, columns, newColumn, options = {}) {
+    columns = [...columns];
+    options = structuredClone(options);
+    queueOp(simpleTable, {
+        kind: "fusable",
+        method: "concatenate()",
+        parameters: { columns, newColumn, options },
+        needsSchema: true,
+        values: typeof options.separator === "string"
+            ? [options.separator]
+            : undefined,
+        buildSelect: (input, types) => {
+            assertNewColumns(types, [newColumn], "concatenate()");
+            const expression = typeof options.separator === "string"
+                ? `CONCAT_WS(?, ${columns
+                    .map((d) => `${quoteIdentifier(d)}`)
+                    .join(", ")})`
+                : `CONCAT(${columns.map((d) => `${quoteIdentifier(d)}`).join(", ")})`;
+            return `SELECT *, CAST(${expression} AS VARCHAR) AS ${quoteIdentifier(newColumn)} FROM ${input}`;
+        },
+    });
+}