@flowblade/sqlduck 0.6.0 → 0.8.0

package/dist/index.cjs

@@ -21,35 +21,96 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 let _duckdb_node_api = require("@duckdb/node-api");
 let zod = require("zod");
 zod = __toESM(zod);
-const toDuckValue = (value) => {
-	if (value instanceof Date) return new _duckdb_node_api.DuckDBTimestampValue(BigInt(value.getTime() * 1e3));
-	return value === void 0 ? null : value;
+var DuckExec = class {
+	#conn;
+	constructor(duckConn) {
+		this.#conn = duckConn;
+	}
+	getRowObjectJS = async (sql) => {
+		return (await this.#conn.run(sql)).getRowObjectsJS();
+	};
+	getRowObjectJson = async (sql) => {
+		return (await this.#conn.run(sql)).getRowObjectsJson();
+	};
+	getOneRowObjectJS = async (sql) => {
+		const rows = await this.getRowObjectJS(sql);
+		if (rows.length === 0) return null;
+		this.#ensureOneRow(rows);
+		return rows[0];
+	};
+	getOneRowObjectJson = async (sql) => {
+		const rows = await this.getRowObjectJson(sql);
+		if (rows.length === 0) return null;
+		this.#ensureOneRow(rows);
+		return rows[0];
+	};
+	#ensureOneRow = (rows) => {
+		if (rows.length > 1) throw new Error("Expected one row, but got multiple rows");
+	};
 };
-async function* rowsToColumnsChunks(rows, chunkSize) {
-	if (!Number.isSafeInteger(chunkSize) || chunkSize <= 0) throw new Error(`chunkSize must be a positive integer, got ${chunkSize}`);
-	const first = await rows.next();
-	if (first.done) return;
-	const keys = Object.keys(first.value);
-	let columns = keys.map(() => []);
-	let rowsInChunk = 0;
-	keys.forEach((k, i) => columns[i].push(toDuckValue(first.value[k])));
-	rowsInChunk++;
-	if (rowsInChunk >= chunkSize) {
-		yield columns;
-		columns = keys.map(() => []);
-		rowsInChunk = 0;
+const duckMemoryTags = [
+	"BASE_TABLE",
+	"HASH_TABLE",
+	"PARQUET_READER",
+	"CSV_READER",
+	"ORDER_BY",
+	"ART_INDEX",
+	"COLUMN_DATA",
+	"METADATA",
+	"OVERFLOW_STRINGS",
+	"IN_MEMORY_TABLE",
+	"ALLOCATOR",
+	"EXTENSION",
+	"TRANSACTION",
+	"EXTERNAL_FILE_CACHE"
+];
+const orderByParams = {
+	memory_usage_bytes_desc: "memory_usage_bytes DESC",
+	tag_desc: "tag DESC",
+	tag_asc: "tag ASC"
+};
+var DuckMemory = class {
+	#conn;
+	#exec;
+	constructor(duckdbConn) {
+		this.#conn = duckdbConn;
+		this.#exec = new DuckExec(duckdbConn);
 	}
-	for await (const row of rows) {
-		keys.forEach((k, i) => columns[i].push(toDuckValue(row[k])));
-		rowsInChunk++;
-		if (rowsInChunk >= chunkSize) {
-			yield columns;
-			columns = keys.map(() => []);
-			rowsInChunk = 0;
+	getAll = async (params) => {
+		const { orderBy } = params ?? {};
+		const query = this.#applyOrderBy(`SELECT tag, memory_usage_bytes, temporary_storage_bytes 
+             FROM duckdb_memory() as m`, orderBy);
+		return (await this.#conn.run(query)).getRowObjectsJS();
+	};
+	getByTag = async (tag) => {
+		if (!duckMemoryTags.includes(tag)) throw new Error(`Invalid DuckDB memory tag: ${tag}`);
+		const query = `SELECT tag, memory_usage_bytes, temporary_storage_bytes
+                   FROM duckdb_memory() as m
+                   WHERE tag = '${tag}'`;
+		return this.#exec.getOneRowObjectJS(query);
+	};
+	getSummary = async () => {
+		const rows = await this.getAll();
+		const summaryInBytes = {
+			total: 0n,
+			totalTemp: 0n
+		};
+		for (const row of rows) {
+			summaryInBytes.total += row.memory_usage_bytes;
+			summaryInBytes.totalTemp += row.temporary_storage_bytes;
 		}
-	}
-	if (rowsInChunk > 0) yield columns;
-}
+		return {
+			totalMB: Math.round(Number(summaryInBytes.total / 1048576n)),
+			totalTempMB: Math.round(Number(summaryInBytes.totalTemp / 1048576n))
+		};
+	};
+	#applyOrderBy = (query, orderBy) => {
+		if (orderBy === void 0) return query;
+		const orderByClause = orderByParams[orderBy];
+		if (orderByClause === void 0) throw new Error(`Invalid orderBy parameter: ${orderBy}`);
+		return `${query} ORDER BY ${orderByClause}`;
+	};
+};
 const isOnDataAppendedAsyncCb = (v) => {
 	return v.constructor.name === "AsyncFunction";
 };
@@ -135,6 +196,35 @@ const createTableFromZod = async (params) => {
 		columnTypes
 	};
 };
+const toDuckValue = (value) => {
+	if (value instanceof Date) return new _duckdb_node_api.DuckDBTimestampValue(BigInt(value.getTime() * 1e3));
+	return value === void 0 ? null : value;
+};
+async function* rowsToColumnsChunks(rows, chunkSize) {
+	if (!Number.isSafeInteger(chunkSize) || chunkSize <= 0) throw new Error(`chunkSize must be a positive integer, got ${chunkSize}`);
+	const first = await rows.next();
+	if (first.done) return;
+	const keys = Object.keys(first.value);
+	let columns = keys.map(() => []);
+	let rowsInChunk = 0;
+	keys.forEach((k, i) => columns[i].push(toDuckValue(first.value[k])));
+	rowsInChunk++;
+	if (rowsInChunk >= chunkSize) {
+		yield columns;
+		columns = keys.map(() => []);
+		rowsInChunk = 0;
+	}
+	for await (const row of rows) {
+		keys.forEach((k, i) => columns[i].push(toDuckValue(row[k])));
+		rowsInChunk++;
+		if (rowsInChunk >= chunkSize) {
+			yield columns;
+			columns = keys.map(() => []);
+			rowsInChunk = 0;
+		}
+	}
+	if (rowsInChunk > 0) yield columns;
+}
 var SqlDuck = class {
 	#duck;
 	#logger;
@@ -223,6 +313,7 @@ const zodCodecs = {
 		encode: BigInt
 	})
 };
+exports.DuckMemory = DuckMemory;
 exports.SqlDuck = SqlDuck;
 exports.Table = Table;
 exports.getTableCreateFromZod = getTableCreateFromZod;

package/dist/index.d.cts

@@ -21,6 +21,34 @@ type OnDataAppendedSyncCb = (stats: OnDataAppendedStats) => void;
 type OnDataAppendedAsyncCb = (stats: OnDataAppendedStats) => Promise<void>;
 type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
 //#endregion
+//#region src/helpers/duck-memory.d.ts
+declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE"];
+type DuckMemoryTag = (typeof duckMemoryTags)[number];
+type DuckMemoryRow = {
+  tag: DuckMemoryTag;
+  memory_usage_bytes: bigint;
+  temporary_storage_bytes: bigint;
+};
+declare const orderByParams: {
+  memory_usage_bytes_desc: string;
+  tag_desc: string;
+  tag_asc: string;
+};
+type OrderByParams = keyof typeof orderByParams;
+type DuckMemorySummary = {
+  totalMB: number;
+  totalTempMB: number;
+};
+declare class DuckMemory {
+  #private;
+  constructor(duckdbConn: DuckDBConnection);
+  getAll: (params?: {
+    orderBy?: OrderByParams;
+  }) => Promise<DuckMemoryRow[]>;
+  getByTag: (tag: DuckMemoryTag) => Promise<DuckMemoryRow | null>;
+  getSummary: () => Promise<DuckMemorySummary>;
+}
+//#endregion
 //#region src/table/table.d.ts
 /**
  * Fully qualified table information
@@ -158,4 +186,4 @@ declare const zodCodecs: {
   readonly bigintToString: z.ZodCodec<z.ZodBigInt, z.ZodString>;
 };
 //#endregion
-export { type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, getTableCreateFromZod, zodCodecs };
+export { DuckMemory, DuckMemoryTag, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, getTableCreateFromZod, zodCodecs };

package/dist/index.d.mts

@@ -21,6 +21,34 @@ type OnDataAppendedSyncCb = (stats: OnDataAppendedStats) => void;
 type OnDataAppendedAsyncCb = (stats: OnDataAppendedStats) => Promise<void>;
 type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
 //#endregion
+//#region src/helpers/duck-memory.d.ts
+declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE"];
+type DuckMemoryTag = (typeof duckMemoryTags)[number];
+type DuckMemoryRow = {
+  tag: DuckMemoryTag;
+  memory_usage_bytes: bigint;
+  temporary_storage_bytes: bigint;
+};
+declare const orderByParams: {
+  memory_usage_bytes_desc: string;
+  tag_desc: string;
+  tag_asc: string;
+};
+type OrderByParams = keyof typeof orderByParams;
+type DuckMemorySummary = {
+  totalMB: number;
+  totalTempMB: number;
+};
+declare class DuckMemory {
+  #private;
+  constructor(duckdbConn: DuckDBConnection);
+  getAll: (params?: {
+    orderBy?: OrderByParams;
+  }) => Promise<DuckMemoryRow[]>;
+  getByTag: (tag: DuckMemoryTag) => Promise<DuckMemoryRow | null>;
+  getSummary: () => Promise<DuckMemorySummary>;
+}
+//#endregion
 //#region src/table/table.d.ts
 /**
  * Fully qualified table information
@@ -158,4 +186,4 @@ declare const zodCodecs: {
   readonly bigintToString: z.ZodCodec<z.ZodBigInt, z.ZodString>;
 };
 //#endregion
-export { type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, getTableCreateFromZod, zodCodecs };
+export { DuckMemory, type DuckMemoryTag, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, getTableCreateFromZod, zodCodecs };

package/dist/index.mjs

@@ -1,34 +1,95 @@
 import { BIGINT, DuckDBDataChunk, DuckDBTimestampValue, INTEGER, TIMESTAMP, VARCHAR } from "@duckdb/node-api";
 import * as z from "zod";
-const toDuckValue = (value) => {
-	if (value instanceof Date) return new DuckDBTimestampValue(BigInt(value.getTime() * 1e3));
-	return value === void 0 ? null : value;
+var DuckExec = class {
+	#conn;
+	constructor(duckConn) {
+		this.#conn = duckConn;
+	}
+	getRowObjectJS = async (sql) => {
+		return (await this.#conn.run(sql)).getRowObjectsJS();
+	};
+	getRowObjectJson = async (sql) => {
+		return (await this.#conn.run(sql)).getRowObjectsJson();
+	};
+	getOneRowObjectJS = async (sql) => {
+		const rows = await this.getRowObjectJS(sql);
+		if (rows.length === 0) return null;
+		this.#ensureOneRow(rows);
+		return rows[0];
+	};
+	getOneRowObjectJson = async (sql) => {
+		const rows = await this.getRowObjectJson(sql);
+		if (rows.length === 0) return null;
+		this.#ensureOneRow(rows);
+		return rows[0];
+	};
+	#ensureOneRow = (rows) => {
+		if (rows.length > 1) throw new Error("Expected one row, but got multiple rows");
+	};
 };
-async function* rowsToColumnsChunks(rows, chunkSize) {
-	if (!Number.isSafeInteger(chunkSize) || chunkSize <= 0) throw new Error(`chunkSize must be a positive integer, got ${chunkSize}`);
-	const first = await rows.next();
-	if (first.done) return;
-	const keys = Object.keys(first.value);
-	let columns = keys.map(() => []);
-	let rowsInChunk = 0;
-	keys.forEach((k, i) => columns[i].push(toDuckValue(first.value[k])));
-	rowsInChunk++;
-	if (rowsInChunk >= chunkSize) {
-		yield columns;
-		columns = keys.map(() => []);
-		rowsInChunk = 0;
+const duckMemoryTags = [
+	"BASE_TABLE",
+	"HASH_TABLE",
+	"PARQUET_READER",
+	"CSV_READER",
+	"ORDER_BY",
+	"ART_INDEX",
+	"COLUMN_DATA",
+	"METADATA",
+	"OVERFLOW_STRINGS",
+	"IN_MEMORY_TABLE",
+	"ALLOCATOR",
+	"EXTENSION",
+	"TRANSACTION",
+	"EXTERNAL_FILE_CACHE"
+];
+const orderByParams = {
+	memory_usage_bytes_desc: "memory_usage_bytes DESC",
+	tag_desc: "tag DESC",
+	tag_asc: "tag ASC"
+};
+var DuckMemory = class {
+	#conn;
+	#exec;
+	constructor(duckdbConn) {
+		this.#conn = duckdbConn;
+		this.#exec = new DuckExec(duckdbConn);
 	}
-	for await (const row of rows) {
-		keys.forEach((k, i) => columns[i].push(toDuckValue(row[k])));
-		rowsInChunk++;
-		if (rowsInChunk >= chunkSize) {
-			yield columns;
-			columns = keys.map(() => []);
-			rowsInChunk = 0;
+	getAll = async (params) => {
+		const { orderBy } = params ?? {};
+		const query = this.#applyOrderBy(`SELECT tag, memory_usage_bytes, temporary_storage_bytes 
+             FROM duckdb_memory() as m`, orderBy);
+		return (await this.#conn.run(query)).getRowObjectsJS();
+	};
+	getByTag = async (tag) => {
+		if (!duckMemoryTags.includes(tag)) throw new Error(`Invalid DuckDB memory tag: ${tag}`);
+		const query = `SELECT tag, memory_usage_bytes, temporary_storage_bytes
+                   FROM duckdb_memory() as m
+                   WHERE tag = '${tag}'`;
+		return this.#exec.getOneRowObjectJS(query);
+	};
+	getSummary = async () => {
+		const rows = await this.getAll();
+		const summaryInBytes = {
+			total: 0n,
+			totalTemp: 0n
+		};
+		for (const row of rows) {
+			summaryInBytes.total += row.memory_usage_bytes;
+			summaryInBytes.totalTemp += row.temporary_storage_bytes;
 		}
-	}
-	if (rowsInChunk > 0) yield columns;
-}
+		return {
+			totalMB: Math.round(Number(summaryInBytes.total / 1048576n)),
+			totalTempMB: Math.round(Number(summaryInBytes.totalTemp / 1048576n))
+		};
+	};
+	#applyOrderBy = (query, orderBy) => {
+		if (orderBy === void 0) return query;
+		const orderByClause = orderByParams[orderBy];
+		if (orderByClause === void 0) throw new Error(`Invalid orderBy parameter: ${orderBy}`);
+		return `${query} ORDER BY ${orderByClause}`;
+	};
+};
 const isOnDataAppendedAsyncCb = (v) => {
 	return v.constructor.name === "AsyncFunction";
 };
@@ -114,6 +175,35 @@ const createTableFromZod = async (params) => {
 		columnTypes
 	};
 };
+const toDuckValue = (value) => {
+	if (value instanceof Date) return new DuckDBTimestampValue(BigInt(value.getTime() * 1e3));
+	return value === void 0 ? null : value;
+};
+async function* rowsToColumnsChunks(rows, chunkSize) {
+	if (!Number.isSafeInteger(chunkSize) || chunkSize <= 0) throw new Error(`chunkSize must be a positive integer, got ${chunkSize}`);
+	const first = await rows.next();
+	if (first.done) return;
+	const keys = Object.keys(first.value);
+	let columns = keys.map(() => []);
+	let rowsInChunk = 0;
+	keys.forEach((k, i) => columns[i].push(toDuckValue(first.value[k])));
+	rowsInChunk++;
+	if (rowsInChunk >= chunkSize) {
+		yield columns;
+		columns = keys.map(() => []);
+		rowsInChunk = 0;
+	}
+	for await (const row of rows) {
+		keys.forEach((k, i) => columns[i].push(toDuckValue(row[k])));
+		rowsInChunk++;
+		if (rowsInChunk >= chunkSize) {
+			yield columns;
+			columns = keys.map(() => []);
+			rowsInChunk = 0;
+		}
+	}
+	if (rowsInChunk > 0) yield columns;
+}
 var SqlDuck = class {
 	#duck;
 	#logger;
@@ -202,4 +292,4 @@ const zodCodecs = {
 		encode: BigInt
 	})
 };
-export { SqlDuck, Table, getTableCreateFromZod, zodCodecs };
+export { DuckMemory, SqlDuck, Table, getTableCreateFromZod, zodCodecs };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flowblade/sqlduck",
-  "version": "0.6.0",
+  "version": "0.8.0",
   "type": "module",
   "sideEffects": false,
   "exports": {
@@ -33,6 +33,12 @@
     "build-release": "yarn build && rimraf ./_release && yarn pack && mkdir ./_release && tar zxvf ./package.tgz --directory ./_release && rm ./package.tgz",
     "docgen": "run-s docgen-typedoc",
     "docgen-typedoc": "rimraf ./docs/api && typedoc --plugin typedoc-plugin-markdown --out ./docs/api",
+    "bench": "vitest bench --run",
+    "bench-bun": "bun --bun run vitest bench --run",
+    "bench-mitata": "node --experimental-strip-types --expose-gc ./bench/stream.mitata.ts ",
+    "bench-mitata-bun": "bun --expose-gc ./bench/stream.mitata.ts ",
+    "bench-codspeed": "cross-env CODSPEED=1 vitest bench --run",
+    "bench-watch": "vitest bench",
     "test": "vitest run",
     "test-unit": "vitest run",
     "test-unit-bun": "bun --bun run vitest run",
@@ -49,60 +55,62 @@
   },
   "dependencies": {
     "@flowblade/core": "^0.2.25",
-    "@flowblade/source-duckdb": "^0.17.3",
-    "@flowblade/sql-tag": "^0.3.0",
+    "@flowblade/source-duckdb": "^0.18.0",
+    "@flowblade/sql-tag": "^0.3.1",
     "@standard-schema/spec": "^1.1.0",
     "p-mutex": "^1.0.0",
     "valibot": "^1.2.0",
-    "zod": "^4.3.5"
+    "zod": "^4.3.6"
   },
   "peerDependencies": {
-    "@duckdb/node-api": "^1.4.3-r.3"
+    "@duckdb/node-api": "^1.4.4-r.1"
   },
   "devDependencies": {
-    "@belgattitude/eslint-config-bases": "8.8.0",
-    "@dotenvx/dotenvx": "1.51.4",
-    "@duckdb/node-api": "1.4.3-r.3",
+    "@belgattitude/eslint-config-bases": "8.9.0",
+    "@dotenvx/dotenvx": "1.52.0",
+    "@duckdb/node-api": "1.4.4-r.1",
     "@faker-js/faker": "10.2.0",
-    "@flowblade/source-kysely": "^1.2.1",
+    "@flowblade/source-kysely": "^1.2.2",
     "@httpx/assert": "0.16.7",
+    "@mitata/counters": "0.0.8",
     "@size-limit/esbuild": "12.0.0",
     "@size-limit/file": "12.0.0",
     "@testcontainers/mssqlserver": "11.11.0",
     "@total-typescript/ts-reset": "0.6.1",
     "@traversable/zod": "0.0.57",
-    "@types/node": "25.0.6",
-    "@typescript-eslint/eslint-plugin": "8.53.0",
-    "@typescript-eslint/parser": "8.53.0",
-    "@vitest/coverage-v8": "4.0.17",
-    "@vitest/ui": "4.0.17",
+    "@types/node": "25.0.10",
+    "@typescript-eslint/eslint-plugin": "8.54.0",
+    "@typescript-eslint/parser": "8.54.0",
+    "@vitest/coverage-v8": "4.0.18",
+    "@vitest/ui": "4.0.18",
     "ansis": "4.2.0",
     "browserslist-to-esbuild": "2.1.1",
-    "core-js": "3.47.0",
+    "core-js": "3.48.0",
     "cross-env": "10.1.0",
-    "es-check": "9.5.3",
+    "es-check": "9.5.4",
     "esbuild": "0.27.2",
     "eslint": "8.57.1",
     "execa": "9.6.1",
     "is-in-ci": "2.0.0",
-    "kysely": "0.28.9",
+    "kysely": "0.28.10",
+    "mitata": "1.0.34",
     "npm-run-all2": "8.0.4",
-    "prettier": "3.7.4",
-    "publint": "0.3.16",
+    "prettier": "3.8.1",
+    "publint": "0.3.17",
     "regexp.escape": "2.0.1",
     "rimraf": "6.1.2",
     "size-limit": "12.0.0",
-    "sql-formatter": "15.6.12",
+    "sql-formatter": "15.7.0",
     "tarn": "3.0.2",
     "tedious": "19.2.0",
     "testcontainers": "11.11.0",
-    "tsdown": "0.19.0",
+    "tsdown": "0.20.1",
     "tsx": "4.21.0",
-    "typedoc": "0.28.15",
+    "typedoc": "0.28.16",
     "typedoc-plugin-markdown": "4.9.0",
     "typescript": "5.9.3",
-    "vite-tsconfig-paths": "6.0.4",
-    "vitest": "4.0.17"
+    "vite-tsconfig-paths": "6.0.5",
+    "vitest": "4.0.18"
   },
   "files": [
     "dist"