@flowblade/sqlduck 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/dist/index.cjs +94 -3
  2. package/dist/index.d.cts +35 -8
  3. package/dist/index.d.mts +35 -8
  4. package/dist/index.mjs +94 -4
  5. package/package.json +18 -18
package/dist/index.cjs CHANGED
@@ -21,6 +21,96 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
21
21
  let _duckdb_node_api = require("@duckdb/node-api");
22
22
  let zod = require("zod");
23
23
  zod = __toESM(zod);
24
+ var DuckExec = class {
25
+ #conn;
26
+ constructor(duckConn) {
27
+ this.#conn = duckConn;
28
+ }
29
+ getRowObjectJS = async (sql) => {
30
+ return (await this.#conn.run(sql)).getRowObjectsJS();
31
+ };
32
+ getRowObjectJson = async (sql) => {
33
+ return (await this.#conn.run(sql)).getRowObjectsJson();
34
+ };
35
+ getOneRowObjectJS = async (sql) => {
36
+ const rows = await this.getRowObjectJS(sql);
37
+ if (rows.length === 0) return null;
38
+ this.#ensureOneRow(rows);
39
+ return rows[0];
40
+ };
41
+ getOneRowObjectJson = async (sql) => {
42
+ const rows = await this.getRowObjectJson(sql);
43
+ if (rows.length === 0) return null;
44
+ this.#ensureOneRow(rows);
45
+ return rows[0];
46
+ };
47
+ #ensureOneRow = (rows) => {
48
+ if (rows.length > 1) throw new Error("Expected one row, but got multiple rows");
49
+ };
50
+ };
51
+ const duckMemoryTags = [
52
+ "BASE_TABLE",
53
+ "HASH_TABLE",
54
+ "PARQUET_READER",
55
+ "CSV_READER",
56
+ "ORDER_BY",
57
+ "ART_INDEX",
58
+ "COLUMN_DATA",
59
+ "METADATA",
60
+ "OVERFLOW_STRINGS",
61
+ "IN_MEMORY_TABLE",
62
+ "ALLOCATOR",
63
+ "EXTENSION",
64
+ "TRANSACTION",
65
+ "EXTERNAL_FILE_CACHE"
66
+ ];
67
+ const orderByParams = {
68
+ memory_usage_bytes_desc: "memory_usage_bytes DESC",
69
+ tag_desc: "tag DESC",
70
+ tag_asc: "tag ASC"
71
+ };
72
+ var DuckMemory = class {
73
+ #conn;
74
+ #exec;
75
+ constructor(duckdbConn) {
76
+ this.#conn = duckdbConn;
77
+ this.#exec = new DuckExec(duckdbConn);
78
+ }
79
+ getAll = async (params) => {
80
+ const { orderBy } = params ?? {};
81
+ const query = this.#applyOrderBy(`SELECT tag, memory_usage_bytes, temporary_storage_bytes
82
+ FROM duckdb_memory() as m`, orderBy);
83
+ return (await this.#conn.run(query)).getRowObjectsJS();
84
+ };
85
+ getByTag = async (tag) => {
86
+ if (!duckMemoryTags.includes(tag)) throw new Error(`Invalid DuckDB memory tag: ${tag}`);
87
+ const query = `SELECT tag, memory_usage_bytes, temporary_storage_bytes
88
+ FROM duckdb_memory() as m
89
+ WHERE tag = '${tag}'`;
90
+ return this.#exec.getOneRowObjectJS(query);
91
+ };
92
+ getSummary = async () => {
93
+ const rows = await this.getAll();
94
+ const summaryInBytes = {
95
+ total: 0n,
96
+ totalTemp: 0n
97
+ };
98
+ for (const row of rows) {
99
+ summaryInBytes.total += row.memory_usage_bytes;
100
+ summaryInBytes.totalTemp += row.temporary_storage_bytes;
101
+ }
102
+ return {
103
+ totalMB: Math.round(Number(summaryInBytes.total / 1048576n)),
104
+ totalTempMB: Math.round(Number(summaryInBytes.totalTemp / 1048576n))
105
+ };
106
+ };
107
+ #applyOrderBy = (query, orderBy) => {
108
+ if (orderBy === void 0) return query;
109
+ const orderByClause = orderByParams[orderBy];
110
+ if (orderByClause === void 0) throw new Error(`Invalid orderBy parameter: ${orderBy}`);
111
+ return `${query} ORDER BY ${orderByClause}`;
112
+ };
113
+ };
24
114
  const toDuckValue = (value) => {
25
115
  if (value instanceof Date) return new _duckdb_node_api.DuckDBTimestampValue(BigInt(value.getTime() * 1e3));
26
116
  return value === void 0 ? null : value;
@@ -59,14 +149,14 @@ const createOnDataAppendedCollector = () => {
59
149
  return (currentTotalRows) => {
60
150
  const cbTimeMs = Math.round(Date.now() - lastCallbackTimeStart);
61
151
  const cbTotalRows = currentTotalRows - appendedTotalRows;
62
- const payload = {
63
- rowsCount: cbTotalRows,
152
+ const stats = {
153
+ totalRows: currentTotalRows,
64
154
  timeMs: cbTimeMs,
65
155
  rowsPerSecond: Math.round(cbTotalRows / cbTimeMs * 1e3)
66
156
  };
67
157
  appendedTotalRows = currentTotalRows;
68
158
  lastCallbackTimeStart = Date.now();
69
- return payload;
159
+ return stats;
70
160
  };
71
161
  };
72
162
  const createMap = {
@@ -223,6 +313,7 @@ const zodCodecs = {
223
313
  encode: BigInt
224
314
  })
225
315
  };
316
+ exports.DuckMemory = DuckMemory;
226
317
  exports.SqlDuck = SqlDuck;
227
318
  exports.Table = Table;
228
319
  exports.getTableCreateFromZod = getTableCreateFromZod;
package/dist/index.d.cts CHANGED
@@ -3,24 +3,52 @@ import * as z from "zod";
3
3
  import { ZodObject } from "zod";
4
4
 
5
5
  //#region src/appender/data-appender-callback.d.ts
6
- type OnDataAppendedParams = {
6
+ type OnDataAppendedStats = {
7
7
  /**
8
- * Total number of rows appended so far
8
+ * Total number of rows appended so far (all batches included)
9
9
  */
10
- rowsCount: number;
10
+ totalRows: number;
11
11
  /**
12
12
  * Time taken to append the last batch in milliseconds
13
13
  */
14
14
  timeMs: number;
15
15
  /**
16
- * Estimated rows per seconds
16
+ * Estimated rows per seconds based on the current batch
17
17
  */
18
18
  rowsPerSecond: number;
19
19
  };
20
- type OnDataAppendedSyncCb = (params: OnDataAppendedParams) => void;
21
- type OnDataAppendedAsyncCb = (params: OnDataAppendedParams) => Promise<void>;
20
+ type OnDataAppendedSyncCb = (stats: OnDataAppendedStats) => void;
21
+ type OnDataAppendedAsyncCb = (stats: OnDataAppendedStats) => Promise<void>;
22
22
  type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
23
23
  //#endregion
24
+ //#region src/helpers/duck-memory.d.ts
25
+ declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE"];
26
+ type DuckMemoryTag = (typeof duckMemoryTags)[number];
27
+ type DuckMemoryRow = {
28
+ tag: DuckMemoryTag;
29
+ memory_usage_bytes: bigint;
30
+ temporary_storage_bytes: bigint;
31
+ };
32
+ declare const orderByParams: {
33
+ memory_usage_bytes_desc: string;
34
+ tag_desc: string;
35
+ tag_asc: string;
36
+ };
37
+ type OrderByParams = keyof typeof orderByParams;
38
+ type DuckMemorySummary = {
39
+ totalMB: number;
40
+ totalTempMB: number;
41
+ };
42
+ declare class DuckMemory {
43
+ #private;
44
+ constructor(duckdbConn: DuckDBConnection);
45
+ getAll: (params?: {
46
+ orderBy?: OrderByParams;
47
+ }) => Promise<DuckMemoryRow[]>;
48
+ getByTag: (tag: DuckMemoryTag) => Promise<DuckMemoryRow | null>;
49
+ getSummary: () => Promise<DuckMemorySummary>;
50
+ }
51
+ //#endregion
24
52
  //#region src/table/table.d.ts
25
53
  /**
26
54
  * Fully qualified table information
@@ -140,7 +168,6 @@ declare class SqlDuck {
140
168
  * onDataAppended: ({ total }) => {
141
169
  * console.log(`Appended ${total} rows so far`);
142
170
  * },
143
- * onDataAppendedBatchSize: 4096, // Call onDataAppended every 4096 rows
144
171
  * createOptions: {
145
172
  * create: 'CREATE_OR_REPLACE',
146
173
  * },
@@ -159,4 +186,4 @@ declare const zodCodecs: {
159
186
  readonly bigintToString: z.ZodCodec<z.ZodBigInt, z.ZodString>;
160
187
  };
161
188
  //#endregion
162
- export { SqlDuck, type SqlDuckParams, Table, getTableCreateFromZod, zodCodecs };
189
+ export { DuckMemory, DuckMemoryTag, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, getTableCreateFromZod, zodCodecs };
package/dist/index.d.mts CHANGED
@@ -3,24 +3,52 @@ import * as z from "zod";
3
3
  import { ZodObject } from "zod";
4
4
 
5
5
  //#region src/appender/data-appender-callback.d.ts
6
- type OnDataAppendedParams = {
6
+ type OnDataAppendedStats = {
7
7
  /**
8
- * Total number of rows appended so far
8
+ * Total number of rows appended so far (all batches included)
9
9
  */
10
- rowsCount: number;
10
+ totalRows: number;
11
11
  /**
12
12
  * Time taken to append the last batch in milliseconds
13
13
  */
14
14
  timeMs: number;
15
15
  /**
16
- * Estimated rows per seconds
16
+ * Estimated rows per seconds based on the current batch
17
17
  */
18
18
  rowsPerSecond: number;
19
19
  };
20
- type OnDataAppendedSyncCb = (params: OnDataAppendedParams) => void;
21
- type OnDataAppendedAsyncCb = (params: OnDataAppendedParams) => Promise<void>;
20
+ type OnDataAppendedSyncCb = (stats: OnDataAppendedStats) => void;
21
+ type OnDataAppendedAsyncCb = (stats: OnDataAppendedStats) => Promise<void>;
22
22
  type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
23
23
  //#endregion
24
+ //#region src/helpers/duck-memory.d.ts
25
+ declare const duckMemoryTags: readonly ["BASE_TABLE", "HASH_TABLE", "PARQUET_READER", "CSV_READER", "ORDER_BY", "ART_INDEX", "COLUMN_DATA", "METADATA", "OVERFLOW_STRINGS", "IN_MEMORY_TABLE", "ALLOCATOR", "EXTENSION", "TRANSACTION", "EXTERNAL_FILE_CACHE"];
26
+ type DuckMemoryTag = (typeof duckMemoryTags)[number];
27
+ type DuckMemoryRow = {
28
+ tag: DuckMemoryTag;
29
+ memory_usage_bytes: bigint;
30
+ temporary_storage_bytes: bigint;
31
+ };
32
+ declare const orderByParams: {
33
+ memory_usage_bytes_desc: string;
34
+ tag_desc: string;
35
+ tag_asc: string;
36
+ };
37
+ type OrderByParams = keyof typeof orderByParams;
38
+ type DuckMemorySummary = {
39
+ totalMB: number;
40
+ totalTempMB: number;
41
+ };
42
+ declare class DuckMemory {
43
+ #private;
44
+ constructor(duckdbConn: DuckDBConnection);
45
+ getAll: (params?: {
46
+ orderBy?: OrderByParams;
47
+ }) => Promise<DuckMemoryRow[]>;
48
+ getByTag: (tag: DuckMemoryTag) => Promise<DuckMemoryRow | null>;
49
+ getSummary: () => Promise<DuckMemorySummary>;
50
+ }
51
+ //#endregion
24
52
  //#region src/table/table.d.ts
25
53
  /**
26
54
  * Fully qualified table information
@@ -140,7 +168,6 @@ declare class SqlDuck {
140
168
  * onDataAppended: ({ total }) => {
141
169
  * console.log(`Appended ${total} rows so far`);
142
170
  * },
143
- * onDataAppendedBatchSize: 4096, // Call onDataAppended every 4096 rows
144
171
  * createOptions: {
145
172
  * create: 'CREATE_OR_REPLACE',
146
173
  * },
@@ -159,4 +186,4 @@ declare const zodCodecs: {
159
186
  readonly bigintToString: z.ZodCodec<z.ZodBigInt, z.ZodString>;
160
187
  };
161
188
  //#endregion
162
- export { SqlDuck, type SqlDuckParams, Table, getTableCreateFromZod, zodCodecs };
189
+ export { DuckMemory, type DuckMemoryTag, type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, getTableCreateFromZod, zodCodecs };
package/dist/index.mjs CHANGED
@@ -1,5 +1,95 @@
1
1
  import { BIGINT, DuckDBDataChunk, DuckDBTimestampValue, INTEGER, TIMESTAMP, VARCHAR } from "@duckdb/node-api";
2
2
  import * as z from "zod";
3
+ var DuckExec = class {
4
+ #conn;
5
+ constructor(duckConn) {
6
+ this.#conn = duckConn;
7
+ }
8
+ getRowObjectJS = async (sql) => {
9
+ return (await this.#conn.run(sql)).getRowObjectsJS();
10
+ };
11
+ getRowObjectJson = async (sql) => {
12
+ return (await this.#conn.run(sql)).getRowObjectsJson();
13
+ };
14
+ getOneRowObjectJS = async (sql) => {
15
+ const rows = await this.getRowObjectJS(sql);
16
+ if (rows.length === 0) return null;
17
+ this.#ensureOneRow(rows);
18
+ return rows[0];
19
+ };
20
+ getOneRowObjectJson = async (sql) => {
21
+ const rows = await this.getRowObjectJson(sql);
22
+ if (rows.length === 0) return null;
23
+ this.#ensureOneRow(rows);
24
+ return rows[0];
25
+ };
26
+ #ensureOneRow = (rows) => {
27
+ if (rows.length > 1) throw new Error("Expected one row, but got multiple rows");
28
+ };
29
+ };
30
+ const duckMemoryTags = [
31
+ "BASE_TABLE",
32
+ "HASH_TABLE",
33
+ "PARQUET_READER",
34
+ "CSV_READER",
35
+ "ORDER_BY",
36
+ "ART_INDEX",
37
+ "COLUMN_DATA",
38
+ "METADATA",
39
+ "OVERFLOW_STRINGS",
40
+ "IN_MEMORY_TABLE",
41
+ "ALLOCATOR",
42
+ "EXTENSION",
43
+ "TRANSACTION",
44
+ "EXTERNAL_FILE_CACHE"
45
+ ];
46
+ const orderByParams = {
47
+ memory_usage_bytes_desc: "memory_usage_bytes DESC",
48
+ tag_desc: "tag DESC",
49
+ tag_asc: "tag ASC"
50
+ };
51
+ var DuckMemory = class {
52
+ #conn;
53
+ #exec;
54
+ constructor(duckdbConn) {
55
+ this.#conn = duckdbConn;
56
+ this.#exec = new DuckExec(duckdbConn);
57
+ }
58
+ getAll = async (params) => {
59
+ const { orderBy } = params ?? {};
60
+ const query = this.#applyOrderBy(`SELECT tag, memory_usage_bytes, temporary_storage_bytes
61
+ FROM duckdb_memory() as m`, orderBy);
62
+ return (await this.#conn.run(query)).getRowObjectsJS();
63
+ };
64
+ getByTag = async (tag) => {
65
+ if (!duckMemoryTags.includes(tag)) throw new Error(`Invalid DuckDB memory tag: ${tag}`);
66
+ const query = `SELECT tag, memory_usage_bytes, temporary_storage_bytes
67
+ FROM duckdb_memory() as m
68
+ WHERE tag = '${tag}'`;
69
+ return this.#exec.getOneRowObjectJS(query);
70
+ };
71
+ getSummary = async () => {
72
+ const rows = await this.getAll();
73
+ const summaryInBytes = {
74
+ total: 0n,
75
+ totalTemp: 0n
76
+ };
77
+ for (const row of rows) {
78
+ summaryInBytes.total += row.memory_usage_bytes;
79
+ summaryInBytes.totalTemp += row.temporary_storage_bytes;
80
+ }
81
+ return {
82
+ totalMB: Math.round(Number(summaryInBytes.total / 1048576n)),
83
+ totalTempMB: Math.round(Number(summaryInBytes.totalTemp / 1048576n))
84
+ };
85
+ };
86
+ #applyOrderBy = (query, orderBy) => {
87
+ if (orderBy === void 0) return query;
88
+ const orderByClause = orderByParams[orderBy];
89
+ if (orderByClause === void 0) throw new Error(`Invalid orderBy parameter: ${orderBy}`);
90
+ return `${query} ORDER BY ${orderByClause}`;
91
+ };
92
+ };
3
93
  const toDuckValue = (value) => {
4
94
  if (value instanceof Date) return new DuckDBTimestampValue(BigInt(value.getTime() * 1e3));
5
95
  return value === void 0 ? null : value;
@@ -38,14 +128,14 @@ const createOnDataAppendedCollector = () => {
38
128
  return (currentTotalRows) => {
39
129
  const cbTimeMs = Math.round(Date.now() - lastCallbackTimeStart);
40
130
  const cbTotalRows = currentTotalRows - appendedTotalRows;
41
- const payload = {
42
- rowsCount: cbTotalRows,
131
+ const stats = {
132
+ totalRows: currentTotalRows,
43
133
  timeMs: cbTimeMs,
44
134
  rowsPerSecond: Math.round(cbTotalRows / cbTimeMs * 1e3)
45
135
  };
46
136
  appendedTotalRows = currentTotalRows;
47
137
  lastCallbackTimeStart = Date.now();
48
- return payload;
138
+ return stats;
49
139
  };
50
140
  };
51
141
  const createMap = {
@@ -202,4 +292,4 @@ const zodCodecs = {
202
292
  encode: BigInt
203
293
  })
204
294
  };
205
- export { SqlDuck, Table, getTableCreateFromZod, zodCodecs };
295
+ export { DuckMemory, SqlDuck, Table, getTableCreateFromZod, zodCodecs };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@flowblade/sqlduck",
3
- "version": "0.5.0",
3
+ "version": "0.7.0",
4
4
  "type": "module",
5
5
  "sideEffects": false,
6
6
  "exports": {
@@ -49,8 +49,8 @@
49
49
  },
50
50
  "dependencies": {
51
51
  "@flowblade/core": "^0.2.25",
52
- "@flowblade/source-duckdb": "^0.17.2",
53
- "@flowblade/sql-tag": "^0.2.0",
52
+ "@flowblade/source-duckdb": "^0.17.3",
53
+ "@flowblade/sql-tag": "^0.3.0",
54
54
  "@standard-schema/spec": "^1.1.0",
55
55
  "p-mutex": "^1.0.0",
56
56
  "valibot": "^1.2.0",
@@ -60,49 +60,49 @@
60
60
  "@duckdb/node-api": "^1.4.3-r.3"
61
61
  },
62
62
  "devDependencies": {
63
- "@belgattitude/eslint-config-bases": "8.8.0",
63
+ "@belgattitude/eslint-config-bases": "8.9.0",
64
64
  "@dotenvx/dotenvx": "1.51.4",
65
65
  "@duckdb/node-api": "1.4.3-r.3",
66
66
  "@faker-js/faker": "10.2.0",
67
- "@flowblade/source-kysely": "^1.2.0",
67
+ "@flowblade/source-kysely": "^1.2.2",
68
68
  "@httpx/assert": "0.16.7",
69
69
  "@size-limit/esbuild": "12.0.0",
70
70
  "@size-limit/file": "12.0.0",
71
71
  "@testcontainers/mssqlserver": "11.11.0",
72
72
  "@total-typescript/ts-reset": "0.6.1",
73
73
  "@traversable/zod": "0.0.57",
74
- "@types/node": "25.0.3",
75
- "@typescript-eslint/eslint-plugin": "8.52.0",
76
- "@typescript-eslint/parser": "8.52.0",
77
- "@vitest/coverage-v8": "4.0.16",
78
- "@vitest/ui": "4.0.16",
74
+ "@types/node": "25.0.9",
75
+ "@typescript-eslint/eslint-plugin": "8.53.0",
76
+ "@typescript-eslint/parser": "8.53.0",
77
+ "@vitest/coverage-v8": "4.0.17",
78
+ "@vitest/ui": "4.0.17",
79
79
  "ansis": "4.2.0",
80
80
  "browserslist-to-esbuild": "2.1.1",
81
81
  "core-js": "3.47.0",
82
82
  "cross-env": "10.1.0",
83
- "es-check": "9.5.3",
83
+ "es-check": "9.5.4",
84
84
  "esbuild": "0.27.2",
85
85
  "eslint": "8.57.1",
86
86
  "execa": "9.6.1",
87
87
  "is-in-ci": "2.0.0",
88
- "kysely": "0.28.9",
88
+ "kysely": "0.28.10",
89
89
  "npm-run-all2": "8.0.4",
90
- "prettier": "3.7.4",
90
+ "prettier": "3.8.0",
91
91
  "publint": "0.3.16",
92
92
  "regexp.escape": "2.0.1",
93
93
  "rimraf": "6.1.2",
94
94
  "size-limit": "12.0.0",
95
- "sql-formatter": "15.6.12",
95
+ "sql-formatter": "15.7.0",
96
96
  "tarn": "3.0.2",
97
97
  "tedious": "19.2.0",
98
98
  "testcontainers": "11.11.0",
99
- "tsdown": "0.18.4",
99
+ "tsdown": "0.19.0",
100
100
  "tsx": "4.21.0",
101
- "typedoc": "0.28.15",
101
+ "typedoc": "0.28.16",
102
102
  "typedoc-plugin-markdown": "4.9.0",
103
103
  "typescript": "5.9.3",
104
- "vite-tsconfig-paths": "6.0.3",
105
- "vitest": "4.0.16"
104
+ "vite-tsconfig-paths": "6.0.4",
105
+ "vitest": "4.0.17"
106
106
  },
107
107
  "files": [
108
108
  "dist"