@flowblade/sqlduck 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/dist/index.cjs +26 -5
  2. package/dist/index.d.cts +22 -16
  3. package/dist/index.d.mts +22 -16
  4. package/dist/index.mjs +26 -5
  5. package/package.json +12 -12
package/dist/index.cjs CHANGED
@@ -50,6 +50,25 @@ async function* rowsToColumnsChunks(rows, chunkSize) {
50
50
  }
51
51
  if (rowsInChunk > 0) yield columns;
52
52
  }
53
+ const isOnDataAppendedAsyncCb = (v) => {
54
+ return v.constructor.name === "AsyncFunction";
55
+ };
56
+ const createOnDataAppendedCollector = () => {
57
+ let lastCallbackTimeStart = Date.now();
58
+ let appendedTotalRows = 0;
59
+ return (currentTotalRows) => {
60
+ const cbTimeMs = Math.round(Date.now() - lastCallbackTimeStart);
61
+ const cbTotalRows = currentTotalRows - appendedTotalRows;
62
+ const stats = {
63
+ totalRows: currentTotalRows,
64
+ timeMs: cbTimeMs,
65
+ rowsPerSecond: Math.round(cbTotalRows / cbTimeMs * 1e3)
66
+ };
67
+ appendedTotalRows = currentTotalRows;
68
+ lastCallbackTimeStart = Date.now();
69
+ return stats;
70
+ };
71
+ };
53
72
  const createMap = {
54
73
  CREATE: "CREATE TABLE",
55
74
  CREATE_OR_REPLACE: "CREATE OR REPLACE TABLE",
@@ -124,10 +143,8 @@ var SqlDuck = class {
124
143
  this.#logger = params.logger;
125
144
  }
126
145
  toTable = async (params) => {
127
- const { table, schema, chunkSize = 2048, rowStream, createOptions, onDataAppended, onDataAppendedBatchSize } = params;
146
+ const { table, schema, chunkSize = 2048, rowStream, createOptions, onDataAppended } = params;
128
147
  if (!Number.isSafeInteger(chunkSize) || chunkSize < 1 || chunkSize > 2048) throw new Error("chunkSize must be a number between 1 and 2048");
129
- const callbackBatchSize = onDataAppendedBatchSize ?? chunkSize;
130
- if (!Number.isSafeInteger(callbackBatchSize) || callbackBatchSize < 1) throw new Error("onDataAppendedBatchSize must be a number greater than 0");
131
148
  const timeStart = Date.now();
132
149
  const { columnTypes, ddl } = await createTableFromZod({
133
150
  conn: this.#duck,
@@ -138,6 +155,7 @@ var SqlDuck = class {
138
155
  const appender = await this.#duck.createAppender(table.tableName, table.schemaName, table.databaseName);
139
156
  const chunkTypes = columnTypes.map((v) => v[1]);
140
157
  let totalRows = 0;
158
+ const dataAppendedCollector = createOnDataAppendedCollector();
141
159
  const columnStream = rowsToColumnsChunks(rowStream, chunkSize);
142
160
  for await (const dataChunk of columnStream) {
143
161
  const chunk = _duckdb_node_api.DuckDBDataChunk.create(chunkTypes);
@@ -146,9 +164,12 @@ var SqlDuck = class {
146
164
  chunk.setColumns(dataChunk);
147
165
  appender.appendDataChunk(chunk);
148
166
  appender.flushSync();
149
- if (onDataAppended !== void 0 && totalRows % callbackBatchSize === 0) onDataAppended({ total: totalRows });
167
+ if (onDataAppended !== void 0) {
168
+ const payload = dataAppendedCollector(totalRows);
169
+ if (isOnDataAppendedAsyncCb(onDataAppended)) await onDataAppended(payload);
170
+ else onDataAppended(payload);
171
+ }
150
172
  }
151
- if (onDataAppended !== void 0 && totalRows % callbackBatchSize !== 0) onDataAppended({ total: totalRows });
152
173
  return {
153
174
  timeMs: Math.round(Date.now() - timeStart),
154
175
  totalRows,
package/dist/index.d.cts CHANGED
@@ -2,6 +2,25 @@ import { DuckDBConnection, DuckDBType } from "@duckdb/node-api";
2
2
  import * as z from "zod";
3
3
  import { ZodObject } from "zod";
4
4
 
5
+ //#region src/appender/data-appender-callback.d.ts
6
+ type OnDataAppendedStats = {
7
+ /**
8
+ * Total number of rows appended so far (all batches included)
9
+ */
10
+ totalRows: number;
11
+ /**
12
+ * Time taken to append the last batch in milliseconds
13
+ */
14
+ timeMs: number;
15
+ /**
16
+ * Estimated rows per seconds based on the current batch
17
+ */
18
+ rowsPerSecond: number;
19
+ };
20
+ type OnDataAppendedSyncCb = (stats: OnDataAppendedStats) => void;
21
+ type OnDataAppendedAsyncCb = (stats: OnDataAppendedStats) => Promise<void>;
22
+ type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
23
+ //#endregion
5
24
  //#region src/table/table.d.ts
6
25
  /**
7
26
  * Fully qualified table information
@@ -71,21 +90,9 @@ type ToTableParams<TSchema extends TableSchemaZod> = {
71
90
  */
72
91
  createOptions?: TableCreateOptions;
73
92
  /**
74
- * Callback called each time data is appended to the table
75
- * See also `onDataAppendedBatchSize` to limit the number of calls
76
- * when appending a lot of data
77
- */
78
- onDataAppended?: (params: {
79
- /**
80
- * Total number of rows appended so far
81
- */
82
- total: number;
83
- }) => void;
84
- /**
85
- * Number of rows appended before calling `onDataAppended` callback
86
- * @default chunkSize
93
+ * Callback called each time a datachunk is appended to the table
87
94
  */
88
- onDataAppendedBatchSize?: number;
95
+ onDataAppended?: OnDataAppendedCb;
89
96
  };
90
97
  type ToTableResult = {
91
98
  /**
@@ -133,7 +140,6 @@ declare class SqlDuck {
133
140
  * onDataAppended: ({ total }) => {
134
141
  * console.log(`Appended ${total} rows so far`);
135
142
  * },
136
- * onDataAppendedBatchSize: 4096, // Call onDataAppended every 4096 rows
137
143
  * createOptions: {
138
144
  * create: 'CREATE_OR_REPLACE',
139
145
  * },
@@ -152,4 +158,4 @@ declare const zodCodecs: {
152
158
  readonly bigintToString: z.ZodCodec<z.ZodBigInt, z.ZodString>;
153
159
  };
154
160
  //#endregion
155
- export { SqlDuck, type SqlDuckParams, Table, getTableCreateFromZod, zodCodecs };
161
+ export { type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, getTableCreateFromZod, zodCodecs };
package/dist/index.d.mts CHANGED
@@ -2,6 +2,25 @@ import { DuckDBConnection, DuckDBType } from "@duckdb/node-api";
2
2
  import * as z from "zod";
3
3
  import { ZodObject } from "zod";
4
4
 
5
+ //#region src/appender/data-appender-callback.d.ts
6
+ type OnDataAppendedStats = {
7
+ /**
8
+ * Total number of rows appended so far (all batches included)
9
+ */
10
+ totalRows: number;
11
+ /**
12
+ * Time taken to append the last batch in milliseconds
13
+ */
14
+ timeMs: number;
15
+ /**
16
+ * Estimated rows per seconds based on the current batch
17
+ */
18
+ rowsPerSecond: number;
19
+ };
20
+ type OnDataAppendedSyncCb = (stats: OnDataAppendedStats) => void;
21
+ type OnDataAppendedAsyncCb = (stats: OnDataAppendedStats) => Promise<void>;
22
+ type OnDataAppendedCb = OnDataAppendedSyncCb | OnDataAppendedAsyncCb;
23
+ //#endregion
5
24
  //#region src/table/table.d.ts
6
25
  /**
7
26
  * Fully qualified table information
@@ -71,21 +90,9 @@ type ToTableParams<TSchema extends TableSchemaZod> = {
71
90
  */
72
91
  createOptions?: TableCreateOptions;
73
92
  /**
74
- * Callback called each time data is appended to the table
75
- * See also `onDataAppendedBatchSize` to limit the number of calls
76
- * when appending a lot of data
77
- */
78
- onDataAppended?: (params: {
79
- /**
80
- * Total number of rows appended so far
81
- */
82
- total: number;
83
- }) => void;
84
- /**
85
- * Number of rows appended before calling `onDataAppended` callback
86
- * @default chunkSize
93
+ * Callback called each time a datachunk is appended to the table
87
94
  */
88
- onDataAppendedBatchSize?: number;
95
+ onDataAppended?: OnDataAppendedCb;
89
96
  };
90
97
  type ToTableResult = {
91
98
  /**
@@ -133,7 +140,6 @@ declare class SqlDuck {
133
140
  * onDataAppended: ({ total }) => {
134
141
  * console.log(`Appended ${total} rows so far`);
135
142
  * },
136
- * onDataAppendedBatchSize: 4096, // Call onDataAppended every 4096 rows
137
143
  * createOptions: {
138
144
  * create: 'CREATE_OR_REPLACE',
139
145
  * },
@@ -152,4 +158,4 @@ declare const zodCodecs: {
152
158
  readonly bigintToString: z.ZodCodec<z.ZodBigInt, z.ZodString>;
153
159
  };
154
160
  //#endregion
155
- export { SqlDuck, type SqlDuckParams, Table, getTableCreateFromZod, zodCodecs };
161
+ export { type OnDataAppendedCb, type OnDataAppendedStats, SqlDuck, type SqlDuckParams, Table, type ToTableParams, getTableCreateFromZod, zodCodecs };
package/dist/index.mjs CHANGED
@@ -29,6 +29,25 @@ async function* rowsToColumnsChunks(rows, chunkSize) {
29
29
  }
30
30
  if (rowsInChunk > 0) yield columns;
31
31
  }
32
+ const isOnDataAppendedAsyncCb = (v) => {
33
+ return v.constructor.name === "AsyncFunction";
34
+ };
35
+ const createOnDataAppendedCollector = () => {
36
+ let lastCallbackTimeStart = Date.now();
37
+ let appendedTotalRows = 0;
38
+ return (currentTotalRows) => {
39
+ const cbTimeMs = Math.round(Date.now() - lastCallbackTimeStart);
40
+ const cbTotalRows = currentTotalRows - appendedTotalRows;
41
+ const stats = {
42
+ totalRows: currentTotalRows,
43
+ timeMs: cbTimeMs,
44
+ rowsPerSecond: Math.round(cbTotalRows / cbTimeMs * 1e3)
45
+ };
46
+ appendedTotalRows = currentTotalRows;
47
+ lastCallbackTimeStart = Date.now();
48
+ return stats;
49
+ };
50
+ };
32
51
  const createMap = {
33
52
  CREATE: "CREATE TABLE",
34
53
  CREATE_OR_REPLACE: "CREATE OR REPLACE TABLE",
@@ -103,10 +122,8 @@ var SqlDuck = class {
103
122
  this.#logger = params.logger;
104
123
  }
105
124
  toTable = async (params) => {
106
- const { table, schema, chunkSize = 2048, rowStream, createOptions, onDataAppended, onDataAppendedBatchSize } = params;
125
+ const { table, schema, chunkSize = 2048, rowStream, createOptions, onDataAppended } = params;
107
126
  if (!Number.isSafeInteger(chunkSize) || chunkSize < 1 || chunkSize > 2048) throw new Error("chunkSize must be a number between 1 and 2048");
108
- const callbackBatchSize = onDataAppendedBatchSize ?? chunkSize;
109
- if (!Number.isSafeInteger(callbackBatchSize) || callbackBatchSize < 1) throw new Error("onDataAppendedBatchSize must be a number greater than 0");
110
127
  const timeStart = Date.now();
111
128
  const { columnTypes, ddl } = await createTableFromZod({
112
129
  conn: this.#duck,
@@ -117,6 +134,7 @@ var SqlDuck = class {
117
134
  const appender = await this.#duck.createAppender(table.tableName, table.schemaName, table.databaseName);
118
135
  const chunkTypes = columnTypes.map((v) => v[1]);
119
136
  let totalRows = 0;
137
+ const dataAppendedCollector = createOnDataAppendedCollector();
120
138
  const columnStream = rowsToColumnsChunks(rowStream, chunkSize);
121
139
  for await (const dataChunk of columnStream) {
122
140
  const chunk = DuckDBDataChunk.create(chunkTypes);
@@ -125,9 +143,12 @@ var SqlDuck = class {
125
143
  chunk.setColumns(dataChunk);
126
144
  appender.appendDataChunk(chunk);
127
145
  appender.flushSync();
128
- if (onDataAppended !== void 0 && totalRows % callbackBatchSize === 0) onDataAppended({ total: totalRows });
146
+ if (onDataAppended !== void 0) {
147
+ const payload = dataAppendedCollector(totalRows);
148
+ if (isOnDataAppendedAsyncCb(onDataAppended)) await onDataAppended(payload);
149
+ else onDataAppended(payload);
150
+ }
129
151
  }
130
- if (onDataAppended !== void 0 && totalRows % callbackBatchSize !== 0) onDataAppended({ total: totalRows });
131
152
  return {
132
153
  timeMs: Math.round(Date.now() - timeStart),
133
154
  totalRows,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@flowblade/sqlduck",
3
- "version": "0.4.0",
3
+ "version": "0.6.0",
4
4
  "type": "module",
5
5
  "sideEffects": false,
6
6
  "exports": {
@@ -49,8 +49,8 @@
49
49
  },
50
50
  "dependencies": {
51
51
  "@flowblade/core": "^0.2.25",
52
- "@flowblade/source-duckdb": "^0.17.2",
53
- "@flowblade/sql-tag": "^0.2.0",
52
+ "@flowblade/source-duckdb": "^0.17.3",
53
+ "@flowblade/sql-tag": "^0.3.0",
54
54
  "@standard-schema/spec": "^1.1.0",
55
55
  "p-mutex": "^1.0.0",
56
56
  "valibot": "^1.2.0",
@@ -64,18 +64,18 @@
64
64
  "@dotenvx/dotenvx": "1.51.4",
65
65
  "@duckdb/node-api": "1.4.3-r.3",
66
66
  "@faker-js/faker": "10.2.0",
67
- "@flowblade/source-kysely": "^1.2.0",
67
+ "@flowblade/source-kysely": "^1.2.1",
68
68
  "@httpx/assert": "0.16.7",
69
69
  "@size-limit/esbuild": "12.0.0",
70
70
  "@size-limit/file": "12.0.0",
71
71
  "@testcontainers/mssqlserver": "11.11.0",
72
72
  "@total-typescript/ts-reset": "0.6.1",
73
73
  "@traversable/zod": "0.0.57",
74
- "@types/node": "25.0.3",
75
- "@typescript-eslint/eslint-plugin": "8.52.0",
76
- "@typescript-eslint/parser": "8.52.0",
77
- "@vitest/coverage-v8": "4.0.16",
78
- "@vitest/ui": "4.0.16",
74
+ "@types/node": "25.0.6",
75
+ "@typescript-eslint/eslint-plugin": "8.53.0",
76
+ "@typescript-eslint/parser": "8.53.0",
77
+ "@vitest/coverage-v8": "4.0.17",
78
+ "@vitest/ui": "4.0.17",
79
79
  "ansis": "4.2.0",
80
80
  "browserslist-to-esbuild": "2.1.1",
81
81
  "core-js": "3.47.0",
@@ -96,13 +96,13 @@
96
96
  "tarn": "3.0.2",
97
97
  "tedious": "19.2.0",
98
98
  "testcontainers": "11.11.0",
99
- "tsdown": "0.18.3",
99
+ "tsdown": "0.19.0",
100
100
  "tsx": "4.21.0",
101
101
  "typedoc": "0.28.15",
102
102
  "typedoc-plugin-markdown": "4.9.0",
103
103
  "typescript": "5.9.3",
104
- "vite-tsconfig-paths": "6.0.3",
105
- "vitest": "4.0.16"
104
+ "vite-tsconfig-paths": "6.0.4",
105
+ "vitest": "4.0.17"
106
106
  },
107
107
  "files": [
108
108
  "dist"