@flowblade/sqlduck 0.0.6 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/dist/index.cjs +21 -7
  2. package/dist/index.d.cts +9 -3
  3. package/dist/index.d.mts +9 -3
  4. package/dist/index.mjs +21 -7
  5. package/package.json +4 -4
package/dist/index.cjs CHANGED
@@ -57,7 +57,7 @@ const createMap = {
57
57
  };
58
58
  const getTableCreateFromZod = (table, schema, options) => {
59
59
  const { create = "CREATE" } = options ?? {};
60
- const fqTable = table.getFullyQualifiedTableName();
60
+ const fqTable = table.getFullName();
61
61
  const json = schema.toJSONSchema({ target: "openapi-3.0" });
62
62
  const columns = [];
63
63
  if (json.properties === void 0) throw new TypeError("Schema must have at least one property");
@@ -109,7 +109,7 @@ const createTableFromZod = async (params) => {
109
109
  try {
110
110
  await conn.run(ddl);
111
111
  } catch (e) {
112
- throw new Error(`Failed to create table '${table.getFullyQualifiedTableName()}': ${e.message}`, { cause: e });
112
+ throw new Error(`Failed to create table '${table.getFullName()}': ${e.message}`, { cause: e });
113
113
  }
114
114
  return {
115
115
  ddl,
@@ -125,33 +125,47 @@ var SqlDuck = class {
125
125
  }
126
126
  toTable = async (params) => {
127
127
  const { table, schema, chunkSize, rowStream, createOptions } = params;
128
+ const timeStart = Date.now();
128
129
  const { columnTypes } = await createTableFromZod({
129
130
  conn: this.#duck,
130
131
  schema,
131
132
  table,
132
133
  options: createOptions
133
134
  });
134
- const appender = await this.#duck.createAppender(table.fqTable.name, table.fqTable.schema, table.fqTable.database);
135
+ const appender = await this.#duck.createAppender(table.tableName, table.schemaName, table.databaseName);
135
136
  const chunkTypes = columnTypes.map((v) => v[1]);
136
- const columnStream = rowsToColumnsChunks(rowStream, chunkSize ?? 2048);
137
+ const chunkLimit = chunkSize ?? 2048;
138
+ let totalRows = 0;
139
+ const columnStream = rowsToColumnsChunks(rowStream, chunkLimit);
137
140
  for await (const dataChunk of columnStream) {
138
141
  const chunk = _duckdb_node_api.DuckDBDataChunk.create(chunkTypes);
139
142
  if (this.#logger) this.#logger(`Inserting chunk of ${dataChunk.length} rows`);
143
+ totalRows += dataChunk?.[0]?.length ?? 0;
140
144
  chunk.setColumns(dataChunk);
141
145
  appender.appendDataChunk(chunk);
142
146
  appender.flushSync();
143
147
  }
148
+ return {
149
+ timeMs: Math.round(Date.now() - timeStart),
150
+ totalRows
151
+ };
144
152
  };
145
153
  };
146
154
  var Table = class Table {
147
155
  #fqTable;
148
- get fqTable() {
149
- return this.#fqTable;
156
+ get tableName() {
157
+ return this.#fqTable.name;
158
+ }
159
+ get schemaName() {
160
+ return this.#fqTable.schema;
161
+ }
162
+ get databaseName() {
163
+ return this.#fqTable.database;
150
164
  }
151
165
  constructor(fqTableOrName) {
152
166
  this.#fqTable = typeof fqTableOrName === "string" ? { name: fqTableOrName } : fqTableOrName;
153
167
  }
154
- getFullyQualifiedTableName = (options) => {
168
+ getFullName = (options) => {
155
169
  const { defaultDatabase, defaultSchema } = options ?? {};
156
170
  const { name, database = defaultDatabase, schema = defaultSchema } = this.#fqTable;
157
171
  return [
package/dist/index.d.cts CHANGED
@@ -13,13 +13,15 @@ type FQTable = {
13
13
  };
14
14
  declare class Table {
15
15
  #private;
16
- get fqTable(): FQTable;
16
+ get tableName(): string;
17
+ get schemaName(): string | undefined;
18
+ get databaseName(): string | undefined;
17
19
  constructor(fqTableOrName: FQTable | string);
18
20
  /**
19
21
  * Return fully qualified table name by concatenating
20
22
  * database, schema and table with a 'dot' separator.
21
23
  */
22
- getFullyQualifiedTableName: (options?: {
24
+ getFullName: (options?: {
23
25
  defaultDatabase?: string;
24
26
  defaultSchema?: string;
25
27
  }) => string;
@@ -51,10 +53,14 @@ type ToTableParams<TSchema extends TableSchemaZod> = {
51
53
  chunkSize?: number;
52
54
  createOptions?: TableCreateOptions;
53
55
  };
56
+ type ToTableStats = {
57
+ timeMs: number;
58
+ totalRows: number;
59
+ };
54
60
  declare class SqlDuck {
55
61
  #private;
56
62
  constructor(params: SqlDuckParams);
57
- toTable: <TSchema extends ZodObject>(params: ToTableParams<TSchema>) => Promise<undefined>;
63
+ toTable: <TSchema extends ZodObject>(params: ToTableParams<TSchema>) => Promise<ToTableStats>;
58
64
  }
59
65
  //#endregion
60
66
  //#region src/utils/zod-codecs.d.ts
package/dist/index.d.mts CHANGED
@@ -13,13 +13,15 @@ type FQTable = {
13
13
  };
14
14
  declare class Table {
15
15
  #private;
16
- get fqTable(): FQTable;
16
+ get tableName(): string;
17
+ get schemaName(): string | undefined;
18
+ get databaseName(): string | undefined;
17
19
  constructor(fqTableOrName: FQTable | string);
18
20
  /**
19
21
  * Return fully qualified table name by concatenating
20
22
  * database, schema and table with a 'dot' separator.
21
23
  */
22
- getFullyQualifiedTableName: (options?: {
24
+ getFullName: (options?: {
23
25
  defaultDatabase?: string;
24
26
  defaultSchema?: string;
25
27
  }) => string;
@@ -51,10 +53,14 @@ type ToTableParams<TSchema extends TableSchemaZod> = {
51
53
  chunkSize?: number;
52
54
  createOptions?: TableCreateOptions;
53
55
  };
56
+ type ToTableStats = {
57
+ timeMs: number;
58
+ totalRows: number;
59
+ };
54
60
  declare class SqlDuck {
55
61
  #private;
56
62
  constructor(params: SqlDuckParams);
57
- toTable: <TSchema extends ZodObject>(params: ToTableParams<TSchema>) => Promise<undefined>;
63
+ toTable: <TSchema extends ZodObject>(params: ToTableParams<TSchema>) => Promise<ToTableStats>;
58
64
  }
59
65
  //#endregion
60
66
  //#region src/utils/zod-codecs.d.ts
package/dist/index.mjs CHANGED
@@ -36,7 +36,7 @@ const createMap = {
36
36
  };
37
37
  const getTableCreateFromZod = (table, schema, options) => {
38
38
  const { create = "CREATE" } = options ?? {};
39
- const fqTable = table.getFullyQualifiedTableName();
39
+ const fqTable = table.getFullName();
40
40
  const json = schema.toJSONSchema({ target: "openapi-3.0" });
41
41
  const columns = [];
42
42
  if (json.properties === void 0) throw new TypeError("Schema must have at least one property");
@@ -88,7 +88,7 @@ const createTableFromZod = async (params) => {
88
88
  try {
89
89
  await conn.run(ddl);
90
90
  } catch (e) {
91
- throw new Error(`Failed to create table '${table.getFullyQualifiedTableName()}': ${e.message}`, { cause: e });
91
+ throw new Error(`Failed to create table '${table.getFullName()}': ${e.message}`, { cause: e });
92
92
  }
93
93
  return {
94
94
  ddl,
@@ -104,33 +104,47 @@ var SqlDuck = class {
104
104
  }
105
105
  toTable = async (params) => {
106
106
  const { table, schema, chunkSize, rowStream, createOptions } = params;
107
+ const timeStart = Date.now();
107
108
  const { columnTypes } = await createTableFromZod({
108
109
  conn: this.#duck,
109
110
  schema,
110
111
  table,
111
112
  options: createOptions
112
113
  });
113
- const appender = await this.#duck.createAppender(table.fqTable.name, table.fqTable.schema, table.fqTable.database);
114
+ const appender = await this.#duck.createAppender(table.tableName, table.schemaName, table.databaseName);
114
115
  const chunkTypes = columnTypes.map((v) => v[1]);
115
- const columnStream = rowsToColumnsChunks(rowStream, chunkSize ?? 2048);
116
+ const chunkLimit = chunkSize ?? 2048;
117
+ let totalRows = 0;
118
+ const columnStream = rowsToColumnsChunks(rowStream, chunkLimit);
116
119
  for await (const dataChunk of columnStream) {
117
120
  const chunk = DuckDBDataChunk.create(chunkTypes);
118
121
  if (this.#logger) this.#logger(`Inserting chunk of ${dataChunk.length} rows`);
122
+ totalRows += dataChunk?.[0]?.length ?? 0;
119
123
  chunk.setColumns(dataChunk);
120
124
  appender.appendDataChunk(chunk);
121
125
  appender.flushSync();
122
126
  }
127
+ return {
128
+ timeMs: Math.round(Date.now() - timeStart),
129
+ totalRows
130
+ };
123
131
  };
124
132
  };
125
133
  var Table = class Table {
126
134
  #fqTable;
127
- get fqTable() {
128
- return this.#fqTable;
135
+ get tableName() {
136
+ return this.#fqTable.name;
137
+ }
138
+ get schemaName() {
139
+ return this.#fqTable.schema;
140
+ }
141
+ get databaseName() {
142
+ return this.#fqTable.database;
129
143
  }
130
144
  constructor(fqTableOrName) {
131
145
  this.#fqTable = typeof fqTableOrName === "string" ? { name: fqTableOrName } : fqTableOrName;
132
146
  }
133
- getFullyQualifiedTableName = (options) => {
147
+ getFullName = (options) => {
134
148
  const { defaultDatabase, defaultSchema } = options ?? {};
135
149
  const { name, database = defaultDatabase, schema = defaultSchema } = this.#fqTable;
136
150
  return [
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@flowblade/sqlduck",
3
- "version": "0.0.6",
3
+ "version": "0.1.0",
4
4
  "type": "module",
5
5
  "sideEffects": false,
6
6
  "exports": {
@@ -46,8 +46,8 @@
46
46
  "check-size": "size-limit"
47
47
  },
48
48
  "dependencies": {
49
- "@flowblade/core": "^0.2.24",
50
- "@flowblade/source-duckdb": "^0.17.0",
49
+ "@flowblade/core": "^0.2.25",
50
+ "@flowblade/source-duckdb": "^0.17.1",
51
51
  "@flowblade/sql-tag": "^0.2.0",
52
52
  "@standard-schema/spec": "^1.1.0",
53
53
  "p-mutex": "^1.0.0",
@@ -62,7 +62,7 @@
62
62
  "@dotenvx/dotenvx": "1.51.2",
63
63
  "@duckdb/node-api": "1.4.3-r.2",
64
64
  "@faker-js/faker": "10.1.0",
65
- "@flowblade/source-kysely": "^1.1.0",
65
+ "@flowblade/source-kysely": "^1.2.0",
66
66
  "@httpx/assert": "0.16.7",
67
67
  "@size-limit/esbuild": "12.0.0",
68
68
  "@size-limit/file": "12.0.0",