drizzle-sybase 4.0.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +264 -7
- package/THIRD_PARTY_NOTICES.md +25 -0
- package/dist/bulk.d.ts +164 -0
- package/dist/bulk.d.ts.map +1 -0
- package/dist/bulk.js +1135 -0
- package/dist/bulk.js.map +1 -0
- package/dist/checks.d.ts +15 -0
- package/dist/checks.d.ts.map +1 -0
- package/dist/checks.js +19 -0
- package/dist/checks.js.map +1 -0
- package/dist/codecs.d.ts +1 -1
- package/dist/codecs.d.ts.map +1 -1
- package/dist/codecs.js +10 -2
- package/dist/codecs.js.map +1 -1
- package/dist/columns/index.d.ts +9 -1
- package/dist/columns/index.d.ts.map +1 -1
- package/dist/columns/index.js +5 -5
- package/dist/columns/index.js.map +1 -1
- package/dist/connection.d.ts +75 -12
- package/dist/connection.d.ts.map +1 -1
- package/dist/connection.js +216 -43
- package/dist/connection.js.map +1 -1
- package/dist/db.d.ts +51 -26
- package/dist/db.d.ts.map +1 -1
- package/dist/db.js +56 -2
- package/dist/db.js.map +1 -1
- package/dist/dialect.d.ts +4 -2
- package/dist/dialect.d.ts.map +1 -1
- package/dist/dialect.js +41 -15
- package/dist/dialect.js.map +1 -1
- package/dist/errors.d.ts +82 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +90 -4
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +5 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/introspect/codegen.d.ts.map +1 -1
- package/dist/introspect/codegen.js +22 -12
- package/dist/introspect/codegen.js.map +1 -1
- package/dist/introspect/fetch.d.ts.map +1 -1
- package/dist/introspect/fetch.js +24 -8
- package/dist/introspect/fetch.js.map +1 -1
- package/dist/introspect/types.d.ts +5 -0
- package/dist/introspect/types.d.ts.map +1 -1
- package/dist/native/index.d.ts +97 -9
- package/dist/native/index.d.ts.map +1 -1
- package/dist/native/index.js +17 -0
- package/dist/native/index.js.map +1 -1
- package/dist/native/sybase_native.darwin-arm64.node +0 -0
- package/dist/native/sybase_native.darwin-x64.node +0 -0
- package/dist/native/sybase_native.linux-arm64.node +0 -0
- package/dist/native/sybase_native.linux-x64.node +0 -0
- package/dist/native/sybase_native.win32-x64.node +0 -0
- package/dist/pool.d.ts +24 -2
- package/dist/pool.d.ts.map +1 -1
- package/dist/pool.js +117 -37
- package/dist/pool.js.map +1 -1
- package/dist/query-builders/insert.d.ts.map +1 -1
- package/dist/query-builders/insert.js +5 -0
- package/dist/query-builders/insert.js.map +1 -1
- package/dist/query-builders/select.d.ts +5 -0
- package/dist/query-builders/select.d.ts.map +1 -1
- package/dist/query-builders/select.js +16 -0
- package/dist/query-builders/select.js.map +1 -1
- package/dist/result.d.ts +20 -7
- package/dist/result.d.ts.map +1 -1
- package/dist/result.js +35 -17
- package/dist/result.js.map +1 -1
- package/dist/session.d.ts +25 -2
- package/dist/session.d.ts.map +1 -1
- package/dist/session.js +134 -11
- package/dist/session.js.map +1 -1
- package/dist/table.d.ts.map +1 -1
- package/dist/table.js +5 -2
- package/dist/table.js.map +1 -1
- package/licenses/FreeTDS-COPYING_LIB.txt +481 -0
- package/package.json +8 -4
- package/patches/freetds-1.5.17-sybase-bulk.patch +592 -0
- package/scripts/build-native.ts +34 -0
- package/scripts/build-win.ps1 +49 -8
- package/scripts/build.sh +101 -30
- package/scripts/smoke-native.ts +67 -0
- package/scripts/smoke-package.sh +79 -0
- package/scripts/verify-published.ts +29 -5
- package/src/native/binding.c +3325 -322
- package/vendor/freetds-1.5.17.tar.gz +0 -0
package/dist/bulk.js
ADDED
|
@@ -0,0 +1,1135 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bulk copy into a table, in process: db-lib's bcp API, driven chunk by chunk.
|
|
3
|
+
*
|
|
4
|
+
* This module turns the caller's rows into chunks for the native layer and
|
|
5
|
+
* keeps the books — which batch is open, what the server confirmed — so that
|
|
6
|
+
* the result, and every failure, says exactly what was committed. The native
|
|
7
|
+
* side (`bulkBegin` / `bulkSend` in binding.c) converts and checks each value;
|
|
8
|
+
* the connection (`SybaseConnection.bulkInsert`) supplies the I/O, the
|
|
9
|
+
* deadline, and discards itself when a copy fails part way.
|
|
10
|
+
*/
|
|
11
|
+
import { checkInteger, MAX_INT32 } from "./checks.js";
|
|
12
|
+
import { formatSybaseDateTime } from "./datetime.js";
|
|
13
|
+
import { messageOf, SybaseBulkCopyError } from "./errors.js";
|
|
14
|
+
// ---------------------------------------------------------------------------
|
|
15
|
+
// Validation
|
|
16
|
+
// ---------------------------------------------------------------------------
|
|
17
|
+
/** A plain identifier, as `is_table_name` in binding.c accepts it. */
|
|
18
|
+
const IDENTIFIER = /^[A-Za-z_@#][A-Za-z0-9_@#$]*$/;
|
|
19
|
+
/**
|
|
20
|
+
* Whether `table` is `table`, `owner.table`, `db..table` or `db.owner.table` —
|
|
21
|
+
* a `#temp` table only on its own: it is in the session's temporary database,
|
|
22
|
+
* which a qualified name would look past.
|
|
23
|
+
*/
|
|
24
|
+
const isTableName = (table) => {
|
|
25
|
+
if (table.length === 0 || table.length > 92) {
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
const parts = table.split(".");
|
|
29
|
+
if (parts.length > 3 || (parts.length > 1 && parts.some(part => part.startsWith("#")))) {
|
|
30
|
+
return false;
|
|
31
|
+
}
|
|
32
|
+
return parts.every((part, i) => IDENTIFIER.test(part) || (parts.length === 3 && i === 1 && part === ""));
|
|
33
|
+
};
|
|
34
|
+
/**
|
|
35
|
+
* Reject bad options before a connection is taken for them.
|
|
36
|
+
*
|
|
37
|
+
* @internal
|
|
38
|
+
*/
|
|
39
|
+
export const validateBulkInsertOptions = (options) => {
|
|
40
|
+
if (typeof options !== "object" || options === null) {
|
|
41
|
+
throw new TypeError("bulkInsert: expected an options object");
|
|
42
|
+
}
|
|
43
|
+
if (typeof options.table !== "string" || !isTableName(options.table)) {
|
|
44
|
+
throw new TypeError("bulkInsert: table must be `table`, `owner.table`, `db..table` or `db.owner.table` of plain " +
|
|
45
|
+
"identifiers (letters, digits, _ @ # $), at most 92 characters — a #temp table on its " +
|
|
46
|
+
`own; got ${String(options.table)}`);
|
|
47
|
+
}
|
|
48
|
+
if (options.columns !== undefined &&
|
|
49
|
+
!(Array.isArray(options.columns) &&
|
|
50
|
+
options.columns.length > 0 &&
|
|
51
|
+
options.columns.every(name => typeof name === "string" && name !== ""))) {
|
|
52
|
+
throw new TypeError("bulkInsert: columns must be a non-empty array of column names");
|
|
53
|
+
}
|
|
54
|
+
const rows = options.rows;
|
|
55
|
+
if (!(typeof rows === "string" || rows instanceof Uint8Array || iterationOf(rows) !== undefined)) {
|
|
56
|
+
throw new TypeError("bulkInsert: rows must be TSV (a string or Buffer), or an iterable of rows or TSV pieces");
|
|
57
|
+
}
|
|
58
|
+
if (options.batchSize !== "all") {
|
|
59
|
+
checkInteger("batchSize", options.batchSize, 1, MAX_INT32);
|
|
60
|
+
}
|
|
61
|
+
checkInteger("expectedRows", options.expectedRows, 0, Number.MAX_SAFE_INTEGER);
|
|
62
|
+
checkInteger("maxErrors", options.maxErrors, 0, Number.MAX_SAFE_INTEGER);
|
|
63
|
+
checkInteger("timeoutSeconds", options.timeoutSeconds, 1, Math.floor(MAX_INT32 / 2000));
|
|
64
|
+
checkInteger("hardTimeoutMs", options.hardTimeoutMs, 1, MAX_INT32);
|
|
65
|
+
for (const flag of ["keepIdentity", "ignoreDefaults", "allowConstraintBypass"]) {
|
|
66
|
+
if (options[flag] !== undefined && typeof options[flag] !== "boolean") {
|
|
67
|
+
throw new TypeError(`bulkInsert: ${flag} must be a boolean`);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
// ---------------------------------------------------------------------------
|
|
72
|
+
// Encoding values
|
|
73
|
+
// ---------------------------------------------------------------------------
|
|
74
|
+
/** Why a value cannot be copied; becomes a rejected row. */
|
|
75
|
+
class BulkValueError extends Error {
|
|
76
|
+
}
|
|
77
|
+
const describe = (value) => value instanceof Date
|
|
78
|
+
? "a Date"
|
|
79
|
+
: value instanceof Uint8Array
|
|
80
|
+
? "bytes"
|
|
81
|
+
: Array.isArray(value)
|
|
82
|
+
? "an array"
|
|
83
|
+
: `a ${typeof value}`;
|
|
84
|
+
const refuse = (value, column, accepts) => {
|
|
85
|
+
throw new BulkValueError(`is ${describe(value)}; a ${column.type} column takes ${accepts}`);
|
|
86
|
+
};
|
|
87
|
+
const HEX = Array.from({ length: 256 }, (_, i) => i.toString(16).padStart(2, "0"));
|
|
88
|
+
/**
|
|
89
|
+
* A finite number as a decimal literal without an exponent: `String` writes
|
|
90
|
+
* 1e-7 and 1e21, which the exact decimal parser does not take. The digits are
|
|
91
|
+
* those of `String` — the shortest that read back as the same double — moved.
|
|
92
|
+
*/
|
|
93
|
+
const plainDecimal = (value) => {
|
|
94
|
+
const text = String(value);
|
|
95
|
+
const e = text.indexOf("e");
|
|
96
|
+
if (e < 0) {
|
|
97
|
+
return text;
|
|
98
|
+
}
|
|
99
|
+
const sign = text.startsWith("-") ? "-" : "";
|
|
100
|
+
const mantissa = text.slice(sign.length, e);
|
|
101
|
+
const dot = mantissa.indexOf(".");
|
|
102
|
+
const digits = mantissa.replace(".", "");
|
|
103
|
+
const point = (dot < 0 ? mantissa.length : dot) + Number(text.slice(e + 1));
|
|
104
|
+
return point <= 0
|
|
105
|
+
? `${sign}0.${"0".repeat(-point)}${digits}`
|
|
106
|
+
: point >= digits.length
|
|
107
|
+
? `${sign}${digits}${"0".repeat(point - digits.length)}`
|
|
108
|
+
: `${sign}${digits.slice(0, point)}.${digits.slice(point)}`;
|
|
109
|
+
};
|
|
110
|
+
/**
|
|
111
|
+
* How `value` is iterated, as `for await` would (GetMethod): through
|
|
112
|
+
* `Symbol.asyncIterator` unless it is null or undefined, else through
|
|
113
|
+
* `Symbol.iterator`; `undefined` if it is not iterable.
|
|
114
|
+
*/
|
|
115
|
+
const iterationOf = (value) => {
|
|
116
|
+
if (typeof value !== "object" || value === null) {
|
|
117
|
+
return undefined;
|
|
118
|
+
}
|
|
119
|
+
const source = value;
|
|
120
|
+
const asyncMethod = source[Symbol.asyncIterator];
|
|
121
|
+
if (asyncMethod !== undefined && asyncMethod !== null) {
|
|
122
|
+
return typeof asyncMethod === "function" ? "async" : undefined;
|
|
123
|
+
}
|
|
124
|
+
return typeof source[Symbol.iterator] === "function" ? "sync" : undefined;
|
|
125
|
+
};
|
|
126
|
+
/** How many newlines `text` holds. */
|
|
127
|
+
const countNewlines = (text) => {
|
|
128
|
+
let n = 0;
|
|
129
|
+
if (typeof text === "string") {
|
|
130
|
+
for (let at = text.indexOf("\n"); at !== -1; at = text.indexOf("\n", at + 1)) {
|
|
131
|
+
n++;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
else {
|
|
135
|
+
for (let at = text.indexOf(10); at !== -1; at = text.indexOf(10, at + 1)) {
|
|
136
|
+
n++;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
return n;
|
|
140
|
+
};
|
|
141
|
+
/**
|
|
142
|
+
* Refuse text with a lone surrogate: encoded to UTF-8 it would silently become
|
|
143
|
+
* U+FFFD. (Value rows reach the native side as UTF-16, which refuses it there.)
|
|
144
|
+
*/
|
|
145
|
+
const checkWellFormed = (text) => {
|
|
146
|
+
if (!text.isWellFormed()) {
|
|
147
|
+
throw new TypeError("bulkInsert: the TSV has an unpaired surrogate: it is not valid text");
|
|
148
|
+
}
|
|
149
|
+
};
|
|
150
|
+
/**
|
|
151
|
+
* Longest value accepted for a column that is not text or binary, in
|
|
152
|
+
* characters (BULK_TOKEN_MAX - 1 in binding.c, which refuses any longer).
|
|
153
|
+
* Refused here too, before a long one is copied into a chunk: the lengths of
|
|
154
|
+
* text and binary values are bounded by their columns, so a row's size is
|
|
155
|
+
* bounded by its table's.
|
|
156
|
+
*/
|
|
157
|
+
const MAX_TOKEN_LENGTH = 127;
|
|
158
|
+
/** The types whose values are tokens (numbers, dates, bits) rather than text or bytes. */
|
|
159
|
+
const TOKEN_TYPES = new Set([
|
|
160
|
+
"tinyint",
|
|
161
|
+
"smallint",
|
|
162
|
+
"int",
|
|
163
|
+
"bigint",
|
|
164
|
+
"money",
|
|
165
|
+
"smallmoney",
|
|
166
|
+
"numeric",
|
|
167
|
+
"decimal",
|
|
168
|
+
"float",
|
|
169
|
+
"real",
|
|
170
|
+
"bit",
|
|
171
|
+
"datetime",
|
|
172
|
+
"smalldatetime"
|
|
173
|
+
]);
|
|
174
|
+
/** How the values of a column are given to the native layer, by its type. */
|
|
175
|
+
const encoderFor = (column, timeZone) => {
|
|
176
|
+
switch (column.type) {
|
|
177
|
+
case "tinyint":
|
|
178
|
+
case "smallint":
|
|
179
|
+
case "int":
|
|
180
|
+
case "bigint":
|
|
181
|
+
return value => {
|
|
182
|
+
if (value == null) {
|
|
183
|
+
return null;
|
|
184
|
+
}
|
|
185
|
+
if (typeof value === "string" || typeof value === "bigint") {
|
|
186
|
+
return String(value);
|
|
187
|
+
}
|
|
188
|
+
if (typeof value === "number" && Number.isSafeInteger(value)) {
|
|
189
|
+
return String(value);
|
|
190
|
+
}
|
|
191
|
+
return refuse(value, column, "an integer: a safe-integer number, a bigint or a digit string");
|
|
192
|
+
};
|
|
193
|
+
case "money":
|
|
194
|
+
case "smallmoney":
|
|
195
|
+
// Not a bigint: amounts are commonly held as integers of ten-thousandths
|
|
196
|
+
// as well as of whole units, and the two differ by 10,000 — a guess
|
|
197
|
+
// either way would store some callers' amounts wrong, silently.
|
|
198
|
+
return value => {
|
|
199
|
+
if (value == null) {
|
|
200
|
+
return null;
|
|
201
|
+
}
|
|
202
|
+
if (typeof value === "string") {
|
|
203
|
+
return value;
|
|
204
|
+
}
|
|
205
|
+
return refuse(value, column, 'a decimal string ("12.3456"); a bigint is refused, as its unit would be a guess');
|
|
206
|
+
};
|
|
207
|
+
case "numeric":
|
|
208
|
+
case "decimal": {
|
|
209
|
+
// A number holds 15–17 significant digits: fine for a narrow numeric,
|
|
210
|
+
// silently wrong for the rest (as in the query builders).
|
|
211
|
+
const numbers = column.precision <= 15;
|
|
212
|
+
return value => {
|
|
213
|
+
if (value == null) {
|
|
214
|
+
return null;
|
|
215
|
+
}
|
|
216
|
+
if (typeof value === "string" || typeof value === "bigint") {
|
|
217
|
+
return String(value);
|
|
218
|
+
}
|
|
219
|
+
if (numbers && typeof value === "number" && Number.isFinite(value)) {
|
|
220
|
+
return plainDecimal(value);
|
|
221
|
+
}
|
|
222
|
+
return refuse(value, column, numbers
|
|
223
|
+
? "a decimal string, a bigint or a number"
|
|
224
|
+
: "a decimal string or a bigint, not a number");
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
case "float":
|
|
228
|
+
case "real":
|
|
229
|
+
return value => {
|
|
230
|
+
if (value == null) {
|
|
231
|
+
return null;
|
|
232
|
+
}
|
|
233
|
+
if (typeof value === "string" || (typeof value === "number" && Number.isFinite(value))) {
|
|
234
|
+
return String(value);
|
|
235
|
+
}
|
|
236
|
+
return refuse(value, column, "a finite number or a numeric string");
|
|
237
|
+
};
|
|
238
|
+
case "bit":
|
|
239
|
+
return value => {
|
|
240
|
+
if (value == null) {
|
|
241
|
+
return null;
|
|
242
|
+
}
|
|
243
|
+
if (typeof value === "boolean") {
|
|
244
|
+
return value ? "1" : "0";
|
|
245
|
+
}
|
|
246
|
+
if (value === 0 || value === 1 || value === "0" || value === "1") {
|
|
247
|
+
return String(value);
|
|
248
|
+
}
|
|
249
|
+
return refuse(value, column, "a boolean, 0 or 1");
|
|
250
|
+
};
|
|
251
|
+
case "datetime":
|
|
252
|
+
case "smalldatetime":
|
|
253
|
+
return value => {
|
|
254
|
+
if (value == null) {
|
|
255
|
+
return null;
|
|
256
|
+
}
|
|
257
|
+
if (typeof value === "string") {
|
|
258
|
+
return value;
|
|
259
|
+
}
|
|
260
|
+
if (value instanceof Date) {
|
|
261
|
+
try {
|
|
262
|
+
return formatSybaseDateTime(value, timeZone);
|
|
263
|
+
}
|
|
264
|
+
catch (err) {
|
|
265
|
+
throw new BulkValueError(err instanceof Error ? err.message : String(err));
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
return refuse(value, column, "a Date or a YYYY-MM-DD HH:MM:SS.mmm string");
|
|
269
|
+
};
|
|
270
|
+
case "binary":
|
|
271
|
+
case "varbinary":
|
|
272
|
+
return value => {
|
|
273
|
+
if (value == null) {
|
|
274
|
+
return null;
|
|
275
|
+
}
|
|
276
|
+
if (value instanceof Uint8Array) {
|
|
277
|
+
if (value.length === 0) {
|
|
278
|
+
throw new BulkValueError("is empty: ASE has no zero-length binary value");
|
|
279
|
+
}
|
|
280
|
+
if (value.length > column.length) {
|
|
281
|
+
throw new BulkValueError(`is ${value.length} bytes; the column holds ${column.length}`);
|
|
282
|
+
}
|
|
283
|
+
let hex = "";
|
|
284
|
+
for (const byte of value) {
|
|
285
|
+
hex += HEX[byte];
|
|
286
|
+
}
|
|
287
|
+
return hex;
|
|
288
|
+
}
|
|
289
|
+
return refuse(value, column, "a Uint8Array or Buffer");
|
|
290
|
+
};
|
|
291
|
+
default:
|
|
292
|
+
// char, varchar (reported as char), nchar, nvarchar
|
|
293
|
+
return value => {
|
|
294
|
+
if (value == null) {
|
|
295
|
+
return null;
|
|
296
|
+
}
|
|
297
|
+
if (typeof value === "string") {
|
|
298
|
+
// Every UTF-16 unit takes a byte or more in the server's character
|
|
299
|
+
// set, so a value with more (trailing blanks aside, which ASE drops)
|
|
300
|
+
// cannot fit: refused here, before it is copied to the native side.
|
|
301
|
+
let end = value.length;
|
|
302
|
+
while (end > column.length && value.charCodeAt(end - 1) === 0x20) {
|
|
303
|
+
end--;
|
|
304
|
+
}
|
|
305
|
+
if (end > column.length) {
|
|
306
|
+
throw new BulkValueError(`is at least ${end} bytes in the server's character set; the column holds ${column.length}`);
|
|
307
|
+
}
|
|
308
|
+
return end === value.length ? value : value.slice(0, end);
|
|
309
|
+
}
|
|
310
|
+
return refuse(value, column, "a string");
|
|
311
|
+
};
|
|
312
|
+
}
|
|
313
|
+
};
|
|
314
|
+
// ---------------------------------------------------------------------------
|
|
315
|
+
// Reading rows into chunks
|
|
316
|
+
// ---------------------------------------------------------------------------
|
|
317
|
+
/** Most input rows per native call. */
|
|
318
|
+
const CHUNK_ROWS = 10000;
|
|
319
|
+
/** Roughly the most data per native call: 4 MiB of TSV, 2M UTF-16 units of values. */
|
|
320
|
+
const CHUNK_BYTES = 4 * 1024 * 1024;
|
|
321
|
+
/** TSV gathered from a streamed source before a chunk is cut from it. */
|
|
322
|
+
const CHUNK_FILL = 1024 * 1024;
|
|
323
|
+
/**
|
|
324
|
+
* Longest TSV row accepted: a bound on what one row may make the reader buffer.
|
|
325
|
+
* Far beyond a real ASE row (one fits in a page) even as hex text.
|
|
326
|
+
*/
|
|
327
|
+
const MAX_TSV_ROW_BYTES = CHUNK_BYTES;
|
|
328
|
+
/** Size of the blocks small streamed TSV pieces are copied into. */
|
|
329
|
+
const TSV_BLOCK_BYTES = 64 * 1024;
|
|
330
|
+
/** Input rows read between checks of the deadline. */
|
|
331
|
+
const DEADLINE_CHECK_ROWS = 1024;
|
|
332
|
+
const CHUNK_UNITS = 2 * 1024 * 1024;
|
|
333
|
+
/** Field length marking NULL (BULK_NULL in binding.c). */
|
|
334
|
+
const NULL_LENGTH = 0xffffffff;
|
|
335
|
+
/** Pulls the caller's rows and cuts them into chunks. */
|
|
336
|
+
class RowReader {
|
|
337
|
+
columns;
|
|
338
|
+
encoders;
|
|
339
|
+
maxErrors;
|
|
340
|
+
checkDeadline;
|
|
341
|
+
sync;
|
|
342
|
+
async;
|
|
343
|
+
mode;
|
|
344
|
+
/** Input rows read so far. */
|
|
345
|
+
rowNumber = 0;
|
|
346
|
+
/** TSV not yet cut into rows (from `pendingStart`): a string, or bytes. */
|
|
347
|
+
pending = "";
|
|
348
|
+
pendingStart = 0;
|
|
349
|
+
/** TSV read but not yet joined onto `pending`, then what of `block` is not yet in it. */
|
|
350
|
+
pieces = [];
|
|
351
|
+
piecesBytes = 0;
|
|
352
|
+
/**
|
|
353
|
+
* Where small pieces are copied to, back to back: a source of tiny pieces
|
|
354
|
+
* would otherwise hold one Buffer per piece. `block[blockStart, blockFill)`
|
|
355
|
+
* is buffered but not yet in `pieces`.
|
|
356
|
+
*/
|
|
357
|
+
block;
|
|
358
|
+
blockStart = 0;
|
|
359
|
+
blockFill = 0;
|
|
360
|
+
/** Complete rows buffered (in `pending` from `pendingStart`, and in `pieces`): their newlines. */
|
|
361
|
+
newlines = 0;
|
|
362
|
+
/** A high surrogate that ended a string piece, waiting for its other half. */
|
|
363
|
+
highSurrogate = "";
|
|
364
|
+
sourceDone = false;
|
|
365
|
+
/** Rows refused so far by the reader. */
|
|
366
|
+
refusedCount = 0;
|
|
367
|
+
/** Rows refused but not yet handed over in a chunk (see takeRefused). */
|
|
368
|
+
refused = [];
|
|
369
|
+
/** A value row read but not yet placed in a chunk. */
|
|
370
|
+
heldRow;
|
|
371
|
+
/** Per column: whether its values are tokens (see MAX_TOKEN_LENGTH). */
|
|
372
|
+
tokens;
|
|
373
|
+
holding = false;
|
|
374
|
+
constructor(rows, columns, encoders,
|
|
375
|
+
/** The copy fails once the reader alone refuses more rows than this. */
|
|
376
|
+
maxErrors,
|
|
377
|
+
/** Throws once the copy's deadline has passed; see BulkIo.checkDeadline. */
|
|
378
|
+
checkDeadline) {
|
|
379
|
+
this.columns = columns;
|
|
380
|
+
this.encoders = encoders;
|
|
381
|
+
this.maxErrors = maxErrors;
|
|
382
|
+
this.checkDeadline = checkDeadline;
|
|
383
|
+
this.tokens = columns.map(column => TOKEN_TYPES.has(column.type));
|
|
384
|
+
if (typeof rows === "string" || rows instanceof Uint8Array) {
|
|
385
|
+
if (typeof rows === "string") {
|
|
386
|
+
checkWellFormed(rows);
|
|
387
|
+
}
|
|
388
|
+
this.mode = "tsv";
|
|
389
|
+
this.pending =
|
|
390
|
+
typeof rows === "string"
|
|
391
|
+
? rows
|
|
392
|
+
: Buffer.from(rows.buffer, rows.byteOffset, rows.byteLength);
|
|
393
|
+
this.sourceDone = true;
|
|
394
|
+
}
|
|
395
|
+
else if (iterationOf(rows) === "async") {
|
|
396
|
+
this.async = rows[Symbol.asyncIterator]();
|
|
397
|
+
}
|
|
398
|
+
else {
|
|
399
|
+
this.sync = rows[Symbol.iterator]();
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
/** Let the source clean up after a failed copy (e.g. close a file stream). Once. */
|
|
403
|
+
close() {
|
|
404
|
+
const sync = this.sync;
|
|
405
|
+
const async = this.async;
|
|
406
|
+
this.sync = undefined;
|
|
407
|
+
this.async = undefined;
|
|
408
|
+
try {
|
|
409
|
+
void Promise.resolve(async?.return?.()).catch(() => { });
|
|
410
|
+
sync?.return?.();
|
|
411
|
+
}
|
|
412
|
+
catch {
|
|
413
|
+
// The source's own cleanup failing changes nothing here.
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
async pull() {
|
|
417
|
+
if (this.sync) {
|
|
418
|
+
return this.sync.next();
|
|
419
|
+
}
|
|
420
|
+
if (this.async) {
|
|
421
|
+
return this.async.next();
|
|
422
|
+
}
|
|
423
|
+
return { done: true, value: undefined }; // closed
|
|
424
|
+
}
|
|
425
|
+
/**
|
|
426
|
+
* The rows refused since the last chunk. A failed read hands them over this
|
|
427
|
+
* way, so that the copy's error still lists them.
|
|
428
|
+
*/
|
|
429
|
+
takeRefused() {
|
|
430
|
+
const refused = this.refused;
|
|
431
|
+
this.refused = [];
|
|
432
|
+
return refused;
|
|
433
|
+
}
|
|
434
|
+
/**
|
|
435
|
+
* Known to have no more rows: the source has ended and nothing it gave is
|
|
436
|
+
* left unread — as soon as the last chunk is cut from TSV given whole. (Not
|
|
437
|
+
* knowing does not mean there are more: an iterated source is known to have
|
|
438
|
+
* ended only once read past its last row.)
|
|
439
|
+
*/
|
|
440
|
+
get exhausted() {
|
|
441
|
+
return (this.sourceDone &&
|
|
442
|
+
!this.holding &&
|
|
443
|
+
this.refused.length === 0 &&
|
|
444
|
+
this.pendingStart >= this.pending.length &&
|
|
445
|
+
this.piecesBytes === 0 &&
|
|
446
|
+
this.blockFill === this.blockStart &&
|
|
447
|
+
this.highSurrogate === "");
|
|
448
|
+
}
|
|
449
|
+
/**
|
|
450
|
+
* The next chunk of at most `maxRows` input rows, or `null` at the end — and
|
|
451
|
+
* once the reader alone has refused more than `maxErrors` rows: the chunk
|
|
452
|
+
* holding the one past the limit is never sent (the copy fails as it is
|
|
453
|
+
* counted), so there is nothing more to read.
|
|
454
|
+
*/
|
|
455
|
+
async next(maxRows) {
|
|
456
|
+
if (this.refusedCount > this.maxErrors) {
|
|
457
|
+
return null;
|
|
458
|
+
}
|
|
459
|
+
if (this.mode === undefined) {
|
|
460
|
+
if (this.sourceDone) {
|
|
461
|
+
return null;
|
|
462
|
+
}
|
|
463
|
+
const first = await this.pull();
|
|
464
|
+
if (first.done) {
|
|
465
|
+
this.sourceDone = true;
|
|
466
|
+
return null;
|
|
467
|
+
}
|
|
468
|
+
if (Array.isArray(first.value)) {
|
|
469
|
+
this.mode = "values";
|
|
470
|
+
this.heldRow = first.value;
|
|
471
|
+
this.holding = true;
|
|
472
|
+
}
|
|
473
|
+
else {
|
|
474
|
+
this.mode = "tsv";
|
|
475
|
+
this.addTsv(first.value);
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
return this.mode === "values" ? this.nextValues(maxRows) : this.nextTsv(maxRows);
|
|
479
|
+
}
|
|
480
|
+
// --- Value rows ---
|
|
481
|
+
async nextValues(maxRows) {
|
|
482
|
+
const parts = [];
|
|
483
|
+
const lens = [];
|
|
484
|
+
let units = 0;
|
|
485
|
+
let rows = 0;
|
|
486
|
+
let inputRows = 0;
|
|
487
|
+
let firstRow = 0;
|
|
488
|
+
const width = this.columns.length;
|
|
489
|
+
const fields = Array.from({ length: width });
|
|
490
|
+
// Lengths count too (4 bytes, two units, each): a wide row of NULLs is
|
|
491
|
+
// all lengths and no text.
|
|
492
|
+
while (inputRows < maxRows && units + lens.length * 2 < CHUNK_UNITS) {
|
|
493
|
+
let row;
|
|
494
|
+
if (this.holding) {
|
|
495
|
+
row = this.heldRow;
|
|
496
|
+
this.holding = false;
|
|
497
|
+
this.heldRow = undefined;
|
|
498
|
+
}
|
|
499
|
+
else {
|
|
500
|
+
if (this.sourceDone) {
|
|
501
|
+
break;
|
|
502
|
+
}
|
|
503
|
+
const item = await this.pull();
|
|
504
|
+
if (item.done) {
|
|
505
|
+
this.sourceDone = true;
|
|
506
|
+
break;
|
|
507
|
+
}
|
|
508
|
+
row = item.value;
|
|
509
|
+
}
|
|
510
|
+
const rowNumber = ++this.rowNumber;
|
|
511
|
+
inputRows++;
|
|
512
|
+
if (inputRows % DEADLINE_CHECK_ROWS === 0) {
|
|
513
|
+
this.checkDeadline();
|
|
514
|
+
}
|
|
515
|
+
const problem = this.encodeRow(row, width, fields);
|
|
516
|
+
if (problem) {
|
|
517
|
+
this.refused.push({
|
|
518
|
+
rejected: { row: rowNumber, ...problem.rejected },
|
|
519
|
+
cause: problem.cause
|
|
520
|
+
});
|
|
521
|
+
this.refusedCount++;
|
|
522
|
+
// Rows of a chunk are numbered consecutively: end it at the gap — and
|
|
523
|
+
// read no further once past the limit (see next).
|
|
524
|
+
if (rows > 0 || this.refusedCount > this.maxErrors) {
|
|
525
|
+
break;
|
|
526
|
+
}
|
|
527
|
+
continue;
|
|
528
|
+
}
|
|
529
|
+
if (rows === 0) {
|
|
530
|
+
firstRow = rowNumber;
|
|
531
|
+
}
|
|
532
|
+
for (const field of fields) {
|
|
533
|
+
if (field === null) {
|
|
534
|
+
lens.push(NULL_LENGTH);
|
|
535
|
+
}
|
|
536
|
+
else {
|
|
537
|
+
parts.push(field);
|
|
538
|
+
lens.push(field.length);
|
|
539
|
+
units += field.length;
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
rows++;
|
|
543
|
+
}
|
|
544
|
+
if (inputRows === 0) {
|
|
545
|
+
return null;
|
|
546
|
+
}
|
|
547
|
+
return {
|
|
548
|
+
payload: { text: parts.join(""), lens: Uint32Array.from(lens) },
|
|
549
|
+
rows,
|
|
550
|
+
firstRow: rows > 0 ? firstRow : this.rowNumber + 1,
|
|
551
|
+
inputRows,
|
|
552
|
+
endsBatch: false,
|
|
553
|
+
refused: this.takeRefused()
|
|
554
|
+
};
|
|
555
|
+
}
|
|
556
|
+
/** Encode `row` into `fields`; what is wrong with it, if anything. */
|
|
557
|
+
encodeRow(row, width, fields) {
|
|
558
|
+
if (!Array.isArray(row)) {
|
|
559
|
+
// TSV among value rows is a source mixing the two — not a bad row, which
|
|
560
|
+
// maxErrors could skip: the copy stops, as for an array among TSV pieces.
|
|
561
|
+
if (typeof row === "string" || row instanceof Uint8Array) {
|
|
562
|
+
throw new TypeError(`bulkInsert: a source of value rows yielded ${describe(row)}; ` +
|
|
563
|
+
"rows cannot mix TSV and arrays");
|
|
564
|
+
}
|
|
565
|
+
const message = `is ${describe(row)}, not an array of values`;
|
|
566
|
+
return { rejected: { message }, cause: new TypeError(message) };
|
|
567
|
+
}
|
|
568
|
+
if (row.length !== width) {
|
|
569
|
+
const message = `has ${row.length} values; ${width} columns are copied`;
|
|
570
|
+
return { rejected: { message }, cause: new TypeError(message) };
|
|
571
|
+
}
|
|
572
|
+
for (let k = 0; k < width; k++) {
|
|
573
|
+
try {
|
|
574
|
+
const field = this.encoders[k](row[k]);
|
|
575
|
+
if (field !== null && field.length > MAX_TOKEN_LENGTH && this.tokens[k]) {
|
|
576
|
+
throw new BulkValueError(`is ${field.length} characters long: no ${this.columns[k].type} value is written ` +
|
|
577
|
+
`with more than ${MAX_TOKEN_LENGTH}`);
|
|
578
|
+
}
|
|
579
|
+
fields[k] = field;
|
|
580
|
+
}
|
|
581
|
+
catch (err) {
|
|
582
|
+
if (!(err instanceof BulkValueError)) {
|
|
583
|
+
throw err;
|
|
584
|
+
}
|
|
585
|
+
return {
|
|
586
|
+
rejected: { column: this.columns[k].name, message: `value ${err.message}` },
|
|
587
|
+
cause: new TypeError(`column '${this.columns[k].name}': value ${err.message}`)
|
|
588
|
+
};
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
return undefined;
|
|
592
|
+
}
|
|
593
|
+
// --- TSV ---
|
|
594
|
+
addTsv(piece) {
|
|
595
|
+
if (typeof piece !== "string" && !(piece instanceof Uint8Array)) {
|
|
596
|
+
throw new TypeError(`bulkInsert: a TSV source yielded ${describe(piece)}; expected strings or Buffers ` +
|
|
597
|
+
"(rows cannot mix TSV and arrays)");
|
|
598
|
+
}
|
|
599
|
+
if (typeof piece === "string") {
|
|
600
|
+
// A string may be split between the two halves of a surrogate pair: the
|
|
601
|
+
// high one waits for the next piece.
|
|
602
|
+
let text = this.highSurrogate + piece;
|
|
603
|
+
this.highSurrogate = "";
|
|
604
|
+
const last = text.charCodeAt(text.length - 1);
|
|
605
|
+
if (last >= 0xd800 && last <= 0xdbff) {
|
|
606
|
+
this.highSurrogate = text.slice(-1);
|
|
607
|
+
text = text.slice(0, -1);
|
|
608
|
+
}
|
|
609
|
+
checkWellFormed(text);
|
|
610
|
+
this.addBytes(Buffer.byteLength(text, "utf8"), countNewlines(text), (target, at) => target.write(text, at, "utf8"));
|
|
611
|
+
}
|
|
612
|
+
else if (piece.byteLength > 0) {
|
|
613
|
+
this.checkNoHalfCharacter();
|
|
614
|
+
// Copied: the source may reuse its buffer for the next piece, and this
|
|
615
|
+
// one is held until a chunk is cut from it.
|
|
616
|
+
this.addBytes(piece.byteLength, countNewlines(piece), (target, at) => target.set(piece, at));
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
/** Buffer `length` bytes that `write` puts at `target[at]`. */
|
|
620
|
+
addBytes(length, newlines, write) {
|
|
621
|
+
if (length === 0) {
|
|
622
|
+
return;
|
|
623
|
+
}
|
|
624
|
+
if (length >= TSV_BLOCK_BYTES) {
|
|
625
|
+
this.flushBlock();
|
|
626
|
+
const own = Buffer.allocUnsafe(length);
|
|
627
|
+
write(own, 0);
|
|
628
|
+
this.pieces.push(own);
|
|
629
|
+
}
|
|
630
|
+
else {
|
|
631
|
+
if (!this.block || this.blockFill + length > this.block.length) {
|
|
632
|
+
this.flushBlock();
|
|
633
|
+
this.block = Buffer.allocUnsafe(TSV_BLOCK_BYTES);
|
|
634
|
+
this.blockStart = this.blockFill = 0;
|
|
635
|
+
}
|
|
636
|
+
write(this.block, this.blockFill);
|
|
637
|
+
this.blockFill += length;
|
|
638
|
+
}
|
|
639
|
+
this.piecesBytes += length;
|
|
640
|
+
this.newlines += newlines;
|
|
641
|
+
}
|
|
642
|
+
/** Move what `block` holds that is not yet in `pieces` there, keeping order. */
|
|
643
|
+
flushBlock() {
|
|
644
|
+
if (this.block && this.blockFill > this.blockStart) {
|
|
645
|
+
this.pieces.push(this.block.subarray(this.blockStart, this.blockFill));
|
|
646
|
+
this.blockStart = this.blockFill;
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
checkNoHalfCharacter() {
|
|
650
|
+
if (this.highSurrogate) {
|
|
651
|
+
throw new TypeError(`bulkInsert: row ${this.rowNumber + 1} of the TSV has an unpaired surrogate (U+` +
|
|
652
|
+
`${this.highSurrogate.charCodeAt(0).toString(16).toUpperCase()}) at the end of a string piece`);
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
/**
|
|
656
|
+
* Join the pieces read since the last chunk onto what is left of `pending`, in
|
|
657
|
+
* one copy — none when all there is is one piece: it is already this reader's
|
|
658
|
+
* own (see addTsv), and a block is only ever written past what it holds.
|
|
659
|
+
*/
|
|
660
|
+
materialize() {
|
|
661
|
+
if (this.piecesBytes === 0) {
|
|
662
|
+
return;
|
|
663
|
+
}
|
|
664
|
+
this.flushBlock();
|
|
665
|
+
const rest = typeof this.pending === "string"
|
|
666
|
+
? Buffer.from(this.pending.slice(this.pendingStart), "utf8")
|
|
667
|
+
: this.pending.subarray(this.pendingStart);
|
|
668
|
+
this.pending =
|
|
669
|
+
rest.length === 0 && this.pieces.length === 1
|
|
670
|
+
? this.pieces[0]
|
|
671
|
+
: Buffer.concat([rest, ...this.pieces]);
|
|
672
|
+
this.pendingStart = 0;
|
|
673
|
+
this.pieces = [];
|
|
674
|
+
this.piecesBytes = 0;
|
|
675
|
+
}
|
|
676
|
+
newlineAt(from) {
|
|
677
|
+
return typeof this.pending === "string"
|
|
678
|
+
? this.pending.indexOf("\n", from)
|
|
679
|
+
: this.pending.indexOf(10, from);
|
|
680
|
+
}
|
|
681
|
+
async pullTsv() {
|
|
682
|
+
this.checkDeadline();
|
|
683
|
+
const item = await this.pull();
|
|
684
|
+
if (item.done) {
|
|
685
|
+
this.sourceDone = true;
|
|
686
|
+
this.checkNoHalfCharacter();
|
|
687
|
+
}
|
|
688
|
+
else {
|
|
689
|
+
this.addTsv(item.value);
|
|
690
|
+
}
|
|
691
|
+
}
|
|
692
|
+
rowTooLong(withNewline, row = this.rowNumber + 1) {
|
|
693
|
+
return new TypeError(`row ${row} of the TSV is longer than ${MAX_TSV_ROW_BYTES} bytes` +
|
|
694
|
+
(withNewline ? "" : " with no newline: rows must end with \\n"));
|
|
695
|
+
}
|
|
696
|
+
async nextTsv(maxRows) {
|
|
697
|
+
for (;;) {
|
|
698
|
+
// Read on until the rows wanted are buffered, gathering a worthwhile
|
|
699
|
+
// amount meanwhile so that a source of small pieces does not make one
|
|
700
|
+
// native call per piece — but no further once they are: a source that
|
|
701
|
+
// pauses must not hold back a batch it has already given in full. With no
|
|
702
|
+
// complete row buffered, pieces are only counted, not joined and rescanned.
|
|
703
|
+
const buffered = this.pending.length - this.pendingStart + this.piecesBytes;
|
|
704
|
+
if (!this.sourceDone &&
|
|
705
|
+
this.newlines < maxRows &&
|
|
706
|
+
(buffered < CHUNK_FILL || this.newlines === 0)) {
|
|
707
|
+
if (this.newlines === 0 && buffered > MAX_TSV_ROW_BYTES) {
|
|
708
|
+
throw this.rowTooLong(false);
|
|
709
|
+
}
|
|
710
|
+
await this.pullTsv();
|
|
711
|
+
continue;
|
|
712
|
+
}
|
|
713
|
+
this.materialize();
|
|
714
|
+
const start = this.pendingStart;
|
|
715
|
+
// A string is sent as UTF-8, which only sizing each row exactly tells —
|
|
716
|
+
// a call per row. Its units are a lower bound of its bytes: the rows are
|
|
717
|
+
// found by those first, and the chunk measured once; only when it may
|
|
718
|
+
// break a limit are they found again row by row. Either way the same rows.
|
|
719
|
+
const { end, rows } = typeof this.pending === "string"
|
|
720
|
+
? (this.scanRows(start, maxRows, false) ?? this.scanRows(start, maxRows, true))
|
|
721
|
+
: this.scanRows(start, maxRows, true);
|
|
722
|
+
if (rows > 0) {
|
|
723
|
+
return this.cutTsv(start, end, rows);
|
|
724
|
+
}
|
|
725
|
+
if (this.sourceDone) {
|
|
726
|
+
return null;
|
|
727
|
+
}
|
|
728
|
+
// No row ends within what is buffered: read on (see above).
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
/**
|
|
732
|
+
* The rows of the next chunk, from `start`: up to `maxRows`, and — but for
|
|
733
|
+
* the first — no more than CHUNK_BYTES of what is sent. `exact` sizes each
|
|
734
|
+
* row as sent (a string's UTF-8); otherwise a string's rows are sized by
|
|
735
|
+
* their units, and `undefined` is returned when that cannot settle it: a row
|
|
736
|
+
* that may be too long, or a chunk past CHUNK_BYTES once measured.
|
|
737
|
+
*/
|
|
738
|
+
scanRows(start, maxRows, exact) {
|
|
739
|
+
const pending = this.pending;
|
|
740
|
+
const length = pending.length;
|
|
741
|
+
const isString = typeof pending === "string";
|
|
742
|
+
let end = start;
|
|
743
|
+
let rows = 0;
|
|
744
|
+
let bytes = 0;
|
|
745
|
+
while (rows < maxRows) {
|
|
746
|
+
let next = this.newlineAt(end) + 1;
|
|
747
|
+
if (next === 0) {
|
|
748
|
+
// No newline left: once the source has ended, the rest is the last row.
|
|
749
|
+
if (!this.sourceDone || end === length) {
|
|
750
|
+
break;
|
|
751
|
+
}
|
|
752
|
+
next = length;
|
|
753
|
+
}
|
|
754
|
+
let rowBytes = next - end;
|
|
755
|
+
if (isString) {
|
|
756
|
+
if (exact) {
|
|
757
|
+
rowBytes = Buffer.byteLength(pending.slice(end, next), "utf8");
|
|
758
|
+
}
|
|
759
|
+
else if (rowBytes * 3 > MAX_TSV_ROW_BYTES) {
|
|
760
|
+
return undefined; // its UTF-8 may be past the limit
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
if (rowBytes > MAX_TSV_ROW_BYTES) {
|
|
764
|
+
throw this.rowTooLong(true, this.rowNumber + rows + 1);
|
|
765
|
+
}
|
|
766
|
+
// Past the size limit: stop before this row, unless it is the first.
|
|
767
|
+
if (rows > 0 && bytes + rowBytes > CHUNK_BYTES) {
|
|
768
|
+
break;
|
|
769
|
+
}
|
|
770
|
+
bytes += rowBytes;
|
|
771
|
+
end = next;
|
|
772
|
+
rows++;
|
|
773
|
+
}
|
|
774
|
+
if (isString &&
|
|
775
|
+
!exact &&
|
|
776
|
+
rows > 1 &&
|
|
777
|
+
Buffer.byteLength(pending.slice(start, end), "utf8") > CHUNK_BYTES) {
|
|
778
|
+
return undefined;
|
|
779
|
+
}
|
|
780
|
+
return { end, rows };
|
|
781
|
+
}
|
|
782
|
+
cutTsv(start, end, rows) {
|
|
783
|
+
const tsv = typeof this.pending === "string"
|
|
784
|
+
? this.pending.slice(start, end)
|
|
785
|
+
: this.pending.subarray(start, end);
|
|
786
|
+
this.pendingStart = end;
|
|
787
|
+
// Each row cut ended at a newline, but for a last one without (the source
|
|
788
|
+
// has ended then, and the count no longer matters).
|
|
789
|
+
this.newlines = Math.max(0, this.newlines - rows);
|
|
790
|
+
const firstRow = this.rowNumber + 1;
|
|
791
|
+
this.rowNumber += rows;
|
|
792
|
+
return { payload: { tsv }, rows, firstRow, inputRows: rows, endsBatch: false, refused: [] };
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
const COMMIT_NONE = 0;
|
|
796
|
+
const COMMIT_BATCH = 1;
|
|
797
|
+
const COMMIT_DONE = 2;
|
|
798
|
+
/** Most rejected rows listed in a result or error. */
|
|
799
|
+
const MAX_REJECTED_LISTED = 1000;
|
|
800
|
+
/**
|
|
801
|
+
* Run a whole copy over `io`.
|
|
802
|
+
*
|
|
803
|
+
* @internal
|
|
804
|
+
*/
|
|
805
|
+
export const copyRows = async (io, options) => {
|
|
806
|
+
const startedAt = performance.now();
|
|
807
|
+
const batchSize = options.batchSize === undefined || options.batchSize === "all" ? Infinity : options.batchSize;
|
|
808
|
+
const maxErrors = options.maxErrors ?? 0;
|
|
809
|
+
const expected = options.expectedRows;
|
|
810
|
+
const batches = [];
|
|
811
|
+
const rejected = [];
|
|
812
|
+
let rejectedCount = 0;
|
|
813
|
+
/** Rows the server has confirmed committed. */
|
|
814
|
+
let committedRows = 0;
|
|
815
|
+
/** Rows sent in the open batch (accepted by the native side). */
|
|
816
|
+
let openRows = 0;
|
|
817
|
+
/** Rows read and handed to the native side so far, rejected or not. */
|
|
818
|
+
let readRows = 0;
|
|
819
|
+
/** Lets the caller's source clean up (e.g. close a file stream); set once rows are read. */
|
|
820
|
+
let closeReader = () => { };
|
|
821
|
+
/** Whether the connection has been given up — by a failure, or a commit that discarded it. */
|
|
822
|
+
let abandoned = false;
|
|
823
|
+
const abandon = () => {
|
|
824
|
+
abandoned = true;
|
|
825
|
+
io.abandon();
|
|
826
|
+
};
|
|
827
|
+
const fail = (message, details) => {
|
|
828
|
+
closeReader();
|
|
829
|
+
if (details.abandon) {
|
|
830
|
+
abandon();
|
|
831
|
+
}
|
|
832
|
+
const doubt = details.rowsInDoubt ?? 0;
|
|
833
|
+
const committed = committedRows === 0
|
|
834
|
+
? "No rows were committed"
|
|
835
|
+
: `${committedRows} row${committedRows === 1 ? " was" : "s were"} committed by ` +
|
|
836
|
+
`${batches.length} earlier batch${batches.length === 1 ? "" : "es"}`;
|
|
837
|
+
const tail = doubt > 0
|
|
838
|
+
? `${committed}; the commit of the last batch (${doubt} rows) was under way when the ` +
|
|
839
|
+
"answer was lost: those rows may or may not have been committed"
|
|
840
|
+
: committedRows === 0
|
|
841
|
+
? committed
|
|
842
|
+
: `${committed}; nothing else was`;
|
|
843
|
+
// Server messages end in line breaks.
|
|
844
|
+
const reason = message.trim().replace(/\.$/, "");
|
|
845
|
+
throw new SybaseBulkCopyError(`Bulk copy into ${options.table} failed: ${reason}. ${tail}.`, {
|
|
846
|
+
rowsCommitted: committedRows,
|
|
847
|
+
rowsInDoubt: doubt,
|
|
848
|
+
batches: batches.slice(),
|
|
849
|
+
rejected: byRow(rejected),
|
|
850
|
+
rejectedCount,
|
|
851
|
+
row: details.row,
|
|
852
|
+
column: details.column,
|
|
853
|
+
// As it is, not only as this failure left it: a commit before it may have
|
|
854
|
+
// discarded the connection already.
|
|
855
|
+
connectionDead: abandoned,
|
|
856
|
+
cause: details.cause
|
|
857
|
+
});
|
|
858
|
+
};
|
|
859
|
+
const recordRejects = (list, count) => {
|
|
860
|
+
rejectedCount += count;
|
|
861
|
+
for (const item of list) {
|
|
862
|
+
if (rejected.length < MAX_REJECTED_LISTED) {
|
|
863
|
+
rejected.push(item);
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
};
|
|
867
|
+
// --- Open the copy ---
|
|
868
|
+
let begun;
|
|
869
|
+
// Checked on the clock before every native call: reading the caller's rows
|
|
870
|
+
// may have run past the deadline before its timer could fire.
|
|
871
|
+
if (io.checkDeadline()) {
|
|
872
|
+
return fail("the deadline passed", { cause: io.deadlineError(), abandon: true });
|
|
873
|
+
}
|
|
874
|
+
try {
|
|
875
|
+
begun = await io.race(io.begin({
|
|
876
|
+
table: options.table,
|
|
877
|
+
columns: options.columns,
|
|
878
|
+
keepIdentity: options.keepIdentity,
|
|
879
|
+
ignoreDefaults: options.ignoreDefaults,
|
|
880
|
+
allowConstraintBypass: options.allowConstraintBypass,
|
|
881
|
+
timeoutSeconds: options.timeoutSeconds
|
|
882
|
+
}));
|
|
883
|
+
}
|
|
884
|
+
catch (err) {
|
|
885
|
+
const failure = err;
|
|
886
|
+
return fail(failure?.message ?? String(err), {
|
|
887
|
+
cause: io.toCause(err),
|
|
888
|
+
abandon: io.expired || failure?.reusable !== true
|
|
889
|
+
});
|
|
890
|
+
}
|
|
891
|
+
// From here on every failure goes through `fail`: the copy is open.
|
|
892
|
+
let reader;
|
|
893
|
+
try {
|
|
894
|
+
const encoders = begun.columns.map(column => encoderFor(column, io.timeZone));
|
|
895
|
+
reader = new RowReader(options.rows, begun.columns, encoders, maxErrors, () => {
|
|
896
|
+
if (io.checkDeadline()) {
|
|
897
|
+
throw io.deadlineError();
|
|
898
|
+
}
|
|
899
|
+
});
|
|
900
|
+
}
|
|
901
|
+
catch (err) {
|
|
902
|
+
return fail(`reading the rows failed: ${messageOf(err)}`, {
|
|
903
|
+
cause: err,
|
|
904
|
+
abandon: true
|
|
905
|
+
});
|
|
906
|
+
}
|
|
907
|
+
closeReader = () => reader.close();
|
|
908
|
+
/** Rows read into the batch being planned (planning runs ahead of sending). */
|
|
909
|
+
let planned = 0;
|
|
910
|
+
const read = async () => {
|
|
911
|
+
const chunk = await reader.next(Math.min(CHUNK_ROWS, batchSize - planned));
|
|
912
|
+
if (chunk) {
|
|
913
|
+
planned += chunk.inputRows;
|
|
914
|
+
chunk.endsBatch = planned >= batchSize;
|
|
915
|
+
if (chunk.endsBatch) {
|
|
916
|
+
planned = 0;
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
return chunk;
|
|
920
|
+
};
|
|
921
|
+
/**
|
|
922
|
+
* Read the next chunk, settling rather than throwing: a read runs ahead while
|
|
923
|
+
* the previous chunk is being sent, and its failure must not act — close the
|
|
924
|
+
* connection — before that send has finished.
|
|
925
|
+
*/
|
|
926
|
+
const readAhead = () => io.race(read()).then(chunk => ({ chunk }), (error) => ({ error }));
|
|
927
|
+
/** The chunk read, or the copy's failure if reading failed. */
|
|
928
|
+
const settleRead = (read) => {
|
|
929
|
+
if ("chunk" in read) {
|
|
930
|
+
return read.chunk;
|
|
931
|
+
}
|
|
932
|
+
const err = read.error;
|
|
933
|
+
// Rows refused before the read failed are still listed (and may themselves
|
|
934
|
+
// be past the limit: that is then the failure reported).
|
|
935
|
+
admitRefused(reader.takeRefused());
|
|
936
|
+
if (io.expired) {
|
|
937
|
+
return fail("the deadline passed", { cause: io.deadlineError(), abandon: true });
|
|
938
|
+
}
|
|
939
|
+
return fail(`reading the rows failed: ${messageOf(err)}`, {
|
|
940
|
+
cause: err,
|
|
941
|
+
abandon: true
|
|
942
|
+
});
|
|
943
|
+
};
|
|
944
|
+
const send = async (payload, rows, firstRow, commit) => {
|
|
945
|
+
// Past the deadline the connection is closed (now, if the timer has not yet
|
|
946
|
+
// run): nothing is sent after it.
|
|
947
|
+
if (io.checkDeadline()) {
|
|
948
|
+
return fail("the deadline passed", { cause: io.deadlineError(), abandon: true });
|
|
949
|
+
}
|
|
950
|
+
let result;
|
|
951
|
+
try {
|
|
952
|
+
result = await io.race(io.send({
|
|
953
|
+
...payload,
|
|
954
|
+
rows,
|
|
955
|
+
firstRow,
|
|
956
|
+
commit,
|
|
957
|
+
maxRejects: Math.max(0, maxErrors - rejectedCount)
|
|
958
|
+
}));
|
|
959
|
+
}
|
|
960
|
+
catch (err) {
|
|
961
|
+
const failure = (typeof err === "object" && err !== null ? err : {});
|
|
962
|
+
recordRejects(failure.rejected ?? [], failure.rejectedCount ?? 0);
|
|
963
|
+
// A native failure says what this call committed, even when it failed
|
|
964
|
+
// after committing (its deadline passed during the commit).
|
|
965
|
+
const committedNow = typeof failure.committed === "number" ? failure.committed : 0;
|
|
966
|
+
const sentNow = failure.rowsSent;
|
|
967
|
+
openRows += sentNow ?? rows;
|
|
968
|
+
if (committedNow > 0) {
|
|
969
|
+
batches.push({ rows: committedNow });
|
|
970
|
+
committedRows += committedNow;
|
|
971
|
+
openRows = 0;
|
|
972
|
+
}
|
|
973
|
+
// The commit was under way and its answer was lost: the native side says
|
|
974
|
+
// so when the connection broke; when the deadline passed here with the
|
|
975
|
+
// call still running, it cannot be known either.
|
|
976
|
+
// Without an `outcome` (the deadline passed with the call running, or
|
|
977
|
+
// its answer could not be handed over) the commit is not known either.
|
|
978
|
+
const inDoubt = committedNow === 0 &&
|
|
979
|
+
commit !== COMMIT_NONE &&
|
|
980
|
+
failure.outcome !== "none" &&
|
|
981
|
+
failure.outcome !== "failed";
|
|
982
|
+
return fail(failure.message ?? String(err), {
|
|
983
|
+
cause: io.toCause(err),
|
|
984
|
+
rowsInDoubt: inDoubt ? openRows : 0,
|
|
985
|
+
row: failure.row,
|
|
986
|
+
column: failure.column,
|
|
987
|
+
abandon: true
|
|
988
|
+
});
|
|
989
|
+
}
|
|
990
|
+
recordRejects(result.rejected, result.rejectedCount);
|
|
991
|
+
openRows += result.rowsSent;
|
|
992
|
+
if (commit !== COMMIT_NONE) {
|
|
993
|
+
if (typeof result.committed !== "number") {
|
|
994
|
+
// The commit's answer did not arrive in full: whether it committed
|
|
995
|
+
// cannot be read off it.
|
|
996
|
+
return fail("the server's answer to the commit could not be read", {
|
|
997
|
+
rowsInDoubt: openRows,
|
|
998
|
+
abandon: true
|
|
999
|
+
});
|
|
1000
|
+
}
|
|
1001
|
+
const confirmed = result.committed;
|
|
1002
|
+
const sent = openRows;
|
|
1003
|
+
if (confirmed > 0) {
|
|
1004
|
+
batches.push({ rows: confirmed });
|
|
1005
|
+
committedRows += confirmed;
|
|
1006
|
+
}
|
|
1007
|
+
openRows = 0;
|
|
1008
|
+
if (confirmed !== sent) {
|
|
1009
|
+
fail(`the server reported ${confirmed} rows committed for a batch of ${sent} rows sent`, {
|
|
1010
|
+
abandon: true
|
|
1011
|
+
});
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
1014
|
+
if (result.discard) {
|
|
1015
|
+
// Committed, but the session could not be put back as it was (identity
|
|
1016
|
+
// insert, timeout): the connection must not be used again.
|
|
1017
|
+
abandon();
|
|
1018
|
+
}
|
|
1019
|
+
};
|
|
1020
|
+
/** Rows copied so far, as far as is known before the next commit. */
|
|
1021
|
+
const copiedSoFar = () => (maxErrors > 0 ? committedRows + openRows : readRows);
|
|
1022
|
+
/**
|
|
1023
|
+
* Fail before a commit that would break `expectedRows`: as soon as more rows
|
|
1024
|
+
* than expected are seen, and before the last commit if there are fewer.
|
|
1025
|
+
* Without `maxErrors` every row read is copied, so this runs before the rows
|
|
1026
|
+
* are sent; with it, once they are (the commit then goes in a call of its own).
|
|
1027
|
+
*/
|
|
1028
|
+
const checkExpected = (total, last) => {
|
|
1029
|
+
if (expected === undefined) {
|
|
1030
|
+
return;
|
|
1031
|
+
}
|
|
1032
|
+
if (total > expected) {
|
|
1033
|
+
fail(`expectedRows is ${expected}, but the input has more rows`, { abandon: true });
|
|
1034
|
+
}
|
|
1035
|
+
if (last && total !== expected) {
|
|
1036
|
+
fail(`expectedRows is ${expected}, but ${total} rows would be copied`, { abandon: true });
|
|
1037
|
+
}
|
|
1038
|
+
};
|
|
1039
|
+
/** Count rows the reader refused, failing past `maxErrors`: before their chunk is sent. */
|
|
1040
|
+
const admitRefused = (refused) => {
|
|
1041
|
+
for (const { rejected: item, cause } of refused) {
|
|
1042
|
+
recordRejects([item], 1);
|
|
1043
|
+
if (rejectedCount > maxErrors) {
|
|
1044
|
+
fail(`row ${item.row}${item.column ? `, column '${item.column}'` : ""}: ${item.message}`, {
|
|
1045
|
+
cause,
|
|
1046
|
+
row: item.row,
|
|
1047
|
+
column: item.column,
|
|
1048
|
+
abandon: true
|
|
1049
|
+
});
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
};
|
|
1053
|
+
const EMPTY = { text: "", lens: new Uint32Array(0) };
|
|
1054
|
+
let current = settleRead(await readAhead());
|
|
1055
|
+
if (current === null) {
|
|
1056
|
+
// Nothing to send: end the copy, so the connection can be used again.
|
|
1057
|
+
await send(EMPTY, 0, 1, COMMIT_DONE);
|
|
1058
|
+
if (expected !== undefined && expected !== 0) {
|
|
1059
|
+
fail(`expectedRows is ${expected}, but the input has no rows`, { abandon: false });
|
|
1060
|
+
}
|
|
1061
|
+
return {
|
|
1062
|
+
rowsCopied: 0,
|
|
1063
|
+
batches,
|
|
1064
|
+
rejected,
|
|
1065
|
+
rejectedCount,
|
|
1066
|
+
elapsedMs: Math.round(performance.now() - startedAt)
|
|
1067
|
+
};
|
|
1068
|
+
}
|
|
1069
|
+
// The read after `current`, running while `current` is sent (the native side
|
|
1070
|
+
// works on a worker thread). At most this one read is in flight.
|
|
1071
|
+
let pending = readAhead();
|
|
1072
|
+
const takeNext = async () => {
|
|
1073
|
+
const next = settleRead(await pending);
|
|
1074
|
+
pending = next ? readAhead() : null;
|
|
1075
|
+
return next;
|
|
1076
|
+
};
|
|
1077
|
+
while (current) {
|
|
1078
|
+
// Counted first: whatever happens to the reads after it — a failure, a
|
|
1079
|
+
// source that stalls — must not hide or delay what this chunk refused.
|
|
1080
|
+
admitRefused(current.refused);
|
|
1081
|
+
// A full batch is committed as soon as it is sent; a partial one has to
|
|
1082
|
+
// wait for the next read, to learn whether the input has ended. So does a
|
|
1083
|
+
// full one under expectedRows: if it is the last, the count must be checked
|
|
1084
|
+
// before it commits, not after.
|
|
1085
|
+
let next;
|
|
1086
|
+
let commit = COMMIT_BATCH;
|
|
1087
|
+
// Known last (the reader has nothing more, as for TSV given whole): the
|
|
1088
|
+
// next read returns at once, and the batch is committed with the copy's
|
|
1089
|
+
// end — not followed by a round trip for an empty one.
|
|
1090
|
+
if (!current.endsBatch || expected !== undefined || reader.exhausted) {
|
|
1091
|
+
next = await takeNext();
|
|
1092
|
+
commit = next === null ? COMMIT_DONE : current.endsBatch ? COMMIT_BATCH : COMMIT_NONE;
|
|
1093
|
+
}
|
|
1094
|
+
const separateCommit = commit !== COMMIT_NONE && expected !== undefined && maxErrors > 0;
|
|
1095
|
+
if (maxErrors === 0) {
|
|
1096
|
+
checkExpected(readRows + current.inputRows, commit === COMMIT_DONE);
|
|
1097
|
+
}
|
|
1098
|
+
await send(current.payload, current.rows, current.firstRow, separateCommit ? COMMIT_NONE : commit);
|
|
1099
|
+
readRows += current.inputRows;
|
|
1100
|
+
// With maxErrors, how many rows are copied is known only once sent: more
|
|
1101
|
+
// than expected fails now, not after the rest has been read and sent too.
|
|
1102
|
+
if (maxErrors > 0 && !separateCommit) {
|
|
1103
|
+
checkExpected(copiedSoFar(), false);
|
|
1104
|
+
}
|
|
1105
|
+
if (separateCommit) {
|
|
1106
|
+
checkExpected(copiedSoFar(), commit === COMMIT_DONE);
|
|
1107
|
+
await send(EMPTY, 0, current.firstRow, commit);
|
|
1108
|
+
}
|
|
1109
|
+
if (next === undefined) {
|
|
1110
|
+
next = await takeNext();
|
|
1111
|
+
if (next === null) {
|
|
1112
|
+
// The input ended exactly at a batch boundary: end the copy, with an
|
|
1113
|
+
// empty last batch.
|
|
1114
|
+
checkExpected(copiedSoFar(), true);
|
|
1115
|
+
await send(EMPTY, 0, readRows + 1, COMMIT_DONE);
|
|
1116
|
+
}
|
|
1117
|
+
}
|
|
1118
|
+
current = next;
|
|
1119
|
+
}
|
|
1120
|
+
if (expected !== undefined && committedRows !== expected) {
|
|
1121
|
+
fail(`expectedRows is ${expected}, but the server reported ${committedRows} rows committed`, {
|
|
1122
|
+
abandon: false
|
|
1123
|
+
});
|
|
1124
|
+
}
|
|
1125
|
+
return {
|
|
1126
|
+
rowsCopied: committedRows,
|
|
1127
|
+
batches,
|
|
1128
|
+
rejected: byRow(rejected),
|
|
1129
|
+
rejectedCount,
|
|
1130
|
+
elapsedMs: Math.round(performance.now() - startedAt)
|
|
1131
|
+
};
|
|
1132
|
+
};
|
|
1133
|
+
/** Rejected rows in input order: a chunk's are found by the reader and by the native side. */
|
|
1134
|
+
const byRow = (list) => list.slice().sort((a, b) => a.row - b.row);
|
|
1135
|
+
//# sourceMappingURL=bulk.js.map
|