inibase 2.0.1 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +211 -2
- package/dist/expression.d.ts +159 -0
- package/dist/expression.js +495 -0
- package/dist/file.d.ts +20 -1
- package/dist/file.js +211 -16
- package/dist/index.d.ts +177 -1
- package/dist/index.js +1414 -150
- package/dist/journal.d.ts +109 -0
- package/dist/journal.js +263 -0
- package/dist/utils.js +54 -0
- package/package.json +7 -2
package/dist/index.js
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import "dotenv/config";
|
|
2
|
-
import { randomBytes, scryptSync } from "node:crypto";
|
|
2
|
+
import { randomBytes, randomUUID, scryptSync } from "node:crypto";
|
|
3
3
|
import { appendFileSync, existsSync, readFileSync } from "node:fs";
|
|
4
|
-
import { glob, mkdir, readdir, readFile, rename, rm, unlink, writeFile, } from "node:fs/promises";
|
|
5
|
-
import { join, parse } from "node:path";
|
|
4
|
+
import { glob, mkdir, readdir, readFile, rename, rm, stat, unlink, writeFile, } from "node:fs/promises";
|
|
5
|
+
import { basename, join, parse, resolve } from "node:path";
|
|
6
6
|
import { inspect } from "node:util";
|
|
7
7
|
import Inison from "inison";
|
|
8
|
+
import { buildFieldIndex, collectFieldDeps, parseExpression, resolveExpression, resolveFramePath, topoSortComputedFields, } from "./expression.js";
|
|
8
9
|
import * as File from "./file.js";
|
|
10
|
+
import { DatabaseJournal, Journal } from "./journal.js";
|
|
9
11
|
import * as Utils from "./utils.js";
|
|
10
12
|
import * as UtilsServer from "./utils.server.js";
|
|
11
13
|
export const ERROR_CODES = [
|
|
@@ -22,6 +24,15 @@ export const ERROR_CODES = [
|
|
|
22
24
|
"TABLE_NOT_EXISTS",
|
|
23
25
|
"INVALID_REGEX_MATCH",
|
|
24
26
|
"INVALID_NAME",
|
|
27
|
+
"COMPUTED_FIELD_SYNTAX",
|
|
28
|
+
"COMPUTED_FIELD_UNKNOWN_FIELD",
|
|
29
|
+
"COMPUTED_FIELD_INVALID_LINK",
|
|
30
|
+
"COMPUTED_FIELD_INVALID_TARGET",
|
|
31
|
+
"COMPUTED_FIELD_CONFLICT",
|
|
32
|
+
"COMPUTED_FIELD_CYCLE",
|
|
33
|
+
"COMPUTED_FIELD_SETTABLE",
|
|
34
|
+
"COMPUTED_FIELD_DANGLING_LINK",
|
|
35
|
+
"COMPUTED_FIELD_ARITHMETIC",
|
|
25
36
|
];
|
|
26
37
|
// hide ExperimentalWarning glob()
|
|
27
38
|
// Guard against non-Node environments (e.g. accidental import in a browser bundle)
|
|
@@ -30,6 +41,75 @@ if (typeof process !== "undefined" &&
|
|
|
30
41
|
process.removeAllListeners("warning");
|
|
31
42
|
}
|
|
32
43
|
export const globalConfig = {};
|
|
44
|
+
/**
|
|
45
|
+
* Batch-scoped link-hop reader for computed-field evaluation: a dry pass
|
|
46
|
+
* records every (table, column, id) triple the expressions need, each
|
|
47
|
+
* distinct triple is resolved exactly once by the engine's `readLinkedRow`
|
|
48
|
+
* (deduplicated across rows and fields of the batch), and subsequent passes
|
|
49
|
+
* are served from a warm per-(table, column) cache.
|
|
50
|
+
*/
|
|
51
|
+
class LinkedRowReader {
|
|
52
|
+
readRow;
|
|
53
|
+
pending = new Map();
|
|
54
|
+
cache = new Map();
|
|
55
|
+
constructor(readRow) {
|
|
56
|
+
this.readRow = readRow;
|
|
57
|
+
}
|
|
58
|
+
get hasPending() {
|
|
59
|
+
return this.pending.size > 0;
|
|
60
|
+
}
|
|
61
|
+
record(table, column, id) {
|
|
62
|
+
const key = `${table}\u0000${column}`;
|
|
63
|
+
let ids = this.pending.get(key);
|
|
64
|
+
if (!ids)
|
|
65
|
+
this.pending.set(key, (ids = new Set()));
|
|
66
|
+
ids.add(id);
|
|
67
|
+
}
|
|
68
|
+
async read(table, column, id) {
|
|
69
|
+
const key = `${table}\u0000${column}`;
|
|
70
|
+
let map = this.cache.get(key);
|
|
71
|
+
if (!map) {
|
|
72
|
+
map = await this.resolve(key, this.pending.get(key) ?? new Set());
|
|
73
|
+
this.pending.delete(key);
|
|
74
|
+
this.cache.set(key, map);
|
|
75
|
+
}
|
|
76
|
+
return map.get(id) ?? null;
|
|
77
|
+
}
|
|
78
|
+
/** Resolve every pending bucket now (called between the collection pass
|
|
79
|
+
* and the real evaluation pass). */
|
|
80
|
+
async resolveAll() {
|
|
81
|
+
for (const [key, ids] of this.pending)
|
|
82
|
+
this.cache.set(key, await this.resolve(key, ids));
|
|
83
|
+
this.pending.clear();
|
|
84
|
+
}
|
|
85
|
+
async resolve(key, ids) {
|
|
86
|
+
const separator = key.indexOf("\u0000");
|
|
87
|
+
const table = key.slice(0, separator);
|
|
88
|
+
const column = key.slice(separator + 1);
|
|
89
|
+
const map = new Map();
|
|
90
|
+
// One engine read per distinct triple; the buckets' triples are
|
|
91
|
+
// independent so they resolve concurrently. A missing row caches as
|
|
92
|
+
// null so repeated dangling hops fail once, exactly like a direct
|
|
93
|
+
// readLinkedRow would per hop.
|
|
94
|
+
await Promise.all(Array.from(ids).map(async (id) => {
|
|
95
|
+
map.set(id, await this.readRow(table, column, id));
|
|
96
|
+
}));
|
|
97
|
+
return map;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
/** True when an expression tree contains any link hop (multi-segment path). */
|
|
101
|
+
function astHasHops(node) {
|
|
102
|
+
switch (node.kind) {
|
|
103
|
+
case "num":
|
|
104
|
+
return false;
|
|
105
|
+
case "path":
|
|
106
|
+
return node.ids.length > 1;
|
|
107
|
+
case "bin":
|
|
108
|
+
return astHasHops(node.left) || astHasHops(node.right);
|
|
109
|
+
case "fn":
|
|
110
|
+
return astHasHops(node.arg);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
33
113
|
/**
|
|
34
114
|
* @param {string} database - Database name
|
|
35
115
|
* @param {string} [mainFolder="."] - Main folder path
|
|
@@ -45,9 +125,22 @@ export default class Inibase {
|
|
|
45
125
|
* resolve numeric ids to line numbers arithmetically instead of scanning
|
|
46
126
|
* the id file. Set false by any partial row deletion. */
|
|
47
127
|
idDensity = new Map();
|
|
128
|
+
/** Per-table computed-plan cache (null = table has no computed fields).
|
|
129
|
+
* A plan depends only on the table's persisted schema (compiled ASTs are
|
|
130
|
+
* id-based and rename-proof), so it is rebuilt only when the schema
|
|
131
|
+
* changes: see the invalidation in `getTable`'s reload branch,
|
|
132
|
+
* `createTable` and `updateTableLocked`. Bounded by the number of tables. */
|
|
133
|
+
computedPlanCache = new Map();
|
|
48
134
|
databasePath;
|
|
49
135
|
uniqueMap;
|
|
50
136
|
schemaFileExtension = process.env.INIBASE_SCHEMA_EXTENSION ?? "json";
|
|
137
|
+
/**
|
|
138
|
+
* Open database transaction (see begin/commit/rollback). Holds the
|
|
139
|
+
* database lock (`<db>/.tmp/.locked`) for its whole lifetime and the
|
|
140
|
+
* per-table writer lock of every table it mutates, so mutations stage into
|
|
141
|
+
* the database journal and publish only on commit().
|
|
142
|
+
*/
|
|
143
|
+
transaction = null;
|
|
51
144
|
constructor(database, mainFolder = ".", language = "en") {
|
|
52
145
|
this.language = language;
|
|
53
146
|
this.validateName(database);
|
|
@@ -128,6 +221,11 @@ export default class Inibase {
|
|
|
128
221
|
*/
|
|
129
222
|
async createTable(tableName, schema, config) {
|
|
130
223
|
this.validateName(tableName);
|
|
224
|
+
// DDL does not participate in the write-ahead journal: schema surgery
|
|
225
|
+
// inside a transaction would escape the atomic publish/rollback scope.
|
|
226
|
+
if (this.transaction)
|
|
227
|
+
throw this.createError("INVALID_PARAMETERS");
|
|
228
|
+
await this.ensureDatabaseRecovered();
|
|
131
229
|
if (schema)
|
|
132
230
|
this.validateSchema(schema);
|
|
133
231
|
const tablePath = join(this.databasePath, tableName);
|
|
@@ -145,32 +243,44 @@ export default class Inibase {
|
|
|
145
243
|
};
|
|
146
244
|
if (config) {
|
|
147
245
|
if (config.compression)
|
|
148
|
-
await
|
|
246
|
+
await File.write(join(tablePath, ".compression.config"), "");
|
|
149
247
|
if (config.cache)
|
|
150
|
-
await
|
|
248
|
+
await File.write(join(tablePath, ".cache.config"), "");
|
|
151
249
|
if (config.prepend)
|
|
152
|
-
await
|
|
250
|
+
await File.write(join(tablePath, ".prepend.config"), "");
|
|
153
251
|
if (config.decodeID)
|
|
154
|
-
await
|
|
252
|
+
await File.write(join(tablePath, ".decodeID.config"), "");
|
|
155
253
|
}
|
|
156
254
|
if (schema) {
|
|
157
255
|
const lastSchemaID = { value: 0 };
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
await
|
|
256
|
+
// Compile computed expressions only after ids are assigned (the
|
|
257
|
+
// persisted AST carries ids, making it rename-proof), and before
|
|
258
|
+
// the schema is written so a bad expression never lands on disk.
|
|
259
|
+
schema = await this.compileComputedFields(Utils.addIdToSchema(schema, lastSchemaID));
|
|
260
|
+
await File.write(join(tablePath, `schema.${this.schemaFileExtension}`), this.schemaFileExtension === "json"
|
|
261
|
+
? JSON.stringify(schema, null, 2)
|
|
262
|
+
: Inison.stringify(schema));
|
|
263
|
+
await File.write(join(tablePath, `${lastSchemaID.value}.schema`), "");
|
|
162
264
|
}
|
|
163
265
|
else
|
|
164
|
-
await
|
|
165
|
-
await
|
|
266
|
+
await File.write(join(tablePath, "0.schema"), "");
|
|
267
|
+
await File.write(join(tablePath, "0-0.pagination"), "");
|
|
268
|
+
// Make the new table's metadata durable before acknowledging creation.
|
|
269
|
+
await File.syncDir(tablePath);
|
|
270
|
+
await File.syncDir(join(tablePath, ".tmp"));
|
|
166
271
|
this.idDensity.set(tableName, true);
|
|
272
|
+
// Defensive: a table of the same name may have had a cached plan from
|
|
273
|
+
// before a delete+recreate.
|
|
274
|
+
this.computedPlanCache.delete(tableName);
|
|
167
275
|
}
|
|
168
276
|
// Function to replace the string in one schema file
|
|
169
277
|
async replaceStringInFile(filePath, targetString, replaceString) {
|
|
170
278
|
const data = await readFile(filePath, "utf8");
|
|
171
279
|
if (data.includes(targetString)) {
|
|
172
280
|
const updatedContent = data.replaceAll(targetString, replaceString);
|
|
173
|
-
|
|
281
|
+
// File.write fsyncs the replacement so a link-update after a table
|
|
282
|
+
// rename survives a crash.
|
|
283
|
+
await File.write(filePath, updatedContent);
|
|
174
284
|
}
|
|
175
285
|
}
|
|
176
286
|
/**
|
|
@@ -182,12 +292,35 @@ export default class Inibase {
|
|
|
182
292
|
*/
|
|
183
293
|
async updateTable(tableName, schema, config) {
|
|
184
294
|
this.validateName(tableName);
|
|
295
|
+
// DDL does not participate in the write-ahead journal: schema surgery
|
|
296
|
+
// inside a transaction would escape the atomic publish/rollback scope.
|
|
297
|
+
if (this.transaction)
|
|
298
|
+
throw this.createError("INVALID_PARAMETERS");
|
|
299
|
+
await this.ensureDatabaseRecovered();
|
|
185
300
|
if (config?.name)
|
|
186
301
|
this.validateName(config.name);
|
|
187
302
|
const table = await this.getTable(tableName);
|
|
188
303
|
if (!table)
|
|
189
304
|
return;
|
|
190
305
|
const tablePath = join(this.databasePath, tableName);
|
|
306
|
+
// DDL is serialized with DML writers on the same per-table lock, so a
|
|
307
|
+
// post/put/delete can never interleave with schema/file surgery.
|
|
308
|
+
try {
|
|
309
|
+
await File.lock(join(tablePath, ".tmp"));
|
|
310
|
+
await this.updateTableLocked(tableName, table, tablePath, schema, config);
|
|
311
|
+
}
|
|
312
|
+
finally {
|
|
313
|
+
await File.unlock(join(tablePath, ".tmp"));
|
|
314
|
+
// Renaming the table moves its .tmp (and with it the lock file)
|
|
315
|
+
// to the new directory, so the unlock above only released the old
|
|
316
|
+
// path. Release the lock at its new location too, or the renamed
|
|
317
|
+
// table is left with a perpetual live-owner lock no writer can
|
|
318
|
+
// steal.
|
|
319
|
+
if (config?.name && config.name !== tableName)
|
|
320
|
+
await File.unlock(join(join(this.databasePath, config.name), ".tmp"));
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
async updateTableLocked(tableName, table, tablePath, schema, config) {
|
|
191
324
|
if (schema) {
|
|
192
325
|
this.validateSchema(schema);
|
|
193
326
|
// remove id from schema
|
|
@@ -199,6 +332,18 @@ export default class Inibase {
|
|
|
199
332
|
value: schemaIdFilePath ? Number(parse(schemaIdFilePath).name) : 0,
|
|
200
333
|
};
|
|
201
334
|
schema = Utils.addIdToSchema(schema, lastSchemaID);
|
|
335
|
+
// Compile computed expressions now that ids are assigned (the
|
|
336
|
+
// persisted AST only stores ids, so later renames never retarget
|
|
337
|
+
// an expression). Compilation happens before any file surgery so
|
|
338
|
+
// a bad expression aborts the migration cleanly.
|
|
339
|
+
schema = await this.compileComputedFields(schema);
|
|
340
|
+
// Row count is needed for the computed backfill below, so read the
|
|
341
|
+
// pagination before the schema is rewritten.
|
|
342
|
+
let totalLines = 0;
|
|
343
|
+
for await (const paginationFileName of glob("*.pagination", {
|
|
344
|
+
cwd: tablePath,
|
|
345
|
+
}))
|
|
346
|
+
totalLines = parse(paginationFileName).name.split("-").map(Number)[1];
|
|
202
347
|
// if schema file exists, update columns files names based on field id
|
|
203
348
|
if ((await File.isExists(join(tablePath, `schema.${this.schemaFileExtension}`))) &&
|
|
204
349
|
table.schema?.length) {
|
|
@@ -214,13 +359,55 @@ export default class Inibase {
|
|
|
214
359
|
}
|
|
215
360
|
}));
|
|
216
361
|
}
|
|
217
|
-
|
|
362
|
+
// Computed-field backfill: a computed expression that is added or
|
|
363
|
+
// changed by this migration must be (re)derived for every existing
|
|
364
|
+
// row. Evaluation happens BEFORE the schema is rewritten, so a
|
|
365
|
+
// failed migration (dangling link, arithmetic error, unknown
|
|
366
|
+
// field, ...) leaves the old schema and column values untouched.
|
|
367
|
+
const oldComputed = new Map((table.schema ?? [])
|
|
368
|
+
.filter((field) => typeof field.computed !== "undefined")
|
|
369
|
+
.map((field) => [field.key, this.computedExprOf(field)]));
|
|
370
|
+
const changedComputedKeys = new Set(totalLines > 0
|
|
371
|
+
? schema
|
|
372
|
+
.filter((field) => typeof field.computed !== "undefined" &&
|
|
373
|
+
oldComputed.get(field.key) !== this.computedExprOf(field))
|
|
374
|
+
.map((field) => field.key)
|
|
375
|
+
: []);
|
|
376
|
+
const backfillReplacers = changedComputedKeys.size
|
|
377
|
+
? await (async () => {
|
|
378
|
+
const plan = await this.buildComputedPlan(tableName, schema);
|
|
379
|
+
if (!plan)
|
|
380
|
+
return undefined;
|
|
381
|
+
const lineNumbers = Array.from({ length: totalLines }, (_, index) => index + 1);
|
|
382
|
+
const existing = await this.processSchemaData(tableName, schema.filter((field) => typeof field.computed === "undefined"), lineNumbers);
|
|
383
|
+
return this.evaluateComputedRows(tableName, plan, existing);
|
|
384
|
+
})()
|
|
385
|
+
: undefined;
|
|
386
|
+
// Publish the backfilled values before the schema write.
|
|
387
|
+
if (backfillReplacers) {
|
|
388
|
+
const backfillRenameList = [];
|
|
389
|
+
for (const key of changedComputedKeys) {
|
|
390
|
+
const lines = {};
|
|
391
|
+
for (const [line, values] of Object.entries(backfillReplacers))
|
|
392
|
+
if (Object.hasOwn(values, key))
|
|
393
|
+
lines[Number(line)] = File.encode(values[key]);
|
|
394
|
+
if (!Object.keys(lines).length)
|
|
395
|
+
continue;
|
|
396
|
+
backfillRenameList.push(await File.replace(join(tablePath, `${key}${this.getFileExtension(tableName)}`), lines));
|
|
397
|
+
}
|
|
398
|
+
for (const [tmp, live] of backfillRenameList)
|
|
399
|
+
if (tmp && live)
|
|
400
|
+
await rename(tmp, live);
|
|
401
|
+
}
|
|
402
|
+
// Data-bearing schema writes go through File.write so they are
|
|
403
|
+
// fsynced before updateTable returns.
|
|
404
|
+
await File.write(join(tablePath, `schema.${this.schemaFileExtension}`), this.schemaFileExtension === "json"
|
|
218
405
|
? JSON.stringify(schema, null, 2)
|
|
219
406
|
: Inison.stringify(schema));
|
|
220
407
|
if (schemaIdFilePath)
|
|
221
408
|
await rename(schemaIdFilePath, join(tablePath, `${lastSchemaID.value}.schema`));
|
|
222
409
|
else
|
|
223
|
-
await
|
|
410
|
+
await File.write(join(tablePath, `${lastSchemaID.value}.schema`), "");
|
|
224
411
|
// Fields added by this migration have no backing file yet. If the
|
|
225
412
|
// first post after the migration writes such a file from scratch it
|
|
226
413
|
// starts at line 1 and every existing row becomes misaligned (the
|
|
@@ -228,11 +415,6 @@ export default class Inibase {
|
|
|
228
415
|
// files padded with one empty line per existing row so appends keep
|
|
229
416
|
// line-aligned with the other column files (decode("") is
|
|
230
417
|
// undefined/null, matching the "no value yet" semantics).
|
|
231
|
-
let totalLines = 0;
|
|
232
|
-
for await (const paginationFileName of glob("*.pagination", {
|
|
233
|
-
cwd: tablePath,
|
|
234
|
-
}))
|
|
235
|
-
totalLines = parse(paginationFileName).name.split("-").map(Number)[1];
|
|
236
418
|
await Promise.allSettled(schema.map(async ({ key }) => {
|
|
237
419
|
const filePath = join(tablePath, `${key}${this.getFileExtension(tableName)}`);
|
|
238
420
|
if (!(await File.isExists(filePath)))
|
|
@@ -242,26 +424,31 @@ export default class Inibase {
|
|
|
242
424
|
if (config) {
|
|
243
425
|
if (config.compression !== undefined &&
|
|
244
426
|
config.compression !== table.config.compression) {
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
427
|
+
// Toggle compression crash-safely: the shell only decompresses
|
|
428
|
+
// to a temp file (streamed), the publish step fsyncs each temp
|
|
429
|
+
// before renaming it over the original, then the config marker
|
|
430
|
+
// is fsynced. A crash mid-toggle leaves valid (uncompressed or
|
|
431
|
+
// compressed) files, never a truncated one.
|
|
432
|
+
const toggleFiles = (await readdir(tablePath)).filter((name) => config.compression
|
|
433
|
+
? name.endsWith(this.fileExtension) &&
|
|
434
|
+
!name.endsWith(`${this.fileExtension}.gz`)
|
|
435
|
+
: name.endsWith(`${this.fileExtension}.gz`));
|
|
436
|
+
for (const name of toggleFiles) {
|
|
437
|
+
const src = join(tablePath, name);
|
|
438
|
+
const tmp = `${src}.recompressed`;
|
|
439
|
+
const target = config.compression ? `${src}.gz` : src.slice(0, -3); // strip ".gz"
|
|
440
|
+
await UtilsServer.exec(`${config.compression ? "gzip" : "gunzip"} -c ${File.escapeShellPath(src)} > ${File.escapeShellPath(tmp)}`);
|
|
441
|
+
await File.syncFile(tmp);
|
|
442
|
+
await rename(tmp, target);
|
|
443
|
+
}
|
|
257
444
|
if (config.compression)
|
|
258
|
-
await
|
|
445
|
+
await File.write(join(tablePath, ".compression.config"), "");
|
|
259
446
|
else
|
|
260
447
|
await unlink(join(tablePath, ".compression.config"));
|
|
261
448
|
}
|
|
262
449
|
if (config.cache !== undefined && config.cache !== table.config.cache) {
|
|
263
450
|
if (config.cache)
|
|
264
|
-
await
|
|
451
|
+
await File.write(join(tablePath, ".cache.config"), "");
|
|
265
452
|
else {
|
|
266
453
|
await this.clearCache(tableName);
|
|
267
454
|
await unlink(join(tablePath, ".cache.config"));
|
|
@@ -270,12 +457,17 @@ export default class Inibase {
|
|
|
270
457
|
if (config.decodeID !== undefined &&
|
|
271
458
|
config.decodeID !== table.config.decodeID) {
|
|
272
459
|
if (config.decodeID)
|
|
273
|
-
await
|
|
460
|
+
await File.write(join(tablePath, ".decodeID.config"), "");
|
|
274
461
|
else
|
|
275
462
|
await unlink(join(tablePath, ".decodeID.config"));
|
|
276
463
|
}
|
|
277
464
|
if (config.prepend !== undefined &&
|
|
278
465
|
config.prepend !== table.config.prepend) {
|
|
466
|
+
// Reverse every column file so the "first" row stays first after
|
|
467
|
+
// toggling prepend. The (streaming) shell only writes `<file>.reversed`
|
|
468
|
+
// temp files; the publish step below fsyncs each temp before renaming
|
|
469
|
+
// it into place, so a crash mid-toggle never leaves a half-reversed
|
|
470
|
+
// live file (the old file stays valid until the rename).
|
|
279
471
|
await UtilsServer.execFile("find", [
|
|
280
472
|
tableName,
|
|
281
473
|
"-type",
|
|
@@ -286,24 +478,34 @@ export default class Inibase {
|
|
|
286
478
|
"sh",
|
|
287
479
|
"-c",
|
|
288
480
|
`for file; do ${config.compression
|
|
289
|
-
? `zcat "$file" | ${process.platform === "darwin" ? "tail -r" : "tac"} | gzip > "$file.reversed"
|
|
290
|
-
: `${process.platform === "darwin" ? "tail -r" : "tac"} "$file" > "$file.reversed"
|
|
481
|
+
? `zcat "$file" | ${process.platform === "darwin" ? "tail -r" : "tac"} | gzip > "$file.reversed"`
|
|
482
|
+
: `${process.platform === "darwin" ? "tail -r" : "tac"} "$file" > "$file.reversed"`}; done`,
|
|
291
483
|
"_",
|
|
292
484
|
"{}",
|
|
293
485
|
"+",
|
|
294
486
|
], { cwd: this.databasePath });
|
|
487
|
+
const reversedSuffix = `${this.fileExtension}${config.compression ? ".gz" : ""}.reversed`;
|
|
488
|
+
for (const fileName of await readdir(tablePath)) {
|
|
489
|
+
if (!fileName.endsWith(reversedSuffix))
|
|
490
|
+
continue;
|
|
491
|
+
const reversedPath = join(tablePath, fileName);
|
|
492
|
+
await File.syncFile(reversedPath);
|
|
493
|
+
await rename(reversedPath, join(tablePath, fileName.slice(0, -".reversed".length)));
|
|
494
|
+
}
|
|
295
495
|
if (config.prepend)
|
|
296
|
-
await
|
|
496
|
+
await File.write(join(tablePath, ".prepend.config"), "");
|
|
297
497
|
else
|
|
298
498
|
await unlink(join(tablePath, ".prepend.config"));
|
|
299
499
|
}
|
|
300
500
|
if (config.name) {
|
|
301
501
|
await rename(tablePath, join(this.databasePath, config.name));
|
|
302
|
-
// replace table name in other linked tables (relationship)
|
|
502
|
+
// replace table name in other linked tables (relationship).
|
|
503
|
+
// glob() returns paths relative to `cwd`, so resolve them
|
|
504
|
+
// against the database path before touching the files.
|
|
303
505
|
for await (const schemaPath of glob(`**/schema.${this.schemaFileExtension}`, {
|
|
304
506
|
cwd: this.databasePath,
|
|
305
507
|
}))
|
|
306
|
-
await this.replaceStringInFile(schemaPath,
|
|
508
|
+
await this.replaceStringInFile(resolve(this.databasePath, schemaPath),
|
|
307
509
|
// TODO: escape caracters in table name
|
|
308
510
|
this.schemaFileExtension === "json"
|
|
309
511
|
? `"table": "${tableName}"`
|
|
@@ -312,7 +514,17 @@ export default class Inibase {
|
|
|
312
514
|
: `table:${config.name}`);
|
|
313
515
|
}
|
|
314
516
|
}
|
|
517
|
+
// Flush the directory entries touched by this DDL (renames, unlinks,
|
|
518
|
+
// fresh config markers) before updateTable returns so the schema/config
|
|
519
|
+
// change survives a crash.
|
|
520
|
+
await File.syncDir(config?.name ? this.databasePath : tablePath);
|
|
315
521
|
globalConfig[this.databasePath].tables?.delete(tableName);
|
|
522
|
+
// The schema on disk changed under the DDL lock: drop the cached plan
|
|
523
|
+
// (also under the old name when the table was renamed) so the next DML
|
|
524
|
+
// rebuilds from the reloaded schema.
|
|
525
|
+
this.computedPlanCache.delete(tableName);
|
|
526
|
+
if (config?.name && config.name !== tableName)
|
|
527
|
+
this.computedPlanCache.delete(config.name);
|
|
316
528
|
}
|
|
317
529
|
/**
|
|
318
530
|
* Get table schema and config
|
|
@@ -338,6 +550,9 @@ export default class Inibase {
|
|
|
338
550
|
},
|
|
339
551
|
timestamp: await File.getFileDate(join(tablePath, `schema.${this.schemaFileExtension}`)),
|
|
340
552
|
});
|
|
553
|
+
// A reload means the on-disk schema changed (create/update/external
|
|
554
|
+
// edit): the cached computed plan is derived from it.
|
|
555
|
+
this.computedPlanCache.delete(tableName);
|
|
341
556
|
return globalConfig[this.databasePath].tables?.get(tableName);
|
|
342
557
|
}
|
|
343
558
|
async getTableSchema(tableName) {
|
|
@@ -356,10 +571,14 @@ export default class Inibase {
|
|
|
356
571
|
otherSchemaFileExtension === "json"
|
|
357
572
|
? JSON.parse(schemaFile)
|
|
358
573
|
: Inison.unstringify(schemaFile);
|
|
359
|
-
|
|
574
|
+
// Mirror the legacy schema file into the preferred extension and
|
|
575
|
+
// drop the old one (fsync-backed; this read-path migration must
|
|
576
|
+
// survive a crash).
|
|
577
|
+
await File.write(join(tablePath, `schema.${this.schemaFileExtension}`), this.schemaFileExtension === "json"
|
|
360
578
|
? JSON.stringify(schema, null, 2)
|
|
361
579
|
: Inison.stringify(schema));
|
|
362
580
|
await unlink(join(tablePath, `schema.${otherSchemaFileExtension}`));
|
|
581
|
+
await File.syncDir(tablePath);
|
|
363
582
|
}
|
|
364
583
|
else
|
|
365
584
|
schemaFile = await readFile(join(tablePath, `schema.${this.schemaFileExtension}`), "utf8");
|
|
@@ -406,6 +625,13 @@ export default class Inibase {
|
|
|
406
625
|
}
|
|
407
626
|
if (Utils.isObject(data)) {
|
|
408
627
|
for (const field of schema) {
|
|
628
|
+
// Computed columns are never user-settable: their value is
|
|
629
|
+
// always derived by the engine at write time.
|
|
630
|
+
if (typeof field.computed !== "undefined") {
|
|
631
|
+
if (Object.hasOwn(data, field.key))
|
|
632
|
+
throw this.createError("COMPUTED_FIELD_SETTABLE", field.key);
|
|
633
|
+
continue;
|
|
634
|
+
}
|
|
409
635
|
if (!Object.hasOwn(data, field.key) ||
|
|
410
636
|
data[field.key] === null ||
|
|
411
637
|
data[field.key] === undefined ||
|
|
@@ -789,6 +1015,454 @@ export default class Inibase {
|
|
|
789
1015
|
}
|
|
790
1016
|
return RETURN;
|
|
791
1017
|
}
|
|
1018
|
+
/* -----------------------------------------------------------------------
|
|
1019
|
+
* Computed fields
|
|
1020
|
+
* --------------------------------------------------------------------- */
|
|
1021
|
+
/**
|
|
1022
|
+
* Extract the raw expression from a field's `computed` property (string or
|
|
1023
|
+
* persisted `{ expr, ast }` spec).
|
|
1024
|
+
*/
|
|
1025
|
+
computedExprOf(field) {
|
|
1026
|
+
const computed = field.computed;
|
|
1027
|
+
return typeof computed === "string" ? computed : (computed?.expr ?? "");
|
|
1028
|
+
}
|
|
1029
|
+
/**
|
|
1030
|
+
* Compile every `computed` expression of a schema (which must already have
|
|
1031
|
+
* ids assigned) into its persisted `{ expr, ast }` form, in dependency
|
|
1032
|
+
* order. Throws `COMPUTED_FIELD_CYCLE` on cyclic fields. Used by DDL so the
|
|
1033
|
+
* on-disk schema always carries compiled ASTs.
|
|
1034
|
+
*/
|
|
1035
|
+
async compileComputedFields(schema) {
|
|
1036
|
+
const computedFields = schema.filter((field) => typeof field.computed !== "undefined");
|
|
1037
|
+
if (!computedFields.length)
|
|
1038
|
+
return schema;
|
|
1039
|
+
const index = buildFieldIndex(schema);
|
|
1040
|
+
const ctx = {
|
|
1041
|
+
language: this.language,
|
|
1042
|
+
ownKey: "",
|
|
1043
|
+
index,
|
|
1044
|
+
getTableIndex: async (target) => this.tableFieldIndex(target),
|
|
1045
|
+
};
|
|
1046
|
+
const resolved = [];
|
|
1047
|
+
for (const field of computedFields) {
|
|
1048
|
+
// Computed fields are evaluated from the row of the table they
|
|
1049
|
+
// belong to, so v1 restricts them to top-level schema fields.
|
|
1050
|
+
const ref = index.get(field.id);
|
|
1051
|
+
if (!ref || ref.arrayAncestor)
|
|
1052
|
+
throw this.createError("COMPUTED_FIELD_INVALID_TARGET", field.key);
|
|
1053
|
+
ctx.ownKey = field.key;
|
|
1054
|
+
const raw = parseExpression(this.computedExprOf(field), this.language, field.key);
|
|
1055
|
+
const { ast, deps } = await resolveExpression(raw, ctx);
|
|
1056
|
+
resolved.push({ field, ast, deps });
|
|
1057
|
+
}
|
|
1058
|
+
const ordered = topoSortComputedFields(resolved.map(({ field, deps }) => ({
|
|
1059
|
+
id: field.id,
|
|
1060
|
+
key: field.key,
|
|
1061
|
+
deps,
|
|
1062
|
+
})), this.language);
|
|
1063
|
+
const specByKey = new Map();
|
|
1064
|
+
for (const meta of ordered) {
|
|
1065
|
+
const entry = resolved.find(({ field }) => field.id === meta.id);
|
|
1066
|
+
if (!entry)
|
|
1067
|
+
continue;
|
|
1068
|
+
specByKey.set(meta.key, {
|
|
1069
|
+
expr: this.computedExprOf(entry.field),
|
|
1070
|
+
ast: entry.ast,
|
|
1071
|
+
});
|
|
1072
|
+
}
|
|
1073
|
+
return schema.map((field) => typeof field.computed !== "undefined"
|
|
1074
|
+
? { ...field, computed: specByKey.get(field.key) }
|
|
1075
|
+
: field);
|
|
1076
|
+
}
|
|
1077
|
+
/**
|
|
1078
|
+
* Build the evaluation plan (topological order + id index) for a table's
|
|
1079
|
+
* computed fields. Throws on cycles or unresolvable expressions.
|
|
1080
|
+
*/
|
|
1081
|
+
async buildComputedPlan(tableName, schema) {
|
|
1082
|
+
// Plans only depend on the persisted schema (compiled, id-based ASTs),
|
|
1083
|
+
// so cache per table and invalidate on schema changes. The explicit
|
|
1084
|
+
// `schema` argument (backfill during updateTable) always bypasses the
|
|
1085
|
+
// cache — the caller owns that freshly-migrated schema.
|
|
1086
|
+
if (schema === undefined) {
|
|
1087
|
+
const cached = this.computedPlanCache.get(tableName);
|
|
1088
|
+
if (cached !== undefined)
|
|
1089
|
+
return cached;
|
|
1090
|
+
}
|
|
1091
|
+
const s = schema ?? globalConfig[this.databasePath].tables?.get(tableName)?.schema;
|
|
1092
|
+
if (!s)
|
|
1093
|
+
return null;
|
|
1094
|
+
const index = buildFieldIndex(s);
|
|
1095
|
+
const fields = [];
|
|
1096
|
+
const indexCache = new Map();
|
|
1097
|
+
const ctx = {
|
|
1098
|
+
language: this.language,
|
|
1099
|
+
ownKey: "",
|
|
1100
|
+
index,
|
|
1101
|
+
getTableIndex: async (target) => {
|
|
1102
|
+
let cached = indexCache.get(target);
|
|
1103
|
+
if (!cached) {
|
|
1104
|
+
cached = await this.tableFieldIndex(target);
|
|
1105
|
+
if (cached)
|
|
1106
|
+
indexCache.set(target, cached);
|
|
1107
|
+
}
|
|
1108
|
+
return cached;
|
|
1109
|
+
},
|
|
1110
|
+
};
|
|
1111
|
+
for (const field of s) {
|
|
1112
|
+
if (typeof field.computed === "undefined")
|
|
1113
|
+
continue;
|
|
1114
|
+
const computed = field.computed;
|
|
1115
|
+
if (typeof computed === "string") {
|
|
1116
|
+
ctx.ownKey = field.key;
|
|
1117
|
+
const raw = parseExpression(computed, this.language, field.key);
|
|
1118
|
+
const { ast, deps } = await resolveExpression(raw, ctx);
|
|
1119
|
+
fields.push({
|
|
1120
|
+
id: field.id,
|
|
1121
|
+
key: field.key,
|
|
1122
|
+
ast,
|
|
1123
|
+
deps,
|
|
1124
|
+
});
|
|
1125
|
+
}
|
|
1126
|
+
else
|
|
1127
|
+
fields.push({
|
|
1128
|
+
id: field.id,
|
|
1129
|
+
key: field.key,
|
|
1130
|
+
ast: computed.ast,
|
|
1131
|
+
deps: collectFieldDeps(computed.ast),
|
|
1132
|
+
});
|
|
1133
|
+
}
|
|
1134
|
+
if (!fields.length) {
|
|
1135
|
+
if (schema === undefined)
|
|
1136
|
+
this.computedPlanCache.set(tableName, null);
|
|
1137
|
+
return null;
|
|
1138
|
+
}
|
|
1139
|
+
const ordered = topoSortComputedFields(fields.map(({ id, key, deps }) => ({ id, key, deps })), this.language);
|
|
1140
|
+
const byId = new Map(fields.map((f) => [f.id, f]));
|
|
1141
|
+
const planFields = ordered.map((meta) => byId.get(meta.id));
|
|
1142
|
+
const plan = {
|
|
1143
|
+
fields: planFields,
|
|
1144
|
+
index,
|
|
1145
|
+
hasHops: planFields.some((field) => astHasHops(field.ast)),
|
|
1146
|
+
};
|
|
1147
|
+
if (schema === undefined)
|
|
1148
|
+
this.computedPlanCache.set(tableName, plan);
|
|
1149
|
+
return plan;
|
|
1150
|
+
}
|
|
1151
|
+
/** Id index of a table's schema (link targets are re-resolved at write
|
|
1152
|
+
* time, so renames never retarget a compiled expression). */
|
|
1153
|
+
async tableFieldIndex(tableName) {
|
|
1154
|
+
const schema = await this.getTableSchema(tableName);
|
|
1155
|
+
return schema ? buildFieldIndex(schema) : undefined;
|
|
1156
|
+
}
|
|
1157
|
+
/**
|
|
1158
|
+
* Evaluate a batch of (merged) rows against the table's computed fields.
|
|
1159
|
+
* Returns `lineNo -> { computedKey -> value }` in dependency order.
|
|
1160
|
+
*
|
|
1161
|
+
* Batched link-hop reads: when the plan contains any link hop, a dry
|
|
1162
|
+
* collection pass records every (table, column, id) triple the rows'
|
|
1163
|
+
* expressions need (no file I/O), each distinct triple is then resolved
|
|
1164
|
+
* exactly once — deduplicated across rows and fields — and a final pass
|
|
1165
|
+
* evaluates against the warm cache. Plans without hops run the single
|
|
1166
|
+
* evaluation pass unchanged.
|
|
1167
|
+
*/
|
|
1168
|
+
async evaluateComputedRows(tableName, plan, rows) {
|
|
1169
|
+
const out = {};
|
|
1170
|
+
const indexCache = new Map([
|
|
1171
|
+
[tableName, plan.index],
|
|
1172
|
+
]);
|
|
1173
|
+
const entries = Object.entries(rows);
|
|
1174
|
+
const reader = plan.hasHops ? this.createLinkReader() : null;
|
|
1175
|
+
if (reader) {
|
|
1176
|
+
for (const [, row] of entries)
|
|
1177
|
+
await this.evaluateRowComputed(tableName, plan, row, indexCache, reader, true);
|
|
1178
|
+
if (reader.hasPending)
|
|
1179
|
+
await reader.resolveAll();
|
|
1180
|
+
}
|
|
1181
|
+
for (const [line, row] of entries)
|
|
1182
|
+
out[Number(line)] = await this.evaluateRowComputed(tableName, plan, row, indexCache, reader ?? undefined);
|
|
1183
|
+
return out;
|
|
1184
|
+
}
|
|
1185
|
+
createLinkReader() {
|
|
1186
|
+
return new LinkedRowReader((table, column, id) => this.readLinkedRow(table, id, column));
|
|
1187
|
+
}
|
|
1188
|
+
/** Evaluate every computed field of one row (topological order) and merge
|
|
1189
|
+
* the results back into the row so dependent fields see them. */
|
|
1190
|
+
async evaluateRowComputed(tableName, plan, row, indexCache, links, collect = false) {
|
|
1191
|
+
const out = {};
|
|
1192
|
+
// One structured frame per row, shared by every computed field: paths
|
|
1193
|
+
// resolve against the row in place (direct key-path reads), so no
|
|
1194
|
+
// flattened copy is ever built. Freshly evaluated values are written
|
|
1195
|
+
// back into the row — computed fields are top-level keys, so dependent
|
|
1196
|
+
// fields see earlier results through the same key-path reads.
|
|
1197
|
+
const env = {
|
|
1198
|
+
table: tableName,
|
|
1199
|
+
index: plan.index,
|
|
1200
|
+
frame: row,
|
|
1201
|
+
row,
|
|
1202
|
+
strip: "",
|
|
1203
|
+
ownKey: "",
|
|
1204
|
+
indexCache,
|
|
1205
|
+
links,
|
|
1206
|
+
collect,
|
|
1207
|
+
};
|
|
1208
|
+
for (const field of plan.fields) {
|
|
1209
|
+
env.strip = "";
|
|
1210
|
+
env.ownKey = field.key;
|
|
1211
|
+
const value = await this.evaluateNode(field.ast, env);
|
|
1212
|
+
out[field.key] = value;
|
|
1213
|
+
row[field.key] = value;
|
|
1214
|
+
}
|
|
1215
|
+
return out;
|
|
1216
|
+
}
|
|
1217
|
+
async evaluateNode(node, env) {
|
|
1218
|
+
switch (node.kind) {
|
|
1219
|
+
case "num":
|
|
1220
|
+
return node.value;
|
|
1221
|
+
case "path":
|
|
1222
|
+
return this.evaluatePath(node, env);
|
|
1223
|
+
case "bin": {
|
|
1224
|
+
const a = await this.evaluateNode(node.left, env);
|
|
1225
|
+
const b = await this.evaluateNode(node.right, env);
|
|
1226
|
+
return this.applyBinaryOp(node.op, a, b, env.ownKey, env.collect);
|
|
1227
|
+
}
|
|
1228
|
+
case "fn": {
|
|
1229
|
+
const arrayRef = env.index.get(node.arrayFieldId);
|
|
1230
|
+
const arrayKey = arrayRef?.key;
|
|
1231
|
+
const raw = arrayKey ? env.row[arrayKey] : undefined;
|
|
1232
|
+
// formatData turns a missing/empty array-of-objects into `{}`;
|
|
1233
|
+
// treat anything that is not an actual array as empty.
|
|
1234
|
+
const elements = Array.isArray(raw) ? raw : [];
|
|
1235
|
+
const values = [];
|
|
1236
|
+
// Reuse the env object across elements (only frame/strip
|
|
1237
|
+
// change) to avoid allocating a frame per element. Elements
|
|
1238
|
+
// are walked in place via direct key-path reads, so nothing is
|
|
1239
|
+
// flattened or cached.
|
|
1240
|
+
const savedFrame = env.frame;
|
|
1241
|
+
const savedStrip = env.strip;
|
|
1242
|
+
const stripPrefix = arrayKey ? `${arrayKey}.` : "";
|
|
1243
|
+
for (const element of elements) {
|
|
1244
|
+
if (!Utils.isObject(element))
|
|
1245
|
+
continue;
|
|
1246
|
+
env.frame = element;
|
|
1247
|
+
env.strip = stripPrefix;
|
|
1248
|
+
const value = await this.evaluateNode(node.arg, env);
|
|
1249
|
+
if (value === null || value === undefined)
|
|
1250
|
+
continue;
|
|
1251
|
+
const num = this.coerceNumber(value, env.ownKey);
|
|
1252
|
+
values.push(num);
|
|
1253
|
+
}
|
|
1254
|
+
env.frame = savedFrame;
|
|
1255
|
+
env.strip = savedStrip;
|
|
1256
|
+
return this.aggregate(node.name, values, elements.length, env.ownKey, env.collect);
|
|
1257
|
+
}
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
1260
|
+
/** Evaluate a path (`ids`, dot-separated link hops) against the current
|
|
1261
|
+
* frame. Returns `null` when an intermediate link value is missing. */
|
|
1262
|
+
async evaluatePath(node, env) {
|
|
1263
|
+
const ids = node.ids;
|
|
1264
|
+
let table = env.table;
|
|
1265
|
+
let index = env.index;
|
|
1266
|
+
let frame = env.frame;
|
|
1267
|
+
let strip = env.strip;
|
|
1268
|
+
let value;
|
|
1269
|
+
let ref;
|
|
1270
|
+
for (let i = 0; i < ids.length; i++) {
|
|
1271
|
+
ref = index.get(ids[i]);
|
|
1272
|
+
if (!ref)
|
|
1273
|
+
throw this.createError("COMPUTED_FIELD_UNKNOWN_FIELD", [
|
|
1274
|
+
env.ownKey,
|
|
1275
|
+
ids[i],
|
|
1276
|
+
]);
|
|
1277
|
+
const key = strip && ref.key.startsWith(strip)
|
|
1278
|
+
? ref.key.slice(strip.length)
|
|
1279
|
+
: ref.key;
|
|
1280
|
+
if (i === 0) {
|
|
1281
|
+
// Direct key-path read: walk the structured frame in place.
|
|
1282
|
+
value = resolveFramePath(frame, key);
|
|
1283
|
+
strip = "";
|
|
1284
|
+
}
|
|
1285
|
+
else {
|
|
1286
|
+
// `value` is the id of the row this hop lives in.
|
|
1287
|
+
if (value === undefined ||
|
|
1288
|
+
value === null ||
|
|
1289
|
+
value === "" ||
|
|
1290
|
+
(typeof value === "object" && value !== null))
|
|
1291
|
+
return null;
|
|
1292
|
+
// A transaction cannot read a dependency table it has already
|
|
1293
|
+
// staged (no read-your-writes past its commit point).
|
|
1294
|
+
if (this.transaction?.tables.has(table))
|
|
1295
|
+
throw this.createError("INVALID_PARAMETERS");
|
|
1296
|
+
// Batched link-hop reads: during the collection pass a hop
|
|
1297
|
+
// only records its (table, column, id) triple; the real pass
|
|
1298
|
+
// is served from the reader's warm cache. Without a reader
|
|
1299
|
+
// this is the plain per-hop engine read.
|
|
1300
|
+
const row = env.links
|
|
1301
|
+
? env.collect
|
|
1302
|
+
? (env.links.record(table, ref.key, value), null)
|
|
1303
|
+
: await env.links.read(table, ref.key, value)
|
|
1304
|
+
: await this.readLinkedRow(table, value, ref.key);
|
|
1305
|
+
if (!env.collect &&
|
|
1306
|
+
(row === undefined || row === null))
|
|
1307
|
+
throw this.createError("COMPUTED_FIELD_DANGLING_LINK", [
|
|
1308
|
+
env.ownKey,
|
|
1309
|
+
table,
|
|
1310
|
+
]);
|
|
1311
|
+
// The linked row is the next frame, walked in place (no
|
|
1312
|
+
// flattened copy). readLinkedRow already fetched only the
|
|
1313
|
+
// `ref.key` column, so the frame is minimal.
|
|
1314
|
+
frame = row;
|
|
1315
|
+
value = row ? resolveFramePath(frame, ref.key) : null;
|
|
1316
|
+
index = (await this.indexFor(table, env.indexCache)) ?? index;
|
|
1317
|
+
}
|
|
1318
|
+
if (i < ids.length - 1) {
|
|
1319
|
+
if (ref.field.type !== "table" || typeof ref.field.table !== "string")
|
|
1320
|
+
throw this.createError("COMPUTED_FIELD_INVALID_LINK", [
|
|
1321
|
+
env.ownKey,
|
|
1322
|
+
ids[i + 1],
|
|
1323
|
+
]);
|
|
1324
|
+
table = ref.field.table;
|
|
1325
|
+
index = (await this.indexFor(table, env.indexCache)) ?? index;
|
|
1326
|
+
}
|
|
1327
|
+
}
|
|
1328
|
+
return value === undefined ? null : value;
|
|
1329
|
+
}
|
|
1330
|
+
/** Id index lookup with a shared per-evaluation cache. */
|
|
1331
|
+
async indexFor(tableName, cache) {
|
|
1332
|
+
const cached = cache.get(tableName);
|
|
1333
|
+
if (cached)
|
|
1334
|
+
return cached;
|
|
1335
|
+
const index = await this.tableFieldIndex(tableName);
|
|
1336
|
+
if (index)
|
|
1337
|
+
cache.set(tableName, index);
|
|
1338
|
+
return index;
|
|
1339
|
+
}
|
|
1340
|
+
/** Read a single column of one linked row via `get`, or null when the
|
|
1341
|
+
* row does not exist (dangling link). */
|
|
1342
|
+
async readLinkedRow(tableName, id, column) {
|
|
1343
|
+
const row = await this.get(tableName, id, { columns: [column] }, true);
|
|
1344
|
+
return row ?? null;
|
|
1345
|
+
}
|
|
1346
|
+
coerceNumber(value, ownKey) {
|
|
1347
|
+
const num = Number(value);
|
|
1348
|
+
if (!Number.isFinite(num))
|
|
1349
|
+
throw this.createError("COMPUTED_FIELD_ARITHMETIC", [
|
|
1350
|
+
ownKey,
|
|
1351
|
+
`non-numeric value '${String(value)}'`,
|
|
1352
|
+
]);
|
|
1353
|
+
return num;
|
|
1354
|
+
}
|
|
1355
|
+
applyBinaryOp(op, a, b, ownKey, collect = false) {
|
|
1356
|
+
if (a === null || a === undefined || b === null || b === undefined) {
|
|
1357
|
+
// Collection passes must tolerate unresolved (null) link hops:
|
|
1358
|
+
// the pass only discovers which links are needed, so a neutral
|
|
1359
|
+
// result is fine. Real evaluation still throws.
|
|
1360
|
+
if (collect)
|
|
1361
|
+
return 0;
|
|
1362
|
+
throw this.createError("COMPUTED_FIELD_ARITHMETIC", [
|
|
1363
|
+
ownKey,
|
|
1364
|
+
"missing or null operand",
|
|
1365
|
+
]);
|
|
1366
|
+
}
|
|
1367
|
+
const an = this.coerceNumber(a, ownKey);
|
|
1368
|
+
const bn = this.coerceNumber(b, ownKey);
|
|
1369
|
+
switch (op) {
|
|
1370
|
+
case "add":
|
|
1371
|
+
return an + bn;
|
|
1372
|
+
case "sub":
|
|
1373
|
+
return an - bn;
|
|
1374
|
+
case "mul":
|
|
1375
|
+
return an * bn;
|
|
1376
|
+
case "div":
|
|
1377
|
+
if (bn === 0)
|
|
1378
|
+
throw this.createError("COMPUTED_FIELD_ARITHMETIC", [
|
|
1379
|
+
ownKey,
|
|
1380
|
+
"division by zero",
|
|
1381
|
+
]);
|
|
1382
|
+
return an / bn;
|
|
1383
|
+
case "mod":
|
|
1384
|
+
if (bn === 0)
|
|
1385
|
+
throw this.createError("COMPUTED_FIELD_ARITHMETIC", [
|
|
1386
|
+
ownKey,
|
|
1387
|
+
"modulo by zero",
|
|
1388
|
+
]);
|
|
1389
|
+
return an % bn;
|
|
1390
|
+
}
|
|
1391
|
+
}
|
|
1392
|
+
aggregate(name, values, elementCount, ownKey, collect = false) {
|
|
1393
|
+
switch (name) {
|
|
1394
|
+
case "count":
|
|
1395
|
+
// Aggregates iterate the row's actual array, so count is the
|
|
1396
|
+
// element count (regardless of value presence).
|
|
1397
|
+
return elementCount;
|
|
1398
|
+
case "sum":
|
|
1399
|
+
return values.reduce((acc, v) => acc + v, 0);
|
|
1400
|
+
case "avg": {
|
|
1401
|
+
if (!values.length) {
|
|
1402
|
+
// Collection passes may have skipped every value of an
|
|
1403
|
+
// array whose elements only held unresolved link hops.
|
|
1404
|
+
if (collect)
|
|
1405
|
+
return 0;
|
|
1406
|
+
throw this.createError("COMPUTED_FIELD_ARITHMETIC", [
|
|
1407
|
+
ownKey,
|
|
1408
|
+
"average over no values",
|
|
1409
|
+
]);
|
|
1410
|
+
}
|
|
1411
|
+
return values.reduce((acc, v) => acc + v, 0) / values.length;
|
|
1412
|
+
}
|
|
1413
|
+
case "min": {
|
|
1414
|
+
if (!values.length) {
|
|
1415
|
+
if (collect)
|
|
1416
|
+
return 0;
|
|
1417
|
+
throw this.createError("COMPUTED_FIELD_ARITHMETIC", [
|
|
1418
|
+
ownKey,
|
|
1419
|
+
"min over no values",
|
|
1420
|
+
]);
|
|
1421
|
+
}
|
|
1422
|
+
return Math.min(...values);
|
|
1423
|
+
}
|
|
1424
|
+
case "max": {
|
|
1425
|
+
if (!values.length) {
|
|
1426
|
+
if (collect)
|
|
1427
|
+
return 0;
|
|
1428
|
+
throw this.createError("COMPUTED_FIELD_ARITHMETIC", [
|
|
1429
|
+
ownKey,
|
|
1430
|
+
"max over no values",
|
|
1431
|
+
]);
|
|
1432
|
+
}
|
|
1433
|
+
return Math.max(...values);
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
}
|
|
1437
|
+
/** Merge evaluated per-line computed values into a `pathesContents` map
|
|
1438
|
+
* as line-numbered replace records (encoded cells). */
|
|
1439
|
+
mergeComputedLineRecords(tableName, plan, evaluated, pathesContents, existing) {
|
|
1440
|
+
for (const field of plan.fields) {
|
|
1441
|
+
const path = join(this.databasePath, tableName, `${field.key}${this.getFileExtension(tableName)}`);
|
|
1442
|
+
const entries = Object.entries(evaluated);
|
|
1443
|
+
// When the pre-update rows are available, compare each column's
|
|
1444
|
+
// re-evaluated line values against the stored ones (deterministic
|
|
1445
|
+
// encode). A column that is identical for every evaluated line is
|
|
1446
|
+
// left untouched — no content buffer, no File.replace, no fsync.
|
|
1447
|
+
if (existing) {
|
|
1448
|
+
let matches = true;
|
|
1449
|
+
for (const [line, values] of entries) {
|
|
1450
|
+
const old = existing[Number(line)]?.[field.key];
|
|
1451
|
+
const fresh = File.encode(values[field.key]);
|
|
1452
|
+
if (old === undefined || File.encode(old) !== fresh) {
|
|
1453
|
+
matches = false;
|
|
1454
|
+
break;
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
if (matches)
|
|
1458
|
+
continue;
|
|
1459
|
+
}
|
|
1460
|
+
const content = pathesContents[path] ?? {};
|
|
1461
|
+
for (const [line, values] of entries)
|
|
1462
|
+
content[Number(line)] = File.encode(values[field.key]);
|
|
1463
|
+
pathesContents[path] = content;
|
|
1464
|
+
}
|
|
1465
|
+
}
|
|
792
1466
|
// Helper function to determine if a field is simple
|
|
793
1467
|
isSimpleField(fieldType) {
|
|
794
1468
|
const complexTypes = ["array", "object", "table"];
|
|
@@ -1264,11 +1938,387 @@ export default class Inibase {
|
|
|
1264
1938
|
await rm(cacheFolderPath, { recursive: true, force: true });
|
|
1265
1939
|
await mkdir(cacheFolderPath);
|
|
1266
1940
|
}
|
|
1941
|
+
/**
|
|
1942
|
+
* Commit a multi-file mutation crash-atomically:
|
|
1943
|
+
* 1. fsync every freshly-written temp file;
|
|
1944
|
+
* 2. write the journal `begin` entry and fsync it;
|
|
1945
|
+
* 3. rename the pagination metadata file first (atomic publication point:
|
|
1946
|
+
* the row count flips in a single rename, which is what lock-free
|
|
1947
|
+
* readers observe) and then swap each live file aside (backup) and
|
|
1948
|
+
* rename the temp into place;
|
|
1949
|
+
* 4. write the journal `commit` marker and fsync it;
|
|
1950
|
+
* 5. discard backups/temps + the journal, fsync the directories.
|
|
1951
|
+
*
|
|
1952
|
+
* On any failure before `commit`, the journal is rolled back so the table
|
|
1953
|
+
* is left exactly as it was. `renameList` entries are [tempPath, livePath]
|
|
1954
|
+
* pairs; a null tempPath means a pure removal (live file taken out).
|
|
1955
|
+
*/
|
|
1956
|
+
async commitFiles(tablePath, renameList, pagination) {
|
|
1957
|
+
const txn = randomUUID();
|
|
1958
|
+
const ops = [];
|
|
1959
|
+
// Backups are parked in a per-transaction subdirectory
|
|
1960
|
+
// (.tmp/backup/<txn>/), so a later transaction can never collide with
|
|
1961
|
+
// the leftovers of an earlier crashed one.
|
|
1962
|
+
const backupDir = join(tablePath, ".tmp", "backup", txn);
|
|
1963
|
+
await mkdir(backupDir, { recursive: true });
|
|
1964
|
+
for (const [tmp, live] of renameList) {
|
|
1965
|
+
if (!live)
|
|
1966
|
+
continue;
|
|
1967
|
+
ops.push({
|
|
1968
|
+
live,
|
|
1969
|
+
backup: join(backupDir, basename(live)),
|
|
1970
|
+
tmp,
|
|
1971
|
+
existed: await File.isExists(live),
|
|
1972
|
+
});
|
|
1973
|
+
}
|
|
1974
|
+
const journal = new Journal(tablePath, txn);
|
|
1975
|
+
try {
|
|
1976
|
+
// Make every replacement durable before it can be published.
|
|
1977
|
+
await Promise.allSettled(ops
|
|
1978
|
+
.filter((op) => op.tmp)
|
|
1979
|
+
.map(async (op) => File.syncFile(op.tmp)));
|
|
1980
|
+
await journal.begin(ops, pagination);
|
|
1981
|
+
// Publish: the pagination rename is the atomic publication point
|
|
1982
|
+
// (row count flips in one rename) and MUST come first — readers
|
|
1983
|
+
// that snapshot file identities detect the flip and retry. Then
|
|
1984
|
+
// park each original and move the replacement in.
|
|
1985
|
+
if (pagination)
|
|
1986
|
+
await rename(pagination.from, pagination.to);
|
|
1987
|
+
for (const op of ops) {
|
|
1988
|
+
if (op.existed)
|
|
1989
|
+
await rename(op.live, op.backup);
|
|
1990
|
+
if (op.tmp)
|
|
1991
|
+
await rename(op.tmp, op.live);
|
|
1992
|
+
}
|
|
1993
|
+
await journal.commit();
|
|
1994
|
+
}
|
|
1995
|
+
catch (error) {
|
|
1996
|
+
await journal.rollback().catch(() => { });
|
|
1997
|
+
throw error;
|
|
1998
|
+
}
|
|
1999
|
+
finally {
|
|
2000
|
+
await Promise.allSettled(ops.map((op) => unlink(op.backup).catch(() => { })));
|
|
2001
|
+
await rm(backupDir, { recursive: true, force: true }).catch(() => { });
|
|
2002
|
+
await journal.dispose();
|
|
2003
|
+
await unlink(journal.path).catch(() => { });
|
|
2004
|
+
// Make the renames durable before acknowledging the commit.
|
|
2005
|
+
await File.syncDir(join(tablePath, ".tmp"));
|
|
2006
|
+
await File.syncDir(tablePath);
|
|
2007
|
+
}
|
|
2008
|
+
}
|
|
2009
|
+
/**
|
|
2010
|
+
* Runs crash recovery for a table (and any crashed database transaction)
|
|
2011
|
+
* before a read. Mutation paths get the same guarantee implicitly (the
|
|
2012
|
+
* writer lock runs recovery on acquire); reads call this explicitly
|
|
2013
|
+
* because they never take the table lock.
|
|
2014
|
+
*/
|
|
2015
|
+
async ensureTableRecovered(tableName) {
|
|
2016
|
+
const tablePath = join(this.databasePath, tableName);
|
|
2017
|
+
if (await File.isExists(join(tablePath, ".tmp", "journal.jsonl"))) {
|
|
2018
|
+
await File.lock(join(tablePath, ".tmp"));
|
|
2019
|
+
await File.unlock(join(tablePath, ".tmp"));
|
|
2020
|
+
}
|
|
2021
|
+
// A database journal exists only while a live transaction holds the
|
|
2022
|
+
// database lock, or after one crashed. Recovery must never roll back a
|
|
2023
|
+
// live transaction, so acquire the database lock non-blocking here:
|
|
2024
|
+
// on success the previous owner is gone (recovery ran); on failure a
|
|
2025
|
+
// live transaction owns the lock across processes and readers simply
|
|
2026
|
+
// proceed against the committed state.
|
|
2027
|
+
const dbTmp = join(this.databasePath, ".tmp");
|
|
2028
|
+
if (await File.isExists(join(dbTmp, "journal.jsonl"))) {
|
|
2029
|
+
if (await File.tryLock(dbTmp))
|
|
2030
|
+
await File.unlock(dbTmp);
|
|
2031
|
+
}
|
|
2032
|
+
}
|
|
2033
|
+
/**
|
|
2034
|
+
* Blocking database-journal recovery, used by mutation paths (writers are
|
|
2035
|
+
* serialized with live transactions on the database lock anyway).
|
|
2036
|
+
*/
|
|
2037
|
+
async ensureDatabaseRecovered() {
|
|
2038
|
+
const dbTmp = join(this.databasePath, ".tmp");
|
|
2039
|
+
if (await File.isExists(join(dbTmp, "journal.jsonl"))) {
|
|
2040
|
+
await File.lock(dbTmp);
|
|
2041
|
+
await File.unlock(dbTmp);
|
|
2042
|
+
}
|
|
2043
|
+
}
|
|
2044
|
+
async ensureDatabaseTmpDir() {
|
|
2045
|
+
await mkdir(join(this.databasePath, ".tmp"), { recursive: true });
|
|
2046
|
+
}
|
|
2047
|
+
/** Staged per-table entry of the open transaction, or null when none. */
|
|
2048
|
+
txnTableEntry(tableName) {
|
|
2049
|
+
const txn = this.transaction;
|
|
2050
|
+
if (!txn)
|
|
2051
|
+
return null;
|
|
2052
|
+
let entry = txn.tables.get(tableName);
|
|
2053
|
+
if (!entry) {
|
|
2054
|
+
entry = {
|
|
2055
|
+
locked: false,
|
|
2056
|
+
paginationFrom: "",
|
|
2057
|
+
lastId: 0,
|
|
2058
|
+
total: 0,
|
|
2059
|
+
staged: [],
|
|
2060
|
+
};
|
|
2061
|
+
txn.tables.set(tableName, entry);
|
|
2062
|
+
}
|
|
2063
|
+
return entry;
|
|
2064
|
+
}
|
|
2065
|
+
/** Lock a table for the open transaction (idempotent per transaction). */
|
|
2066
|
+
async ensureTxnLock(tableName) {
|
|
2067
|
+
const entry = this.txnTableEntry(tableName);
|
|
2068
|
+
if (!entry || entry.locked)
|
|
2069
|
+
return;
|
|
2070
|
+
await File.lock(join(this.databasePath, tableName, ".tmp"));
|
|
2071
|
+
entry.locked = true;
|
|
2072
|
+
}
|
|
2073
|
+
/**
|
|
2074
|
+
* Resolve the pagination state a DML op should build on. Outside a
|
|
2075
|
+
* transaction this reads the live pagination file (as before); inside a
|
|
2076
|
+
* transaction the first touch reads it once and the entry keeps the staged
|
|
2077
|
+
* id/count so chained ops (guarded to one per table) and commit() stay
|
|
2078
|
+
* consistent without publishing anything early.
|
|
2079
|
+
*/
|
|
2080
|
+
async resolvePagination(tableName) {
|
|
2081
|
+
const tablePath = join(this.databasePath, tableName);
|
|
2082
|
+
const entry = this.txnTableEntry(tableName);
|
|
2083
|
+
if (entry?.paginationFrom) {
|
|
2084
|
+
return {
|
|
2085
|
+
filePath: entry.paginationFrom,
|
|
2086
|
+
lastId: entry.lastId,
|
|
2087
|
+
total: entry.total,
|
|
2088
|
+
};
|
|
2089
|
+
}
|
|
2090
|
+
let paginationFilePath = "";
|
|
2091
|
+
for await (const fileName of glob("*.pagination", { cwd: tablePath }))
|
|
2092
|
+
paginationFilePath = join(tablePath, fileName);
|
|
2093
|
+
const [lastId, total] = parse(paginationFilePath)
|
|
2094
|
+
.name.split("-")
|
|
2095
|
+
.map(Number);
|
|
2096
|
+
if (entry) {
|
|
2097
|
+
entry.paginationFrom = paginationFilePath;
|
|
2098
|
+
entry.lastId = lastId;
|
|
2099
|
+
entry.total = total;
|
|
2100
|
+
}
|
|
2101
|
+
return { filePath: paginationFilePath, lastId, total };
|
|
2102
|
+
}
|
|
2103
|
+
/**
|
|
2104
|
+
* Stage one table mutation into the open transaction: fsync its temps and
|
|
2105
|
+
* append an `op` entry to the database journal (no live file is touched;
|
|
2106
|
+
* commit() performs the actual renames). One staged mutation per table per
|
|
2107
|
+
* transaction (multi-table atomicity; a second touch of the same table
|
|
2108
|
+
* would need read-your-writes composition).
|
|
2109
|
+
*/
|
|
2110
|
+
async stageTxnOp(tableName, renameList, pagination) {
|
|
2111
|
+
const txn = this.transaction;
|
|
2112
|
+
const entry = this.txnTableEntry(tableName);
|
|
2113
|
+
if (!txn || !entry)
|
|
2114
|
+
throw this.createError("INVALID_PARAMETERS");
|
|
2115
|
+
if (entry.staged.length)
|
|
2116
|
+
throw this.createError("INVALID_PARAMETERS");
|
|
2117
|
+
const backupDir = join(this.databasePath, ".tmp", "backup", txn.id);
|
|
2118
|
+
await mkdir(backupDir, { recursive: true });
|
|
2119
|
+
const ops = [];
|
|
2120
|
+
for (const [tmp, live] of renameList) {
|
|
2121
|
+
if (!live)
|
|
2122
|
+
continue;
|
|
2123
|
+
// Benchmarks must be unique within the whole transaction: the same
|
|
2124
|
+
// column basename can exist in several tables, and rollback
|
|
2125
|
+
// restores by path. Namespace per table (one staged op per table).
|
|
2126
|
+
ops.push({
|
|
2127
|
+
live,
|
|
2128
|
+
backup: join(backupDir, `${entry.staged.length}-${tableName}-${basename(live)}`),
|
|
2129
|
+
tmp,
|
|
2130
|
+
existed: await File.isExists(live),
|
|
2131
|
+
});
|
|
2132
|
+
}
|
|
2133
|
+
// Make every replacement durable before the journal records intent.
|
|
2134
|
+
await Promise.allSettled(ops
|
|
2135
|
+
.filter((op) => op.tmp)
|
|
2136
|
+
.map(async (op) => File.syncFile(op.tmp)));
|
|
2137
|
+
await txn.journal.op(ops, pagination);
|
|
2138
|
+
entry.staged.push({ ops, pagination });
|
|
2139
|
+
if (pagination)
|
|
2140
|
+
entry.paginationFrom = pagination.to;
|
|
2141
|
+
}
|
|
2142
|
+
/**
|
|
2143
|
+
* Begin a database transaction. Mutations issued while the transaction is
|
|
2144
|
+
* open (post/put/delete, including cascade deletes) are staged into the
|
|
2145
|
+
* database journal and published atomically at commit(); rollback()
|
|
2146
|
+
* discards them without touching any live file.
|
|
2147
|
+
*
|
|
2148
|
+
* @param tables Optional table names to pre-lock at begin() in sorted
|
|
2149
|
+
* order (the deadlock-free way to span tables). Tables not listed are
|
|
2150
|
+
* locked on first touch, in first-touch order.
|
|
2151
|
+
*/
|
|
2152
|
+
async begin(tables = []) {
|
|
2153
|
+
if (this.transaction)
|
|
2154
|
+
throw this.createError("INVALID_PARAMETERS");
|
|
2155
|
+
await this.ensureDatabaseTmpDir();
|
|
2156
|
+
// The database lock is the transaction mutex: it serializes
|
|
2157
|
+
// transactions and its acquisition runs crash recovery on any journal
|
|
2158
|
+
// left behind by a crashed transaction.
|
|
2159
|
+
await File.lock(join(this.databasePath, ".tmp"));
|
|
2160
|
+
const uniqueTables = [...new Set(tables)].sort();
|
|
2161
|
+
const acquired = [];
|
|
2162
|
+
try {
|
|
2163
|
+
// Validate every listed table before locking anything.
|
|
2164
|
+
for (const name of uniqueTables) {
|
|
2165
|
+
this.validateName(name);
|
|
2166
|
+
await this.getTable(name); // throws TABLE_NOT_EXISTS
|
|
2167
|
+
}
|
|
2168
|
+
const id = randomUUID();
|
|
2169
|
+
this.transaction = {
|
|
2170
|
+
id,
|
|
2171
|
+
journal: new DatabaseJournal(this.databasePath, id),
|
|
2172
|
+
tables: new Map(),
|
|
2173
|
+
};
|
|
2174
|
+
await this.transaction.journal.begin(uniqueTables);
|
|
2175
|
+
for (const name of uniqueTables) {
|
|
2176
|
+
await File.lock(join(this.databasePath, name, ".tmp"));
|
|
2177
|
+
acquired.push(name);
|
|
2178
|
+
this.transaction.tables.set(name, {
|
|
2179
|
+
locked: true,
|
|
2180
|
+
paginationFrom: "",
|
|
2181
|
+
lastId: 0,
|
|
2182
|
+
total: 0,
|
|
2183
|
+
staged: [],
|
|
2184
|
+
});
|
|
2185
|
+
}
|
|
2186
|
+
}
|
|
2187
|
+
catch (error) {
|
|
2188
|
+
// Release only the table locks this process actually took (an
|
|
2189
|
+
// unlock of a never-acquired path could unlink another process's
|
|
2190
|
+
// lock file).
|
|
2191
|
+
for (const name of acquired)
|
|
2192
|
+
await File.unlock(join(this.databasePath, name, ".tmp")).catch(() => { });
|
|
2193
|
+
this.transaction = null;
|
|
2194
|
+
await File.unlock(join(this.databasePath, ".tmp")).catch(() => { });
|
|
2195
|
+
throw error;
|
|
2196
|
+
}
|
|
2197
|
+
}
|
|
2198
|
+
/**
|
|
2199
|
+
* Publish every staged mutation atomically: per table (sorted), the
|
|
2200
|
+
* pagination rename comes first (the atomic publication point readers
|
|
2201
|
+
* observe) and then live->backup + tmp->live swaps, before a single fsynced
|
|
2202
|
+
* `commit` marker makes the whole transaction durable. A crash at any
|
|
2203
|
+
* point is recovered by the journal rule (no marker -> roll back all
|
|
2204
|
+
* tables, marker -> roll forward all tables).
|
|
2205
|
+
*/
|
|
2206
|
+
async commit() {
|
|
2207
|
+
const txn = this.transaction;
|
|
2208
|
+
if (!txn)
|
|
2209
|
+
throw this.createError("INVALID_PARAMETERS");
|
|
2210
|
+
try {
|
|
2211
|
+
for (const tableName of [...txn.tables.keys()].sort()) {
|
|
2212
|
+
const entry = txn.tables.get(tableName);
|
|
2213
|
+
if (!entry)
|
|
2214
|
+
continue;
|
|
2215
|
+
for (const { ops, pagination } of entry.staged) {
|
|
2216
|
+
if (pagination)
|
|
2217
|
+
await rename(pagination.from, pagination.to);
|
|
2218
|
+
for (const op of ops) {
|
|
2219
|
+
if (op.existed)
|
|
2220
|
+
await rename(op.live, op.backup);
|
|
2221
|
+
if (op.tmp)
|
|
2222
|
+
await rename(op.tmp, op.live);
|
|
2223
|
+
}
|
|
2224
|
+
}
|
|
2225
|
+
}
|
|
2226
|
+
await txn.journal.commit();
|
|
2227
|
+
// Clean the fast-path leftovers; recovery owns any crash leftovers.
|
|
2228
|
+
await txn.journal.dispose();
|
|
2229
|
+
await unlink(txn.journal.path).catch(() => { });
|
|
2230
|
+
await rm(join(this.databasePath, ".tmp", "backup", txn.id), {
|
|
2231
|
+
recursive: true,
|
|
2232
|
+
force: true,
|
|
2233
|
+
}).catch(() => { });
|
|
2234
|
+
await File.syncDir(join(this.databasePath, ".tmp"));
|
|
2235
|
+
await File.syncDir(this.databasePath);
|
|
2236
|
+
for (const tableName of txn.tables.keys()) {
|
|
2237
|
+
await File.syncDir(join(this.databasePath, tableName));
|
|
2238
|
+
await File.syncDir(join(this.databasePath, tableName, ".tmp"));
|
|
2239
|
+
}
|
|
2240
|
+
}
|
|
2241
|
+
catch (error) {
|
|
2242
|
+
await txn.journal.rollback().catch(() => { });
|
|
2243
|
+
throw error;
|
|
2244
|
+
}
|
|
2245
|
+
finally {
|
|
2246
|
+
for (const tableName of [...txn.tables.keys()].sort().reverse())
|
|
2247
|
+
await File.unlock(join(this.databasePath, tableName, ".tmp"));
|
|
2248
|
+
await File.unlock(join(this.databasePath, ".tmp"));
|
|
2249
|
+
this.transaction = null;
|
|
2250
|
+
}
|
|
2251
|
+
}
|
|
2252
|
+
/**
|
|
2253
|
+
* Discard the open transaction: temps and the journal are removed and no
|
|
2254
|
+
* live file is touched (nothing is published before commit()).
|
|
2255
|
+
*/
|
|
2256
|
+
async rollback() {
|
|
2257
|
+
const txn = this.transaction;
|
|
2258
|
+
if (!txn)
|
|
2259
|
+
throw this.createError("INVALID_PARAMETERS");
|
|
2260
|
+
try {
|
|
2261
|
+
await txn.journal.rollback().catch(() => { });
|
|
2262
|
+
}
|
|
2263
|
+
finally {
|
|
2264
|
+
for (const tableName of [...txn.tables.keys()].sort().reverse())
|
|
2265
|
+
await File.unlock(join(this.databasePath, tableName, ".tmp"));
|
|
2266
|
+
await File.unlock(join(this.databasePath, ".tmp"));
|
|
2267
|
+
this.transaction = null;
|
|
2268
|
+
}
|
|
2269
|
+
}
|
|
2270
|
+
/**
|
|
2271
|
+
* Snapshot the identity (dev:inode:mtime:size) of every column file and
|
|
2272
|
+
* the pagination file. Reading data and then re-verifying this snapshot
|
|
2273
|
+
* lets lock-free readers detect an in-flight writer commit and retry
|
|
2274
|
+
* instead of returning a torn row set.
|
|
2275
|
+
*/
|
|
2276
|
+
async snapshotTableFiles(tableName) {
|
|
2277
|
+
const tablePath = join(this.databasePath, tableName);
|
|
2278
|
+
const extension = this.getFileExtension(tableName);
|
|
2279
|
+
const snapshot = new Map();
|
|
2280
|
+
for (const fileName of await readdir(tablePath).catch(() => [])) {
|
|
2281
|
+
if (!fileName.endsWith(extension) && !fileName.endsWith(".pagination"))
|
|
2282
|
+
continue;
|
|
2283
|
+
const filePath = join(tablePath, fileName);
|
|
2284
|
+
const fileStat = await stat(filePath).catch(() => null);
|
|
2285
|
+
if (fileStat)
|
|
2286
|
+
snapshot.set(filePath, `${fileStat.dev}:${fileStat.ino}:${fileStat.mtimeMs}:${fileStat.size}`);
|
|
2287
|
+
}
|
|
2288
|
+
return snapshot;
|
|
2289
|
+
}
|
|
2290
|
+
/** True when every snapshotted file is still present and unchanged. */
|
|
2291
|
+
async verifyTableFiles(snapshot) {
|
|
2292
|
+
for (const [filePath, identity] of snapshot) {
|
|
2293
|
+
const fileStat = await stat(filePath).catch(() => null);
|
|
2294
|
+
if (!fileStat ||
|
|
2295
|
+
`${fileStat.dev}:${fileStat.ino}:${fileStat.mtimeMs}:${fileStat.size}` !==
|
|
2296
|
+
identity)
|
|
2297
|
+
return false;
|
|
2298
|
+
}
|
|
2299
|
+
return true;
|
|
2300
|
+
}
|
|
1267
2301
|
async get(tableName, where, options = {
|
|
1268
2302
|
page: 1,
|
|
1269
2303
|
perPage: 15,
|
|
1270
2304
|
}, onlyOne, onlyLinesNumbers, _whereIsLinesNumbers) {
|
|
1271
2305
|
this.validateName(tableName);
|
|
2306
|
+
await this.ensureTableRecovered(tableName);
|
|
2307
|
+
// Lock-free reads with optimistic retry: snapshot the identity of every
|
|
2308
|
+
// column + pagination file, run the read, then verify nothing changed
|
|
2309
|
+
// mid-scan. A writer commit flips at least one file identity, so a torn
|
|
2310
|
+
// read is detected and re-run instead of being returned.
|
|
2311
|
+
for (let attempt = 0;; attempt++) {
|
|
2312
|
+
const snapshot = await this.snapshotTableFiles(tableName);
|
|
2313
|
+
const result = await this.getOnce(tableName, where, options, onlyOne, onlyLinesNumbers, _whereIsLinesNumbers);
|
|
2314
|
+
if (attempt === 2 || (await this.verifyTableFiles(snapshot)))
|
|
2315
|
+
return result;
|
|
2316
|
+
}
|
|
2317
|
+
}
|
|
2318
|
+
async getOnce(tableName, where, options = {
|
|
2319
|
+
page: 1,
|
|
2320
|
+
perPage: 15,
|
|
2321
|
+
}, onlyOne, onlyLinesNumbers, _whereIsLinesNumbers) {
|
|
1272
2322
|
const tablePath = join(this.databasePath, tableName);
|
|
1273
2323
|
// Ensure options.columns is an array
|
|
1274
2324
|
if (options.columns) {
|
|
@@ -1316,9 +2366,11 @@ export default class Inibase {
|
|
|
1316
2366
|
.concat(options.sort)
|
|
1317
2367
|
.map((column) => [column, true]);
|
|
1318
2368
|
let cacheKey = "";
|
|
1319
|
-
// Criteria
|
|
2369
|
+
// Criteria. The sort cache is versioned by the pagination row count
|
|
2370
|
+
// (see the criteria-cache note) so stale sorted line numbers from
|
|
2371
|
+
// before a post/delete are never replayed.
|
|
1320
2372
|
if (globalConfig[this.databasePath].tables?.get(tableName)?.config.cache)
|
|
1321
|
-
cacheKey = UtilsServer.hashString(inspect(sortArray, { sorted: true }));
|
|
2373
|
+
cacheKey = UtilsServer.hashString(inspect([sortArray, pagination[1]], { sorted: true }));
|
|
1322
2374
|
if (where) {
|
|
1323
2375
|
const lineNumbers = await this.get(tableName, where, undefined, undefined, true);
|
|
1324
2376
|
if (!lineNumbers?.length)
|
|
@@ -1507,7 +2559,11 @@ export default class Inibase {
|
|
|
1507
2559
|
let cachedFilePath = "";
|
|
1508
2560
|
// Criteria
|
|
1509
2561
|
if (globalConfig[this.databasePath].tables?.get(tableName)?.config.cache) {
|
|
1510
|
-
|
|
2562
|
+
// Cache entries are versioned by the pagination row count so a
|
|
2563
|
+
// stale cache written before a post/delete (in this process or
|
|
2564
|
+
// another) is detectable: the candidate filename simply stops
|
|
2565
|
+
// matching and the cache is rebuilt.
|
|
2566
|
+
cachedFilePath = join(tablePath, ".cache", `${UtilsServer.hashString(inspect(where, { sorted: true }))}-${pagination[1]}${this.fileExtension}`);
|
|
1511
2567
|
if (await File.isExists(cachedFilePath)) {
|
|
1512
2568
|
const cachedItems = (await readFile(cachedFilePath, "utf8")).split(",");
|
|
1513
2569
|
if (!this.totalItems.has(`${tableName}-*`))
|
|
@@ -1571,37 +2627,55 @@ export default class Inibase {
|
|
|
1571
2627
|
? options.columns
|
|
1572
2628
|
: [options.columns]));
|
|
1573
2629
|
const tablePath = join(this.databasePath, tableName);
|
|
2630
|
+
if (!this.transaction)
|
|
2631
|
+
await this.ensureDatabaseRecovered();
|
|
1574
2632
|
await this.getTable(tableName);
|
|
1575
2633
|
if (!globalConfig[this.databasePath].tables?.get(tableName)?.schema)
|
|
1576
2634
|
throw this.createError("NO_SCHEMA", tableName);
|
|
1577
2635
|
if (!returnPostedData)
|
|
1578
2636
|
returnPostedData = false;
|
|
1579
2637
|
let clonedData = structuredClone(data);
|
|
1580
|
-
const keys = UtilsServer.hashString(Object.keys(Array.isArray(clonedData) ? clonedData[0] : clonedData).join("."));
|
|
1581
2638
|
await this.validateTableData(tableName, clonedData);
|
|
1582
2639
|
const renameList = [];
|
|
2640
|
+
let txnStaged = false;
|
|
1583
2641
|
try {
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
.
|
|
1590
|
-
|
|
1591
|
-
|
|
2642
|
+
// Inside a transaction the table lock is taken once per txn (and
|
|
2643
|
+
// held until commit/rollback); otherwise the usual writer lock.
|
|
2644
|
+
if (this.transaction)
|
|
2645
|
+
await this.ensureTxnLock(tableName);
|
|
2646
|
+
else
|
|
2647
|
+
await File.lock(join(tablePath, ".tmp"));
|
|
2648
|
+
const { filePath: paginationFilePath, lastId, total: _totalItems, } = await this.resolvePagination(tableName);
|
|
2649
|
+
let lastIdValue = lastId;
|
|
2650
|
+
if (!this.transaction)
|
|
2651
|
+
this.totalItems.set(`${tableName}-*`, _totalItems);
|
|
1592
2652
|
if (Utils.isArrayOfObjects(clonedData))
|
|
1593
2653
|
for (let index = 0; index < clonedData.length; index++) {
|
|
1594
2654
|
const element = clonedData[index];
|
|
1595
|
-
element.id = ++
|
|
2655
|
+
element.id = ++lastIdValue;
|
|
1596
2656
|
element.createdAt = Date.now();
|
|
1597
2657
|
element.updatedAt = undefined;
|
|
1598
2658
|
}
|
|
1599
2659
|
else {
|
|
1600
|
-
clonedData.id = ++
|
|
2660
|
+
clonedData.id = ++lastIdValue;
|
|
1601
2661
|
clonedData.createdAt = Date.now();
|
|
1602
2662
|
clonedData.updatedAt = undefined;
|
|
1603
2663
|
}
|
|
1604
2664
|
clonedData = this.formatData(clonedData, globalConfig[this.databasePath].tables?.get(tableName)?.schema ?? [], false);
|
|
2665
|
+
// Derived columns: evaluate every computed expression against the
|
|
2666
|
+
// final formatted row (the row is pieced back in place so helpers
|
|
2667
|
+
// iterate the real array values and dependent fields see each
|
|
2668
|
+
// other). Evaluation happens before any file is touched.
|
|
2669
|
+
const computedPlan = await this.buildComputedPlan(tableName);
|
|
2670
|
+
if (computedPlan) {
|
|
2671
|
+
const rows = Array.isArray(clonedData)
|
|
2672
|
+
? clonedData
|
|
2673
|
+
: [clonedData];
|
|
2674
|
+
// Evaluate every row against the shared plan in one batch:
|
|
2675
|
+
// link hops are read once per distinct (table, column, id)
|
|
2676
|
+
// across the whole post instead of once per hop.
|
|
2677
|
+
await this.evaluateComputedRows(tableName, computedPlan, Object.fromEntries(rows.map((row, index) => [index, row])));
|
|
2678
|
+
}
|
|
1605
2679
|
const pathesContents = this.joinPathesContents(tableName, globalConfig[this.databasePath].tables?.get(tableName)?.config.prepend
|
|
1606
2680
|
? Array.isArray(clonedData)
|
|
1607
2681
|
? clonedData.toReversed()
|
|
@@ -1611,15 +2685,34 @@ export default class Inibase {
|
|
|
1611
2685
|
.prepend
|
|
1612
2686
|
? await File.prepend(path, content)
|
|
1613
2687
|
: await File.append(path, content))));
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
2688
|
+
const newTotal = _totalItems + (Array.isArray(data) ? data.length : 1);
|
|
2689
|
+
const pagination = {
|
|
2690
|
+
from: paginationFilePath,
|
|
2691
|
+
to: join(tablePath, `${lastIdValue}-${newTotal}.pagination`),
|
|
2692
|
+
};
|
|
2693
|
+
if (this.transaction) {
|
|
2694
|
+
// Stage: journal the intent (fsynced op entry); the live files
|
|
2695
|
+
// only change when commit() publishes.
|
|
2696
|
+
await this.stageTxnOp(tableName, renameList, pagination);
|
|
2697
|
+
txnStaged = true;
|
|
2698
|
+
const stagedEntry = this.txnTableEntry(tableName);
|
|
2699
|
+
if (stagedEntry) {
|
|
2700
|
+
stagedEntry.lastId = lastIdValue;
|
|
2701
|
+
stagedEntry.total = newTotal;
|
|
2702
|
+
}
|
|
2703
|
+
}
|
|
2704
|
+
else {
|
|
2705
|
+
// Crash-atomic commit: journal + backup swap + pagination rename.
|
|
2706
|
+
await this.commitFiles(tablePath, renameList, pagination);
|
|
2707
|
+
this.totalItems.set(`${tableName}-*`, newTotal);
|
|
2708
|
+
if (globalConfig[this.databasePath].tables?.get(tableName)?.config.cache)
|
|
2709
|
+
await this.clearCache(tableName);
|
|
2710
|
+
}
|
|
2711
|
+
if (returnPostedData) {
|
|
2712
|
+
if (this.transaction)
|
|
2713
|
+
// No read-your-writes yet: return the formatted staged rows
|
|
2714
|
+
// (ids + defaults) instead of a committed-state read.
|
|
2715
|
+
return (Array.isArray(clonedData) ? clonedData : clonedData);
|
|
1623
2716
|
return this.get(tableName, globalConfig[this.databasePath].tables?.get(tableName)?.config.prepend
|
|
1624
2717
|
? Array.isArray(clonedData)
|
|
1625
2718
|
? clonedData.map((_, index) => index + 1).toReversed()
|
|
@@ -1630,6 +2723,7 @@ export default class Inibase {
|
|
|
1630
2723
|
.toReversed()
|
|
1631
2724
|
: this.totalItems.get(`${tableName}-*`), options, !Utils.isArrayOfObjects(clonedData), // return only one item if data is not array of objects
|
|
1632
2725
|
undefined, true);
|
|
2726
|
+
}
|
|
1633
2727
|
return Array.isArray(clonedData)
|
|
1634
2728
|
? (globalConfig[this.databasePath].tables?.get(tableName)?.config
|
|
1635
2729
|
.prepend
|
|
@@ -1638,11 +2732,22 @@ export default class Inibase {
|
|
|
1638
2732
|
: UtilsServer.encodeID(clonedData.id);
|
|
1639
2733
|
}
|
|
1640
2734
|
finally {
|
|
1641
|
-
if (
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
2735
|
+
if (this.transaction) {
|
|
2736
|
+
// Staged temps belong to the journal op; commit()/rollback()
|
|
2737
|
+
// owns them. Temps from a failed pre-stage attempt are cleaned
|
|
2738
|
+
// here so nothing leaks.
|
|
2739
|
+
if (!txnStaged && renameList.length)
|
|
2740
|
+
await Promise.allSettled(renameList
|
|
2741
|
+
.filter((pair) => Boolean(pair[1]))
|
|
2742
|
+
.map(async ([tempPath, _]) => unlink(tempPath)));
|
|
2743
|
+
}
|
|
2744
|
+
else {
|
|
2745
|
+
if (renameList.length)
|
|
2746
|
+
await Promise.allSettled(renameList
|
|
2747
|
+
.filter((pair) => Boolean(pair[1]))
|
|
2748
|
+
.map(async ([tempPath, _]) => unlink(tempPath)));
|
|
2749
|
+
await File.unlock(join(tablePath, ".tmp"));
|
|
2750
|
+
}
|
|
1646
2751
|
}
|
|
1647
2752
|
}
|
|
1648
2753
|
async put(tableName, data, where, options = {
|
|
@@ -1650,12 +2755,15 @@ export default class Inibase {
|
|
|
1650
2755
|
perPage: 15,
|
|
1651
2756
|
}, returnUpdatedData, _whereIsLinesNumbers) {
|
|
1652
2757
|
const renameList = [];
|
|
2758
|
+
let txnStaged = false;
|
|
1653
2759
|
this.validateName(tableName);
|
|
1654
2760
|
if (options.columns)
|
|
1655
2761
|
this.validateColumns((Array.isArray(options.columns)
|
|
1656
2762
|
? options.columns
|
|
1657
2763
|
: [options.columns]));
|
|
1658
2764
|
const tablePath = join(this.databasePath, tableName);
|
|
2765
|
+
if (!this.transaction)
|
|
2766
|
+
await this.ensureDatabaseRecovered();
|
|
1659
2767
|
await this.throwErrorIfTableEmpty(tableName);
|
|
1660
2768
|
let clonedData = structuredClone(data);
|
|
1661
2769
|
if (!where) {
|
|
@@ -1677,27 +2785,74 @@ export default class Inibase {
|
|
|
1677
2785
|
...(({ id, ...restOfData }) => restOfData)(clonedData),
|
|
1678
2786
|
updatedAt: Date.now(),
|
|
1679
2787
|
});
|
|
2788
|
+
// Derived columns: a where-less put rewrites every row, so each
|
|
2789
|
+
// computed expression must be re-evaluated against the existing
|
|
2790
|
+
// row overlaid with the payload. The plan itself is schema-only.
|
|
2791
|
+
const computedPlan = await this.buildComputedPlan(tableName);
|
|
1680
2792
|
try {
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
2793
|
+
if (this.transaction)
|
|
2794
|
+
await this.ensureTxnLock(tableName);
|
|
2795
|
+
else
|
|
2796
|
+
await File.lock(join(tablePath, ".tmp"));
|
|
2797
|
+
const { total } = await this.resolvePagination(tableName);
|
|
2798
|
+
if (computedPlan) {
|
|
2799
|
+
const lineNumbers = Array.from({ length: total }, (_, index) => index + 1);
|
|
2800
|
+
const schema = globalConfig[this.databasePath].tables?.get(tableName)?.schema ??
|
|
2801
|
+
[];
|
|
2802
|
+
// Read every stored column (computed ones included) so the
|
|
2803
|
+
// merge can skip columns whose re-evaluated values are
|
|
2804
|
+
// unchanged — no rewrite, no fsync for them.
|
|
2805
|
+
const existing = await this.processSchemaData(tableName, schema, lineNumbers);
|
|
2806
|
+
const payloadRows = (Array.isArray(clonedData) ? clonedData : [clonedData]).map((row) => Object.fromEntries(Object.entries(row).filter(([, v]) => v !== "undefined")));
|
|
2807
|
+
const mergedRows = {};
|
|
2808
|
+
for (let index = 0; index < lineNumbers.length; index++) {
|
|
2809
|
+
const line = lineNumbers[index];
|
|
2810
|
+
mergedRows[line] = {
|
|
2811
|
+
...(existing[line] ?? {}),
|
|
2812
|
+
...(payloadRows[index % payloadRows.length] ?? {}),
|
|
2813
|
+
updatedAt: Date.now(),
|
|
2814
|
+
};
|
|
2815
|
+
}
|
|
2816
|
+
const evaluated = await this.evaluateComputedRows(tableName, computedPlan, mergedRows);
|
|
2817
|
+
this.mergeComputedLineRecords(tableName, computedPlan, evaluated, pathesContents, existing);
|
|
2818
|
+
}
|
|
2819
|
+
await Promise.allSettled(Object.entries(pathesContents).map(async ([path, content]) => renameList.push(await File.replace(path, content, total))));
|
|
2820
|
+
if (this.transaction) {
|
|
2821
|
+
// Stage instead of publishing: row count is unchanged so
|
|
2822
|
+
// there is no pagination rename to journal.
|
|
2823
|
+
await this.stageTxnOp(tableName, renameList, null);
|
|
2824
|
+
txnStaged = true;
|
|
2825
|
+
}
|
|
2826
|
+
else {
|
|
2827
|
+
// Crash-atomic commit (row count unchanged -> no pagination rename).
|
|
2828
|
+
await this.commitFiles(tablePath, renameList, null);
|
|
2829
|
+
if (globalConfig[this.databasePath].tables?.get(tableName)?.config.cache)
|
|
2830
|
+
await this.clearCache(tableName);
|
|
2831
|
+
}
|
|
2832
|
+
if (returnUpdatedData) {
|
|
2833
|
+
if (this.transaction)
|
|
2834
|
+
// Reading the committed state would miss the staged
|
|
2835
|
+
// write (no read-your-writes yet).
|
|
2836
|
+
throw this.createError("INVALID_PARAMETERS");
|
|
1693
2837
|
return await this.get(tableName, undefined, options);
|
|
2838
|
+
}
|
|
1694
2839
|
}
|
|
1695
2840
|
finally {
|
|
1696
|
-
if (
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
2841
|
+
if (this.transaction) {
|
|
2842
|
+
// Staged temps belong to the journal op; commit()/rollback()
|
|
2843
|
+
// owns them.
|
|
2844
|
+
if (!txnStaged && renameList.length)
|
|
2845
|
+
await Promise.allSettled(renameList
|
|
2846
|
+
.filter((pair) => Boolean(pair[1]))
|
|
2847
|
+
.map(async ([tempPath, _]) => unlink(tempPath)));
|
|
2848
|
+
}
|
|
2849
|
+
else {
|
|
2850
|
+
if (renameList.length)
|
|
2851
|
+
await Promise.allSettled(renameList
|
|
2852
|
+
.filter((pair) => Boolean(pair[1]))
|
|
2853
|
+
.map(async ([tempPath, _]) => unlink(tempPath)));
|
|
2854
|
+
await File.unlock(join(tablePath, ".tmp"));
|
|
2855
|
+
}
|
|
1701
2856
|
}
|
|
1702
2857
|
}
|
|
1703
2858
|
else if (((Array.isArray(where) && where.every(Utils.isNumber)) ||
|
|
@@ -1720,26 +2875,70 @@ export default class Inibase {
|
|
|
1720
2875
|
return obj;
|
|
1721
2876
|
}, {}),
|
|
1722
2877
|
]));
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
2878
|
+
// Derived columns re-evaluate the target lines: existing values +
|
|
2879
|
+
// payload overlay feed the expressions, results go back per line.
|
|
2880
|
+
const computedPlan = await this.buildComputedPlan(tableName);
|
|
1726
2881
|
try {
|
|
1727
|
-
|
|
2882
|
+
// One global lock per table serializes every writer; inside a
|
|
2883
|
+
// transaction the lock is held for the whole txn.
|
|
2884
|
+
if (this.transaction)
|
|
2885
|
+
await this.ensureTxnLock(tableName);
|
|
2886
|
+
else
|
|
2887
|
+
await File.lock(join(tablePath, ".tmp"));
|
|
2888
|
+
if (computedPlan) {
|
|
2889
|
+
const whereLines = Array.isArray(where) ? where : [where];
|
|
2890
|
+
const schema = globalConfig[this.databasePath].tables?.get(tableName)?.schema ??
|
|
2891
|
+
[];
|
|
2892
|
+
// Read every stored column (computed ones included) so the
|
|
2893
|
+
// merge can skip columns whose re-evaluated values are
|
|
2894
|
+
// unchanged — no rewrite, no fsync for them.
|
|
2895
|
+
const existing = await this.processSchemaData(tableName, schema, whereLines);
|
|
2896
|
+
const payloadRows = (Array.isArray(clonedData) ? clonedData : [clonedData]).map((row) => Object.fromEntries(Object.entries(row).filter(([, v]) => v !== "undefined")));
|
|
2897
|
+
const evaluated = {};
|
|
2898
|
+
const indexCache = new Map([
|
|
2899
|
+
[tableName, computedPlan.index],
|
|
2900
|
+
]);
|
|
2901
|
+
for (let index = 0; index < whereLines.length; index++) {
|
|
2902
|
+
const line = whereLines[index];
|
|
2903
|
+
const merged = {
|
|
2904
|
+
...(existing[line] ?? {}),
|
|
2905
|
+
...(payloadRows[index] ?? {}),
|
|
2906
|
+
updatedAt: Date.now(),
|
|
2907
|
+
};
|
|
2908
|
+
evaluated[line] = await this.evaluateRowComputed(tableName, computedPlan, merged, indexCache);
|
|
2909
|
+
}
|
|
2910
|
+
this.mergeComputedLineRecords(tableName, computedPlan, evaluated, pathesContents, existing);
|
|
2911
|
+
}
|
|
1728
2912
|
await Promise.allSettled(Object.entries(pathesContents).map(async ([path, content]) => renameList.push(await File.replace(path, content))));
|
|
1729
|
-
|
|
1730
|
-
.
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
2913
|
+
if (this.transaction) {
|
|
2914
|
+
await this.stageTxnOp(tableName, renameList, null);
|
|
2915
|
+
txnStaged = true;
|
|
2916
|
+
}
|
|
2917
|
+
else {
|
|
2918
|
+
await this.commitFiles(tablePath, renameList, null);
|
|
2919
|
+
if (globalConfig[this.databasePath].tables?.get(tableName)?.config.cache)
|
|
2920
|
+
await this.clearCache(tableName);
|
|
2921
|
+
}
|
|
2922
|
+
if (returnUpdatedData) {
|
|
2923
|
+
if (this.transaction)
|
|
2924
|
+
throw this.createError("INVALID_PARAMETERS");
|
|
1735
2925
|
return this.get(tableName, where, options, !Array.isArray(where), undefined, true);
|
|
2926
|
+
}
|
|
1736
2927
|
}
|
|
1737
2928
|
finally {
|
|
1738
|
-
if (
|
|
1739
|
-
|
|
1740
|
-
.
|
|
1741
|
-
|
|
1742
|
-
|
|
2929
|
+
if (this.transaction) {
|
|
2930
|
+
if (!txnStaged && renameList.length)
|
|
2931
|
+
await Promise.allSettled(renameList
|
|
2932
|
+
.filter((pair) => Boolean(pair[1]))
|
|
2933
|
+
.map(async ([tempPath, _]) => unlink(tempPath)));
|
|
2934
|
+
}
|
|
2935
|
+
else {
|
|
2936
|
+
if (renameList.length)
|
|
2937
|
+
await Promise.allSettled(renameList
|
|
2938
|
+
.filter((pair) => Boolean(pair[1]))
|
|
2939
|
+
.map(async ([tempPath, _]) => unlink(tempPath)));
|
|
2940
|
+
await File.unlock(join(tablePath, ".tmp"));
|
|
2941
|
+
}
|
|
1743
2942
|
}
|
|
1744
2943
|
}
|
|
1745
2944
|
else if ((!_whereIsLinesNumbers &&
|
|
@@ -1777,37 +2976,63 @@ export default class Inibase {
|
|
|
1777
2976
|
*/
|
|
1778
2977
|
async delete(tableName, where, _whereIsLinesNumbers, _cascadeGuard) {
|
|
1779
2978
|
this.validateName(tableName);
|
|
2979
|
+
if (!this.transaction)
|
|
2980
|
+
await this.ensureDatabaseRecovered();
|
|
1780
2981
|
const tablePath = join(this.databasePath, tableName);
|
|
1781
2982
|
await this.throwErrorIfTableEmpty(tableName);
|
|
1782
2983
|
if (!where) {
|
|
2984
|
+
let txnStaged = false;
|
|
2985
|
+
// Crash-atomic truncate: park every column file (pure removal
|
|
2986
|
+
// ops) and publish the empty row count in one journaled commit.
|
|
2987
|
+
const renameList = [];
|
|
1783
2988
|
try {
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
2989
|
+
if (this.transaction)
|
|
2990
|
+
await this.ensureTxnLock(tableName);
|
|
2991
|
+
else
|
|
2992
|
+
await File.lock(join(tablePath, ".tmp"));
|
|
2993
|
+
const files = (await readdir(tablePath)) ?? [];
|
|
2994
|
+
renameList.push(...files
|
|
2995
|
+
.filter((fileName) => fileName.endsWith(this.getFileExtension(tableName)))
|
|
2996
|
+
.map((file) => [null, join(tablePath, file)]));
|
|
2997
|
+
const { filePath: paginationFilePath, lastId, total, } = await this.resolvePagination(tableName);
|
|
2998
|
+
const pagination = {
|
|
2999
|
+
from: paginationFilePath,
|
|
3000
|
+
to: join(tablePath, `${lastId}-0.pagination`),
|
|
3001
|
+
};
|
|
3002
|
+
if (this.transaction) {
|
|
3003
|
+
await this.stageTxnOp(tableName, renameList, pagination);
|
|
3004
|
+
txnStaged = true;
|
|
3005
|
+
const stagedEntry = this.txnTableEntry(tableName);
|
|
3006
|
+
if (stagedEntry)
|
|
3007
|
+
stagedEntry.total = 0;
|
|
3008
|
+
}
|
|
3009
|
+
else {
|
|
3010
|
+
await this.commitFiles(tablePath, renameList, pagination);
|
|
3011
|
+
if (globalConfig[this.databasePath].tables?.get(tableName)?.config.cache)
|
|
3012
|
+
await this.clearCache(tableName);
|
|
3013
|
+
}
|
|
1801
3014
|
this.idDensity.set(tableName, true);
|
|
1802
3015
|
// Deleting every row must also delete rows that reference them.
|
|
1803
|
-
if (
|
|
1804
|
-
const allLines = Array.from({ length:
|
|
3016
|
+
if (total) {
|
|
3017
|
+
const allLines = Array.from({ length: total }, (_, i) => i + 1);
|
|
1805
3018
|
await this.cascadeDelete(tableName, allLines, new Set());
|
|
1806
3019
|
}
|
|
1807
3020
|
return true;
|
|
1808
3021
|
}
|
|
1809
3022
|
finally {
|
|
1810
|
-
|
|
3023
|
+
if (this.transaction) {
|
|
3024
|
+
if (!txnStaged && renameList.length)
|
|
3025
|
+
await Promise.allSettled(renameList
|
|
3026
|
+
.filter((pair) => Boolean(pair[1]))
|
|
3027
|
+
.map(async ([tempPath, _]) => unlink(tempPath)));
|
|
3028
|
+
}
|
|
3029
|
+
else {
|
|
3030
|
+
if (renameList.length)
|
|
3031
|
+
await Promise.allSettled(renameList
|
|
3032
|
+
.filter((pair) => Boolean(pair[1]))
|
|
3033
|
+
.map(async ([tempPath, _]) => unlink(tempPath)));
|
|
3034
|
+
await File.unlock(join(tablePath, ".tmp"));
|
|
3035
|
+
}
|
|
1811
3036
|
}
|
|
1812
3037
|
}
|
|
1813
3038
|
if (((Array.isArray(where) && where.every(Utils.isNumber)) ||
|
|
@@ -1819,46 +3044,77 @@ export default class Inibase {
|
|
|
1819
3044
|
const files = (await readdir(tablePath))?.filter((fileName) => fileName.endsWith(this.getFileExtension(tableName)));
|
|
1820
3045
|
if (files.length) {
|
|
1821
3046
|
const renameList = [];
|
|
3047
|
+
let txnStaged = false;
|
|
1822
3048
|
try {
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
pagination = parse(paginationFileName)
|
|
1831
|
-
.name.split("-")
|
|
1832
|
-
.map(Number);
|
|
1833
|
-
}
|
|
1834
|
-
if (pagination[1] &&
|
|
1835
|
-
pagination[1] - (Array.isArray(where) ? where.length : 1) > 0) {
|
|
3049
|
+
if (this.transaction)
|
|
3050
|
+
await this.ensureTxnLock(tableName);
|
|
3051
|
+
else
|
|
3052
|
+
await File.lock(join(tablePath, ".tmp"));
|
|
3053
|
+
const { filePath: paginationFilePath, lastId, total, } = await this.resolvePagination(tableName);
|
|
3054
|
+
const remaining = total - (Array.isArray(where) ? where.length : 1);
|
|
3055
|
+
if (total && remaining > 0) {
|
|
1836
3056
|
this.idDensity.set(tableName, false);
|
|
1837
3057
|
await Promise.allSettled(files.map(async (file) => renameList.push(await File.remove(join(tablePath, file), where))));
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
3058
|
+
const pagination = {
|
|
3059
|
+
from: paginationFilePath,
|
|
3060
|
+
to: join(tablePath, `${lastId}-${remaining}.pagination`),
|
|
3061
|
+
};
|
|
3062
|
+
if (this.transaction) {
|
|
3063
|
+
await this.stageTxnOp(tableName, renameList, pagination);
|
|
3064
|
+
txnStaged = true;
|
|
3065
|
+
const stagedEntry = this.txnTableEntry(tableName);
|
|
3066
|
+
if (stagedEntry)
|
|
3067
|
+
stagedEntry.total = remaining;
|
|
3068
|
+
}
|
|
3069
|
+
else {
|
|
3070
|
+
await this.commitFiles(tablePath, renameList, pagination);
|
|
3071
|
+
}
|
|
1841
3072
|
}
|
|
1842
3073
|
else {
|
|
1843
3074
|
this.idDensity.set(tableName, true);
|
|
1844
|
-
|
|
3075
|
+
// Deleting every remaining row: pure removals.
|
|
3076
|
+
const truncateList = (await readdir(tablePath))
|
|
1845
3077
|
?.filter((fileName) => fileName.endsWith(this.getFileExtension(tableName)))
|
|
1846
|
-
.map(
|
|
3078
|
+
.map((file) => [null, join(tablePath, file)]);
|
|
3079
|
+
const pagination = {
|
|
3080
|
+
from: paginationFilePath,
|
|
3081
|
+
to: join(tablePath, `${lastId}-0.pagination`),
|
|
3082
|
+
};
|
|
3083
|
+
if (this.transaction) {
|
|
3084
|
+
await this.stageTxnOp(tableName, truncateList, pagination);
|
|
3085
|
+
txnStaged = true;
|
|
3086
|
+
const stagedEntry = this.txnTableEntry(tableName);
|
|
3087
|
+
if (stagedEntry)
|
|
3088
|
+
stagedEntry.total = 0;
|
|
3089
|
+
}
|
|
3090
|
+
else {
|
|
3091
|
+
await this.commitFiles(tablePath, truncateList, pagination);
|
|
3092
|
+
}
|
|
1847
3093
|
}
|
|
1848
|
-
|
|
3094
|
+
// Cache still describes the committed state while a
|
|
3095
|
+
// transaction is open, so only clear it outside one.
|
|
3096
|
+
if (!this.transaction &&
|
|
3097
|
+
globalConfig[this.databasePath].tables?.get(tableName)?.config.cache)
|
|
1849
3098
|
await this.clearCache(tableName);
|
|
1850
|
-
await rename(paginationFilePath, join(tablePath, `${pagination[0]}-${pagination[1] - (Array.isArray(where) ? where.length : 1)}.pagination`));
|
|
1851
3099
|
// Cascade: rows in other tables referencing the deleted rows
|
|
1852
3100
|
// (via `table`-typed fields) are removed too.
|
|
1853
3101
|
await this.cascadeDelete(tableName, Array.isArray(where) ? where : [where], _cascadeGuard ?? new Set());
|
|
1854
3102
|
return true;
|
|
1855
3103
|
}
|
|
1856
3104
|
finally {
|
|
1857
|
-
if (
|
|
1858
|
-
|
|
1859
|
-
.
|
|
1860
|
-
|
|
1861
|
-
|
|
3105
|
+
if (this.transaction) {
|
|
3106
|
+
if (!txnStaged && renameList.length)
|
|
3107
|
+
await Promise.allSettled(renameList
|
|
3108
|
+
.filter((pair) => Boolean(pair[1]))
|
|
3109
|
+
.map(async ([tempPath, _]) => unlink(tempPath)));
|
|
3110
|
+
}
|
|
3111
|
+
else {
|
|
3112
|
+
if (renameList.length)
|
|
3113
|
+
await Promise.allSettled(renameList
|
|
3114
|
+
.filter((pair) => Boolean(pair[1]))
|
|
3115
|
+
.map(async ([tempPath, _]) => unlink(tempPath)));
|
|
3116
|
+
await File.unlock(join(tablePath, ".tmp"));
|
|
3117
|
+
}
|
|
1862
3118
|
}
|
|
1863
3119
|
}
|
|
1864
3120
|
}
|
|
@@ -1928,8 +3184,12 @@ export default class Inibase {
|
|
|
1928
3184
|
for (const l of found)
|
|
1929
3185
|
matching.add(l);
|
|
1930
3186
|
}
|
|
1931
|
-
catch {
|
|
3187
|
+
catch (error) {
|
|
1932
3188
|
// Unreadable/unsupported column -> skip this reference.
|
|
3189
|
+
// Inside a transaction a broken reference must abort the
|
|
3190
|
+
// whole cascade (all-or-nothing).
|
|
3191
|
+
if (this.transaction)
|
|
3192
|
+
throw error;
|
|
1933
3193
|
}
|
|
1934
3194
|
}
|
|
1935
3195
|
const toDelete = [...matching].filter((line) => {
|
|
@@ -1944,8 +3204,12 @@ export default class Inibase {
|
|
|
1944
3204
|
try {
|
|
1945
3205
|
await this.delete(candidateName, toDelete, true, guard);
|
|
1946
3206
|
}
|
|
1947
|
-
catch {
|
|
1948
|
-
// Cascade is best-effort: never break
|
|
3207
|
+
catch (error) {
|
|
3208
|
+
// Cascade is best-effort outside a transaction: never break
|
|
3209
|
+
// the parent delete. Inside a transaction a cascade failure
|
|
3210
|
+
// must abort the whole txn (all-or-nothing).
|
|
3211
|
+
if (this.transaction)
|
|
3212
|
+
throw error;
|
|
1949
3213
|
}
|
|
1950
3214
|
}
|
|
1951
3215
|
}
|