@tiledesk/tiledesk-server 3.1.0 → 3.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/models/dataTable.js +16 -0
- package/models/location.js +6 -12
- package/models/request.js +65 -61
- package/package.json +3 -3
- package/routes/dataTable.js +23 -0
- package/routes/project.js +1 -1
- package/services/dataTableService.js +436 -42
- package/test/dataTablesRoute.test.js +639 -0
- package/utils/dataTableUtils.js +99 -9
|
@@ -2,8 +2,10 @@ const { Table, TableRow } = require('../models/dataTable');
|
|
|
2
2
|
const dataTableUtiles = require('../utils/dataTableUtils');
|
|
3
3
|
const { calculateBsonSize } = require('../utils/dataTableBsonUtils');
|
|
4
4
|
const mongoose = require('mongoose');
|
|
5
|
+
const winston = require('../config/winston');
|
|
5
6
|
|
|
6
7
|
const DEFAULT_TABLE_MAX_SIZE_BYTES = 30 * 1024 * 1024; // 30 MB
|
|
8
|
+
const MAX_BULK_ROWS = process.env.MAX_BULK_ROWS || 100;
|
|
7
9
|
|
|
8
10
|
function getTableMaxSizeBytes() {
|
|
9
11
|
const val = process.env.TABLE_MAX_SIZE_BYTES || DEFAULT_TABLE_MAX_SIZE_BYTES;
|
|
@@ -28,35 +30,221 @@ function getTableStats(table) {
|
|
|
28
30
|
};
|
|
29
31
|
}
|
|
30
32
|
|
|
31
|
-
|
|
33
|
+
/**
|
|
34
|
+
* Rows dropped by the MongoDB TTL monitor never go through this service, so for
|
|
35
|
+
* tables owning a ttl column stats can only drift upwards. Recompute from the
|
|
36
|
+
* real data before refusing a write, instead of rejecting a table that is in
|
|
37
|
+
* fact mostly empty.
|
|
38
|
+
*/
|
|
39
|
+
async function enforceTableSizeLimit(table, delta, id_project, id_table) {
|
|
32
40
|
if (delta <= 0) return;
|
|
33
41
|
const maxSize = getTableMaxSizeBytes();
|
|
34
|
-
|
|
35
|
-
|
|
42
|
+
if (getTableStats(table).sizeBytes + delta <= maxSize) return;
|
|
43
|
+
|
|
44
|
+
if (!dataTableUtiles.findTtlColumn(table.schema || [])) {
|
|
45
|
+
throw new Error('TABLE_SIZE_LIMIT_EXCEEDED');
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const fresh = await recomputeTableStats(id_project, id_table);
|
|
49
|
+
if (fresh.sizeBytes + delta > maxSize) {
|
|
36
50
|
throw new Error('TABLE_SIZE_LIMIT_EXCEEDED');
|
|
37
51
|
}
|
|
38
52
|
}
|
|
39
53
|
|
|
54
|
+
async function recomputeTableStats(id_project, id_table) {
|
|
55
|
+
let rows = 0;
|
|
56
|
+
let sizeBytes = 0;
|
|
57
|
+
|
|
58
|
+
try {
|
|
59
|
+
const agg = await TableRow.aggregate([
|
|
60
|
+
{ $match: { id_project: String(id_project), id_table: String(id_table) } },
|
|
61
|
+
{ $group: { _id: null, rows: { $sum: 1 }, sizeBytes: { $sum: { $bsonSize: '$$ROOT' } } } },
|
|
62
|
+
]);
|
|
63
|
+
if (agg.length > 0) {
|
|
64
|
+
rows = agg[0].rows;
|
|
65
|
+
sizeBytes = agg[0].sizeBytes;
|
|
66
|
+
}
|
|
67
|
+
} catch (err) {
|
|
68
|
+
// $bsonSize requires MongoDB 4.4: sum in the application on older servers
|
|
69
|
+
rows = 0;
|
|
70
|
+
sizeBytes = 0;
|
|
71
|
+
const cursor = TableRow.find({ id_project: id_project, id_table: id_table }).cursor();
|
|
72
|
+
for (let doc = await cursor.next(); doc != null; doc = await cursor.next()) {
|
|
73
|
+
rows += 1;
|
|
74
|
+
sizeBytes += calculateBsonSize(rowToPlainObject(doc));
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
await Table.findOneAndUpdate(
|
|
79
|
+
{ id_project: id_project, _id: id_table },
|
|
80
|
+
{ $set: { 'stats.rows': rows, 'stats.sizeBytes': sizeBytes } }
|
|
81
|
+
);
|
|
82
|
+
return { rows: rows, sizeBytes: sizeBytes };
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Realign the stats of the already formatted tables that own a ttl column, so
|
|
87
|
+
* the caller never sees counters inflated by the rows the MongoDB TTL monitor
|
|
88
|
+
* dropped behind our back. One aggregation covers the whole list, and the
|
|
89
|
+
* refreshed values are written back so the size guard starts from the truth.
|
|
90
|
+
*/
|
|
91
|
+
async function refreshTtlStats(id_project, formattedTables) {
|
|
92
|
+
const ttlTables = formattedTables.filter(function (table) {
|
|
93
|
+
return dataTableUtiles.findTtlColumn(table.schema);
|
|
94
|
+
});
|
|
95
|
+
if (ttlTables.length === 0) return;
|
|
96
|
+
|
|
97
|
+
const totals = {};
|
|
98
|
+
try {
|
|
99
|
+
const agg = await TableRow.aggregate([
|
|
100
|
+
{
|
|
101
|
+
$match: {
|
|
102
|
+
id_project: String(id_project),
|
|
103
|
+
id_table: { $in: ttlTables.map(function (table) { return String(table._id); }) },
|
|
104
|
+
},
|
|
105
|
+
},
|
|
106
|
+
{ $group: { _id: '$id_table', rows: { $sum: 1 }, sizeBytes: { $sum: { $bsonSize: '$$ROOT' } } } },
|
|
107
|
+
]);
|
|
108
|
+
agg.forEach(function (entry) {
|
|
109
|
+
totals[entry._id] = { rows: entry.rows, sizeBytes: entry.sizeBytes };
|
|
110
|
+
});
|
|
111
|
+
} catch (err) {
|
|
112
|
+
// $bsonSize requires MongoDB 4.4: on older servers keep the stored counters
|
|
113
|
+
winston.debug('refreshTtlStats skipped, $bsonSize unavailable: ', err);
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const writes = [];
|
|
118
|
+
ttlTables.forEach(function (table) {
|
|
119
|
+
// A table whose rows all expired is simply missing from the aggregation
|
|
120
|
+
const fresh = totals[String(table._id)] || { rows: 0, sizeBytes: 0 };
|
|
121
|
+
if (fresh.rows === table.stats.rows && fresh.sizeBytes === table.stats.sizeBytes) return;
|
|
122
|
+
table.stats = fresh;
|
|
123
|
+
writes.push({
|
|
124
|
+
updateOne: {
|
|
125
|
+
filter: { id_project: id_project, _id: table._id },
|
|
126
|
+
update: { $set: { 'stats.rows': fresh.rows, 'stats.sizeBytes': fresh.sizeBytes } },
|
|
127
|
+
},
|
|
128
|
+
});
|
|
129
|
+
});
|
|
130
|
+
if (writes.length > 0) await Table.bulkWrite(writes);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** Expiration instant of a row: updatedAt plus the seconds held by the ttl column. */
|
|
134
|
+
function computeExpireAt(schema, validatedData, updatedAt) {
|
|
135
|
+
const ttlCol = dataTableUtiles.findTtlColumn(schema);
|
|
136
|
+
if (!ttlCol) return null;
|
|
137
|
+
const seconds = validatedData ? validatedData[ttlCol.name] : undefined;
|
|
138
|
+
if (typeof seconds !== 'number') return null;
|
|
139
|
+
return new Date(new Date(updatedAt).getTime() + seconds * 1000);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* The ttl seconds a row holds once an update is applied: the value the update
|
|
144
|
+
* carries, or the one already stored when the update leaves the ttl alone.
|
|
145
|
+
* Null means the row has no usable ttl and must not carry a deadline.
|
|
146
|
+
*/
|
|
147
|
+
function effectiveTtlSeconds(existingRow, update, schema) {
|
|
148
|
+
const ttlCol = dataTableUtiles.findTtlColumn(schema);
|
|
149
|
+
if (!ttlCol) return null;
|
|
150
|
+
const incoming = ((update && update.$set) || {})['data.' + ttlCol.name];
|
|
151
|
+
if (typeof incoming === 'number') return incoming;
|
|
152
|
+
const data = (existingRow && existingRow.data) || {};
|
|
153
|
+
return typeof data[ttlCol.name] === 'number' ? data[ttlCol.name] : null;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Add the derived deadline to an update. Since expireAt hangs off updatedAt,
|
|
158
|
+
* any change to the row pushes it forward: the ttl measures idle time, not
|
|
159
|
+
* total lifetime.
|
|
160
|
+
*/
|
|
161
|
+
function withSlidingExpireAt(update, ttlSeconds, hasTtlColumn) {
|
|
162
|
+
if (!hasTtlColumn) return update;
|
|
163
|
+
if (ttlSeconds == null) {
|
|
164
|
+
return Object.assign({}, update, {
|
|
165
|
+
$unset: Object.assign({}, update.$unset, { expireAt: '' }),
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
return Object.assign({}, update, {
|
|
169
|
+
$set: Object.assign({}, update.$set, { expireAt: expireAtFrom(new Date(), ttlSeconds) }),
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* The same rule expressed server side, for when the rows to touch hold
|
|
175
|
+
* different ttl values and a single literal deadline would not fit them all.
|
|
176
|
+
*/
|
|
177
|
+
function slidingExpireAtStage(ttlColumn) {
|
|
178
|
+
const field = '$data.' + ttlColumn.name;
|
|
179
|
+
return {
|
|
180
|
+
$set: {
|
|
181
|
+
updatedAt: '$$NOW',
|
|
182
|
+
expireAt: {
|
|
183
|
+
$cond: [
|
|
184
|
+
{ $in: [{ $type: field }, ['double', 'int', 'long', 'decimal']] },
|
|
185
|
+
{ $add: ['$$NOW', { $multiply: [field, 1000] }] },
|
|
186
|
+
'$$REMOVE',
|
|
187
|
+
],
|
|
188
|
+
},
|
|
189
|
+
},
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function expireAtFrom(updatedAt, seconds) {
|
|
194
|
+
return new Date(new Date(updatedAt).getTime() + seconds * 1000);
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* Realign expireAt on every row of a table after a schema change.
|
|
199
|
+
* Without the unset branch, dropping a ttl column would leave the rows still
|
|
200
|
+
* scheduled for deletion even though the table no longer declares any ttl.
|
|
201
|
+
*/
|
|
202
|
+
async function syncExpireAtForTable(id_project, id_table, ttlColumn) {
|
|
203
|
+
const scope = { id_project: id_project, id_table: id_table };
|
|
204
|
+
|
|
205
|
+
if (!ttlColumn) {
|
|
206
|
+
await TableRow.updateMany(scope, { $unset: { expireAt: '' } });
|
|
207
|
+
} else {
|
|
208
|
+
// The schema change touches the rows, so updatedAt moves and the deadline
|
|
209
|
+
// follows it. Stamping both from $$NOW keeps them in step: reading the
|
|
210
|
+
// stored updatedAt would anchor on the value this very update replaces.
|
|
211
|
+
await TableRow.updateMany(scope, [slidingExpireAtStage(ttlColumn)]);
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
return recomputeTableStats(id_project, id_table);
|
|
215
|
+
}
|
|
216
|
+
|
|
40
217
|
function rowToPlainObject(row) {
|
|
41
218
|
return row.toObject ? row.toObject({ depopulate: true }) : row;
|
|
42
219
|
}
|
|
43
220
|
|
|
44
|
-
function buildInsertRowDoc(id_project, id_table, data, id_row) {
|
|
45
|
-
const now = new Date();
|
|
46
|
-
|
|
221
|
+
function buildInsertRowDoc(id_project, id_table, data, id_row, createdAt, expireAt) {
|
|
222
|
+
const now = createdAt || new Date();
|
|
223
|
+
const doc = {
|
|
47
224
|
_id: id_row || new mongoose.Types.ObjectId(),
|
|
48
225
|
id_project: id_project,
|
|
49
226
|
id_table: id_table,
|
|
50
227
|
data: data,
|
|
51
228
|
createdAt: now,
|
|
52
229
|
updatedAt: now,
|
|
230
|
+
// Mongoose stamps the version key on save. Leaving it out here would make
|
|
231
|
+
// an insert add fewer bytes than the matching delete later subtracts, and
|
|
232
|
+
// the counter would sink below zero on a table with enough churn.
|
|
233
|
+
__v: 0,
|
|
53
234
|
};
|
|
235
|
+
if (expireAt) doc.expireAt = expireAt;
|
|
236
|
+
return doc;
|
|
54
237
|
}
|
|
55
238
|
|
|
56
|
-
function buildUpdatedRowDoc(existingRow, dataPatch) {
|
|
239
|
+
function buildUpdatedRowDoc(existingRow, dataPatch, ttlSeconds, hasTtlColumn) {
|
|
57
240
|
const plain = rowToPlainObject(existingRow);
|
|
58
241
|
plain.data = Object.assign({}, plain.data || {}, dataPatch);
|
|
59
242
|
plain.updatedAt = new Date();
|
|
243
|
+
if (!hasTtlColumn) return plain;
|
|
244
|
+
// Fully derived from the ttl the row ends up with, so a row that loses its
|
|
245
|
+
// ttl value loses the deadline with it
|
|
246
|
+
if (ttlSeconds == null) delete plain.expireAt;
|
|
247
|
+
else plain.expireAt = expireAtFrom(plain.updatedAt, ttlSeconds);
|
|
60
248
|
return plain;
|
|
61
249
|
}
|
|
62
250
|
|
|
@@ -83,12 +271,17 @@ function computeInsertDelta(rowDoc) {
|
|
|
83
271
|
return calculateBsonSize(rowDoc);
|
|
84
272
|
}
|
|
85
273
|
|
|
86
|
-
function computeUpdateDelta(existingRows, update) {
|
|
274
|
+
function computeUpdateDelta(existingRows, update, schema) {
|
|
87
275
|
const dataPatch = dataPatchFromUpdate(update);
|
|
276
|
+
const hasTtl = !!dataTableUtiles.findTtlColumn(schema || []);
|
|
88
277
|
let totalDelta = 0;
|
|
89
278
|
for (let i = 0; i < existingRows.length; i++) {
|
|
90
|
-
const
|
|
91
|
-
|
|
279
|
+
const row = existingRows[i];
|
|
280
|
+
// Rows of the same table can hold different ttl values, so the deadline
|
|
281
|
+
// has to be simulated row by row
|
|
282
|
+
const ttlSeconds = effectiveTtlSeconds(row, update, schema);
|
|
283
|
+
const oldSize = calculateBsonSize(rowToPlainObject(row));
|
|
284
|
+
const newSize = calculateBsonSize(buildUpdatedRowDoc(row, dataPatch, ttlSeconds, hasTtl));
|
|
92
285
|
totalDelta += newSize - oldSize;
|
|
93
286
|
}
|
|
94
287
|
return totalDelta;
|
|
@@ -117,7 +310,7 @@ function formatTable(table) {
|
|
|
117
310
|
|
|
118
311
|
function formatRow(row, schema) {
|
|
119
312
|
const o = row.toObject ? row.toObject() : row;
|
|
120
|
-
|
|
313
|
+
const formatted = {
|
|
121
314
|
_id: o._id,
|
|
122
315
|
id_project: o.id_project,
|
|
123
316
|
id_table: o.id_table,
|
|
@@ -125,6 +318,9 @@ function formatRow(row, schema) {
|
|
|
125
318
|
createdAt: o.createdAt,
|
|
126
319
|
updatedAt: o.updatedAt,
|
|
127
320
|
};
|
|
321
|
+
// The ttl column only holds seconds: surface when the row actually disappears
|
|
322
|
+
if (o.expireAt) formatted.expireAt = o.expireAt;
|
|
323
|
+
return formatted;
|
|
128
324
|
}
|
|
129
325
|
|
|
130
326
|
function formatRowForList(row, schema) {
|
|
@@ -143,7 +339,9 @@ class DataTableService {
|
|
|
143
339
|
|
|
144
340
|
async listTables(id_project) {
|
|
145
341
|
const tables = await Table.find({ id_project }).sort({ createdAt: -1 });
|
|
146
|
-
|
|
342
|
+
const formatted = tables.map(formatTable);
|
|
343
|
+
await refreshTtlStats(id_project, formatted);
|
|
344
|
+
return formatted;
|
|
147
345
|
}
|
|
148
346
|
|
|
149
347
|
async getTable(id_project, id_table) {
|
|
@@ -151,6 +349,7 @@ class DataTableService {
|
|
|
151
349
|
if (!table) return null;
|
|
152
350
|
|
|
153
351
|
const result = formatTable(table);
|
|
352
|
+
await refreshTtlStats(id_project, [result]);
|
|
154
353
|
const rows = await TableRow.find({ id_project: id_project, id_table: id_table })
|
|
155
354
|
.sort({ createdAt: 1 })
|
|
156
355
|
.limit(100);
|
|
@@ -196,7 +395,9 @@ class DataTableService {
|
|
|
196
395
|
update,
|
|
197
396
|
{ new: true }
|
|
198
397
|
);
|
|
199
|
-
|
|
398
|
+
const result = formatTable(renamedOnly);
|
|
399
|
+
await refreshTtlStats(id_project, [result]);
|
|
400
|
+
return result;
|
|
200
401
|
}
|
|
201
402
|
|
|
202
403
|
const currentSchema = table.schema || [];
|
|
@@ -224,7 +425,16 @@ class DataTableService {
|
|
|
224
425
|
update,
|
|
225
426
|
{ new: true }
|
|
226
427
|
);
|
|
227
|
-
|
|
428
|
+
|
|
429
|
+
const result = formatTable(updated);
|
|
430
|
+
if (resolved.ttlAfter || resolved.ttlBefore) {
|
|
431
|
+
result.stats = await syncExpireAtForTable(id_project, id_table, resolved.ttlAfter);
|
|
432
|
+
} else if (resolved.deletes.length > 0 || resolved.renames.length > 0) {
|
|
433
|
+
// Both the $unset and the $rename loops above changed the size of every row
|
|
434
|
+
result.stats = await recomputeTableStats(id_project, id_table);
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
return result;
|
|
228
438
|
}
|
|
229
439
|
|
|
230
440
|
async deleteTable(id_project, id_table) {
|
|
@@ -247,7 +457,12 @@ class DataTableService {
|
|
|
247
457
|
{ schema: schema },
|
|
248
458
|
{ new: true }
|
|
249
459
|
);
|
|
250
|
-
|
|
460
|
+
|
|
461
|
+
// A new column carries no data, so the rows keep their size: only a table
|
|
462
|
+
// owning a ttl column can be holding counters inflated by expired rows.
|
|
463
|
+
const result = formatTable(updated);
|
|
464
|
+
await refreshTtlStats(id_project, [result]);
|
|
465
|
+
return result;
|
|
251
466
|
}
|
|
252
467
|
|
|
253
468
|
async renameColumn(id_project, id_table, columnId, newName) {
|
|
@@ -263,7 +478,11 @@ class DataTableService {
|
|
|
263
478
|
const col = dataTableUtiles.findColumn(schema, columnId);
|
|
264
479
|
if (!col) throw new Error('Column not found');
|
|
265
480
|
|
|
266
|
-
if (col.name === trimmed)
|
|
481
|
+
if (col.name === trimmed) {
|
|
482
|
+
const unchanged = formatTable(table);
|
|
483
|
+
await refreshTtlStats(id_project, [unchanged]);
|
|
484
|
+
return unchanged;
|
|
485
|
+
}
|
|
267
486
|
|
|
268
487
|
if (dataTableUtiles.findColumnByName(schema, trimmed)) {
|
|
269
488
|
throw new Error('Column name already exists: ' + trimmed);
|
|
@@ -285,7 +504,12 @@ class DataTableService {
|
|
|
285
504
|
{ schema: newSchema },
|
|
286
505
|
{ new: true }
|
|
287
506
|
);
|
|
288
|
-
|
|
507
|
+
|
|
508
|
+
const result = formatTable(updated);
|
|
509
|
+
// The $rename above rewrote a key inside every row, so the stored byte
|
|
510
|
+
// count no longer matches what is on disk: a longer name costs more.
|
|
511
|
+
result.stats = await recomputeTableStats(id_project, id_table);
|
|
512
|
+
return result;
|
|
289
513
|
}
|
|
290
514
|
|
|
291
515
|
async deleteColumn(id_project, id_table, columnId) {
|
|
@@ -308,7 +532,16 @@ class DataTableService {
|
|
|
308
532
|
{ schema: newSchema },
|
|
309
533
|
{ new: true }
|
|
310
534
|
);
|
|
311
|
-
|
|
535
|
+
|
|
536
|
+
const result = formatTable(updated);
|
|
537
|
+
if (col.type === dataTableUtiles.COLUMN_TYPES.TTL) {
|
|
538
|
+
result.stats = await syncExpireAtForTable(id_project, id_table, null);
|
|
539
|
+
} else {
|
|
540
|
+
// The $unset above shrank every row: the stored byte count is now too high
|
|
541
|
+
result.stats = await recomputeTableStats(id_project, id_table);
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
return result;
|
|
312
545
|
}
|
|
313
546
|
|
|
314
547
|
async listRows(id_project, id_table) {
|
|
@@ -374,19 +607,99 @@ class DataTableService {
|
|
|
374
607
|
if (body.data === undefined || body.data === null) {
|
|
375
608
|
throw new Error('data is required');
|
|
376
609
|
}
|
|
377
|
-
const
|
|
378
|
-
const
|
|
610
|
+
const schema = table.schema || [];
|
|
611
|
+
const data = dataTableUtiles.validateRowData(body.data, schema);
|
|
612
|
+
const now = new Date();
|
|
613
|
+
const expireAt = computeExpireAt(schema, data, now);
|
|
614
|
+
const rowDoc = buildInsertRowDoc(id_project, id_table, data, body.id_row, now, expireAt);
|
|
379
615
|
const rowSize = computeInsertDelta(rowDoc);
|
|
380
|
-
|
|
616
|
+
await enforceTableSizeLimit(table, rowSize, id_project, id_table);
|
|
381
617
|
|
|
382
618
|
const row = await TableRow.create({
|
|
383
619
|
_id: rowDoc._id,
|
|
384
620
|
id_project: id_project,
|
|
385
621
|
id_table: id_table,
|
|
386
622
|
data: data,
|
|
623
|
+
expireAt: expireAt || undefined,
|
|
387
624
|
});
|
|
388
625
|
await incrementTableStats(id_project, id_table, 1, rowSize);
|
|
389
|
-
return formatRow(row,
|
|
626
|
+
return formatRow(row, schema);
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
async insertRowsByBody(id_project, id_table, body) {
|
|
630
|
+
const table = await Table.findOne({ id_project: id_project, _id: id_table });
|
|
631
|
+
if (!table) return null;
|
|
632
|
+
|
|
633
|
+
const data = body && body.data;
|
|
634
|
+
if (!Array.isArray(data) || data.length === 0) {
|
|
635
|
+
throw new Error('data must be a non-empty array');
|
|
636
|
+
}
|
|
637
|
+
if (data.length > MAX_BULK_ROWS) {
|
|
638
|
+
throw new Error('Invalid data: at most ' + MAX_BULK_ROWS + ' rows per request');
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
const schema = table.schema || [];
|
|
642
|
+
const insertDocs = [];
|
|
643
|
+
let totalSize = 0;
|
|
644
|
+
|
|
645
|
+
// Validate every row before writing anything, so a single invalid row
|
|
646
|
+
// cannot leave the table with a partial batch.
|
|
647
|
+
for (let i = 0; i < data.length; i++) {
|
|
648
|
+
const entry = data[i];
|
|
649
|
+
if (!entry || typeof entry !== 'object' || Array.isArray(entry)) {
|
|
650
|
+
throw new Error('Invalid row at index ' + i + ': each row must be an object');
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
let rowData;
|
|
654
|
+
try {
|
|
655
|
+
rowData = dataTableUtiles.validateRowData(entry, schema);
|
|
656
|
+
} catch (err) {
|
|
657
|
+
throw new Error('Invalid row at index ' + i + ': ' + err.message);
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
const now = new Date();
|
|
661
|
+
const expireAt = computeExpireAt(schema, rowData, now);
|
|
662
|
+
const rowDoc = buildInsertRowDoc(id_project, id_table, rowData, null, now, expireAt);
|
|
663
|
+
totalSize += computeInsertDelta(rowDoc);
|
|
664
|
+
const insertDoc = {
|
|
665
|
+
_id: rowDoc._id,
|
|
666
|
+
id_project: id_project,
|
|
667
|
+
id_table: id_table,
|
|
668
|
+
data: rowData,
|
|
669
|
+
};
|
|
670
|
+
if (expireAt) insertDoc.expireAt = expireAt;
|
|
671
|
+
insertDocs.push(insertDoc);
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
await enforceTableSizeLimit(table, totalSize, id_project, id_table);
|
|
675
|
+
|
|
676
|
+
let created;
|
|
677
|
+
try {
|
|
678
|
+
created = await TableRow.insertMany(insertDocs, { ordered: true });
|
|
679
|
+
} catch (err) {
|
|
680
|
+
await this._reconcileStatsAfterFailedInsert(id_project, id_table, insertDocs);
|
|
681
|
+
throw err;
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
await incrementTableStats(id_project, id_table, created.length, totalSize);
|
|
685
|
+
return created.map(function (r) { return formatRow(r, schema); });
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
/** An ordered insertMany can stop halfway: realign stats on what really landed. */
|
|
689
|
+
async _reconcileStatsAfterFailedInsert(id_project, id_table, insertDocs) {
|
|
690
|
+
const ids = insertDocs.map(function (d) { return d._id; });
|
|
691
|
+
const persisted = await TableRow.find({
|
|
692
|
+
id_project: id_project,
|
|
693
|
+
id_table: id_table,
|
|
694
|
+
_id: { $in: ids },
|
|
695
|
+
});
|
|
696
|
+
if (persisted.length === 0) return;
|
|
697
|
+
|
|
698
|
+
let size = 0;
|
|
699
|
+
for (let i = 0; i < persisted.length; i++) {
|
|
700
|
+
size += calculateBsonSize(rowToPlainObject(persisted[i]));
|
|
701
|
+
}
|
|
702
|
+
await incrementTableStats(id_project, id_table, persisted.length, size);
|
|
390
703
|
}
|
|
391
704
|
|
|
392
705
|
async updateRow(id_project, id_table, id_row, rawData) {
|
|
@@ -403,22 +716,36 @@ class DataTableService {
|
|
|
403
716
|
const filter = { id_project: id_project, id_table: id_table };
|
|
404
717
|
Object.assign(filter, rowFilter);
|
|
405
718
|
|
|
719
|
+
const ttlColumn = dataTableUtiles.findTtlColumn(schema);
|
|
720
|
+
|
|
406
721
|
if (hasConditions(body)) {
|
|
407
722
|
const matches = await TableRow.find(filter).sort({ createdAt: 1 });
|
|
408
723
|
if (matches.length === 0) return undefined;
|
|
409
|
-
const sizeDelta = computeUpdateDelta(matches, update);
|
|
410
|
-
|
|
724
|
+
const sizeDelta = computeUpdateDelta(matches, update, schema);
|
|
725
|
+
await enforceTableSizeLimit(table, sizeDelta, id_project, id_table);
|
|
726
|
+
|
|
727
|
+
// The patch can write the very field the conditions select on, moving a
|
|
728
|
+
// row out of the filter, so everything past this point goes by id
|
|
729
|
+
const ids = matches.map(function (r) { return r._id; });
|
|
411
730
|
await TableRow.updateMany(filter, update);
|
|
731
|
+
// Each matched row may hold its own ttl, so the deadlines are derived on
|
|
732
|
+
// the server from the values the patch above just left in place
|
|
733
|
+
if (ttlColumn) {
|
|
734
|
+
await TableRow.updateMany({ _id: { $in: ids } }, [slidingExpireAtStage(ttlColumn)]);
|
|
735
|
+
}
|
|
736
|
+
|
|
412
737
|
await incrementTableStats(id_project, id_table, 0, sizeDelta);
|
|
413
|
-
const updatedRows = await TableRow.find(
|
|
738
|
+
const updatedRows = await TableRow.find({ _id: { $in: ids } }).sort({ createdAt: 1 });
|
|
414
739
|
return updatedRows.map(function (r) { return formatRow(r, schema); });
|
|
415
740
|
}
|
|
416
741
|
|
|
417
742
|
const existing = await TableRow.findOne(filter);
|
|
418
743
|
if (!existing) return undefined;
|
|
419
|
-
const singleDelta = computeUpdateDelta([existing], update);
|
|
420
|
-
|
|
421
|
-
const
|
|
744
|
+
const singleDelta = computeUpdateDelta([existing], update, schema);
|
|
745
|
+
await enforceTableSizeLimit(table, singleDelta, id_project, id_table);
|
|
746
|
+
const singleUpdate = withSlidingExpireAt(
|
|
747
|
+
update, effectiveTtlSeconds(existing, update, schema), !!ttlColumn);
|
|
748
|
+
const row = await TableRow.findOneAndUpdate(filter, singleUpdate, { new: true });
|
|
422
749
|
await incrementTableStats(id_project, id_table, 0, singleDelta);
|
|
423
750
|
return formatRow(row, schema);
|
|
424
751
|
}
|
|
@@ -434,26 +761,35 @@ class DataTableService {
|
|
|
434
761
|
|
|
435
762
|
const update = dataTableUtiles.buildUpdateSet(body.data, schema);
|
|
436
763
|
const rowData = dataTableUtiles.validateRowData(body.data, schema);
|
|
764
|
+
const hasTtl = !!dataTableUtiles.findTtlColumn(schema);
|
|
765
|
+
|
|
766
|
+
const updateFor = function (existingRow) {
|
|
767
|
+
return withSlidingExpireAt(
|
|
768
|
+
update, effectiveTtlSeconds(existingRow, update, schema), hasTtl);
|
|
769
|
+
};
|
|
437
770
|
|
|
438
771
|
if (body.id_row) {
|
|
439
772
|
const filterById = { id_project: id_project, id_table: id_table, _id: body.id_row };
|
|
440
773
|
const existing = await TableRow.findOne(filterById);
|
|
441
774
|
if (existing) {
|
|
442
|
-
const updateDelta = computeUpdateDelta([existing], update);
|
|
443
|
-
|
|
444
|
-
const updated = await TableRow.findOneAndUpdate(filterById,
|
|
775
|
+
const updateDelta = computeUpdateDelta([existing], update, schema);
|
|
776
|
+
await enforceTableSizeLimit(table, updateDelta, id_project, id_table);
|
|
777
|
+
const updated = await TableRow.findOneAndUpdate(filterById, updateFor(existing), { new: true });
|
|
445
778
|
await incrementTableStats(id_project, id_table, 0, updateDelta);
|
|
446
779
|
return formatRow(updated, schema);
|
|
447
780
|
}
|
|
448
|
-
const
|
|
781
|
+
const insertedAt = new Date();
|
|
782
|
+
const insertExpireAt = computeExpireAt(schema, rowData, insertedAt);
|
|
783
|
+
const insertDoc = buildInsertRowDoc(id_project, id_table, rowData, body.id_row, insertedAt, insertExpireAt);
|
|
449
784
|
const insertSize = computeInsertDelta(insertDoc);
|
|
450
|
-
|
|
785
|
+
await enforceTableSizeLimit(table, insertSize, id_project, id_table);
|
|
451
786
|
try {
|
|
452
787
|
const created = await TableRow.create({
|
|
453
788
|
_id: body.id_row,
|
|
454
789
|
id_project: id_project,
|
|
455
790
|
id_table: id_table,
|
|
456
791
|
data: rowData,
|
|
792
|
+
expireAt: insertExpireAt || undefined,
|
|
457
793
|
});
|
|
458
794
|
await incrementTableStats(id_project, id_table, 1, insertSize);
|
|
459
795
|
return formatRow(created, schema);
|
|
@@ -461,9 +797,9 @@ class DataTableService {
|
|
|
461
797
|
if (createErr.code === 11000) {
|
|
462
798
|
const racedExisting = await TableRow.findOne(filterById);
|
|
463
799
|
if (racedExisting) {
|
|
464
|
-
const racedDelta = computeUpdateDelta([racedExisting], update);
|
|
465
|
-
|
|
466
|
-
const raced = await TableRow.findOneAndUpdate(filterById,
|
|
800
|
+
const racedDelta = computeUpdateDelta([racedExisting], update, schema);
|
|
801
|
+
await enforceTableSizeLimit(table, racedDelta, id_project, id_table);
|
|
802
|
+
const raced = await TableRow.findOneAndUpdate(filterById, updateFor(racedExisting), { new: true });
|
|
467
803
|
await incrementTableStats(id_project, id_table, 0, racedDelta);
|
|
468
804
|
if (raced) return formatRow(raced, schema);
|
|
469
805
|
}
|
|
@@ -478,10 +814,17 @@ class DataTableService {
|
|
|
478
814
|
const matches = await TableRow.find(filter).sort({ createdAt: 1 });
|
|
479
815
|
|
|
480
816
|
if (matches.length === 0) {
|
|
481
|
-
const
|
|
817
|
+
const newAt = new Date();
|
|
818
|
+
const newExpireAt = computeExpireAt(schema, rowData, newAt);
|
|
819
|
+
const newDoc = buildInsertRowDoc(id_project, id_table, rowData, null, newAt, newExpireAt);
|
|
482
820
|
const newSize = computeInsertDelta(newDoc);
|
|
483
|
-
|
|
484
|
-
const inserted = await TableRow.create({
|
|
821
|
+
await enforceTableSizeLimit(table, newSize, id_project, id_table);
|
|
822
|
+
const inserted = await TableRow.create({
|
|
823
|
+
id_project: id_project,
|
|
824
|
+
id_table: id_table,
|
|
825
|
+
data: rowData,
|
|
826
|
+
expireAt: newExpireAt || undefined,
|
|
827
|
+
});
|
|
485
828
|
await incrementTableStats(id_project, id_table, 1, newSize);
|
|
486
829
|
return formatRow(inserted, schema);
|
|
487
830
|
}
|
|
@@ -490,13 +833,64 @@ class DataTableService {
|
|
|
490
833
|
throw new Error('Multiple rows match the conditions');
|
|
491
834
|
}
|
|
492
835
|
|
|
493
|
-
const upsertDelta = computeUpdateDelta(matches, update);
|
|
494
|
-
|
|
495
|
-
const updated = await TableRow.findOneAndUpdate(filter,
|
|
836
|
+
const upsertDelta = computeUpdateDelta(matches, update, schema);
|
|
837
|
+
await enforceTableSizeLimit(table, upsertDelta, id_project, id_table);
|
|
838
|
+
const updated = await TableRow.findOneAndUpdate(filter, updateFor(matches[0]), { new: true });
|
|
496
839
|
await incrementTableStats(id_project, id_table, 0, upsertDelta);
|
|
497
840
|
return formatRow(updated, schema);
|
|
498
841
|
}
|
|
499
842
|
|
|
843
|
+
/**
|
|
844
|
+
* The single upsert with data holding an array of rows. The conditions name
|
|
845
|
+
* the columns that identify a row, and each row of the batch is looked up by
|
|
846
|
+
* the values it carries in those columns.
|
|
847
|
+
*/
|
|
848
|
+
async upsertRowsByBody(id_project, id_table, body) {
|
|
849
|
+
const table = await Table.findOne({ id_project: id_project, _id: id_table });
|
|
850
|
+
if (!table) return null;
|
|
851
|
+
|
|
852
|
+
const rows = body && body.data;
|
|
853
|
+
if (!Array.isArray(rows) || rows.length === 0) {
|
|
854
|
+
throw new Error('data must be a non-empty array');
|
|
855
|
+
}
|
|
856
|
+
if (rows.length > MAX_BULK_ROWS) {
|
|
857
|
+
throw new Error('Invalid data: at most ' + MAX_BULK_ROWS + ' rows per request');
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
const schema = table.schema || [];
|
|
861
|
+
const byConditions = !body.id_row && body.conditions && body.conditions.length;
|
|
862
|
+
|
|
863
|
+
// Validate and bind the whole batch up front: rejecting on the fifth row
|
|
864
|
+
// would otherwise leave the four before it already written.
|
|
865
|
+
const conditionsFor = [];
|
|
866
|
+
for (let i = 0; i < rows.length; i++) {
|
|
867
|
+
const entry = rows[i];
|
|
868
|
+
if (!entry || typeof entry !== 'object' || Array.isArray(entry)) {
|
|
869
|
+
throw new Error('Invalid row at index ' + i + ': each row must be an object');
|
|
870
|
+
}
|
|
871
|
+
try {
|
|
872
|
+
// Binding against the validated data, so the lookup compares the same
|
|
873
|
+
// coerced values that were stored in the first place
|
|
874
|
+
const rowData = dataTableUtiles.validateRowData(entry, schema);
|
|
875
|
+
if (byConditions) {
|
|
876
|
+
conditionsFor.push(dataTableUtiles.bindConditionsToRow(body.conditions, rowData, schema));
|
|
877
|
+
}
|
|
878
|
+
} catch (err) {
|
|
879
|
+
throw new Error('Invalid row at index ' + i + ': ' + err.message);
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
const results = [];
|
|
884
|
+
for (let i = 0; i < rows.length; i++) {
|
|
885
|
+
const single = Object.assign({}, body, { data: rows[i] });
|
|
886
|
+
if (byConditions) single.conditions = conditionsFor[i];
|
|
887
|
+
// Re-reading the table on every row is what keeps the size guard honest:
|
|
888
|
+
// it has to see the bytes the rows before it just added.
|
|
889
|
+
results.push(await this.upsertRowByBody(id_project, id_table, single));
|
|
890
|
+
}
|
|
891
|
+
return results;
|
|
892
|
+
}
|
|
893
|
+
|
|
500
894
|
async deleteRow(id_project, id_table, id_row) {
|
|
501
895
|
return this.deleteRowByBody(id_project, id_table, { id_row: id_row });
|
|
502
896
|
}
|