@tiledesk/tiledesk-server 3.1.0 → 3.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,8 +2,10 @@ const { Table, TableRow } = require('../models/dataTable');
2
2
  const dataTableUtiles = require('../utils/dataTableUtils');
3
3
  const { calculateBsonSize } = require('../utils/dataTableBsonUtils');
4
4
  const mongoose = require('mongoose');
5
+ const winston = require('../config/winston');
5
6
 
6
7
  const DEFAULT_TABLE_MAX_SIZE_BYTES = 30 * 1024 * 1024; // 30 MB
8
+ const MAX_BULK_ROWS = process.env.MAX_BULK_ROWS || 100;
7
9
 
8
10
  function getTableMaxSizeBytes() {
9
11
  const val = process.env.TABLE_MAX_SIZE_BYTES || DEFAULT_TABLE_MAX_SIZE_BYTES;
@@ -28,35 +30,221 @@ function getTableStats(table) {
28
30
  };
29
31
  }
30
32
 
31
- function assertTableSizeLimit(table, delta) {
33
+ /**
34
+ * Rows dropped by the MongoDB TTL monitor never go through this service, so for
35
+ * tables owning a ttl column stats can only drift upwards. Recompute from the
36
+ * real data before refusing a write, instead of rejecting a table that is in
37
+ * fact mostly empty.
38
+ */
39
+ async function enforceTableSizeLimit(table, delta, id_project, id_table) {
32
40
  if (delta <= 0) return;
33
41
  const maxSize = getTableMaxSizeBytes();
34
- const stats = getTableStats(table);
35
- if (stats.sizeBytes + delta > maxSize) {
42
+ if (getTableStats(table).sizeBytes + delta <= maxSize) return;
43
+
44
+ if (!dataTableUtiles.findTtlColumn(table.schema || [])) {
45
+ throw new Error('TABLE_SIZE_LIMIT_EXCEEDED');
46
+ }
47
+
48
+ const fresh = await recomputeTableStats(id_project, id_table);
49
+ if (fresh.sizeBytes + delta > maxSize) {
36
50
  throw new Error('TABLE_SIZE_LIMIT_EXCEEDED');
37
51
  }
38
52
  }
39
53
 
54
+ async function recomputeTableStats(id_project, id_table) {
55
+ let rows = 0;
56
+ let sizeBytes = 0;
57
+
58
+ try {
59
+ const agg = await TableRow.aggregate([
60
+ { $match: { id_project: String(id_project), id_table: String(id_table) } },
61
+ { $group: { _id: null, rows: { $sum: 1 }, sizeBytes: { $sum: { $bsonSize: '$$ROOT' } } } },
62
+ ]);
63
+ if (agg.length > 0) {
64
+ rows = agg[0].rows;
65
+ sizeBytes = agg[0].sizeBytes;
66
+ }
67
+ } catch (err) {
68
+ // $bsonSize requires MongoDB 4.4: sum in the application on older servers
69
+ rows = 0;
70
+ sizeBytes = 0;
71
+ const cursor = TableRow.find({ id_project: id_project, id_table: id_table }).cursor();
72
+ for (let doc = await cursor.next(); doc != null; doc = await cursor.next()) {
73
+ rows += 1;
74
+ sizeBytes += calculateBsonSize(rowToPlainObject(doc));
75
+ }
76
+ }
77
+
78
+ await Table.findOneAndUpdate(
79
+ { id_project: id_project, _id: id_table },
80
+ { $set: { 'stats.rows': rows, 'stats.sizeBytes': sizeBytes } }
81
+ );
82
+ return { rows: rows, sizeBytes: sizeBytes };
83
+ }
84
+
85
+ /**
86
+ * Realign the stats of the already formatted tables that own a ttl column, so
87
+ * the caller never sees counters inflated by the rows the MongoDB TTL monitor
88
+ * dropped behind our back. One aggregation covers the whole list, and the
89
+ * refreshed values are written back so the size guard starts from the truth.
90
+ */
91
+ async function refreshTtlStats(id_project, formattedTables) {
92
+ const ttlTables = formattedTables.filter(function (table) {
93
+ return dataTableUtiles.findTtlColumn(table.schema);
94
+ });
95
+ if (ttlTables.length === 0) return;
96
+
97
+ const totals = {};
98
+ try {
99
+ const agg = await TableRow.aggregate([
100
+ {
101
+ $match: {
102
+ id_project: String(id_project),
103
+ id_table: { $in: ttlTables.map(function (table) { return String(table._id); }) },
104
+ },
105
+ },
106
+ { $group: { _id: '$id_table', rows: { $sum: 1 }, sizeBytes: { $sum: { $bsonSize: '$$ROOT' } } } },
107
+ ]);
108
+ agg.forEach(function (entry) {
109
+ totals[entry._id] = { rows: entry.rows, sizeBytes: entry.sizeBytes };
110
+ });
111
+ } catch (err) {
112
+ // $bsonSize requires MongoDB 4.4: on older servers keep the stored counters
113
+ winston.debug('refreshTtlStats skipped, $bsonSize unavailable: ', err);
114
+ return;
115
+ }
116
+
117
+ const writes = [];
118
+ ttlTables.forEach(function (table) {
119
+ // A table whose rows all expired is simply missing from the aggregation
120
+ const fresh = totals[String(table._id)] || { rows: 0, sizeBytes: 0 };
121
+ if (fresh.rows === table.stats.rows && fresh.sizeBytes === table.stats.sizeBytes) return;
122
+ table.stats = fresh;
123
+ writes.push({
124
+ updateOne: {
125
+ filter: { id_project: id_project, _id: table._id },
126
+ update: { $set: { 'stats.rows': fresh.rows, 'stats.sizeBytes': fresh.sizeBytes } },
127
+ },
128
+ });
129
+ });
130
+ if (writes.length > 0) await Table.bulkWrite(writes);
131
+ }
132
+
133
+ /** Expiration instant of a row: updatedAt plus the seconds held by the ttl column. */
134
+ function computeExpireAt(schema, validatedData, updatedAt) {
135
+ const ttlCol = dataTableUtiles.findTtlColumn(schema);
136
+ if (!ttlCol) return null;
137
+ const seconds = validatedData ? validatedData[ttlCol.name] : undefined;
138
+ if (typeof seconds !== 'number') return null;
139
+ return new Date(new Date(updatedAt).getTime() + seconds * 1000);
140
+ }
141
+
142
+ /**
143
+ * The ttl seconds a row holds once an update is applied: the value the update
144
+ * carries, or the one already stored when the update leaves the ttl alone.
145
+ * Null means the row has no usable ttl and must not carry a deadline.
146
+ */
147
+ function effectiveTtlSeconds(existingRow, update, schema) {
148
+ const ttlCol = dataTableUtiles.findTtlColumn(schema);
149
+ if (!ttlCol) return null;
150
+ const incoming = ((update && update.$set) || {})['data.' + ttlCol.name];
151
+ if (typeof incoming === 'number') return incoming;
152
+ const data = (existingRow && existingRow.data) || {};
153
+ return typeof data[ttlCol.name] === 'number' ? data[ttlCol.name] : null;
154
+ }
155
+
156
+ /**
157
+ * Add the derived deadline to an update. Since expireAt hangs off updatedAt,
158
+ * any change to the row pushes it forward: the ttl measures idle time, not
159
+ * total lifetime.
160
+ */
161
+ function withSlidingExpireAt(update, ttlSeconds, hasTtlColumn) {
162
+ if (!hasTtlColumn) return update;
163
+ if (ttlSeconds == null) {
164
+ return Object.assign({}, update, {
165
+ $unset: Object.assign({}, update.$unset, { expireAt: '' }),
166
+ });
167
+ }
168
+ return Object.assign({}, update, {
169
+ $set: Object.assign({}, update.$set, { expireAt: expireAtFrom(new Date(), ttlSeconds) }),
170
+ });
171
+ }
172
+
173
+ /**
174
+ * The same rule expressed server side, for when the rows to touch hold
175
+ * different ttl values and a single literal deadline would not fit them all.
176
+ */
177
+ function slidingExpireAtStage(ttlColumn) {
178
+ const field = '$data.' + ttlColumn.name;
179
+ return {
180
+ $set: {
181
+ updatedAt: '$$NOW',
182
+ expireAt: {
183
+ $cond: [
184
+ { $in: [{ $type: field }, ['double', 'int', 'long', 'decimal']] },
185
+ { $add: ['$$NOW', { $multiply: [field, 1000] }] },
186
+ '$$REMOVE',
187
+ ],
188
+ },
189
+ },
190
+ };
191
+ }
192
+
193
+ function expireAtFrom(updatedAt, seconds) {
194
+ return new Date(new Date(updatedAt).getTime() + seconds * 1000);
195
+ }
196
+
197
+ /**
198
+ * Realign expireAt on every row of a table after a schema change.
199
+ * Without the unset branch, dropping a ttl column would leave the rows still
200
+ * scheduled for deletion even though the table no longer declares any ttl.
201
+ */
202
+ async function syncExpireAtForTable(id_project, id_table, ttlColumn) {
203
+ const scope = { id_project: id_project, id_table: id_table };
204
+
205
+ if (!ttlColumn) {
206
+ await TableRow.updateMany(scope, { $unset: { expireAt: '' } });
207
+ } else {
208
+ // The schema change touches the rows, so updatedAt moves and the deadline
209
+ // follows it. Stamping both from $$NOW keeps them in step: reading the
210
+ // stored updatedAt would anchor on the value this very update replaces.
211
+ await TableRow.updateMany(scope, [slidingExpireAtStage(ttlColumn)]);
212
+ }
213
+
214
+ return recomputeTableStats(id_project, id_table);
215
+ }
216
+
40
217
  function rowToPlainObject(row) {
41
218
  return row.toObject ? row.toObject({ depopulate: true }) : row;
42
219
  }
43
220
 
44
- function buildInsertRowDoc(id_project, id_table, data, id_row) {
45
- const now = new Date();
46
- return {
221
+ function buildInsertRowDoc(id_project, id_table, data, id_row, createdAt, expireAt) {
222
+ const now = createdAt || new Date();
223
+ const doc = {
47
224
  _id: id_row || new mongoose.Types.ObjectId(),
48
225
  id_project: id_project,
49
226
  id_table: id_table,
50
227
  data: data,
51
228
  createdAt: now,
52
229
  updatedAt: now,
230
+ // Mongoose stamps the version key on save. Leaving it out here would make
231
+ // an insert add fewer bytes than the matching delete later subtracts, and
232
+ // the counter would sink below zero on a table with enough churn.
233
+ __v: 0,
53
234
  };
235
+ if (expireAt) doc.expireAt = expireAt;
236
+ return doc;
54
237
  }
55
238
 
56
- function buildUpdatedRowDoc(existingRow, dataPatch) {
239
+ function buildUpdatedRowDoc(existingRow, dataPatch, ttlSeconds, hasTtlColumn) {
57
240
  const plain = rowToPlainObject(existingRow);
58
241
  plain.data = Object.assign({}, plain.data || {}, dataPatch);
59
242
  plain.updatedAt = new Date();
243
+ if (!hasTtlColumn) return plain;
244
+ // Fully derived from the ttl the row ends up with, so a row that loses its
245
+ // ttl value loses the deadline with it
246
+ if (ttlSeconds == null) delete plain.expireAt;
247
+ else plain.expireAt = expireAtFrom(plain.updatedAt, ttlSeconds);
60
248
  return plain;
61
249
  }
62
250
 
@@ -83,12 +271,17 @@ function computeInsertDelta(rowDoc) {
83
271
  return calculateBsonSize(rowDoc);
84
272
  }
85
273
 
86
- function computeUpdateDelta(existingRows, update) {
274
+ function computeUpdateDelta(existingRows, update, schema) {
87
275
  const dataPatch = dataPatchFromUpdate(update);
276
+ const hasTtl = !!dataTableUtiles.findTtlColumn(schema || []);
88
277
  let totalDelta = 0;
89
278
  for (let i = 0; i < existingRows.length; i++) {
90
- const oldSize = calculateBsonSize(rowToPlainObject(existingRows[i]));
91
- const newSize = calculateBsonSize(buildUpdatedRowDoc(existingRows[i], dataPatch));
279
+ const row = existingRows[i];
280
+ // Rows of the same table can hold different ttl values, so the deadline
281
+ // has to be simulated row by row
282
+ const ttlSeconds = effectiveTtlSeconds(row, update, schema);
283
+ const oldSize = calculateBsonSize(rowToPlainObject(row));
284
+ const newSize = calculateBsonSize(buildUpdatedRowDoc(row, dataPatch, ttlSeconds, hasTtl));
92
285
  totalDelta += newSize - oldSize;
93
286
  }
94
287
  return totalDelta;
@@ -117,7 +310,7 @@ function formatTable(table) {
117
310
 
118
311
  function formatRow(row, schema) {
119
312
  const o = row.toObject ? row.toObject() : row;
120
- return {
313
+ const formatted = {
121
314
  _id: o._id,
122
315
  id_project: o.id_project,
123
316
  id_table: o.id_table,
@@ -125,6 +318,9 @@ function formatRow(row, schema) {
125
318
  createdAt: o.createdAt,
126
319
  updatedAt: o.updatedAt,
127
320
  };
321
+ // The ttl column only holds seconds: surface when the row actually disappears
322
+ if (o.expireAt) formatted.expireAt = o.expireAt;
323
+ return formatted;
128
324
  }
129
325
 
130
326
  function formatRowForList(row, schema) {
@@ -143,7 +339,9 @@ class DataTableService {
143
339
 
144
340
  async listTables(id_project) {
145
341
  const tables = await Table.find({ id_project }).sort({ createdAt: -1 });
146
- return tables.map(formatTable);
342
+ const formatted = tables.map(formatTable);
343
+ await refreshTtlStats(id_project, formatted);
344
+ return formatted;
147
345
  }
148
346
 
149
347
  async getTable(id_project, id_table) {
@@ -151,6 +349,7 @@ class DataTableService {
151
349
  if (!table) return null;
152
350
 
153
351
  const result = formatTable(table);
352
+ await refreshTtlStats(id_project, [result]);
154
353
  const rows = await TableRow.find({ id_project: id_project, id_table: id_table })
155
354
  .sort({ createdAt: 1 })
156
355
  .limit(100);
@@ -196,7 +395,9 @@ class DataTableService {
196
395
  update,
197
396
  { new: true }
198
397
  );
199
- return formatTable(renamedOnly);
398
+ const result = formatTable(renamedOnly);
399
+ await refreshTtlStats(id_project, [result]);
400
+ return result;
200
401
  }
201
402
 
202
403
  const currentSchema = table.schema || [];
@@ -224,7 +425,16 @@ class DataTableService {
224
425
  update,
225
426
  { new: true }
226
427
  );
227
- return formatTable(updated);
428
+
429
+ const result = formatTable(updated);
430
+ if (resolved.ttlAfter || resolved.ttlBefore) {
431
+ result.stats = await syncExpireAtForTable(id_project, id_table, resolved.ttlAfter);
432
+ } else if (resolved.deletes.length > 0 || resolved.renames.length > 0) {
433
+ // Both the $unset and the $rename loops above changed the size of every row
434
+ result.stats = await recomputeTableStats(id_project, id_table);
435
+ }
436
+
437
+ return result;
228
438
  }
229
439
 
230
440
  async deleteTable(id_project, id_table) {
@@ -247,7 +457,12 @@ class DataTableService {
247
457
  { schema: schema },
248
458
  { new: true }
249
459
  );
250
- return formatTable(updated);
460
+
461
+ // A new column carries no data, so the rows keep their size: only a table
462
+ // owning a ttl column can be holding counters inflated by expired rows.
463
+ const result = formatTable(updated);
464
+ await refreshTtlStats(id_project, [result]);
465
+ return result;
251
466
  }
252
467
 
253
468
  async renameColumn(id_project, id_table, columnId, newName) {
@@ -263,7 +478,11 @@ class DataTableService {
263
478
  const col = dataTableUtiles.findColumn(schema, columnId);
264
479
  if (!col) throw new Error('Column not found');
265
480
 
266
- if (col.name === trimmed) return formatTable(table);
481
+ if (col.name === trimmed) {
482
+ const unchanged = formatTable(table);
483
+ await refreshTtlStats(id_project, [unchanged]);
484
+ return unchanged;
485
+ }
267
486
 
268
487
  if (dataTableUtiles.findColumnByName(schema, trimmed)) {
269
488
  throw new Error('Column name already exists: ' + trimmed);
@@ -285,7 +504,12 @@ class DataTableService {
285
504
  { schema: newSchema },
286
505
  { new: true }
287
506
  );
288
- return formatTable(updated);
507
+
508
+ const result = formatTable(updated);
509
+ // The $rename above rewrote a key inside every row, so the stored byte
510
+ // count no longer matches what is on disk: a longer name costs more.
511
+ result.stats = await recomputeTableStats(id_project, id_table);
512
+ return result;
289
513
  }
290
514
 
291
515
  async deleteColumn(id_project, id_table, columnId) {
@@ -308,7 +532,16 @@ class DataTableService {
308
532
  { schema: newSchema },
309
533
  { new: true }
310
534
  );
311
- return formatTable(updated);
535
+
536
+ const result = formatTable(updated);
537
+ if (col.type === dataTableUtiles.COLUMN_TYPES.TTL) {
538
+ result.stats = await syncExpireAtForTable(id_project, id_table, null);
539
+ } else {
540
+ // The $unset above shrank every row: the stored byte count is now too high
541
+ result.stats = await recomputeTableStats(id_project, id_table);
542
+ }
543
+
544
+ return result;
312
545
  }
313
546
 
314
547
  async listRows(id_project, id_table) {
@@ -374,19 +607,99 @@ class DataTableService {
374
607
  if (body.data === undefined || body.data === null) {
375
608
  throw new Error('data is required');
376
609
  }
377
- const data = dataTableUtiles.validateRowData(body.data, table.schema || []);
378
- const rowDoc = buildInsertRowDoc(id_project, id_table, data, body.id_row);
610
+ const schema = table.schema || [];
611
+ const data = dataTableUtiles.validateRowData(body.data, schema);
612
+ const now = new Date();
613
+ const expireAt = computeExpireAt(schema, data, now);
614
+ const rowDoc = buildInsertRowDoc(id_project, id_table, data, body.id_row, now, expireAt);
379
615
  const rowSize = computeInsertDelta(rowDoc);
380
- assertTableSizeLimit(table, rowSize);
616
+ await enforceTableSizeLimit(table, rowSize, id_project, id_table);
381
617
 
382
618
  const row = await TableRow.create({
383
619
  _id: rowDoc._id,
384
620
  id_project: id_project,
385
621
  id_table: id_table,
386
622
  data: data,
623
+ expireAt: expireAt || undefined,
387
624
  });
388
625
  await incrementTableStats(id_project, id_table, 1, rowSize);
389
- return formatRow(row, table.schema || []);
626
+ return formatRow(row, schema);
627
+ }
628
+
629
+ async insertRowsByBody(id_project, id_table, body) {
630
+ const table = await Table.findOne({ id_project: id_project, _id: id_table });
631
+ if (!table) return null;
632
+
633
+ const data = body && body.data;
634
+ if (!Array.isArray(data) || data.length === 0) {
635
+ throw new Error('data must be a non-empty array');
636
+ }
637
+ if (data.length > MAX_BULK_ROWS) {
638
+ throw new Error('Invalid data: at most ' + MAX_BULK_ROWS + ' rows per request');
639
+ }
640
+
641
+ const schema = table.schema || [];
642
+ const insertDocs = [];
643
+ let totalSize = 0;
644
+
645
+ // Validate every row before writing anything, so a single invalid row
646
+ // cannot leave the table with a partial batch.
647
+ for (let i = 0; i < data.length; i++) {
648
+ const entry = data[i];
649
+ if (!entry || typeof entry !== 'object' || Array.isArray(entry)) {
650
+ throw new Error('Invalid row at index ' + i + ': each row must be an object');
651
+ }
652
+
653
+ let rowData;
654
+ try {
655
+ rowData = dataTableUtiles.validateRowData(entry, schema);
656
+ } catch (err) {
657
+ throw new Error('Invalid row at index ' + i + ': ' + err.message);
658
+ }
659
+
660
+ const now = new Date();
661
+ const expireAt = computeExpireAt(schema, rowData, now);
662
+ const rowDoc = buildInsertRowDoc(id_project, id_table, rowData, null, now, expireAt);
663
+ totalSize += computeInsertDelta(rowDoc);
664
+ const insertDoc = {
665
+ _id: rowDoc._id,
666
+ id_project: id_project,
667
+ id_table: id_table,
668
+ data: rowData,
669
+ };
670
+ if (expireAt) insertDoc.expireAt = expireAt;
671
+ insertDocs.push(insertDoc);
672
+ }
673
+
674
+ await enforceTableSizeLimit(table, totalSize, id_project, id_table);
675
+
676
+ let created;
677
+ try {
678
+ created = await TableRow.insertMany(insertDocs, { ordered: true });
679
+ } catch (err) {
680
+ await this._reconcileStatsAfterFailedInsert(id_project, id_table, insertDocs);
681
+ throw err;
682
+ }
683
+
684
+ await incrementTableStats(id_project, id_table, created.length, totalSize);
685
+ return created.map(function (r) { return formatRow(r, schema); });
686
+ }
687
+
688
+ /** An ordered insertMany can stop halfway: realign stats on what really landed. */
689
+ async _reconcileStatsAfterFailedInsert(id_project, id_table, insertDocs) {
690
+ const ids = insertDocs.map(function (d) { return d._id; });
691
+ const persisted = await TableRow.find({
692
+ id_project: id_project,
693
+ id_table: id_table,
694
+ _id: { $in: ids },
695
+ });
696
+ if (persisted.length === 0) return;
697
+
698
+ let size = 0;
699
+ for (let i = 0; i < persisted.length; i++) {
700
+ size += calculateBsonSize(rowToPlainObject(persisted[i]));
701
+ }
702
+ await incrementTableStats(id_project, id_table, persisted.length, size);
390
703
  }
391
704
 
392
705
  async updateRow(id_project, id_table, id_row, rawData) {
@@ -403,22 +716,36 @@ class DataTableService {
403
716
  const filter = { id_project: id_project, id_table: id_table };
404
717
  Object.assign(filter, rowFilter);
405
718
 
719
+ const ttlColumn = dataTableUtiles.findTtlColumn(schema);
720
+
406
721
  if (hasConditions(body)) {
407
722
  const matches = await TableRow.find(filter).sort({ createdAt: 1 });
408
723
  if (matches.length === 0) return undefined;
409
- const sizeDelta = computeUpdateDelta(matches, update);
410
- assertTableSizeLimit(table, sizeDelta);
724
+ const sizeDelta = computeUpdateDelta(matches, update, schema);
725
+ await enforceTableSizeLimit(table, sizeDelta, id_project, id_table);
726
+
727
+ // The patch can write the very field the conditions select on, moving a
728
+ // row out of the filter, so everything past this point goes by id
729
+ const ids = matches.map(function (r) { return r._id; });
411
730
  await TableRow.updateMany(filter, update);
731
+ // Each matched row may hold its own ttl, so the deadlines are derived on
732
+ // the server from the values the patch above just left in place
733
+ if (ttlColumn) {
734
+ await TableRow.updateMany({ _id: { $in: ids } }, [slidingExpireAtStage(ttlColumn)]);
735
+ }
736
+
412
737
  await incrementTableStats(id_project, id_table, 0, sizeDelta);
413
- const updatedRows = await TableRow.find(filter).sort({ createdAt: 1 });
738
+ const updatedRows = await TableRow.find({ _id: { $in: ids } }).sort({ createdAt: 1 });
414
739
  return updatedRows.map(function (r) { return formatRow(r, schema); });
415
740
  }
416
741
 
417
742
  const existing = await TableRow.findOne(filter);
418
743
  if (!existing) return undefined;
419
- const singleDelta = computeUpdateDelta([existing], update);
420
- assertTableSizeLimit(table, singleDelta);
421
- const row = await TableRow.findOneAndUpdate(filter, update, { new: true });
744
+ const singleDelta = computeUpdateDelta([existing], update, schema);
745
+ await enforceTableSizeLimit(table, singleDelta, id_project, id_table);
746
+ const singleUpdate = withSlidingExpireAt(
747
+ update, effectiveTtlSeconds(existing, update, schema), !!ttlColumn);
748
+ const row = await TableRow.findOneAndUpdate(filter, singleUpdate, { new: true });
422
749
  await incrementTableStats(id_project, id_table, 0, singleDelta);
423
750
  return formatRow(row, schema);
424
751
  }
@@ -434,26 +761,35 @@ class DataTableService {
434
761
 
435
762
  const update = dataTableUtiles.buildUpdateSet(body.data, schema);
436
763
  const rowData = dataTableUtiles.validateRowData(body.data, schema);
764
+ const hasTtl = !!dataTableUtiles.findTtlColumn(schema);
765
+
766
+ const updateFor = function (existingRow) {
767
+ return withSlidingExpireAt(
768
+ update, effectiveTtlSeconds(existingRow, update, schema), hasTtl);
769
+ };
437
770
 
438
771
  if (body.id_row) {
439
772
  const filterById = { id_project: id_project, id_table: id_table, _id: body.id_row };
440
773
  const existing = await TableRow.findOne(filterById);
441
774
  if (existing) {
442
- const updateDelta = computeUpdateDelta([existing], update);
443
- assertTableSizeLimit(table, updateDelta);
444
- const updated = await TableRow.findOneAndUpdate(filterById, update, { new: true });
775
+ const updateDelta = computeUpdateDelta([existing], update, schema);
776
+ await enforceTableSizeLimit(table, updateDelta, id_project, id_table);
777
+ const updated = await TableRow.findOneAndUpdate(filterById, updateFor(existing), { new: true });
445
778
  await incrementTableStats(id_project, id_table, 0, updateDelta);
446
779
  return formatRow(updated, schema);
447
780
  }
448
- const insertDoc = buildInsertRowDoc(id_project, id_table, rowData, body.id_row);
781
+ const insertedAt = new Date();
782
+ const insertExpireAt = computeExpireAt(schema, rowData, insertedAt);
783
+ const insertDoc = buildInsertRowDoc(id_project, id_table, rowData, body.id_row, insertedAt, insertExpireAt);
449
784
  const insertSize = computeInsertDelta(insertDoc);
450
- assertTableSizeLimit(table, insertSize);
785
+ await enforceTableSizeLimit(table, insertSize, id_project, id_table);
451
786
  try {
452
787
  const created = await TableRow.create({
453
788
  _id: body.id_row,
454
789
  id_project: id_project,
455
790
  id_table: id_table,
456
791
  data: rowData,
792
+ expireAt: insertExpireAt || undefined,
457
793
  });
458
794
  await incrementTableStats(id_project, id_table, 1, insertSize);
459
795
  return formatRow(created, schema);
@@ -461,9 +797,9 @@ class DataTableService {
461
797
  if (createErr.code === 11000) {
462
798
  const racedExisting = await TableRow.findOne(filterById);
463
799
  if (racedExisting) {
464
- const racedDelta = computeUpdateDelta([racedExisting], update);
465
- assertTableSizeLimit(table, racedDelta);
466
- const raced = await TableRow.findOneAndUpdate(filterById, update, { new: true });
800
+ const racedDelta = computeUpdateDelta([racedExisting], update, schema);
801
+ await enforceTableSizeLimit(table, racedDelta, id_project, id_table);
802
+ const raced = await TableRow.findOneAndUpdate(filterById, updateFor(racedExisting), { new: true });
467
803
  await incrementTableStats(id_project, id_table, 0, racedDelta);
468
804
  if (raced) return formatRow(raced, schema);
469
805
  }
@@ -478,10 +814,17 @@ class DataTableService {
478
814
  const matches = await TableRow.find(filter).sort({ createdAt: 1 });
479
815
 
480
816
  if (matches.length === 0) {
481
- const newDoc = buildInsertRowDoc(id_project, id_table, rowData);
817
+ const newAt = new Date();
818
+ const newExpireAt = computeExpireAt(schema, rowData, newAt);
819
+ const newDoc = buildInsertRowDoc(id_project, id_table, rowData, null, newAt, newExpireAt);
482
820
  const newSize = computeInsertDelta(newDoc);
483
- assertTableSizeLimit(table, newSize);
484
- const inserted = await TableRow.create({ id_project: id_project, id_table: id_table, data: rowData });
821
+ await enforceTableSizeLimit(table, newSize, id_project, id_table);
822
+ const inserted = await TableRow.create({
823
+ id_project: id_project,
824
+ id_table: id_table,
825
+ data: rowData,
826
+ expireAt: newExpireAt || undefined,
827
+ });
485
828
  await incrementTableStats(id_project, id_table, 1, newSize);
486
829
  return formatRow(inserted, schema);
487
830
  }
@@ -490,13 +833,64 @@ class DataTableService {
490
833
  throw new Error('Multiple rows match the conditions');
491
834
  }
492
835
 
493
- const upsertDelta = computeUpdateDelta(matches, update);
494
- assertTableSizeLimit(table, upsertDelta);
495
- const updated = await TableRow.findOneAndUpdate(filter, update, { new: true });
836
+ const upsertDelta = computeUpdateDelta(matches, update, schema);
837
+ await enforceTableSizeLimit(table, upsertDelta, id_project, id_table);
838
+ const updated = await TableRow.findOneAndUpdate(filter, updateFor(matches[0]), { new: true });
496
839
  await incrementTableStats(id_project, id_table, 0, upsertDelta);
497
840
  return formatRow(updated, schema);
498
841
  }
499
842
 
843
+ /**
844
+ * The single upsert with data holding an array of rows. The conditions name
845
+ * the columns that identify a row, and each row of the batch is looked up by
846
+ * the values it carries in those columns.
847
+ */
848
+ async upsertRowsByBody(id_project, id_table, body) {
849
+ const table = await Table.findOne({ id_project: id_project, _id: id_table });
850
+ if (!table) return null;
851
+
852
+ const rows = body && body.data;
853
+ if (!Array.isArray(rows) || rows.length === 0) {
854
+ throw new Error('data must be a non-empty array');
855
+ }
856
+ if (rows.length > MAX_BULK_ROWS) {
857
+ throw new Error('Invalid data: at most ' + MAX_BULK_ROWS + ' rows per request');
858
+ }
859
+
860
+ const schema = table.schema || [];
861
+ const byConditions = !body.id_row && body.conditions && body.conditions.length;
862
+
863
+ // Validate and bind the whole batch up front: rejecting on the fifth row
864
+ // would otherwise leave the four before it already written.
865
+ const conditionsFor = [];
866
+ for (let i = 0; i < rows.length; i++) {
867
+ const entry = rows[i];
868
+ if (!entry || typeof entry !== 'object' || Array.isArray(entry)) {
869
+ throw new Error('Invalid row at index ' + i + ': each row must be an object');
870
+ }
871
+ try {
872
+ // Binding against the validated data, so the lookup compares the same
873
+ // coerced values that were stored in the first place
874
+ const rowData = dataTableUtiles.validateRowData(entry, schema);
875
+ if (byConditions) {
876
+ conditionsFor.push(dataTableUtiles.bindConditionsToRow(body.conditions, rowData, schema));
877
+ }
878
+ } catch (err) {
879
+ throw new Error('Invalid row at index ' + i + ': ' + err.message);
880
+ }
881
+ }
882
+
883
+ const results = [];
884
+ for (let i = 0; i < rows.length; i++) {
885
+ const single = Object.assign({}, body, { data: rows[i] });
886
+ if (byConditions) single.conditions = conditionsFor[i];
887
+ // Re-reading the table on every row is what keeps the size guard honest:
888
+ // it has to see the bytes the rows before it just added.
889
+ results.push(await this.upsertRowByBody(id_project, id_table, single));
890
+ }
891
+ return results;
892
+ }
893
+
500
894
  async deleteRow(id_project, id_table, id_row) {
501
895
  return this.deleteRowByBody(id_project, id_table, { id_row: id_row });
502
896
  }