@tiledesk/tiledesk-server 3.1.0 → 3.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/models/dataTable.js +16 -0
- package/models/location.js +6 -12
- package/models/request.js +65 -61
- package/package.json +3 -3
- package/routes/dataTable.js +23 -0
- package/routes/project.js +1 -1
- package/services/dataTableService.js +436 -42
- package/test/dataTablesRoute.test.js +639 -0
- package/utils/dataTableUtils.js +99 -9
|
@@ -8,6 +8,7 @@ const projectService = require('../services/projectService');
|
|
|
8
8
|
const userService = require('../services/userService');
|
|
9
9
|
var RoleConstants = require("../models/roleConstants");
|
|
10
10
|
const Project_user = require('../models/project_user');
|
|
11
|
+
const { TableRow } = require('../models/dataTable');
|
|
11
12
|
let should = chai.should();
|
|
12
13
|
|
|
13
14
|
let log = false;
|
|
@@ -849,4 +850,642 @@ describe('Data Tables Route', () => {
|
|
|
849
850
|
});
|
|
850
851
|
}).timeout(5000);
|
|
851
852
|
|
|
853
|
+
})
|
|
854
|
+
|
|
855
|
+
describe('Data Tables TTL column', function () {
|
|
856
|
+
|
|
857
|
+
this.timeout(30000);
|
|
858
|
+
|
|
859
|
+
// A ttl column holds seconds; the service mirrors them into the top level
|
|
860
|
+
// expireAt date that carries the MongoDB TTL index.
|
|
861
|
+
async function setupTable(schema) {
|
|
862
|
+
const email = "test-signup-" + Date.now() + "-" + Math.random().toString(36).slice(2) + "@email.com";
|
|
863
|
+
const pwd = "pwd";
|
|
864
|
+
const user = await userService.signup(email, pwd, "Test Firstname", "Test lastname");
|
|
865
|
+
const project = await projectService.create("test-ttl-column", user._id);
|
|
866
|
+
|
|
867
|
+
const res = await chai.request(server)
|
|
868
|
+
.post('/' + project._id + '/tables')
|
|
869
|
+
.auth(email, pwd)
|
|
870
|
+
.send({ name: "events", schema: schema });
|
|
871
|
+
expect(res).to.have.status(200);
|
|
872
|
+
|
|
873
|
+
return {
|
|
874
|
+
email: email,
|
|
875
|
+
pwd: pwd,
|
|
876
|
+
table: res.body,
|
|
877
|
+
url: '/' + project._id + '/tables/' + res.body._id,
|
|
878
|
+
tablesUrl: '/' + project._id + '/tables',
|
|
879
|
+
};
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
const storedRows = (ctx) => TableRow.find({ id_table: String(ctx.table._id) }).lean().exec();
|
|
883
|
+
const secondsBetween = (expireAt, anchor) => (new Date(expireAt) - new Date(anchor)) / 1000;
|
|
884
|
+
|
|
885
|
+
const withTtl = [{ name: "label", type: "string" }, { name: "lifetime", type: "ttl" }];
|
|
886
|
+
|
|
887
|
+
it('accepts a ttl column in the schema', async () => {
|
|
888
|
+
const ctx = await setupTable(withTtl);
|
|
889
|
+
expect(ctx.table.schema).to.be.an('array').that.has.lengthOf(2);
|
|
890
|
+
expect(ctx.table.schema[1].type).to.equal("ttl");
|
|
891
|
+
});
|
|
892
|
+
|
|
893
|
+
it('rejects a schema with more than one ttl column', async () => {
|
|
894
|
+
const email = "test-signup-" + Date.now() + "-" + Math.random().toString(36).slice(2) + "@email.com";
|
|
895
|
+
const pwd = "pwd";
|
|
896
|
+
const user = await userService.signup(email, pwd, "Test Firstname", "Test lastname");
|
|
897
|
+
const project = await projectService.create("test-ttl-duplicate", user._id);
|
|
898
|
+
|
|
899
|
+
const res = await chai.request(server)
|
|
900
|
+
.post('/' + project._id + '/tables')
|
|
901
|
+
.auth(email, pwd)
|
|
902
|
+
.send({ name: "events", schema: [{ name: "first", type: "ttl" }, { name: "second", type: "ttl" }] });
|
|
903
|
+
|
|
904
|
+
expect(res).to.have.status(400);
|
|
905
|
+
expect(res.body.message).to.include('only one ttl column');
|
|
906
|
+
});
|
|
907
|
+
|
|
908
|
+
it('rejects a ttl value that is not a positive integer', async () => {
|
|
909
|
+
const ctx = await setupTable(withTtl);
|
|
910
|
+
|
|
911
|
+
for (const invalid of [0, -5, 1.5, "abc"]) {
|
|
912
|
+
const res = await chai.request(server)
|
|
913
|
+
.post(ctx.url + '/row/insert')
|
|
914
|
+
.auth(ctx.email, ctx.pwd)
|
|
915
|
+
.send({ data: { label: "x", lifetime: invalid } });
|
|
916
|
+
|
|
917
|
+
expect(res, 'lifetime ' + JSON.stringify(invalid)).to.have.status(400);
|
|
918
|
+
expect(res.body.message).to.include('column lifetime');
|
|
919
|
+
}
|
|
920
|
+
});
|
|
921
|
+
|
|
922
|
+
it('rejects a ttl so large that the deadline is not a representable date', async () => {
|
|
923
|
+
const ctx = await setupTable(withTtl);
|
|
924
|
+
|
|
925
|
+
const res = await chai.request(server)
|
|
926
|
+
.post(ctx.url + '/row/insert')
|
|
927
|
+
.auth(ctx.email, ctx.pwd)
|
|
928
|
+
.send({ data: { label: "x", lifetime: 1e15 } });
|
|
929
|
+
|
|
930
|
+
// Without the cap the model refuses the cast and the client gets a 500
|
|
931
|
+
expect(res).to.have.status(400);
|
|
932
|
+
expect(res.body.message).to.include('must not exceed');
|
|
933
|
+
});
|
|
934
|
+
|
|
935
|
+
it('sets expireAt to updatedAt plus the ttl seconds on insert', async () => {
|
|
936
|
+
const ctx = await setupTable(withTtl);
|
|
937
|
+
|
|
938
|
+
const res = await chai.request(server)
|
|
939
|
+
.post(ctx.url + '/row/insert')
|
|
940
|
+
.auth(ctx.email, ctx.pwd)
|
|
941
|
+
.send({ data: { label: "session", lifetime: 3600 } });
|
|
942
|
+
|
|
943
|
+
expect(res).to.have.status(200);
|
|
944
|
+
expect(res.body.expireAt, 'the response surfaces when the row disappears').to.exist;
|
|
945
|
+
expect(secondsBetween(res.body.expireAt, res.body.updatedAt)).to.be.closeTo(3600, 2);
|
|
946
|
+
});
|
|
947
|
+
|
|
948
|
+
it('leaves rows that carry no ttl value without an expireAt', async () => {
|
|
949
|
+
const ctx = await setupTable(withTtl);
|
|
950
|
+
|
|
951
|
+
const res = await chai.request(server)
|
|
952
|
+
.post(ctx.url + '/row/insert')
|
|
953
|
+
.auth(ctx.email, ctx.pwd)
|
|
954
|
+
.send({ data: { label: "permanent" } });
|
|
955
|
+
|
|
956
|
+
expect(res).to.have.status(200);
|
|
957
|
+
expect(res.body.expireAt).to.be.undefined;
|
|
958
|
+
|
|
959
|
+
const rows = await storedRows(ctx);
|
|
960
|
+
expect(rows[0].expireAt, 'nothing for the TTL monitor to collect').to.be.undefined;
|
|
961
|
+
});
|
|
962
|
+
|
|
963
|
+
it('never sets expireAt on a table without a ttl column', async () => {
|
|
964
|
+
const ctx = await setupTable([{ name: "label", type: "string" }, { name: "lifetime", type: "number" }]);
|
|
965
|
+
|
|
966
|
+
const res = await chai.request(server)
|
|
967
|
+
.post(ctx.url + '/row/insert')
|
|
968
|
+
.auth(ctx.email, ctx.pwd)
|
|
969
|
+
.send({ data: { label: "x", lifetime: 3600 } });
|
|
970
|
+
|
|
971
|
+
expect(res).to.have.status(200);
|
|
972
|
+
expect(res.body.expireAt).to.be.undefined;
|
|
973
|
+
});
|
|
974
|
+
|
|
975
|
+
it('re-anchors expireAt on the moment of the update when the ttl changes', async () => {
|
|
976
|
+
const ctx = await setupTable(withTtl);
|
|
977
|
+
|
|
978
|
+
const inserted = await chai.request(server)
|
|
979
|
+
.post(ctx.url + '/row/insert')
|
|
980
|
+
.auth(ctx.email, ctx.pwd)
|
|
981
|
+
.send({ data: { label: "session", lifetime: 100 } });
|
|
982
|
+
expect(inserted).to.have.status(200);
|
|
983
|
+
|
|
984
|
+
await new Promise(resolve => setTimeout(resolve, 1200));
|
|
985
|
+
|
|
986
|
+
const updated = await chai.request(server)
|
|
987
|
+
.put(ctx.url + '/row/update')
|
|
988
|
+
.auth(ctx.email, ctx.pwd)
|
|
989
|
+
.send({ id_row: inserted.body._id, data: { lifetime: 200 } });
|
|
990
|
+
|
|
991
|
+
expect(updated).to.have.status(200);
|
|
992
|
+
expect(secondsBetween(updated.body.expireAt, updated.body.updatedAt)).to.be.closeTo(200, 2);
|
|
993
|
+
// Anchoring on createdAt instead would have kept the deadline where it was
|
|
994
|
+
expect(secondsBetween(updated.body.expireAt, inserted.body.createdAt)).to.be.above(201);
|
|
995
|
+
});
|
|
996
|
+
|
|
997
|
+
it('pushes expireAt forward even when the update leaves the ttl alone', async () => {
|
|
998
|
+
const ctx = await setupTable(withTtl);
|
|
999
|
+
|
|
1000
|
+
const inserted = await chai.request(server)
|
|
1001
|
+
.post(ctx.url + '/row/insert')
|
|
1002
|
+
.auth(ctx.email, ctx.pwd)
|
|
1003
|
+
.send({ data: { label: "session", lifetime: 500 } });
|
|
1004
|
+
|
|
1005
|
+
await new Promise(resolve => setTimeout(resolve, 1500));
|
|
1006
|
+
|
|
1007
|
+
const updated = await chai.request(server)
|
|
1008
|
+
.put(ctx.url + '/row/update')
|
|
1009
|
+
.auth(ctx.email, ctx.pwd)
|
|
1010
|
+
.send({ id_row: inserted.body._id, data: { label: "renamed" } });
|
|
1011
|
+
|
|
1012
|
+
expect(updated).to.have.status(200);
|
|
1013
|
+
// The ttl measures idle time, so touching the row renews the full 500s
|
|
1014
|
+
expect(secondsBetween(updated.body.expireAt, updated.body.updatedAt)).to.be.closeTo(500, 1);
|
|
1015
|
+
expect(new Date(updated.body.expireAt) - new Date(inserted.body.expireAt)).to.be.above(1000);
|
|
1016
|
+
});
|
|
1017
|
+
|
|
1018
|
+
it('returns the updated rows when the patch rewrites the field the conditions select on', async () => {
|
|
1019
|
+
const ctx = await setupTable(withTtl);
|
|
1020
|
+
|
|
1021
|
+
await chai.request(server)
|
|
1022
|
+
.post(ctx.url + '/row/insert')
|
|
1023
|
+
.auth(ctx.email, ctx.pwd)
|
|
1024
|
+
.send({ data: { label: "pending", lifetime: 300 } });
|
|
1025
|
+
|
|
1026
|
+
const res = await chai.request(server)
|
|
1027
|
+
.put(ctx.url + '/row/update')
|
|
1028
|
+
.auth(ctx.email, ctx.pwd)
|
|
1029
|
+
.send({
|
|
1030
|
+
conditions: [{ column: 'label', operator: 'equal', value: 'pending' }],
|
|
1031
|
+
data: { label: "done" },
|
|
1032
|
+
});
|
|
1033
|
+
|
|
1034
|
+
expect(res).to.have.status(200);
|
|
1035
|
+
// Reading the result back through the conditions would find nothing now
|
|
1036
|
+
expect(res.body).to.have.lengthOf(1);
|
|
1037
|
+
expect(res.body[0].data.label).to.equal("done");
|
|
1038
|
+
});
|
|
1039
|
+
|
|
1040
|
+
it('renews the deadline of a row whose ttl column is left untouched by conditions', async () => {
|
|
1041
|
+
const ctx = await setupTable(withTtl);
|
|
1042
|
+
|
|
1043
|
+
const inserted = await chai.request(server)
|
|
1044
|
+
.post(ctx.url + '/row/insert')
|
|
1045
|
+
.auth(ctx.email, ctx.pwd)
|
|
1046
|
+
.send({ data: { label: "batch", lifetime: 400 } });
|
|
1047
|
+
|
|
1048
|
+
await new Promise(resolve => setTimeout(resolve, 1500));
|
|
1049
|
+
|
|
1050
|
+
const res = await chai.request(server)
|
|
1051
|
+
.put(ctx.url + '/row/update')
|
|
1052
|
+
.auth(ctx.email, ctx.pwd)
|
|
1053
|
+
.send({
|
|
1054
|
+
conditions: [{ column: 'label', operator: 'equal', value: 'batch' }],
|
|
1055
|
+
data: { label: "renamed" },
|
|
1056
|
+
});
|
|
1057
|
+
|
|
1058
|
+
expect(res).to.have.status(200);
|
|
1059
|
+
// The ttl value never reaches the server here: it is read back from the row
|
|
1060
|
+
const rows = await storedRows(ctx);
|
|
1061
|
+
expect(secondsBetween(rows[0].expireAt, rows[0].updatedAt)).to.be.closeTo(400, 1);
|
|
1062
|
+
expect(new Date(rows[0].expireAt) - new Date(inserted.body.expireAt)).to.be.above(1000);
|
|
1063
|
+
});
|
|
1064
|
+
|
|
1065
|
+
it('gives every row matched by conditions the same deadline', async () => {
|
|
1066
|
+
const ctx = await setupTable(withTtl);
|
|
1067
|
+
|
|
1068
|
+
const first = await chai.request(server)
|
|
1069
|
+
.post(ctx.url + '/row/insert')
|
|
1070
|
+
.auth(ctx.email, ctx.pwd)
|
|
1071
|
+
.send({ data: { label: "batch", lifetime: 100 } });
|
|
1072
|
+
|
|
1073
|
+
await new Promise(resolve => setTimeout(resolve, 1200));
|
|
1074
|
+
|
|
1075
|
+
const second = await chai.request(server)
|
|
1076
|
+
.post(ctx.url + '/row/insert')
|
|
1077
|
+
.auth(ctx.email, ctx.pwd)
|
|
1078
|
+
.send({ data: { label: "batch", lifetime: 100 } });
|
|
1079
|
+
|
|
1080
|
+
const res = await chai.request(server)
|
|
1081
|
+
.put(ctx.url + '/row/update')
|
|
1082
|
+
.auth(ctx.email, ctx.pwd)
|
|
1083
|
+
.send({
|
|
1084
|
+
must_match: 'all',
|
|
1085
|
+
conditions: [{ column: 'label', operator: 'equal', value: 'batch' }],
|
|
1086
|
+
data: { lifetime: 900 },
|
|
1087
|
+
});
|
|
1088
|
+
|
|
1089
|
+
expect(res).to.have.status(200);
|
|
1090
|
+
expect(res.body).to.be.an('array').that.has.lengthOf(2);
|
|
1091
|
+
|
|
1092
|
+
const byId = {};
|
|
1093
|
+
res.body.forEach(row => { byId[String(row._id)] = row; });
|
|
1094
|
+
// Both rows were touched at the same instant, so they expire together even
|
|
1095
|
+
// though they were created more than a second apart
|
|
1096
|
+
expect(byId[first.body._id].expireAt).to.equal(byId[second.body._id].expireAt);
|
|
1097
|
+
expect(secondsBetween(byId[first.body._id].expireAt, byId[first.body._id].updatedAt)).to.be.closeTo(900, 2);
|
|
1098
|
+
expect(secondsBetween(byId[first.body._id].expireAt, first.body.createdAt)).to.be.above(901);
|
|
1099
|
+
});
|
|
1100
|
+
|
|
1101
|
+
it('sets expireAt on bulk inserted rows', async () => {
|
|
1102
|
+
const ctx = await setupTable(withTtl);
|
|
1103
|
+
|
|
1104
|
+
const res = await chai.request(server)
|
|
1105
|
+
.post(ctx.url + '/rows/insert')
|
|
1106
|
+
.auth(ctx.email, ctx.pwd)
|
|
1107
|
+
.send({ data: [{ label: "a", lifetime: 60 }, { label: "b" }] });
|
|
1108
|
+
|
|
1109
|
+
expect(res).to.have.status(200);
|
|
1110
|
+
expect(res.body).to.be.an('array').that.has.lengthOf(2);
|
|
1111
|
+
expect(secondsBetween(res.body[0].expireAt, res.body[0].updatedAt)).to.be.closeTo(60, 2);
|
|
1112
|
+
expect(res.body[1].expireAt).to.be.undefined;
|
|
1113
|
+
});
|
|
1114
|
+
|
|
1115
|
+
it('sets expireAt on a row created by upsert', async () => {
|
|
1116
|
+
const ctx = await setupTable(withTtl);
|
|
1117
|
+
|
|
1118
|
+
const res = await chai.request(server)
|
|
1119
|
+
.put(ctx.url + '/row/upsert')
|
|
1120
|
+
.auth(ctx.email, ctx.pwd)
|
|
1121
|
+
.send({
|
|
1122
|
+
must_match: 'all',
|
|
1123
|
+
conditions: [{ column: 'label', operator: 'equal', value: 'fresh' }],
|
|
1124
|
+
data: { label: "fresh", lifetime: 120 },
|
|
1125
|
+
});
|
|
1126
|
+
|
|
1127
|
+
expect(res).to.have.status(200);
|
|
1128
|
+
expect(secondsBetween(res.body.expireAt, res.body.updatedAt)).to.be.closeTo(120, 2);
|
|
1129
|
+
});
|
|
1130
|
+
|
|
1131
|
+
it('drops expireAt when the ttl column becomes a number', async () => {
|
|
1132
|
+
const ctx = await setupTable(withTtl);
|
|
1133
|
+
|
|
1134
|
+
await chai.request(server)
|
|
1135
|
+
.post(ctx.url + '/row/insert')
|
|
1136
|
+
.auth(ctx.email, ctx.pwd)
|
|
1137
|
+
.send({ data: { label: "session", lifetime: 300 } });
|
|
1138
|
+
|
|
1139
|
+
const schema = ctx.table.schema.map(c => Object.assign({}, c));
|
|
1140
|
+
schema[1].type = "number";
|
|
1141
|
+
|
|
1142
|
+
const res = await chai.request(server)
|
|
1143
|
+
.put(ctx.url)
|
|
1144
|
+
.auth(ctx.email, ctx.pwd)
|
|
1145
|
+
.send({ schema: schema });
|
|
1146
|
+
|
|
1147
|
+
expect(res).to.have.status(200);
|
|
1148
|
+
const rows = await storedRows(ctx);
|
|
1149
|
+
// Without this the rows would keep vanishing although no ttl is declared
|
|
1150
|
+
expect(rows[0].expireAt).to.be.undefined;
|
|
1151
|
+
});
|
|
1152
|
+
|
|
1153
|
+
it('backfills expireAt when a number column becomes a ttl', async () => {
|
|
1154
|
+
const ctx = await setupTable([{ name: "label", type: "string" }, { name: "lifetime", type: "number" }]);
|
|
1155
|
+
|
|
1156
|
+
await chai.request(server)
|
|
1157
|
+
.post(ctx.url + '/row/insert')
|
|
1158
|
+
.auth(ctx.email, ctx.pwd)
|
|
1159
|
+
.send({ data: { label: "session", lifetime: 300 } });
|
|
1160
|
+
|
|
1161
|
+
// Wide enough a gap that anchoring on createdAt would be visible
|
|
1162
|
+
await new Promise(resolve => setTimeout(resolve, 1500));
|
|
1163
|
+
|
|
1164
|
+
const schema = ctx.table.schema.map(c => Object.assign({}, c));
|
|
1165
|
+
schema[1].type = "ttl";
|
|
1166
|
+
|
|
1167
|
+
const res = await chai.request(server)
|
|
1168
|
+
.put(ctx.url)
|
|
1169
|
+
.auth(ctx.email, ctx.pwd)
|
|
1170
|
+
.send({ schema: schema });
|
|
1171
|
+
|
|
1172
|
+
expect(res).to.have.status(200);
|
|
1173
|
+
const rows = await storedRows(ctx);
|
|
1174
|
+
expect(rows[0].expireAt).to.exist;
|
|
1175
|
+
expect(secondsBetween(rows[0].expireAt, rows[0].updatedAt)).to.be.closeTo(300, 1);
|
|
1176
|
+
});
|
|
1177
|
+
|
|
1178
|
+
it('drops expireAt when the ttl column is deleted', async () => {
|
|
1179
|
+
const ctx = await setupTable(withTtl);
|
|
1180
|
+
|
|
1181
|
+
await chai.request(server)
|
|
1182
|
+
.post(ctx.url + '/row/insert')
|
|
1183
|
+
.auth(ctx.email, ctx.pwd)
|
|
1184
|
+
.send({ data: { label: "session", lifetime: 300 } });
|
|
1185
|
+
|
|
1186
|
+
const res = await chai.request(server)
|
|
1187
|
+
.delete(ctx.url + '/columns/' + ctx.table.schema[1].id)
|
|
1188
|
+
.auth(ctx.email, ctx.pwd);
|
|
1189
|
+
|
|
1190
|
+
expect(res).to.have.status(200);
|
|
1191
|
+
const rows = await storedRows(ctx);
|
|
1192
|
+
expect(rows[0].expireAt).to.be.undefined;
|
|
1193
|
+
});
|
|
1194
|
+
|
|
1195
|
+
it('keeps a live expireAt when the ttl column is only renamed', async () => {
|
|
1196
|
+
const ctx = await setupTable(withTtl);
|
|
1197
|
+
|
|
1198
|
+
const inserted = await chai.request(server)
|
|
1199
|
+
.post(ctx.url + '/row/insert')
|
|
1200
|
+
.auth(ctx.email, ctx.pwd)
|
|
1201
|
+
.send({ data: { label: "session", lifetime: 300 } });
|
|
1202
|
+
|
|
1203
|
+
// Wide enough a gap that anchoring on createdAt would be visible
|
|
1204
|
+
await new Promise(resolve => setTimeout(resolve, 1500));
|
|
1205
|
+
|
|
1206
|
+
const schema = ctx.table.schema.map(c => Object.assign({}, c));
|
|
1207
|
+
schema[1].name = "renamed_lifetime";
|
|
1208
|
+
|
|
1209
|
+
const res = await chai.request(server)
|
|
1210
|
+
.put(ctx.url)
|
|
1211
|
+
.auth(ctx.email, ctx.pwd)
|
|
1212
|
+
.send({ schema: schema });
|
|
1213
|
+
|
|
1214
|
+
expect(res).to.have.status(200);
|
|
1215
|
+
const rows = await storedRows(ctx);
|
|
1216
|
+
// The rename rewrites the row, so the deadline is re-anchored rather than
|
|
1217
|
+
// carried over verbatim, but it still sits a full ttl away
|
|
1218
|
+
expect(secondsBetween(rows[0].expireAt, rows[0].updatedAt)).to.be.closeTo(300, 1);
|
|
1219
|
+
expect(new Date(rows[0].expireAt)).to.be.at.least(new Date(inserted.body.expireAt));
|
|
1220
|
+
expect(rows[0].data.renamed_lifetime).to.equal(300);
|
|
1221
|
+
});
|
|
1222
|
+
|
|
1223
|
+
it('declares the TTL index MongoDB needs to collect expired rows', async () => {
|
|
1224
|
+
// The model creates it without awaiting, so give it a chance to land
|
|
1225
|
+
let ttlIndex;
|
|
1226
|
+
for (let attempt = 0; attempt < 20 && !ttlIndex; attempt++) {
|
|
1227
|
+
const indexes = await TableRow.collection.indexes();
|
|
1228
|
+
ttlIndex = indexes.find(i => i.key && i.key.expireAt === 1);
|
|
1229
|
+
if (!ttlIndex) await new Promise(resolve => setTimeout(resolve, 250));
|
|
1230
|
+
}
|
|
1231
|
+
|
|
1232
|
+
expect(ttlIndex, 'without it the rows would keep an expireAt nothing honours').to.exist;
|
|
1233
|
+
expect(ttlIndex.expireAfterSeconds).to.equal(0);
|
|
1234
|
+
});
|
|
1235
|
+
|
|
1236
|
+
it('reports the real row count after the TTL monitor removed rows', async () => {
|
|
1237
|
+
const ctx = await setupTable(withTtl);
|
|
1238
|
+
|
|
1239
|
+
for (let i = 0; i < 4; i++) {
|
|
1240
|
+
await chai.request(server)
|
|
1241
|
+
.post(ctx.url + '/row/insert')
|
|
1242
|
+
.auth(ctx.email, ctx.pwd)
|
|
1243
|
+
.send({ data: { label: "row" + i, lifetime: 3600 } });
|
|
1244
|
+
}
|
|
1245
|
+
|
|
1246
|
+
const inflated = await chai.request(server).get(ctx.url).auth(ctx.email, ctx.pwd);
|
|
1247
|
+
expect(inflated.body.stats.rows).to.equal(4);
|
|
1248
|
+
|
|
1249
|
+
// The TTL monitor deletes behind the service's back: reproduce that here
|
|
1250
|
+
const victims = (await storedRows(ctx)).slice(0, 3).map(r => r._id);
|
|
1251
|
+
await TableRow.deleteMany({ _id: { $in: victims } });
|
|
1252
|
+
|
|
1253
|
+
const res = await chai.request(server).get(ctx.url).auth(ctx.email, ctx.pwd);
|
|
1254
|
+
expect(res).to.have.status(200);
|
|
1255
|
+
expect(res.body.stats.rows).to.equal(1);
|
|
1256
|
+
expect(res.body.stats.sizeBytes).to.be.above(0).and.below(inflated.body.stats.sizeBytes);
|
|
1257
|
+
});
|
|
1258
|
+
|
|
1259
|
+
})
|
|
1260
|
+
|
|
1261
|
+
describe('Data Tables bulk upsert', function () {
|
|
1262
|
+
|
|
1263
|
+
this.timeout(30000);
|
|
1264
|
+
|
|
1265
|
+
// Same body as the single upsert, except data carries an array of rows and
|
|
1266
|
+
// the conditions only name the columns that identify a row: the value to
|
|
1267
|
+
// match comes from each row of the batch.
|
|
1268
|
+
async function setupTable(schema) {
|
|
1269
|
+
const email = "test-signup-" + Date.now() + "-" + Math.random().toString(36).slice(2) + "@email.com";
|
|
1270
|
+
const pwd = "pwd";
|
|
1271
|
+
const user = await userService.signup(email, pwd, "Test Firstname", "Test lastname");
|
|
1272
|
+
const project = await projectService.create("test-bulk-upsert", user._id);
|
|
1273
|
+
|
|
1274
|
+
const res = await chai.request(server)
|
|
1275
|
+
.post('/' + project._id + '/tables')
|
|
1276
|
+
.auth(email, pwd)
|
|
1277
|
+
.send({ name: "people", schema: schema });
|
|
1278
|
+
expect(res).to.have.status(200);
|
|
1279
|
+
|
|
1280
|
+
return {
|
|
1281
|
+
email: email,
|
|
1282
|
+
pwd: pwd,
|
|
1283
|
+
table: res.body,
|
|
1284
|
+
url: '/' + project._id + '/tables/' + res.body._id,
|
|
1285
|
+
};
|
|
1286
|
+
}
|
|
1287
|
+
|
|
1288
|
+
const people = [{ name: "name", type: "string" }, { name: "score", type: "number" }];
|
|
1289
|
+
const storedRows = (ctx) => TableRow.find({ id_table: String(ctx.table._id) }).lean().exec();
|
|
1290
|
+
const upsert = (ctx, body) => chai.request(server)
|
|
1291
|
+
.put(ctx.url + '/rows/upsert').auth(ctx.email, ctx.pwd).send(body);
|
|
1292
|
+
|
|
1293
|
+
it('inserts every row of the batch when nothing matches the conditions', async () => {
|
|
1294
|
+
const ctx = await setupTable(people);
|
|
1295
|
+
|
|
1296
|
+
const res = await upsert(ctx, {
|
|
1297
|
+
conditions: [{ column: 'name', operator: 'equal' }],
|
|
1298
|
+
data: [{ name: "Anna", score: 1 }, { name: "Bea", score: 2 }],
|
|
1299
|
+
});
|
|
1300
|
+
|
|
1301
|
+
expect(res).to.have.status(200);
|
|
1302
|
+
expect(res.body).to.have.lengthOf(2);
|
|
1303
|
+
expect(res.body.map(r => r.data.name)).to.have.members(["Anna", "Bea"]);
|
|
1304
|
+
expect(await storedRows(ctx)).to.have.lengthOf(2);
|
|
1305
|
+
});
|
|
1306
|
+
|
|
1307
|
+
it('updates the matching row instead of inserting a second one', async () => {
|
|
1308
|
+
const ctx = await setupTable(people);
|
|
1309
|
+
|
|
1310
|
+
await chai.request(server)
|
|
1311
|
+
.post(ctx.url + '/row/insert')
|
|
1312
|
+
.auth(ctx.email, ctx.pwd)
|
|
1313
|
+
.send({ data: { name: "Anna", score: 1 } });
|
|
1314
|
+
|
|
1315
|
+
const res = await upsert(ctx, {
|
|
1316
|
+
conditions: [{ column: 'name', operator: 'equal' }],
|
|
1317
|
+
data: [{ name: "Anna", score: 99 }],
|
|
1318
|
+
});
|
|
1319
|
+
|
|
1320
|
+
expect(res).to.have.status(200);
|
|
1321
|
+
expect(res.body).to.have.lengthOf(1);
|
|
1322
|
+
expect(res.body[0].data.score).to.equal(99);
|
|
1323
|
+
expect(await storedRows(ctx)).to.have.lengthOf(1);
|
|
1324
|
+
});
|
|
1325
|
+
|
|
1326
|
+
it('mixes update and insert within the same batch', async () => {
|
|
1327
|
+
const ctx = await setupTable(people);
|
|
1328
|
+
|
|
1329
|
+
await chai.request(server)
|
|
1330
|
+
.post(ctx.url + '/row/insert')
|
|
1331
|
+
.auth(ctx.email, ctx.pwd)
|
|
1332
|
+
.send({ data: { name: "Anna", score: 1 } });
|
|
1333
|
+
|
|
1334
|
+
// Each row is looked up by its own name, so they do not compete for the
|
|
1335
|
+
// same match: Anna already exists, Bea does not
|
|
1336
|
+
const res = await upsert(ctx, {
|
|
1337
|
+
conditions: [{ column: 'name', operator: 'equal' }],
|
|
1338
|
+
data: [{ name: "Anna", score: 99 }, { name: "Bea", score: 20 }],
|
|
1339
|
+
});
|
|
1340
|
+
|
|
1341
|
+
expect(res).to.have.status(200);
|
|
1342
|
+
expect(res.body[0]._id).to.not.equal(res.body[1]._id);
|
|
1343
|
+
const rows = await storedRows(ctx);
|
|
1344
|
+
expect(rows).to.have.lengthOf(2);
|
|
1345
|
+
const byName = {};
|
|
1346
|
+
rows.forEach(r => { byName[r.data.name] = r.data.score; });
|
|
1347
|
+
expect(byName).to.deep.equal({ Anna: 99, Bea: 20 });
|
|
1348
|
+
});
|
|
1349
|
+
|
|
1350
|
+
it('ignores a literal value written in the conditions', async () => {
|
|
1351
|
+
const ctx = await setupTable(people);
|
|
1352
|
+
|
|
1353
|
+
await chai.request(server)
|
|
1354
|
+
.post(ctx.url + '/row/insert')
|
|
1355
|
+
.auth(ctx.email, ctx.pwd)
|
|
1356
|
+
.send({ data: { name: "Anna", score: 1 } });
|
|
1357
|
+
|
|
1358
|
+
// The value is superseded by the one each row carries, so Bea is inserted
|
|
1359
|
+
// rather than overwriting Anna
|
|
1360
|
+
const res = await upsert(ctx, {
|
|
1361
|
+
conditions: [{ column: 'name', operator: 'equal', value: 'Anna' }],
|
|
1362
|
+
data: [{ name: "Bea", score: 2 }],
|
|
1363
|
+
});
|
|
1364
|
+
|
|
1365
|
+
expect(res).to.have.status(200);
|
|
1366
|
+
const rows = await storedRows(ctx);
|
|
1367
|
+
expect(rows.map(r => r.data.name)).to.have.members(["Anna", "Bea"]);
|
|
1368
|
+
});
|
|
1369
|
+
|
|
1370
|
+
it('refuses a row that carries no value for a match column', async () => {
|
|
1371
|
+
const ctx = await setupTable(people);
|
|
1372
|
+
|
|
1373
|
+
const res = await upsert(ctx, {
|
|
1374
|
+
conditions: [{ column: 'name', operator: 'equal' }],
|
|
1375
|
+
data: [{ name: "Anna", score: 1 }, { score: 2 }],
|
|
1376
|
+
});
|
|
1377
|
+
|
|
1378
|
+
expect(res).to.have.status(400);
|
|
1379
|
+
expect(await storedRows(ctx)).to.be.empty;
|
|
1380
|
+
});
|
|
1381
|
+
|
|
1382
|
+
it('matches on every column when the conditions name more than one', async () => {
|
|
1383
|
+
const ctx = await setupTable(people);
|
|
1384
|
+
|
|
1385
|
+
await chai.request(server)
|
|
1386
|
+
.post(ctx.url + '/row/insert')
|
|
1387
|
+
.auth(ctx.email, ctx.pwd)
|
|
1388
|
+
.send({ data: { name: "Anna", score: 1 } });
|
|
1389
|
+
|
|
1390
|
+
const res = await upsert(ctx, {
|
|
1391
|
+
must_match: 'all',
|
|
1392
|
+
conditions: [{ column: 'name', operator: 'equal' }, { column: 'score', operator: 'equal' }],
|
|
1393
|
+
data: [{ name: "Anna", score: 1 }, { name: "Anna", score: 2 }],
|
|
1394
|
+
});
|
|
1395
|
+
|
|
1396
|
+
expect(res).to.have.status(200);
|
|
1397
|
+
// Same name but a different score, so the second row is a new one
|
|
1398
|
+
const rows = await storedRows(ctx);
|
|
1399
|
+
expect(rows).to.have.lengthOf(2);
|
|
1400
|
+
expect(rows.map(r => r.data.score)).to.have.members([1, 2]);
|
|
1401
|
+
});
|
|
1402
|
+
|
|
1403
|
+
it('keeps the table stats in step with what was written', async () => {
|
|
1404
|
+
const ctx = await setupTable(people);
|
|
1405
|
+
|
|
1406
|
+
await upsert(ctx, {
|
|
1407
|
+
conditions: [{ column: 'name', operator: 'equal' }],
|
|
1408
|
+
data: [{ name: "Anna", score: 1 }, { name: "Bea", score: 2 }, { name: "Carla", score: 3 }],
|
|
1409
|
+
});
|
|
1410
|
+
|
|
1411
|
+
const agg = await TableRow.aggregate([
|
|
1412
|
+
{ $match: { id_table: String(ctx.table._id) } },
|
|
1413
|
+
{ $group: { _id: null, rows: { $sum: 1 }, sizeBytes: { $sum: { $bsonSize: '$$ROOT' } } } },
|
|
1414
|
+
]);
|
|
1415
|
+
const table = await chai.request(server).get(ctx.url).auth(ctx.email, ctx.pwd);
|
|
1416
|
+
expect(table.body.stats.rows).to.equal(agg[0].rows);
|
|
1417
|
+
expect(table.body.stats.sizeBytes).to.equal(agg[0].sizeBytes);
|
|
1418
|
+
});
|
|
1419
|
+
|
|
1420
|
+
it('rejects a data that is not an array', async () => {
|
|
1421
|
+
const ctx = await setupTable(people);
|
|
1422
|
+
|
|
1423
|
+
const res = await upsert(ctx, {
|
|
1424
|
+
conditions: [{ column: 'name', operator: 'equal', value: 'Anna' }],
|
|
1425
|
+
data: { name: "Anna", score: 1 },
|
|
1426
|
+
});
|
|
1427
|
+
|
|
1428
|
+
expect(res).to.have.status(400);
|
|
1429
|
+
});
|
|
1430
|
+
|
|
1431
|
+
it('writes nothing when a row halfway through the batch is invalid', async () => {
|
|
1432
|
+
const ctx = await setupTable(people);
|
|
1433
|
+
|
|
1434
|
+
const res = await upsert(ctx, {
|
|
1435
|
+
conditions: [{ column: 'name', operator: 'equal' }],
|
|
1436
|
+
data: [{ name: "Anna", score: 1 }, { name: "Bea", score: "not a number" }],
|
|
1437
|
+
});
|
|
1438
|
+
|
|
1439
|
+
expect(res).to.have.status(400);
|
|
1440
|
+
expect(await storedRows(ctx)).to.be.empty;
|
|
1441
|
+
});
|
|
1442
|
+
|
|
1443
|
+
it('refuses a batch larger than the per request limit', async () => {
|
|
1444
|
+
const ctx = await setupTable(people);
|
|
1445
|
+
|
|
1446
|
+
const data = [];
|
|
1447
|
+
for (let i = 0; i < 101; i++) data.push({ name: "p" + i, score: i });
|
|
1448
|
+
const res = await upsert(ctx, {
|
|
1449
|
+
conditions: [{ column: 'name', operator: 'equal' }],
|
|
1450
|
+
data: data,
|
|
1451
|
+
});
|
|
1452
|
+
|
|
1453
|
+
expect(res).to.have.status(400);
|
|
1454
|
+
expect(await storedRows(ctx)).to.be.empty;
|
|
1455
|
+
});
|
|
1456
|
+
|
|
1457
|
+
it('reports a conflict when the conditions match more than one row', async () => {
|
|
1458
|
+
const ctx = await setupTable(people);
|
|
1459
|
+
|
|
1460
|
+
for (const score of [1, 2]) {
|
|
1461
|
+
await chai.request(server)
|
|
1462
|
+
.post(ctx.url + '/row/insert')
|
|
1463
|
+
.auth(ctx.email, ctx.pwd)
|
|
1464
|
+
.send({ data: { name: "Anna", score: score } });
|
|
1465
|
+
}
|
|
1466
|
+
|
|
1467
|
+
const res = await upsert(ctx, {
|
|
1468
|
+
conditions: [{ column: 'name', operator: 'equal' }],
|
|
1469
|
+
data: [{ name: "Anna", score: 99 }],
|
|
1470
|
+
});
|
|
1471
|
+
|
|
1472
|
+
expect(res).to.have.status(409);
|
|
1473
|
+
});
|
|
1474
|
+
|
|
1475
|
+
it('gives the inserted rows an expireAt when the table owns a ttl column', async () => {
|
|
1476
|
+
const ctx = await setupTable([{ name: "name", type: "string" }, { name: "lifetime", type: "ttl" }]);
|
|
1477
|
+
|
|
1478
|
+
const res = await upsert(ctx, {
|
|
1479
|
+
conditions: [{ column: 'name', operator: 'equal' }],
|
|
1480
|
+
data: [{ name: "Anna", lifetime: 120 }, { name: "Bea", lifetime: 240 }],
|
|
1481
|
+
});
|
|
1482
|
+
|
|
1483
|
+
expect(res).to.have.status(200);
|
|
1484
|
+
const rows = await storedRows(ctx);
|
|
1485
|
+
const byName = {};
|
|
1486
|
+
rows.forEach(r => { byName[r.data.name] = r; });
|
|
1487
|
+
expect((byName.Anna.expireAt - byName.Anna.updatedAt) / 1000).to.be.closeTo(120, 1);
|
|
1488
|
+
expect((byName.Bea.expireAt - byName.Bea.updatedAt) / 1000).to.be.closeTo(240, 1);
|
|
1489
|
+
});
|
|
1490
|
+
|
|
852
1491
|
})
|