@tiledesk/tiledesk-server 3.0.1 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.circleci/config.yml +1 -1
- package/CHANGELOG.md +34 -0
- package/Dockerfile +3 -2
- package/Dockerfile-en +3 -2
- package/Dockerfile-jobs +3 -2
- package/Dockerfile-profiler +3 -2
- package/README.md +15 -0
- package/app.js +12 -1
- package/bin/quota-reset +68 -0
- package/bin/quota-restore +86 -0
- package/channels/chat21/chat21Handler.js +50 -34
- package/channels/chat21/chat21JoinTracker.js +43 -0
- package/config/database.js +5 -5
- package/config/native_mcp/native_mcp_servers.json +1 -0
- package/controllers/chatbot.controller.js +80 -27
- package/controllers/intents.controller.js +42 -0
- package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-chat21-ionic.md +468 -0
- package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-design-studio.md +1349 -0
- package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-server.md +1694 -0
- package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-tybot.md +789 -0
- package/docs/superpowers/plans/2026-09-29-hitl-known-limits-server.md +83 -0
- package/docs/superpowers/plans/2026-09-29-hitl-known-limits-tybot.md +76 -0
- package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-chat21-ionic.md +97 -0
- package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-design-studio.md +72 -0
- package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-server.md +223 -0
- package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-tybot.md +121 -0
- package/docs/superpowers/plans/2026-09-29-hitl-pre-branch-ports.md +65 -0
- package/docs/superpowers/plans/2026-09-30-flow-start-points-design-studio.md +88 -0
- package/docs/superpowers/plans/2026-09-30-flow-start-points-server.md +103 -0
- package/docs/superpowers/plans/2026-09-30-flow-start-points-tybot.md +43 -0
- package/docs/superpowers/plans/2026-10-01-scheduled-start-point-design-studio.md +80 -0
- package/docs/superpowers/plans/2026-10-01-scheduled-start-point-runtime.md +39 -0
- package/docs/superpowers/plans/2026-10-01-scheduled-start-point-server.md +106 -0
- package/docs/superpowers/specs/2026-09-15-webhook-chatbot-hitl-design.md +715 -0
- package/docs/superpowers/specs/2026-09-30-flow-start-points-design.md +156 -0
- package/docs/superpowers/specs/2026-10-01-scheduled-start-point-design.md +303 -0
- package/docs/superpowers/todo/2026-09-14-webhook-chatbot-shared-code-fixes.md +191 -0
- package/event/botEvent.js +12 -1
- package/event/emailEvent.js +3 -0
- package/event/messageEvent.js +44 -0
- package/jobs.js +10 -0
- package/jobsManager.js +19 -0
- package/middleware/errorHandler.js +6 -2
- package/models/analyticResult.js +1 -1
- package/models/channelConstants.js +3 -2
- package/models/quotaUsage.js +71 -0
- package/models/request.js +6 -0
- package/models/requestSnapshot.js +12 -0
- package/models/routerLogger.js +1 -1
- package/models/webhook.js +44 -0
- package/package.json +7 -5
- package/pubmodules/emailNotification/requestNotification.js +14 -22
- package/pubmodules/events/test/eventRoute.js +7 -6
- package/pubmodules/rules/conciergeBot.js +9 -4
- package/routes/auth.js +1 -12
- package/routes/campaigns.js +0 -13
- package/routes/dataTable.js +0 -3
- package/routes/faq_kb.js +12 -1
- package/routes/messagesRoot.js +0 -1
- package/routes/quotes.js +38 -1
- package/routes/request.js +81 -5
- package/routes/users.js +0 -1
- package/routes/webhook.js +160 -8
- package/routes/webhooks.js +276 -6
- package/routes/widgetLoader.js +6 -3
- package/services/QuoteManager.js +77 -16
- package/services/aiService.js +1 -1
- package/services/chatbotService.js +55 -1
- package/services/departmentService.js +60 -43
- package/services/dolphinSchedulerClient.js +116 -0
- package/services/emailService.js +6 -1
- package/services/hitlParticipantListener.js +260 -0
- package/services/participantInviteService.js +302 -0
- package/services/participantRemoveService.js +79 -0
- package/services/quotaSnapshotService.js +1167 -0
- package/services/requestService.js +102 -144
- package/services/scheduledStart/dolphinBackend.js +246 -0
- package/services/scheduledStart/schedulerBackend.js +101 -0
- package/services/scheduledStart/schedulerServiceClient.js +41 -0
- package/services/scheduledStartPublishListener.js +186 -0
- package/services/scheduledStartRules.js +173 -0
- package/services/scheduledStartService.js +130 -0
- package/services/startPointCleanupListener.js +30 -0
- package/services/startPointService.js +158 -0
- package/services/testWsService.js +1 -4
- package/services/updateRequestSnapshotQueued.js +8 -9
- package/services/webhookConversationService.js +185 -0
- package/services/webhookSettingsValidator.js +61 -0
- package/template/email/checkpointReachedEmail.html +5 -0
- package/test/agentPromptsRoute.test.js +3 -2
- package/test/agentTokensQuota.js +237 -0
- package/test/botEventHitlFallback.js +19 -0
- package/test/chat21HandlerJoinWiring.js +59 -0
- package/test/dataTablesRoute.test.js +10 -2
- package/test/dateUtils.test.js +0 -3
- package/test/departmentService.js +8 -12
- package/test/{fileRoute.js → deprecated/_deprecated_fileRoute} +2 -2
- package/test/{kbsettingsRoute.js → deprecated/_deprecated_kbsettingsRoute} +3 -3
- package/test/faqRoute.js +6 -1
- package/test/faqkbRoute.js +14 -10
- package/test/faqkbV3Route.js +1 -1
- package/test/filepRoute.js +4 -50
- package/test/hitlParticipantListener.js +446 -0
- package/test/hitlQueueWiring.js +191 -0
- package/test/hitlWaitingReset.js +124 -0
- package/test/inviteParticipantsRoute.js +492 -0
- package/test/kbRoute.js +1 -1
- package/test/messageRoute.js +33 -21
- package/test/nativemcp.test.js +7 -1
- package/test/openaiRoute.js +2 -8
- package/test/publishFailLoud.test.js +414 -0
- package/test/quotaSnapshots.js +1044 -0
- package/test/removeParticipantsRoute.js +186 -0
- package/test/requestService.js +24 -16
- package/test/requestServiceRemoveParticipant.js +144 -0
- package/test/requestVisibility.test.js +186 -0
- package/test/scheduledStartFacade.js +184 -0
- package/test/scheduledStartHandover.js +95 -0
- package/test/scheduledStartPublish.js +320 -0
- package/test/scheduledStartRoute.js +163 -0
- package/test/scheduledStartRules.js +177 -0
- package/test/scheduledStartRun.js +299 -0
- package/test/scheduledStartSchedulerBackend.js +116 -0
- package/test/scheduledStartService.js +290 -0
- package/test/startPointsCleanup.js +69 -0
- package/test/startPointsRegression.js +169 -0
- package/test/startPointsRoute.js +158 -0
- package/test/webhookConversationRoute.js +347 -0
- package/test/webhookConversationService.js +261 -0
- package/test/webhookConversationTestUtils.js +55 -0
- package/utils/aiUtils.js +95 -1
- package/utils/intentFieldNames.js +192 -0
- package/utils/quotaCliUtil.js +43 -0
- package/utils/requestVisibility.js +355 -0
- package/websocket/webSocketServer.js +36 -31
- /package/test/{imageRoute.js → deprecated/_deprecated_imageRoute} +0 -0
|
@@ -0,0 +1,1044 @@
|
|
|
1
|
+
process.env.NODE_ENV = 'test';
|
|
2
|
+
process.env.QUOTES_ENABLED = 'true';
|
|
3
|
+
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const { execFile } = require('child_process');
|
|
6
|
+
const mongoose = require('mongoose');
|
|
7
|
+
|
|
8
|
+
const projectService = require('../services/projectService');
|
|
9
|
+
const { QuoteManager } = require('../services/QuoteManager');
|
|
10
|
+
const quotaSnapshotService = require('../services/quotaSnapshotService');
|
|
11
|
+
const QuotaUsage = require('../models/quotaUsage');
|
|
12
|
+
const { TdCache } = require('../utils/TdCache');
|
|
13
|
+
|
|
14
|
+
let chai = require('chai');
|
|
15
|
+
let server = require('../app');
|
|
16
|
+
let expect = chai.expect;
|
|
17
|
+
|
|
18
|
+
const projectMock = require('./mock/projectMock');
|
|
19
|
+
|
|
20
|
+
const tdCache = new TdCache({
|
|
21
|
+
host: process.env.CACHE_REDIS_HOST || '127.0.0.1',
|
|
22
|
+
port: process.env.CACHE_REDIS_PORT || '6379',
|
|
23
|
+
password: process.env.CACHE_REDIS_PASSWORD || undefined
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
const PERIOD = '10/31/2026';
|
|
27
|
+
|
|
28
|
+
function newProjectId() {
|
|
29
|
+
return new mongoose.Types.ObjectId().toString();
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function key(id_project, type, period) {
|
|
33
|
+
return quotaSnapshotService.buildQuotaKey(id_project, type, period || PERIOD);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function redis(command, ...args) {
|
|
37
|
+
return new Promise((resolve, reject) => {
|
|
38
|
+
tdCache.getClient().send_command(command, args, (err, reply) => err ? reject(err) : resolve(reply));
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const S = quotaSnapshotService;
|
|
43
|
+
|
|
44
|
+
function cycle(options) {
|
|
45
|
+
return S.runSnapshotCycle(Object.assign({ tdCache: tdCache, lockTtlSeconds: 30, settleMs: 0 }, options || {}));
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function markDirty(...keys) {
|
|
49
|
+
return redis('sadd', S.DIRTY_KEY, ...keys);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** records the Redis commands sent while fn runs */
|
|
53
|
+
async function recordCommands(fn) {
|
|
54
|
+
const client = tdCache.getClient();
|
|
55
|
+
const original = client.send_command;
|
|
56
|
+
const originalMulti = client.multi;
|
|
57
|
+
const commands = [];
|
|
58
|
+
client.send_command = function (command) {
|
|
59
|
+
commands.push(String(command).toLowerCase());
|
|
60
|
+
return original.apply(this, arguments);
|
|
61
|
+
};
|
|
62
|
+
client.multi = function (queued) {
|
|
63
|
+
commands.push('multi');
|
|
64
|
+
(queued || []).forEach((c) => commands.push(String(c[0]).toLowerCase()));
|
|
65
|
+
const multi = originalMulti.apply(this, arguments);
|
|
66
|
+
const exec = multi.exec;
|
|
67
|
+
multi.exec = function () {
|
|
68
|
+
commands.push('exec');
|
|
69
|
+
return exec.apply(this, arguments);
|
|
70
|
+
};
|
|
71
|
+
return multi;
|
|
72
|
+
};
|
|
73
|
+
try {
|
|
74
|
+
await fn();
|
|
75
|
+
} finally {
|
|
76
|
+
client.send_command = original;
|
|
77
|
+
client.multi = originalMulti;
|
|
78
|
+
}
|
|
79
|
+
return commands;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function mockProject(id_project) {
|
|
83
|
+
let project = JSON.parse(JSON.stringify(projectMock.mockProjectBasicPlan));
|
|
84
|
+
project._id = id_project;
|
|
85
|
+
project.createdAt = new Date(project.createdAt);
|
|
86
|
+
if (project.profile.subStart) project.profile.subStart = new Date(project.profile.subStart);
|
|
87
|
+
if (project.profile.subEnd) project.profile.subEnd = new Date(project.profile.subEnd);
|
|
88
|
+
project.isActiveSubscription = projectMock.mockProjectBasicPlan.isActiveSubscription;
|
|
89
|
+
return project;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
describe('QuotaSnapshots', function () {
|
|
93
|
+
|
|
94
|
+
this.timeout(15000);
|
|
95
|
+
|
|
96
|
+
before(async function () {
|
|
97
|
+
await tdCache.connect();
|
|
98
|
+
await QuotaUsage.init();
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
beforeEach(async function () {
|
|
102
|
+
quotaSnapshotService.resetRestoreMemo();
|
|
103
|
+
await redis('del', quotaSnapshotService.LOCK_KEY);
|
|
104
|
+
await redis('del', quotaSnapshotService.RESTORE_LOCK_KEY);
|
|
105
|
+
// a healthy Redis: the sentinel is present (set at boot by the server)
|
|
106
|
+
await redis('set', quotaSnapshotService.SENTINEL_KEY, 'test');
|
|
107
|
+
await redis('del', S.DIRTY_KEY, S.DIRTY_SAVING_KEY);
|
|
108
|
+
// the weekly full scan is not due
|
|
109
|
+
await redis('set', S.FULL_SCAN_KEY, String(Date.now()));
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
after(function () {
|
|
113
|
+
tdCache.getClient().quit();
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
describe('parseQuotaKey', function () {
|
|
117
|
+
it('parses counter keys and rejects notify, lock and malformed keys', function () {
|
|
118
|
+
expect(quotaSnapshotService.parseQuotaKey('quotes:agent_tokens:64e36f5dbf72263f7c059999:2/29/2024'))
|
|
119
|
+
.to.deep.equal({ type: 'agent_tokens', id_project: '64e36f5dbf72263f7c059999', period: '2/29/2024' });
|
|
120
|
+
expect(quotaSnapshotService.parseQuotaKey('quotes:requests:64e36f5dbf72263f7c059999:2/29/2024:notify:50')).to.equal(null);
|
|
121
|
+
expect(quotaSnapshotService.parseQuotaKey('quotes:snapshot:lock')).to.equal(null);
|
|
122
|
+
expect(quotaSnapshotService.parseQuotaKey('quotes:requests:64e36f5dbf72263f7c059999:not-a-date')).to.equal(null);
|
|
123
|
+
expect(quotaSnapshotService.parseQuotaKey('other:requests:64e36f5dbf72263f7c059999:2/29/2024')).to.equal(null);
|
|
124
|
+
});
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
describe('snapshot', function () {
|
|
128
|
+
|
|
129
|
+
it('saves only the dirty counters and empties the dirty set', async function () {
|
|
130
|
+
const p = newProjectId();
|
|
131
|
+
await redis('set', key(p, 'requests'), '5');
|
|
132
|
+
await redis('set', key(p, 'tokens'), '12.5');
|
|
133
|
+
await redis('set', key(p, 'agent_tokens', '9/30/2026'), '7');
|
|
134
|
+
await redis('set', key(p, 'messages'), '99'); // not dirty: not saved by the hourly cycle
|
|
135
|
+
await redis('set', key(p, 'requests') + ':notify:50', 'true', 'EX', '60');
|
|
136
|
+
await markDirty(key(p, 'requests'), key(p, 'tokens'), key(p, 'agent_tokens', '9/30/2026'));
|
|
137
|
+
|
|
138
|
+
const stats = await cycle();
|
|
139
|
+
expect(stats.error).to.equal(undefined);
|
|
140
|
+
expect(stats.full).to.equal(false);
|
|
141
|
+
expect(stats.dirty).to.equal(3);
|
|
142
|
+
expect(stats.counters).to.equal(3);
|
|
143
|
+
|
|
144
|
+
const docs = await QuotaUsage.find({ id_project: p }).sort({ type: 1 }).lean();
|
|
145
|
+
expect(docs.map((d) => [d.type, d.period, d.value])).to.deep.equal([
|
|
146
|
+
['agent_tokens', '9/30/2026', 7],
|
|
147
|
+
['requests', PERIOD, 5],
|
|
148
|
+
['tokens', PERIOD, 12.5]
|
|
149
|
+
]);
|
|
150
|
+
expect(Object.keys(docs[1].notified)).to.deep.equal(['50']);
|
|
151
|
+
docs.forEach((d) => expect(d.updatedAt).to.be.an.instanceof(Date));
|
|
152
|
+
docs.forEach((d) => expect(d.periodDate).to.be.an.instanceof(Date));
|
|
153
|
+
expect(await redis('exists', S.DIRTY_KEY, S.DIRTY_SAVING_KEY)).to.equal(0);
|
|
154
|
+
expect(await redis('exists', S.LOCK_KEY)).to.equal(0);
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
it('the hourly cycle does not SCAN the keyspace nor read the backup docs back', async function () {
|
|
158
|
+
const p = newProjectId();
|
|
159
|
+
await redis('set', key(p, 'requests'), '5');
|
|
160
|
+
await markDirty(key(p, 'requests'));
|
|
161
|
+
const originalFind = QuotaUsage.find;
|
|
162
|
+
let finds = 0;
|
|
163
|
+
QuotaUsage.find = function () { finds++; return originalFind.apply(this, arguments); };
|
|
164
|
+
let commands;
|
|
165
|
+
try {
|
|
166
|
+
commands = await recordCommands(() => cycle());
|
|
167
|
+
} finally {
|
|
168
|
+
QuotaUsage.find = originalFind;
|
|
169
|
+
}
|
|
170
|
+
expect(commands).to.not.include('scan');
|
|
171
|
+
expect(commands).to.not.include('keys');
|
|
172
|
+
expect(finds).to.equal(0);
|
|
173
|
+
expect((await QuotaUsage.findOne({ id_project: p }).lean()).value).to.equal(5);
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
it('updatedAt changes only when the value or a flag changes (no-op upserts stay no-ops)', async function () {
|
|
177
|
+
const p = newProjectId();
|
|
178
|
+
const k = key(p, 'requests');
|
|
179
|
+
await redis('set', k, '5');
|
|
180
|
+
await markDirty(k);
|
|
181
|
+
const first = await cycle();
|
|
182
|
+
expect(first.settleMs).to.be.a('number');
|
|
183
|
+
expect(first.workMs).to.be.a('number');
|
|
184
|
+
const created = await QuotaUsage.findOne({ id_project: p }).lean();
|
|
185
|
+
expect(created.updatedAt).to.be.an.instanceof(Date);
|
|
186
|
+
|
|
187
|
+
await new Promise((r) => setTimeout(r, 20));
|
|
188
|
+
await markDirty(k); // same value
|
|
189
|
+
const second = await cycle();
|
|
190
|
+
expect(second.modified).to.equal(0);
|
|
191
|
+
let doc = await QuotaUsage.findOne({ id_project: p }).lean();
|
|
192
|
+
expect(doc.updatedAt.getTime()).to.equal(created.updatedAt.getTime());
|
|
193
|
+
|
|
194
|
+
await redis('set', k + ':notify:50', 'true', 'EX', '100');
|
|
195
|
+
await markDirty(k); // new flag
|
|
196
|
+
await cycle();
|
|
197
|
+
doc = await QuotaUsage.findOne({ id_project: p }).lean();
|
|
198
|
+
expect(Object.keys(doc.notified)).to.deep.equal(['50']);
|
|
199
|
+
expect(doc.updatedAt.getTime()).to.be.greaterThan(created.updatedAt.getTime());
|
|
200
|
+
const flagged = doc.updatedAt.getTime();
|
|
201
|
+
|
|
202
|
+
await new Promise((r) => setTimeout(r, 20));
|
|
203
|
+
await redis('set', k, '3'); // lower: no change
|
|
204
|
+
await markDirty(k);
|
|
205
|
+
await cycle();
|
|
206
|
+
doc = await QuotaUsage.findOne({ id_project: p }).lean();
|
|
207
|
+
expect(doc.value).to.equal(5);
|
|
208
|
+
expect(doc.updatedAt.getTime()).to.equal(flagged);
|
|
209
|
+
|
|
210
|
+
await redis('set', k, '8'); // higher
|
|
211
|
+
await markDirty(k);
|
|
212
|
+
await cycle();
|
|
213
|
+
doc = await QuotaUsage.findOne({ id_project: p }).lean();
|
|
214
|
+
expect(doc.value).to.equal(8);
|
|
215
|
+
expect(doc.updatedAt.getTime()).to.be.greaterThan(flagged);
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
it('the weekly full scan of unchanged counters modifies nothing', async function () {
|
|
219
|
+
const p = newProjectId();
|
|
220
|
+
await redis('set', key(p, 'requests'), '5');
|
|
221
|
+
await cycle({ full: true });
|
|
222
|
+
const again = await cycle({ full: true });
|
|
223
|
+
expect(again.modified).to.equal(0);
|
|
224
|
+
expect(again.upserted).to.equal(0);
|
|
225
|
+
});
|
|
226
|
+
|
|
227
|
+
it('uses $max: a lower Redis value never lowers the backup', async function () {
|
|
228
|
+
const p = newProjectId();
|
|
229
|
+
await QuotaUsage.create({ id_project: p, type: 'requests', period: PERIOD, value: 100 });
|
|
230
|
+
await redis('set', key(p, 'requests'), '3');
|
|
231
|
+
await markDirty(key(p, 'requests'));
|
|
232
|
+
await cycle();
|
|
233
|
+
let doc = await QuotaUsage.findOne({ id_project: p, type: 'requests' }).lean();
|
|
234
|
+
expect(doc.value).to.equal(100);
|
|
235
|
+
|
|
236
|
+
await redis('set', key(p, 'requests'), '150');
|
|
237
|
+
await markDirty(key(p, 'requests'));
|
|
238
|
+
await cycle();
|
|
239
|
+
doc = await QuotaUsage.findOne({ id_project: p, type: 'requests' }).lean();
|
|
240
|
+
expect(doc.value).to.equal(150);
|
|
241
|
+
expect(await QuotaUsage.countDocuments({ id_project: p })).to.equal(1);
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
it('a crash between RENAME and DEL is recovered by the next cycle', async function () {
|
|
245
|
+
const p = newProjectId();
|
|
246
|
+
await redis('set', key(p, 'requests'), '9');
|
|
247
|
+
await redis('set', key(p, 'messages'), '4');
|
|
248
|
+
// left by a crashed cycle
|
|
249
|
+
await redis('sadd', S.DIRTY_SAVING_KEY, key(p, 'requests'));
|
|
250
|
+
// changed after the crash
|
|
251
|
+
await markDirty(key(p, 'messages'));
|
|
252
|
+
const stats = await cycle();
|
|
253
|
+
expect(stats.dirty).to.equal(2);
|
|
254
|
+
const docs = await QuotaUsage.find({ id_project: p }).sort({ type: 1 }).lean();
|
|
255
|
+
expect(docs.map((d) => [d.type, d.value])).to.deep.equal([['messages', 4], ['requests', 9]]);
|
|
256
|
+
expect(await redis('exists', S.DIRTY_KEY, S.DIRTY_SAVING_KEY)).to.equal(0);
|
|
257
|
+
});
|
|
258
|
+
|
|
259
|
+
it('changes made during a cycle land in the new dirty set', async function () {
|
|
260
|
+
const qm = new QuoteManager({ tdCache: tdCache });
|
|
261
|
+
const project = mockProject(newProjectId());
|
|
262
|
+
const obj = { createdAt: new Date('2023-10-25T08:45:54.058Z') };
|
|
263
|
+
qm.project = project;
|
|
264
|
+
const k = await qm.generateKey(obj, 'messages');
|
|
265
|
+
await redis('set', key(newProjectId(), 'requests'), '1');
|
|
266
|
+
await markDirty(key(newProjectId(), 'requests'));
|
|
267
|
+
|
|
268
|
+
const running = cycle({ settleMs: 300 });
|
|
269
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
270
|
+
await qm.incrementMessagesCount(project, obj);
|
|
271
|
+
await running;
|
|
272
|
+
expect(await redis('sismember', S.DIRTY_KEY, k)).to.equal(1);
|
|
273
|
+
expect(await redis('exists', S.DIRTY_SAVING_KEY)).to.equal(0);
|
|
274
|
+
});
|
|
275
|
+
|
|
276
|
+
it('skips periods older than the previous one and drops them from the set', async function () {
|
|
277
|
+
const p = newProjectId();
|
|
278
|
+
await redis('set', key(p, 'requests', '1/1/2020'), '5');
|
|
279
|
+
await redis('set', key(p, 'messages'), '5');
|
|
280
|
+
await markDirty(key(p, 'requests', '1/1/2020'), key(p, 'messages'));
|
|
281
|
+
await cycle();
|
|
282
|
+
const docs = await QuotaUsage.find({ id_project: p }).lean();
|
|
283
|
+
expect(docs.map((d) => d.period)).to.deep.equal([PERIOD]);
|
|
284
|
+
expect(await redis('exists', S.DIRTY_KEY, S.DIRTY_SAVING_KEY)).to.equal(0);
|
|
285
|
+
});
|
|
286
|
+
|
|
287
|
+
it('the weekly full scan reconciles a counter changed outside the normal path', async function () {
|
|
288
|
+
const p = newProjectId();
|
|
289
|
+
await redis('set', key(p, 'requests'), '77'); // manual edit: not dirty
|
|
290
|
+
await redis('set', key(p, 'requests') + ':notify:75', 'true', 'EX', '600');
|
|
291
|
+
await cycle();
|
|
292
|
+
expect(await QuotaUsage.countDocuments({ id_project: p })).to.equal(0);
|
|
293
|
+
|
|
294
|
+
await redis('set', S.FULL_SCAN_KEY, String(Date.now() - 8 * 24 * 3600 * 1000));
|
|
295
|
+
const stats = await cycle();
|
|
296
|
+
expect(stats.full).to.equal(true);
|
|
297
|
+
const doc = await QuotaUsage.findOne({ id_project: p }).lean();
|
|
298
|
+
expect(doc.value).to.equal(77);
|
|
299
|
+
expect(Object.keys(doc.notified)).to.deep.equal(['75']);
|
|
300
|
+
expect(Number(await redis('get', S.FULL_SCAN_KEY))).to.be.within(Date.now() - 60000, Date.now());
|
|
301
|
+
});
|
|
302
|
+
|
|
303
|
+
it('the full scan runs on the first cycle (no previous full scan recorded)', async function () {
|
|
304
|
+
await redis('del', S.FULL_SCAN_KEY);
|
|
305
|
+
const stats = await cycle();
|
|
306
|
+
expect(stats.full).to.equal(true);
|
|
307
|
+
});
|
|
308
|
+
|
|
309
|
+
it('a failing bulk batch does not stop the other batches and keeps the set for a retry', async function () {
|
|
310
|
+
const keys = [];
|
|
311
|
+
const msetArgs = [];
|
|
312
|
+
for (let i = 0; i < 700; i++) {
|
|
313
|
+
const k = key(newProjectId(), 'requests');
|
|
314
|
+
keys.push(k);
|
|
315
|
+
msetArgs.push(k, '1');
|
|
316
|
+
}
|
|
317
|
+
await redis('mset', ...msetArgs);
|
|
318
|
+
await markDirty(...keys);
|
|
319
|
+
const original = QuotaUsage.bulkWrite;
|
|
320
|
+
let calls = 0;
|
|
321
|
+
QuotaUsage.bulkWrite = function () {
|
|
322
|
+
calls++;
|
|
323
|
+
if (calls === 1) return Promise.reject(new Error('bulk down'));
|
|
324
|
+
return original.apply(this, arguments);
|
|
325
|
+
};
|
|
326
|
+
let stats;
|
|
327
|
+
try {
|
|
328
|
+
stats = await cycle();
|
|
329
|
+
} finally {
|
|
330
|
+
QuotaUsage.bulkWrite = original;
|
|
331
|
+
}
|
|
332
|
+
expect(calls).to.be.at.least(2); // SSCAN COUNT is a hint: 2 or more batches
|
|
333
|
+
expect(stats.failedBatches).to.equal(1);
|
|
334
|
+
expect(stats.counters).to.be.greaterThan(0);
|
|
335
|
+
// kept for the next cycle, which saves everything
|
|
336
|
+
expect(await redis('scard', S.DIRTY_SAVING_KEY)).to.equal(700);
|
|
337
|
+
const retry = await cycle();
|
|
338
|
+
expect(retry.failedBatches).to.equal(0);
|
|
339
|
+
const ids = keys.map((k) => S.parseQuotaKey(k).id_project);
|
|
340
|
+
expect(await QuotaUsage.countDocuments({ id_project: { $in: ids } })).to.equal(700);
|
|
341
|
+
await redis('del', ...keys);
|
|
342
|
+
});
|
|
343
|
+
|
|
344
|
+
it('the lock prevents a concurrent cycle and is not released by the loser', async function () {
|
|
345
|
+
const p = newProjectId();
|
|
346
|
+
await redis('set', key(p, 'requests'), '5');
|
|
347
|
+
await markDirty(key(p, 'requests'));
|
|
348
|
+
await redis('set', S.LOCK_KEY, 'other-replica', 'EX', '30');
|
|
349
|
+
|
|
350
|
+
const result = await cycle();
|
|
351
|
+
expect(result).to.deep.equal({ skipped: 'locked' });
|
|
352
|
+
expect(await QuotaUsage.countDocuments({ id_project: p })).to.equal(0);
|
|
353
|
+
expect(await redis('get', S.LOCK_KEY)).to.equal('other-replica');
|
|
354
|
+
expect(await redis('sismember', S.DIRTY_KEY, key(p, 'requests'))).to.equal(1);
|
|
355
|
+
await redis('del', S.LOCK_KEY);
|
|
356
|
+
});
|
|
357
|
+
|
|
358
|
+
it('two cycles started together: only one runs', async function () {
|
|
359
|
+
const results = await Promise.all([cycle(), cycle()]);
|
|
360
|
+
expect(results.filter((r) => r.skipped).length).to.equal(1);
|
|
361
|
+
});
|
|
362
|
+
|
|
363
|
+
it('is skipped when QUOTES_ENABLED=false', async function () {
|
|
364
|
+
const p = newProjectId();
|
|
365
|
+
await redis('set', key(p, 'requests'), '5');
|
|
366
|
+
await markDirty(key(p, 'requests'));
|
|
367
|
+
process.env.QUOTES_ENABLED = 'false';
|
|
368
|
+
try {
|
|
369
|
+
const result = await cycle();
|
|
370
|
+
expect(result).to.deep.equal({ skipped: 'quotes_disabled' });
|
|
371
|
+
} finally {
|
|
372
|
+
process.env.QUOTES_ENABLED = 'true';
|
|
373
|
+
}
|
|
374
|
+
expect(await QuotaUsage.countDocuments({ id_project: p })).to.equal(0);
|
|
375
|
+
});
|
|
376
|
+
});
|
|
377
|
+
|
|
378
|
+
describe('dirty marking', function () {
|
|
379
|
+
|
|
380
|
+
const qm = new QuoteManager({ tdCache: tdCache });
|
|
381
|
+
const obj = { createdAt: new Date('2023-10-25T08:45:54.058Z') };
|
|
382
|
+
|
|
383
|
+
async function counterKey(type) {
|
|
384
|
+
const project = mockProject(newProjectId());
|
|
385
|
+
qm.project = project;
|
|
386
|
+
return { project, k: await qm.generateKey(obj, type) };
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
afterEach(function () {
|
|
390
|
+
S.stopWipeDetection();
|
|
391
|
+
});
|
|
392
|
+
|
|
393
|
+
it('every increment path marks its counter dirty in the same round trip (MULTI)', async function () {
|
|
394
|
+
let a = await counterKey('requests');
|
|
395
|
+
const commands = await recordCommands(async () => {
|
|
396
|
+
await qm.incrementRequestsCount(a.project, obj);
|
|
397
|
+
await redis('ping');
|
|
398
|
+
});
|
|
399
|
+
expect(commands).to.include.members(['multi', 'incr', 'sadd', 'exec']);
|
|
400
|
+
expect(await redis('get', a.k)).to.equal('1');
|
|
401
|
+
expect(await redis('sismember', S.DIRTY_KEY, a.k)).to.equal(1);
|
|
402
|
+
|
|
403
|
+
const b = await counterKey('messages');
|
|
404
|
+
await qm.incrementMessagesCount(b.project, obj);
|
|
405
|
+
const c = await counterKey('email');
|
|
406
|
+
await qm.incrementEmailCount(c.project, obj);
|
|
407
|
+
const d = await counterKey('tokens');
|
|
408
|
+
await qm.incrementTokenCount(d.project, Object.assign({ tokens: 10, multiplier: 1.5 }, obj));
|
|
409
|
+
const e = await counterKey('agent_tokens');
|
|
410
|
+
await qm.incrementAgentTokenCount(e.project, Object.assign({ counted: 3 }, obj));
|
|
411
|
+
const f = await counterKey('voice_duration');
|
|
412
|
+
await qm.incrementVoiceDurationCount(f.project, Object.assign({ duration: 4000 }, obj));
|
|
413
|
+
await redis('ping');
|
|
414
|
+
expect(await redis('get', d.k)).to.equal('15');
|
|
415
|
+
expect(await redis('get', e.k)).to.equal('3');
|
|
416
|
+
expect(await redis('get', f.k)).to.equal('4');
|
|
417
|
+
for (const x of [b, c, d, e, f]) {
|
|
418
|
+
expect(await redis('sismember', S.DIRTY_KEY, x.k)).to.equal(1);
|
|
419
|
+
}
|
|
420
|
+
});
|
|
421
|
+
|
|
422
|
+
it('without the watcher every increment marks dirty (no memo)', async function () {
|
|
423
|
+
const a = await counterKey('messages');
|
|
424
|
+
await qm.incrementMessagesCount(a.project, obj);
|
|
425
|
+
await qm.incrementMessagesCount(a.project, obj);
|
|
426
|
+
expect(S.restoreStats.dirtyMarks).to.equal(2);
|
|
427
|
+
});
|
|
428
|
+
|
|
429
|
+
it('with the watcher the memo limits SADDs until a new snapshot generation is seen', async function () {
|
|
430
|
+
// let the unawaited threshold checks of the previous tests finish (they mark their own counters)
|
|
431
|
+
await new Promise((r) => setTimeout(r, 300));
|
|
432
|
+
S.resetRestoreMemo();
|
|
433
|
+
S.startWipeDetection(tdCache, { intervalSeconds: 3600 });
|
|
434
|
+
await S.checkSentinel(tdCache); // a watcher tick: current generation seen
|
|
435
|
+
const a = await counterKey('messages');
|
|
436
|
+
await qm.incrementMessagesCount(a.project, obj);
|
|
437
|
+
await redis('ping'); // the memo is set when the EXEC reply confirms the SADD
|
|
438
|
+
for (let i = 0; i < 4; i++) {
|
|
439
|
+
await qm.incrementMessagesCount(a.project, obj);
|
|
440
|
+
}
|
|
441
|
+
expect(S.restoreStats.dirtyMarks).to.equal(1);
|
|
442
|
+
await redis('ping');
|
|
443
|
+
expect(await redis('get', a.k)).to.equal('5');
|
|
444
|
+
|
|
445
|
+
await cycle({ settleMs: 0 });
|
|
446
|
+
expect(await redis('sismember', S.DIRTY_KEY, a.k)).to.equal(0);
|
|
447
|
+
await S.checkSentinel(tdCache); // next watcher tick: sees the new generation
|
|
448
|
+
await qm.incrementMessagesCount(a.project, obj);
|
|
449
|
+
expect(S.restoreStats.dirtyMarks).to.equal(2);
|
|
450
|
+
expect(await redis('sismember', S.DIRTY_KEY, a.k)).to.equal(1);
|
|
451
|
+
});
|
|
452
|
+
|
|
453
|
+
it('a failed SADD is not memoized and is logged (MULTI reply errors)', async function () {
|
|
454
|
+
const winston = require('../config/winston');
|
|
455
|
+
await new Promise((r) => setTimeout(r, 300));
|
|
456
|
+
S.resetRestoreMemo();
|
|
457
|
+
S.startWipeDetection(tdCache, { intervalSeconds: 3600 });
|
|
458
|
+
await S.checkSentinel(tdCache);
|
|
459
|
+
await redis('set', S.DIRTY_KEY, 'not-a-set'); // SADD -> WRONGTYPE inside EXEC
|
|
460
|
+
const a = await counterKey('messages');
|
|
461
|
+
const originalError = winston.error;
|
|
462
|
+
const logged = [];
|
|
463
|
+
winston.error = function (msg) { logged.push(String(msg)); };
|
|
464
|
+
let commands;
|
|
465
|
+
try {
|
|
466
|
+
commands = await recordCommands(async () => {
|
|
467
|
+
await qm.incrementMessagesCount(a.project, obj);
|
|
468
|
+
await redis('ping');
|
|
469
|
+
await qm.incrementMessagesCount(a.project, obj);
|
|
470
|
+
await redis('ping');
|
|
471
|
+
});
|
|
472
|
+
} finally {
|
|
473
|
+
winston.error = originalError;
|
|
474
|
+
await redis('del', S.DIRTY_KEY);
|
|
475
|
+
}
|
|
476
|
+
expect(commands.filter((c) => c === 'multi').length).to.equal(2);
|
|
477
|
+
expect(logged.some((m) => /sadd/i.test(m))).to.equal(true);
|
|
478
|
+
expect(await redis('get', a.k)).to.equal('2');
|
|
479
|
+
});
|
|
480
|
+
|
|
481
|
+
it('a memo hit sends the plain increment, a miss sends MULTI incr + sadd', async function () {
|
|
482
|
+
await new Promise((r) => setTimeout(r, 300));
|
|
483
|
+
S.resetRestoreMemo();
|
|
484
|
+
S.startWipeDetection(tdCache, { intervalSeconds: 3600 });
|
|
485
|
+
await S.checkSentinel(tdCache);
|
|
486
|
+
const a = await counterKey('tokens');
|
|
487
|
+
const miss = await recordCommands(async () => {
|
|
488
|
+
await qm.incrementTokenCount(a.project, Object.assign({ tokens: 1, multiplier: 1 }, obj));
|
|
489
|
+
await redis('ping');
|
|
490
|
+
});
|
|
491
|
+
const hit = await recordCommands(async () => {
|
|
492
|
+
await qm.incrementTokenCount(a.project, Object.assign({ tokens: 1, multiplier: 1 }, obj));
|
|
493
|
+
await redis('ping');
|
|
494
|
+
});
|
|
495
|
+
expect(miss.filter((c) => ['multi', 'incrbyfloat', 'sadd', 'exec'].includes(c))).to.deep.equal(['multi', 'incrbyfloat', 'sadd', 'exec']);
|
|
496
|
+
expect(hit.filter((c) => ['multi', 'incrbyfloat', 'sadd', 'exec'].includes(c))).to.deep.equal(['incrbyfloat']);
|
|
497
|
+
});
|
|
498
|
+
|
|
499
|
+
it('the memo is used only while the generation read is fresh, and cleared on a read error', async function () {
|
|
500
|
+
await new Promise((r) => setTimeout(r, 300));
|
|
501
|
+
S.resetRestoreMemo();
|
|
502
|
+
S.startWipeDetection(tdCache, { intervalSeconds: 3600 });
|
|
503
|
+
await S.checkSentinel(tdCache);
|
|
504
|
+
const a = await counterKey('messages');
|
|
505
|
+
await qm.incrementMessagesCount(a.project, obj);
|
|
506
|
+
await redis('ping');
|
|
507
|
+
await qm.incrementMessagesCount(a.project, obj);
|
|
508
|
+
expect(S.restoreStats.dirtyMarks).to.equal(1);
|
|
509
|
+
|
|
510
|
+
S.ageGenerationRead(2 * 3600 * 1000); // older than 1.5 x interval: watcher not working
|
|
511
|
+
await qm.incrementMessagesCount(a.project, obj);
|
|
512
|
+
expect(S.restoreStats.dirtyMarks).to.equal(2);
|
|
513
|
+
|
|
514
|
+
await S.checkSentinel(tdCache); // fresh again
|
|
515
|
+
await redis('ping');
|
|
516
|
+
await qm.incrementMessagesCount(a.project, obj);
|
|
517
|
+
expect(S.restoreStats.dirtyMarks).to.equal(2);
|
|
518
|
+
|
|
519
|
+
// the generation read fails: memo cleared, next increment marks again
|
|
520
|
+
const client = tdCache.getClient();
|
|
521
|
+
const original = client.send_command;
|
|
522
|
+
client.send_command = function (command, args, cb) {
|
|
523
|
+
if (String(command).toLowerCase() === 'mget') return cb(new Error('redis down'));
|
|
524
|
+
return original.apply(this, arguments);
|
|
525
|
+
};
|
|
526
|
+
try {
|
|
527
|
+
await S.checkSentinel(tdCache);
|
|
528
|
+
} finally {
|
|
529
|
+
client.send_command = original;
|
|
530
|
+
}
|
|
531
|
+
await qm.incrementMessagesCount(a.project, obj);
|
|
532
|
+
expect(S.restoreStats.dirtyMarks).to.equal(3);
|
|
533
|
+
});
|
|
534
|
+
|
|
535
|
+
it('with the snapshot disabled nothing is marked and the dirty sets are deleted at start', async function () {
|
|
536
|
+
await redis('sadd', S.DIRTY_KEY, 'x');
|
|
537
|
+
await redis('sadd', S.DIRTY_SAVING_KEY, 'y');
|
|
538
|
+
process.env.QUOTA_SNAPSHOT_INTERVAL_MINUTES = '0';
|
|
539
|
+
try {
|
|
540
|
+
S.start({ tdCache: tdCache });
|
|
541
|
+
S.stop();
|
|
542
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
543
|
+
expect(await redis('exists', S.DIRTY_KEY, S.DIRTY_SAVING_KEY)).to.equal(0);
|
|
544
|
+
const a = await counterKey('messages');
|
|
545
|
+
const commands = await recordCommands(async () => {
|
|
546
|
+
await qm.incrementMessagesCount(a.project, obj);
|
|
547
|
+
await redis('ping');
|
|
548
|
+
});
|
|
549
|
+
expect(commands).to.not.include('sadd');
|
|
550
|
+
expect(commands).to.not.include('multi');
|
|
551
|
+
expect(await redis('get', a.k)).to.equal('1');
|
|
552
|
+
} finally {
|
|
553
|
+
delete process.env.QUOTA_SNAPSHOT_INTERVAL_MINUTES;
|
|
554
|
+
S.configureDirtyTracking();
|
|
555
|
+
}
|
|
556
|
+
const b = await counterKey('messages');
|
|
557
|
+
await qm.incrementMessagesCount(b.project, obj);
|
|
558
|
+
await redis('ping');
|
|
559
|
+
expect(await redis('sismember', S.DIRTY_KEY, b.k)).to.equal(1);
|
|
560
|
+
});
|
|
561
|
+
|
|
562
|
+
it('setting a notify flag marks the counter dirty so the flag is backed up', async function () {
|
|
563
|
+
// a counter of the current period (older periods are not snapshotted)
|
|
564
|
+
const project = mockProject(newProjectId());
|
|
565
|
+
const obj = { createdAt: new Date() };
|
|
566
|
+
qm.project = project;
|
|
567
|
+
const a = { project, k: await qm.generateKey(obj, 'email') };
|
|
568
|
+
await redis('set', a.k, '120'); // basic plan: 200 emails, 120 + 1 > 50%
|
|
569
|
+
await qm.incrementEmailCount(a.project, obj);
|
|
570
|
+
let flag = 0;
|
|
571
|
+
for (let i = 0; i < 40 && !flag; i++) {
|
|
572
|
+
flag = await redis('exists', a.k + ':notify:50');
|
|
573
|
+
if (!flag) await new Promise((r) => setTimeout(r, 50));
|
|
574
|
+
}
|
|
575
|
+
expect(flag).to.equal(1);
|
|
576
|
+
await redis('srem', S.DIRTY_KEY, a.k);
|
|
577
|
+
S.resetRestoreMemo();
|
|
578
|
+
// a later threshold on a counter whose increments are memoized still marks it
|
|
579
|
+
await qm.sendEmailIfQuotaExceeded(a.project, obj, 'email', a.k);
|
|
580
|
+
await redis('set', a.k, '160');
|
|
581
|
+
await redis('del', a.k + ':notify:75');
|
|
582
|
+
await qm.sendEmailIfQuotaExceeded(a.project, obj, 'email', a.k);
|
|
583
|
+
expect(await redis('sismember', S.DIRTY_KEY, a.k)).to.equal(1);
|
|
584
|
+
await cycle();
|
|
585
|
+
const doc = await QuotaUsage.findOne(S.parseQuotaKey(a.k)).lean();
|
|
586
|
+
expect(Object.keys(doc.notified).sort()).to.deep.equal(['50', '75']);
|
|
587
|
+
});
|
|
588
|
+
});
|
|
589
|
+
|
|
590
|
+
describe('restore on miss', function () {
|
|
591
|
+
|
|
592
|
+
const qm = new QuoteManager({ tdCache: tdCache });
|
|
593
|
+
const obj = { createdAt: new Date('2023-10-25T08:45:54.058Z') };
|
|
594
|
+
|
|
595
|
+
async function setup(type, value) {
|
|
596
|
+
const p = newProjectId();
|
|
597
|
+
const project = mockProject(p);
|
|
598
|
+
qm.project = project;
|
|
599
|
+
const k = await qm.generateKey(obj, type);
|
|
600
|
+
const parsed = quotaSnapshotService.parseQuotaKey(k);
|
|
601
|
+
await redis('del', k);
|
|
602
|
+
if (value !== undefined) {
|
|
603
|
+
await QuotaUsage.create(Object.assign({ value: value }, parsed));
|
|
604
|
+
}
|
|
605
|
+
return { project, k };
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
it('read path seeds the missing key from the snapshot', async function () {
|
|
609
|
+
const { project, k } = await setup('requests', 42);
|
|
610
|
+
expect(await qm.getCurrentQuote(project, obj, 'requests')).to.equal(42);
|
|
611
|
+
expect(await redis('get', k)).to.equal('42');
|
|
612
|
+
expect(quotaSnapshotService.restoreStats.restored).to.equal(1);
|
|
613
|
+
});
|
|
614
|
+
|
|
615
|
+
it('getAllQuotes and checkQuote seed from the snapshot', async function () {
|
|
616
|
+
const { project, k } = await setup('email', 250);
|
|
617
|
+
const all = await qm.getAllQuotes(project, obj);
|
|
618
|
+
expect(all.email.quote).to.equal(250);
|
|
619
|
+
await redis('del', k);
|
|
620
|
+
quotaSnapshotService.resetRestoreMemo();
|
|
621
|
+
// basic plan email limit is 200
|
|
622
|
+
expect(await qm.checkQuote(project, obj, 'email')).to.equal(false);
|
|
623
|
+
});
|
|
624
|
+
|
|
625
|
+
it('increment path: the increment lands on top of the restored value', async function () {
|
|
626
|
+
let { project, k } = await setup('messages', 10);
|
|
627
|
+
await qm.incrementMessagesCount(project, obj);
|
|
628
|
+
expect(await redis('get', k)).to.equal('11');
|
|
629
|
+
|
|
630
|
+
({ project, k } = await setup('tokens', 100.5));
|
|
631
|
+
await qm.incrementTokenCount(project, Object.assign({ tokens: 10, multiplier: 1 }, obj));
|
|
632
|
+
expect(Number(await redis('get', k))).to.equal(110.5);
|
|
633
|
+
|
|
634
|
+
({ project, k } = await setup('agent_tokens', 1000));
|
|
635
|
+
await qm.incrementAgentTokenCount(project, Object.assign({ counted: 5 }, obj));
|
|
636
|
+
expect(Number(await redis('get', k))).to.equal(1005);
|
|
637
|
+
});
|
|
638
|
+
|
|
639
|
+
it('no snapshot: counts from zero as before', async function () {
|
|
640
|
+
const { project, k } = await setup('messages');
|
|
641
|
+
await qm.incrementMessagesCount(project, obj);
|
|
642
|
+
expect(await redis('get', k)).to.equal('1');
|
|
643
|
+
});
|
|
644
|
+
|
|
645
|
+
it('an existing key is never overwritten by the snapshot', async function () {
|
|
646
|
+
const { project, k } = await setup('messages', 500);
|
|
647
|
+
await redis('set', k, '7');
|
|
648
|
+
expect(await qm.getCurrentQuote(project, obj, 'messages')).to.equal(7);
|
|
649
|
+
expect(quotaSnapshotService.restoreStats.lookups).to.equal(0);
|
|
650
|
+
});
|
|
651
|
+
|
|
652
|
+
it('the memo prevents repeated EXISTS / mongo lookups, also for concurrent calls', async function () {
|
|
653
|
+
const { project, k } = await setup('requests', 42);
|
|
654
|
+
await Promise.all([1, 2, 3, 4, 5].map(() => qm.getCurrentQuote(project, obj, 'requests')));
|
|
655
|
+
await qm.getCurrentQuote(project, obj, 'requests');
|
|
656
|
+
await qm.incrementRequestsCount(project, obj);
|
|
657
|
+
expect(quotaSnapshotService.restoreStats.existsChecks).to.equal(1);
|
|
658
|
+
expect(quotaSnapshotService.restoreStats.lookups).to.equal(1);
|
|
659
|
+
expect(await redis('get', k)).to.equal('43');
|
|
660
|
+
});
|
|
661
|
+
|
|
662
|
+
it('chatbots and kbs (not counted in Redis) cost no I/O', async function () {
|
|
663
|
+
const { project } = await setup('requests', 1);
|
|
664
|
+
await qm.getAllQuotes(project, obj);
|
|
665
|
+
expect(quotaSnapshotService.restoreStats.existsChecks).to.equal(6);
|
|
666
|
+
});
|
|
667
|
+
|
|
668
|
+
it('a mongo failure is remembered: no lookup for 60s', async function () {
|
|
669
|
+
const a = await setup('messages', 10);
|
|
670
|
+
const b = await setup('requests', 10);
|
|
671
|
+
const original = QuotaUsage.findOne;
|
|
672
|
+
let lookups = 0;
|
|
673
|
+
QuotaUsage.findOne = () => { lookups++; throw new Error('mongo down'); };
|
|
674
|
+
try {
|
|
675
|
+
await qm.getCurrentQuote(a.project, obj, 'messages');
|
|
676
|
+
await qm.getCurrentQuote(b.project, obj, 'requests');
|
|
677
|
+
} finally {
|
|
678
|
+
QuotaUsage.findOne = original;
|
|
679
|
+
}
|
|
680
|
+
expect(lookups).to.equal(1);
|
|
681
|
+
});
|
|
682
|
+
|
|
683
|
+
it('mongo failure falls back to the current behaviour', async function () {
|
|
684
|
+
const { project, k } = await setup('messages', 10);
|
|
685
|
+
const original = QuotaUsage.findOne;
|
|
686
|
+
QuotaUsage.findOne = () => { throw new Error('mongo down'); };
|
|
687
|
+
try {
|
|
688
|
+
expect(await qm.getCurrentQuote(project, obj, 'messages')).to.equal(0);
|
|
689
|
+
await qm.incrementMessagesCount(project, obj);
|
|
690
|
+
} finally {
|
|
691
|
+
QuotaUsage.findOne = original;
|
|
692
|
+
}
|
|
693
|
+
expect(await redis('get', k)).to.equal('1');
|
|
694
|
+
expect(quotaSnapshotService.restoreStats.errors).to.be.at.least(1);
|
|
695
|
+
});
|
|
696
|
+
});
|
|
697
|
+
|
|
698
|
+
describe('restore command', function () {
|
|
699
|
+
|
|
700
|
+
async function seed(p, type, value, redisValue, period) {
|
|
701
|
+
await QuotaUsage.create({ id_project: p, type: type, period: period || PERIOD, value: value });
|
|
702
|
+
const k = key(p, type, period);
|
|
703
|
+
await redis('del', k);
|
|
704
|
+
if (redisValue !== undefined) {
|
|
705
|
+
await redis('set', k, String(redisValue));
|
|
706
|
+
}
|
|
707
|
+
return k;
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
it('dry run changes nothing', async function () {
|
|
711
|
+
const p = newProjectId();
|
|
712
|
+
const k1 = await seed(p, 'requests', 7);
|
|
713
|
+
const k2 = await seed(p, 'tokens', 70, 3);
|
|
714
|
+
const { results } = await quotaSnapshotService.restoreFromSnapshots({ tdCache, project: p, period: 'all', dryRun: true });
|
|
715
|
+
expect(results.map((r) => [r.key, r.action])).to.deep.equal([[k1, 'would-set'], [k2, 'skip-exists']]);
|
|
716
|
+
expect(await redis('exists', k1)).to.equal(0);
|
|
717
|
+
|
|
718
|
+
const forced = await quotaSnapshotService.restoreFromSnapshots({ tdCache, project: p, period: 'all', dryRun: true, force: true });
|
|
719
|
+
expect(forced.results.map((r) => r.action)).to.deep.equal(['would-set', 'would-raise']);
|
|
720
|
+
expect(await redis('get', k2)).to.equal('3');
|
|
721
|
+
});
|
|
722
|
+
|
|
723
|
+
it('default mode only creates missing keys (SET NX)', async function () {
|
|
724
|
+
const p = newProjectId();
|
|
725
|
+
const k1 = await seed(p, 'requests', 7);
|
|
726
|
+
const k2 = await seed(p, 'tokens', 70, 3);
|
|
727
|
+
const { summary } = await quotaSnapshotService.restoreFromSnapshots({ tdCache, project: p, period: 'all' });
|
|
728
|
+
expect(summary).to.deep.equal({ 'set': 1, 'skip-exists': 1 });
|
|
729
|
+
expect(await redis('get', k1)).to.equal('7');
|
|
730
|
+
expect(await redis('get', k2)).to.equal('3');
|
|
731
|
+
});
|
|
732
|
+
|
|
733
|
+
it('--force sets max(redis, snapshot)', async function () {
|
|
734
|
+
const p = newProjectId();
|
|
735
|
+
const k1 = await seed(p, 'requests', 7, 3);
|
|
736
|
+
const k2 = await seed(p, 'tokens', 70, 90);
|
|
737
|
+
const k3 = await seed(p, 'email', 5);
|
|
738
|
+
const { results } = await quotaSnapshotService.restoreFromSnapshots({ tdCache, project: p, period: 'all', force: true });
|
|
739
|
+
expect(results.map((r) => r.action)).to.deep.equal(['set', 'raised', 'skip-higher']);
|
|
740
|
+
expect(await redis('get', k1)).to.equal('7');
|
|
741
|
+
expect(await redis('get', k2)).to.equal('90');
|
|
742
|
+
expect(await redis('get', k3)).to.equal('5');
|
|
743
|
+
});
|
|
744
|
+
|
|
745
|
+
it('--project filters the snapshots', async function () {
|
|
746
|
+
const p1 = newProjectId();
|
|
747
|
+
const p2 = newProjectId();
|
|
748
|
+
const k1 = await seed(p1, 'requests', 7);
|
|
749
|
+
const k2 = await seed(p2, 'requests', 8);
|
|
750
|
+
const { results } = await quotaSnapshotService.restoreFromSnapshots({ tdCache, project: p1, period: 'all' });
|
|
751
|
+
expect(results.map((r) => r.key)).to.deep.equal([k1]);
|
|
752
|
+
expect(await redis('exists', k2)).to.equal(0);
|
|
753
|
+
});
|
|
754
|
+
|
|
755
|
+
it('--period current restores only the period that contains now', async function () {
|
|
756
|
+
const savedProject = await projectService.create("quota-snapshot-current", newProjectId());
|
|
757
|
+
const p = savedProject._id.toString();
|
|
758
|
+
|
|
759
|
+
const qm = new QuoteManager({ tdCache: tdCache });
|
|
760
|
+
qm.project = savedProject;
|
|
761
|
+
const currentKey = await qm.generateKey({ createdAt: new Date() }, 'requests');
|
|
762
|
+
const currentPeriod = quotaSnapshotService.parseQuotaKey(currentKey).period;
|
|
763
|
+
|
|
764
|
+
const kCurrent = await seed(p, 'requests', 12, undefined, currentPeriod);
|
|
765
|
+
const kOld = await seed(p, 'requests', 99, undefined, '1/1/2020');
|
|
766
|
+
const { results } = await quotaSnapshotService.restoreFromSnapshots({ tdCache, project: p });
|
|
767
|
+
expect(results.map((r) => [r.key, r.action])).to.deep.equal([[kCurrent, 'set']]);
|
|
768
|
+
expect(await redis('get', kCurrent)).to.equal('12');
|
|
769
|
+
expect(await redis('exists', kOld)).to.equal(0);
|
|
770
|
+
});
|
|
771
|
+
|
|
772
|
+
it('bin/quota-restore runs with the server config', function (done) {
|
|
773
|
+
const p = newProjectId();
|
|
774
|
+
seed(p, 'requests', 7).then((k) => {
|
|
775
|
+
const script = path.join(__dirname, '..', 'bin', 'quota-restore');
|
|
776
|
+
execFile(process.execPath, [script, '--project', p, '--period', 'all', '--dry-run'], { env: Object.assign({}, process.env, { NODE_ENV: 'test' }) }, (err, stdout, stderr) => {
|
|
777
|
+
if (err) return done(new Error(err.message + "\n" + stderr));
|
|
778
|
+
expect(stdout).to.contain('would-set ' + k + ' snapshot=7 redis=(missing)');
|
|
779
|
+
expect(stdout).to.contain('dry run, nothing changed');
|
|
780
|
+
redis('exists', k).then((exists) => {
|
|
781
|
+
expect(exists).to.equal(0);
|
|
782
|
+
done();
|
|
783
|
+
}).catch(done);
|
|
784
|
+
});
|
|
785
|
+
}).catch(done);
|
|
786
|
+
});
|
|
787
|
+
});
|
|
788
|
+
describe('notify flags backup', function () {
|
|
789
|
+
|
|
790
|
+
it('snapshot stores the notified thresholds with their expiry', async function () {
|
|
791
|
+
const p = newProjectId();
|
|
792
|
+
await redis('set', key(p, 'requests'), '600');
|
|
793
|
+
await redis('set', key(p, 'requests') + ':notify:50', 'true', 'EX', '1000');
|
|
794
|
+
await redis('set', key(p, 'requests') + ':notify:75', 'true', 'EX', '2000');
|
|
795
|
+
await markDirty(key(p, 'requests'));
|
|
796
|
+
const before = Date.now();
|
|
797
|
+
await cycle();
|
|
798
|
+
|
|
799
|
+
const doc = await QuotaUsage.findOne({ id_project: p, type: 'requests' }).lean();
|
|
800
|
+
expect(Object.keys(doc.notified).sort()).to.deep.equal(['50', '75']);
|
|
801
|
+
expect(doc.notified['50'].getTime()).to.be.within(before + 990000, Date.now() + 1000000);
|
|
802
|
+
expect(doc.notified['75'].getTime()).to.be.within(before + 1990000, Date.now() + 2000000);
|
|
803
|
+
// a wiped Redis (no flags) never removes them from the backup
|
|
804
|
+
await redis('del', key(p, 'requests') + ':notify:50', key(p, 'requests') + ':notify:75');
|
|
805
|
+
await markDirty(key(p, 'requests'));
|
|
806
|
+
await cycle();
|
|
807
|
+
const after = await QuotaUsage.findOne({ id_project: p, type: 'requests' }).lean();
|
|
808
|
+
expect(Object.keys(after.notified).sort()).to.deep.equal(['50', '75']);
|
|
809
|
+
});
|
|
810
|
+
|
|
811
|
+
it('restore command restores the non expired flags with their remaining TTL', async function () {
|
|
812
|
+
const p = newProjectId();
|
|
813
|
+
const k = key(p, 'requests');
|
|
814
|
+
await QuotaUsage.create({
|
|
815
|
+
id_project: p, type: 'requests', period: PERIOD, value: 600,
|
|
816
|
+
notified: { '50': new Date(Date.now() + 1000 * 1000), '75': new Date(Date.now() - 1000) }
|
|
817
|
+
});
|
|
818
|
+
await redis('del', k, k + ':notify:50', k + ':notify:75');
|
|
819
|
+
await quotaSnapshotService.restoreFromSnapshots({ tdCache, project: p, period: 'all' });
|
|
820
|
+
expect(await redis('get', k)).to.equal('600');
|
|
821
|
+
expect(await redis('get', k + ':notify:50')).to.equal('true');
|
|
822
|
+
expect(await redis('ttl', k + ':notify:50')).to.be.within(990, 1000);
|
|
823
|
+
expect(await redis('exists', k + ':notify:75')).to.equal(0);
|
|
824
|
+
});
|
|
825
|
+
|
|
826
|
+
it('restore on miss restores the flags too', async function () {
|
|
827
|
+
const p = newProjectId();
|
|
828
|
+
const project = mockProject(p);
|
|
829
|
+
const qm = new QuoteManager({ tdCache: tdCache });
|
|
830
|
+
const obj = { createdAt: new Date('2023-10-25T08:45:54.058Z') };
|
|
831
|
+
qm.project = project;
|
|
832
|
+
const k = await qm.generateKey(obj, 'messages');
|
|
833
|
+
await redis('del', k, k + ':notify:50');
|
|
834
|
+
await QuotaUsage.create(Object.assign({ value: 10, notified: { '50': new Date(Date.now() + 500 * 1000) } }, quotaSnapshotService.parseQuotaKey(k)));
|
|
835
|
+
await qm.getCurrentQuote(project, obj, 'messages');
|
|
836
|
+
expect(await redis('get', k + ':notify:50')).to.equal('true');
|
|
837
|
+
});
|
|
838
|
+
|
|
839
|
+
it('invalidateCheckpointKeys clears the backed up flags', async function () {
|
|
840
|
+
const savedProject = await projectService.create("quota-snapshot-invalidate", newProjectId());
|
|
841
|
+
const p = savedProject._id.toString();
|
|
842
|
+
const qm = new QuoteManager({ tdCache: tdCache });
|
|
843
|
+
const obj = { createdAt: new Date() };
|
|
844
|
+
qm.project = savedProject;
|
|
845
|
+
const k = await qm.generateKey(obj, 'requests');
|
|
846
|
+
await QuotaUsage.create(Object.assign({ value: 600, notified: { '50': new Date(Date.now() + 1000000) } }, quotaSnapshotService.parseQuotaKey(k)));
|
|
847
|
+
await qm.invalidateCheckpointKeys(savedProject, obj);
|
|
848
|
+
let doc;
|
|
849
|
+
for (let i = 0; i < 20; i++) {
|
|
850
|
+
doc = await QuotaUsage.findOne({ id_project: p, type: 'requests' }).lean();
|
|
851
|
+
if (!doc.notified) break;
|
|
852
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
853
|
+
}
|
|
854
|
+
expect(doc.notified).to.equal(undefined);
|
|
855
|
+
expect(doc.value).to.equal(600);
|
|
856
|
+
});
|
|
857
|
+
});
|
|
858
|
+
|
|
859
|
+
describe('wipe detection (sentinel)', function () {
|
|
860
|
+
|
|
861
|
+
const qm = new QuoteManager({ tdCache: tdCache });
|
|
862
|
+
|
|
863
|
+
before(async function () {
|
|
864
|
+
await redis('set', quotaSnapshotService.SENTINEL_KEY, 'test');
|
|
865
|
+
quotaSnapshotService.startWipeDetection(tdCache, { intervalSeconds: 3600 });
|
|
866
|
+
await quotaSnapshotService.waitForRestore();
|
|
867
|
+
});
|
|
868
|
+
|
|
869
|
+
after(function () {
|
|
870
|
+
quotaSnapshotService.stopWipeDetection();
|
|
871
|
+
});
|
|
872
|
+
|
|
873
|
+
async function currentSetup(type, value, notified) {
|
|
874
|
+
const savedProject = await projectService.create("quota-sentinel", newProjectId());
|
|
875
|
+
qm.project = savedProject;
|
|
876
|
+
const obj = { createdAt: new Date() };
|
|
877
|
+
const k = await qm.generateKey(obj, type);
|
|
878
|
+
await QuotaUsage.create(Object.assign({ value: value, notified: notified }, quotaSnapshotService.parseQuotaKey(k)));
|
|
879
|
+
return { project: savedProject, obj, k };
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
it('a missing sentinel clears the memo, restores the current period and sets the sentinel again', async function () {
|
|
883
|
+
const a = await currentSetup('requests', 100, { '50': new Date(Date.now() + 100000) });
|
|
884
|
+
const b = await currentSetup('messages', 40);
|
|
885
|
+
const c = await currentSetup('tokens', 5);
|
|
886
|
+
// warm the memo, then wipe
|
|
887
|
+
await qm.getCurrentQuote(a.project, a.obj, 'requests');
|
|
888
|
+
expect(await redis('get', a.k)).to.equal('100');
|
|
889
|
+
await redis('flushall');
|
|
890
|
+
// increments that happened after the wipe, before the detection
|
|
891
|
+
await redis('set', b.k, '3');
|
|
892
|
+
await redis('set', c.k, '9'); // higher than the snapshot: kept as is
|
|
893
|
+
|
|
894
|
+
const result = await quotaSnapshotService.checkSentinel(tdCache, { wait: true });
|
|
895
|
+
expect(result.wiped).to.equal(true);
|
|
896
|
+
expect(result.restore.restored).to.equal(true);
|
|
897
|
+
expect(quotaSnapshotService.restoreStats.memoCleared).to.equal(1);
|
|
898
|
+
expect(await redis('get', a.k)).to.equal('100');
|
|
899
|
+
expect(await redis('get', a.k + ':notify:50')).to.equal('true');
|
|
900
|
+
expect(Number(await redis('get', b.k))).to.equal(43); // snapshot + post-wipe increments
|
|
901
|
+
// the merged value is above the backup: marked dirty for the next snapshot
|
|
902
|
+
expect(await redis('sismember', S.DIRTY_KEY, b.k)).to.equal(1);
|
|
903
|
+
expect(await redis('get', c.k)).to.equal('9');
|
|
904
|
+
expect(await redis('exists', quotaSnapshotService.SENTINEL_KEY)).to.equal(1);
|
|
905
|
+
expect(await redis('exists', quotaSnapshotService.RESTORE_LOCK_KEY)).to.equal(0);
|
|
906
|
+
|
|
907
|
+
// running it again (sentinel lost again, nothing wiped) never double counts
|
|
908
|
+
await redis('del', quotaSnapshotService.SENTINEL_KEY);
|
|
909
|
+
await quotaSnapshotService.checkSentinel(tdCache, { wait: true });
|
|
910
|
+
expect(await redis('get', a.k)).to.equal('100');
|
|
911
|
+
expect(Number(await redis('get', b.k))).to.equal(43);
|
|
912
|
+
});
|
|
913
|
+
|
|
914
|
+
it('only one replica restores: the restore lock held elsewhere skips the restore', async function () {
|
|
915
|
+
const a = await currentSetup('requests', 100);
|
|
916
|
+
await redis('del', a.k, quotaSnapshotService.SENTINEL_KEY);
|
|
917
|
+
await redis('set', quotaSnapshotService.RESTORE_LOCK_KEY, 'other-replica', 'EX', '30');
|
|
918
|
+
|
|
919
|
+
const result = await quotaSnapshotService.checkSentinel(tdCache, { wait: true });
|
|
920
|
+
expect(result.wiped).to.equal(true);
|
|
921
|
+
expect(result.restore).to.deep.equal({ restored: false, reason: 'locked' });
|
|
922
|
+
expect(quotaSnapshotService.restoreStats.memoCleared).to.equal(1);
|
|
923
|
+
expect(await redis('exists', a.k)).to.equal(0);
|
|
924
|
+
expect(await redis('exists', quotaSnapshotService.SENTINEL_KEY)).to.equal(0);
|
|
925
|
+
expect(await redis('get', quotaSnapshotService.RESTORE_LOCK_KEY)).to.equal('other-replica');
|
|
926
|
+
});
|
|
927
|
+
|
|
928
|
+
it('concurrent detections in one process run a single restore', async function () {
|
|
929
|
+
await currentSetup('requests', 100);
|
|
930
|
+
await redis('del', quotaSnapshotService.SENTINEL_KEY);
|
|
931
|
+
const results = await Promise.all([1, 2, 3].map(() => quotaSnapshotService.checkSentinel(tdCache, { wait: true })));
|
|
932
|
+
expect(results.filter((r) => r.restore && r.restore.restored).length).to.equal(1);
|
|
933
|
+
expect(quotaSnapshotService.restoreStats.wipeRestores).to.equal(1);
|
|
934
|
+
});
|
|
935
|
+
|
|
936
|
+
it('an increment during the wipe window is not lost and not double counted', async function () {
|
|
937
|
+
const b = await currentSetup('messages', 10);
|
|
938
|
+
await redis('del', b.k, quotaSnapshotService.SENTINEL_KEY);
|
|
939
|
+
// the counter is missing and the sentinel too: restore on miss must not seed it
|
|
940
|
+
await qm.incrementMessagesCount(b.project, b.obj);
|
|
941
|
+
expect(await redis('get', b.k)).to.equal('1');
|
|
942
|
+
// next timer tick
|
|
943
|
+
await quotaSnapshotService.checkSentinel(tdCache, { wait: true });
|
|
944
|
+
expect(Number(await redis('get', b.k))).to.equal(11);
|
|
945
|
+
expect(await redis('exists', quotaSnapshotService.SENTINEL_KEY)).to.equal(1);
|
|
946
|
+
});
|
|
947
|
+
|
|
948
|
+
it('the hot path never checks the sentinel', async function () {
|
|
949
|
+
const a = await currentSetup('requests', 100);
|
|
950
|
+
for (let i = 0; i < 50; i++) {
|
|
951
|
+
await qm.getCurrentQuote(a.project, a.obj, 'requests');
|
|
952
|
+
}
|
|
953
|
+
expect(quotaSnapshotService.restoreStats.sentinelChecks).to.equal(0);
|
|
954
|
+
expect(quotaSnapshotService.restoreStats.existsChecks).to.equal(1);
|
|
955
|
+
});
|
|
956
|
+
|
|
957
|
+
it('the timer checks the sentinel every interval', async function () {
|
|
958
|
+
quotaSnapshotService.stopWipeDetection();
|
|
959
|
+
quotaSnapshotService.startWipeDetection(tdCache, { intervalSeconds: 0.1 });
|
|
960
|
+
await new Promise((r) => setTimeout(r, 350));
|
|
961
|
+
quotaSnapshotService.stopWipeDetection();
|
|
962
|
+
expect(quotaSnapshotService.restoreStats.sentinelChecks).to.be.within(3, 5);
|
|
963
|
+
quotaSnapshotService.startWipeDetection(tdCache, { intervalSeconds: 3600 });
|
|
964
|
+
});
|
|
965
|
+
});
|
|
966
|
+
describe('reset', function () {
|
|
967
|
+
|
|
968
|
+
async function seeded(value, redisValue, period) {
|
|
969
|
+
const p = newProjectId();
|
|
970
|
+
const k = key(p, 'requests', period);
|
|
971
|
+
await QuotaUsage.create({ id_project: p, type: 'requests', period: period || PERIOD, value: value, notified: { '50': new Date(Date.now() + 100000) } });
|
|
972
|
+
await redis('set', k, String(redisValue));
|
|
973
|
+
await redis('set', k + ':notify:50', 'true', 'EX', '100');
|
|
974
|
+
await redis('set', k + ':notify:75', 'true', 'EX', '100');
|
|
975
|
+
return { p, k };
|
|
976
|
+
}
|
|
977
|
+
|
|
978
|
+
it('sets the counter and the backup to 0 (bypassing $max) and clears the notify flags', async function () {
|
|
979
|
+
const { p, k } = await seeded(500, 520);
|
|
980
|
+
const result = await quotaSnapshotService.resetCounter({ tdCache, project: p, type: 'requests', period: PERIOD });
|
|
981
|
+
expect(result.key).to.equal(k);
|
|
982
|
+
expect(await redis('get', k)).to.equal('0');
|
|
983
|
+
const doc = await QuotaUsage.findOne({ id_project: p, type: 'requests' }).lean();
|
|
984
|
+
expect(doc.value).to.equal(0);
|
|
985
|
+
expect(doc.notified).to.equal(undefined);
|
|
986
|
+
expect(await redis('exists', k + ':notify:50', k + ':notify:75')).to.equal(0);
|
|
987
|
+
// the reset marks it dirty; a following snapshot keeps it at 0
|
|
988
|
+
expect(await redis('sismember', S.DIRTY_KEY, k)).to.equal(1);
|
|
989
|
+
await cycle();
|
|
990
|
+
expect((await QuotaUsage.findOne({ id_project: p, type: 'requests' }).lean()).value).to.equal(0);
|
|
991
|
+
});
|
|
992
|
+
|
|
993
|
+
it('waits for a running snapshot (lock) up to its TTL, reporting progress', async function () {
|
|
994
|
+
const { p, k } = await seeded(500, 520);
|
|
995
|
+
await redis('set', S.LOCK_KEY, 'snapshot-running', 'PX', '1500');
|
|
996
|
+
const progress = [];
|
|
997
|
+
const started = Date.now();
|
|
998
|
+
await quotaSnapshotService.resetCounter({ tdCache, project: p, type: 'requests', period: PERIOD, onWait: (m) => progress.push(m), progressEveryMs: 200 });
|
|
999
|
+
expect(Date.now() - started).to.be.within(1000, 6000);
|
|
1000
|
+
expect(progress.length).to.be.at.least(1);
|
|
1001
|
+
expect(progress[0]).to.match(/snapshot/i);
|
|
1002
|
+
expect(await redis('get', k)).to.equal('0');
|
|
1003
|
+
});
|
|
1004
|
+
|
|
1005
|
+
it('--value sets a given value, --dry-run changes nothing', async function () {
|
|
1006
|
+
const { p, k } = await seeded(500, 520);
|
|
1007
|
+
const dry = await quotaSnapshotService.resetCounter({ tdCache, project: p, type: 'requests', period: PERIOD, value: 42, dryRun: true });
|
|
1008
|
+
expect(dry.redis).to.equal(520);
|
|
1009
|
+
expect(dry.snapshot).to.equal(500);
|
|
1010
|
+
expect(await redis('get', k)).to.equal('520');
|
|
1011
|
+
await quotaSnapshotService.resetCounter({ tdCache, project: p, type: 'requests', period: PERIOD, value: 42 });
|
|
1012
|
+
expect(await redis('get', k)).to.equal('42');
|
|
1013
|
+
expect((await QuotaUsage.findOne({ id_project: p, type: 'requests' }).lean()).value).to.equal(42);
|
|
1014
|
+
});
|
|
1015
|
+
|
|
1016
|
+
it('--period current resolves the period of the project', async function () {
|
|
1017
|
+
const savedProject = await projectService.create("quota-reset-current", newProjectId());
|
|
1018
|
+
const p = savedProject._id.toString();
|
|
1019
|
+
const qm = new QuoteManager({ tdCache: tdCache });
|
|
1020
|
+
qm.project = savedProject;
|
|
1021
|
+
const k = await qm.generateKey({ createdAt: new Date() }, 'tokens');
|
|
1022
|
+
await redis('set', k, '99');
|
|
1023
|
+
const result = await quotaSnapshotService.resetCounter({ tdCache, project: p, type: 'tokens' });
|
|
1024
|
+
expect(result.key).to.equal(k);
|
|
1025
|
+
expect(await redis('get', k)).to.equal('0');
|
|
1026
|
+
});
|
|
1027
|
+
|
|
1028
|
+
it('rejects unknown types and runs from bin/quota-reset', function (done) {
|
|
1029
|
+
quotaSnapshotService.resetCounter({ tdCache, project: newProjectId(), type: 'kbs', period: PERIOD })
|
|
1030
|
+
.then(() => done(new Error('expected a rejection')))
|
|
1031
|
+
.catch(() => seeded(500, 520).then(({ p, k }) => {
|
|
1032
|
+
const script = path.join(__dirname, '..', 'bin', 'quota-reset');
|
|
1033
|
+
execFile(process.execPath, [script, '--project', p, '--type', 'requests', '--period', PERIOD, '--value', '7'], { env: Object.assign({}, process.env, { NODE_ENV: 'test' }) }, (err, stdout, stderr) => {
|
|
1034
|
+
if (err) return done(new Error(err.message + "\n" + stderr));
|
|
1035
|
+
expect(stdout).to.contain(k);
|
|
1036
|
+
redis('get', k).then((v) => {
|
|
1037
|
+
expect(v).to.equal('7');
|
|
1038
|
+
done();
|
|
1039
|
+
}).catch(done);
|
|
1040
|
+
});
|
|
1041
|
+
}).catch(done));
|
|
1042
|
+
});
|
|
1043
|
+
});
|
|
1044
|
+
});
|