@tiledesk/tiledesk-server 3.0.1 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.circleci/config.yml +1 -1
- package/CHANGELOG.md +34 -0
- package/Dockerfile +3 -2
- package/Dockerfile-en +3 -2
- package/Dockerfile-jobs +3 -2
- package/Dockerfile-profiler +3 -2
- package/README.md +15 -0
- package/app.js +12 -1
- package/bin/quota-reset +68 -0
- package/bin/quota-restore +86 -0
- package/channels/chat21/chat21Handler.js +50 -34
- package/channels/chat21/chat21JoinTracker.js +43 -0
- package/config/database.js +5 -5
- package/config/native_mcp/native_mcp_servers.json +1 -0
- package/controllers/chatbot.controller.js +80 -27
- package/controllers/intents.controller.js +42 -0
- package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-chat21-ionic.md +468 -0
- package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-design-studio.md +1349 -0
- package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-server.md +1694 -0
- package/docs/superpowers/plans/2026-09-15-webhook-chatbot-hitl-tybot.md +789 -0
- package/docs/superpowers/plans/2026-09-29-hitl-known-limits-server.md +83 -0
- package/docs/superpowers/plans/2026-09-29-hitl-known-limits-tybot.md +76 -0
- package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-chat21-ionic.md +97 -0
- package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-design-studio.md +72 -0
- package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-server.md +223 -0
- package/docs/superpowers/plans/2026-09-29-hitl-participant-removal-tybot.md +121 -0
- package/docs/superpowers/plans/2026-09-29-hitl-pre-branch-ports.md +65 -0
- package/docs/superpowers/plans/2026-09-30-flow-start-points-design-studio.md +88 -0
- package/docs/superpowers/plans/2026-09-30-flow-start-points-server.md +103 -0
- package/docs/superpowers/plans/2026-09-30-flow-start-points-tybot.md +43 -0
- package/docs/superpowers/plans/2026-10-01-scheduled-start-point-design-studio.md +80 -0
- package/docs/superpowers/plans/2026-10-01-scheduled-start-point-runtime.md +39 -0
- package/docs/superpowers/plans/2026-10-01-scheduled-start-point-server.md +106 -0
- package/docs/superpowers/specs/2026-09-15-webhook-chatbot-hitl-design.md +715 -0
- package/docs/superpowers/specs/2026-09-30-flow-start-points-design.md +156 -0
- package/docs/superpowers/specs/2026-10-01-scheduled-start-point-design.md +303 -0
- package/docs/superpowers/todo/2026-09-14-webhook-chatbot-shared-code-fixes.md +191 -0
- package/event/botEvent.js +12 -1
- package/event/emailEvent.js +3 -0
- package/event/messageEvent.js +44 -0
- package/jobs.js +10 -0
- package/jobsManager.js +19 -0
- package/middleware/errorHandler.js +6 -2
- package/models/analyticResult.js +1 -1
- package/models/channelConstants.js +3 -2
- package/models/quotaUsage.js +71 -0
- package/models/request.js +6 -0
- package/models/requestSnapshot.js +12 -0
- package/models/routerLogger.js +1 -1
- package/models/webhook.js +44 -0
- package/package.json +7 -5
- package/pubmodules/emailNotification/requestNotification.js +14 -22
- package/pubmodules/events/test/eventRoute.js +7 -6
- package/pubmodules/rules/conciergeBot.js +9 -4
- package/routes/auth.js +1 -12
- package/routes/campaigns.js +0 -13
- package/routes/dataTable.js +0 -3
- package/routes/faq_kb.js +12 -1
- package/routes/messagesRoot.js +0 -1
- package/routes/quotes.js +38 -1
- package/routes/request.js +81 -5
- package/routes/users.js +0 -1
- package/routes/webhook.js +160 -8
- package/routes/webhooks.js +276 -6
- package/routes/widgetLoader.js +6 -3
- package/services/QuoteManager.js +77 -16
- package/services/aiService.js +1 -1
- package/services/chatbotService.js +55 -1
- package/services/departmentService.js +60 -43
- package/services/dolphinSchedulerClient.js +116 -0
- package/services/emailService.js +6 -1
- package/services/hitlParticipantListener.js +260 -0
- package/services/participantInviteService.js +302 -0
- package/services/participantRemoveService.js +79 -0
- package/services/quotaSnapshotService.js +1167 -0
- package/services/requestService.js +102 -144
- package/services/scheduledStart/dolphinBackend.js +246 -0
- package/services/scheduledStart/schedulerBackend.js +101 -0
- package/services/scheduledStart/schedulerServiceClient.js +41 -0
- package/services/scheduledStartPublishListener.js +186 -0
- package/services/scheduledStartRules.js +173 -0
- package/services/scheduledStartService.js +130 -0
- package/services/startPointCleanupListener.js +30 -0
- package/services/startPointService.js +158 -0
- package/services/testWsService.js +1 -4
- package/services/updateRequestSnapshotQueued.js +8 -9
- package/services/webhookConversationService.js +185 -0
- package/services/webhookSettingsValidator.js +61 -0
- package/template/email/checkpointReachedEmail.html +5 -0
- package/test/agentPromptsRoute.test.js +3 -2
- package/test/agentTokensQuota.js +237 -0
- package/test/botEventHitlFallback.js +19 -0
- package/test/chat21HandlerJoinWiring.js +59 -0
- package/test/dataTablesRoute.test.js +10 -2
- package/test/dateUtils.test.js +0 -3
- package/test/departmentService.js +8 -12
- package/test/{fileRoute.js → deprecated/_deprecated_fileRoute} +2 -2
- package/test/{kbsettingsRoute.js → deprecated/_deprecated_kbsettingsRoute} +3 -3
- package/test/faqRoute.js +6 -1
- package/test/faqkbRoute.js +14 -10
- package/test/faqkbV3Route.js +1 -1
- package/test/filepRoute.js +4 -50
- package/test/hitlParticipantListener.js +446 -0
- package/test/hitlQueueWiring.js +191 -0
- package/test/hitlWaitingReset.js +124 -0
- package/test/inviteParticipantsRoute.js +492 -0
- package/test/kbRoute.js +1 -1
- package/test/messageRoute.js +33 -21
- package/test/nativemcp.test.js +7 -1
- package/test/openaiRoute.js +2 -8
- package/test/publishFailLoud.test.js +414 -0
- package/test/quotaSnapshots.js +1044 -0
- package/test/removeParticipantsRoute.js +186 -0
- package/test/requestService.js +24 -16
- package/test/requestServiceRemoveParticipant.js +144 -0
- package/test/requestVisibility.test.js +186 -0
- package/test/scheduledStartFacade.js +184 -0
- package/test/scheduledStartHandover.js +95 -0
- package/test/scheduledStartPublish.js +320 -0
- package/test/scheduledStartRoute.js +163 -0
- package/test/scheduledStartRules.js +177 -0
- package/test/scheduledStartRun.js +299 -0
- package/test/scheduledStartSchedulerBackend.js +116 -0
- package/test/scheduledStartService.js +290 -0
- package/test/startPointsCleanup.js +69 -0
- package/test/startPointsRegression.js +169 -0
- package/test/startPointsRoute.js +158 -0
- package/test/webhookConversationRoute.js +347 -0
- package/test/webhookConversationService.js +261 -0
- package/test/webhookConversationTestUtils.js +55 -0
- package/utils/aiUtils.js +95 -1
- package/utils/intentFieldNames.js +192 -0
- package/utils/quotaCliUtil.js +43 -0
- package/utils/requestVisibility.js +355 -0
- package/websocket/webSocketServer.js +36 -31
- /package/test/{imageRoute.js → deprecated/_deprecated_imageRoute} +0 -0
|
@@ -0,0 +1,1167 @@
|
|
|
1
|
+
const os = require('os');
|
|
2
|
+
const crypto = require('crypto');
|
|
3
|
+
const mongoose = require('mongoose');
|
|
4
|
+
const winston = require('../config/winston');
|
|
5
|
+
const QuotaUsage = require('../models/quotaUsage');
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Durable backup of the quota counters that QuoteManager keeps in Redis.
|
|
9
|
+
*
|
|
10
|
+
* - dirty set: every increment (and every notify flag set) adds its counter to "quotes:dirty" in the same
|
|
11
|
+
* round trip (MULTI); while the watcher runs a per-process memo skips the SADD for counters already marked
|
|
12
|
+
* in the current snapshot generation.
|
|
13
|
+
* - snapshot: every QUOTA_SNAPSHOT_INTERVAL_MINUTES one replica (Redis lock) moves the dirty set aside
|
|
14
|
+
* ("quotes:dirty:saving", recovered if a cycle crashed), waits until every process saw the new generation,
|
|
15
|
+
* and upserts those counters of the recent periods (current and previous) into quota_usages with $max on
|
|
16
|
+
* value and on the expiry of each ":notify:<pct>" flag, so a wiped (and not yet restored) Redis never
|
|
17
|
+
* lowers the backup. No keyspace SCAN and no read back; every QUOTA_FULL_SCAN_INTERVAL_HOURS a full
|
|
18
|
+
* SCAN reconciles the counters changed outside the normal path.
|
|
19
|
+
* - wipe detection: the sentinel key "quotes:sentinel" exists while Redis holds the counters. A timer
|
|
20
|
+
* checks it every QUOTA_SENTINEL_CHECK_SECONDS (only in a process where start() ran). When it is
|
|
21
|
+
* missing Redis was wiped: the memo is cleared and one replica (lock "quotes:restore:lock") restores
|
|
22
|
+
* the current period with the "merge" rule, then sets the sentinel again.
|
|
23
|
+
* - restore on miss: before QuoteManager reads or increments a counter whose Redis key does not exist,
|
|
24
|
+
* the key is seeded from quota_usages with SET NX (only while the sentinel exists, when detection runs).
|
|
25
|
+
* - restoreFromSnapshots / resetCounter: used by the wipe restore and by bin/quota-restore, bin/quota-reset.
|
|
26
|
+
*
|
|
27
|
+
* Merge rule: counters only grow and every snapshot is taken from the live key, so after a clean wipe a
|
|
28
|
+
* Redis value lower than its snapshot was recreated by increments made after the wipe: it becomes
|
|
29
|
+
* snapshot + those increments; a value >= snapshot is left untouched. This assumes a clean wipe: a Redis
|
|
30
|
+
* restored from an older RDB/AOF together with a lost sentinel, or a counter lowered by hand, would be
|
|
31
|
+
* double counted (use bin/quota-reset to lower a counter).
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
const KEY_PREFIX = 'quotes:';
|
|
35
|
+
const LOCK_KEY = 'quotes:snapshot:lock';
|
|
36
|
+
const RESTORE_LOCK_KEY = 'quotes:restore:lock';
|
|
37
|
+
const SENTINEL_KEY = 'quotes:sentinel';
|
|
38
|
+
const RESTORE_LOCK_TTL_MS = 60 * 1000; // extended after every batch while the restore runs
|
|
39
|
+
const SCAN_COUNT = 1000;
|
|
40
|
+
const BATCH_SIZE = 500;
|
|
41
|
+
const LOOKUP_TIMEOUT_MS = 2000;
|
|
42
|
+
const MONGO_FAILURE_BACKOFF_MS = 60 * 1000;
|
|
43
|
+
const MEMO_MAX_SIZE = 100000;
|
|
44
|
+
const RECENT_PERIOD_DAYS = 70; // current and previous monthly period
|
|
45
|
+
const NOTIFY_DEFAULT_TTL_MS = 2592000 * 1000; // EX used by QuoteManager for the notify flags
|
|
46
|
+
// expiry = now + PTTL drifts by a few ms between reads: only a later expiry beyond this counts as a change
|
|
47
|
+
const NOTIFY_EXPIRY_TOLERANCE_MS = 60 * 1000;
|
|
48
|
+
const PERIOD_REGEX = /^\d{1,2}\/\d{1,2}\/\d{4}$/;
|
|
49
|
+
const TYPE_REGEX = /^[a-z_]+$/;
|
|
50
|
+
const NOTIFY_REGEX = /^(.+):notify:(\d+)$/;
|
|
51
|
+
const NOTIFY_PCTS = ['50', '75', '95', '100']; // checkpoints of QuoteManager.percentageCalculator
|
|
52
|
+
// counters changed since the last snapshot (SADD with every increment), moved aside by the snapshot cycle
|
|
53
|
+
const DIRTY_KEY = 'quotes:dirty';
|
|
54
|
+
const DIRTY_SAVING_KEY = 'quotes:dirty:saving';
|
|
55
|
+
// bumped every time the dirty set is moved aside: processes clear their "already marked" memo when it changes
|
|
56
|
+
const DIRTY_GEN_KEY = 'quotes:dirty:gen';
|
|
57
|
+
// time (ms) of the last full keyspace reconciliation
|
|
58
|
+
const FULL_SCAN_KEY = 'quotes:fullscan:last';
|
|
59
|
+
// quota types counted in Redis (chatbots, kbs and namespace are counted on mongo by the routes)
|
|
60
|
+
const COUNTED_TYPES = ['requests', 'messages', 'email', 'tokens', 'agent_tokens', 'voice_duration'];
|
|
61
|
+
const NOT_IN_REDIS_TYPES = ['chatbots', 'kbs', 'namespace'];
|
|
62
|
+
|
|
63
|
+
const INSTANCE_TOKEN = os.hostname() + ':' + process.pid + ':' + crypto.randomBytes(6).toString('hex');
|
|
64
|
+
|
|
65
|
+
// Compare-and-delete / compare-and-extend: only if this replica still owns the lock
|
|
66
|
+
const RELEASE_LOCK_SCRIPT = "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) else return 0 end";
|
|
67
|
+
const EXTEND_LOCK_SCRIPT = "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('PEXPIRE', KEYS[1], ARGV[2]) else return 0 end";
|
|
68
|
+
// One call per batch. ARGV[1] = mode (nx | max | merge | read), ARGV[2] = dirty set, ARGV[i+2] = snapshot of KEYS[i].
|
|
69
|
+
// Returns { code, previous value } per key: 0 untouched, 1 set (was missing), 2 raised (max) / merged (marked dirty:
|
|
70
|
+
// the new value is above the backup).
|
|
71
|
+
const RESTORE_BATCH_SCRIPT = [
|
|
72
|
+
"local mode = ARGV[1]",
|
|
73
|
+
"local out = {}",
|
|
74
|
+
"for i, k in ipairs(KEYS) do",
|
|
75
|
+
" local s = ARGV[i + 2]",
|
|
76
|
+
" local cur = redis.call('GET', k)",
|
|
77
|
+
" local code = 0",
|
|
78
|
+
" if mode ~= 'read' then",
|
|
79
|
+
" if not cur then",
|
|
80
|
+
" redis.call('SET', k, s) code = 1",
|
|
81
|
+
" elseif mode == 'max' and (tonumber(cur) or 0) < tonumber(s) then",
|
|
82
|
+
" redis.call('SET', k, s) code = 2 if ARGV[2] ~= '' then redis.call('SADD', ARGV[2], k) end",
|
|
83
|
+
" elseif mode == 'merge' and (tonumber(cur) or 0) < tonumber(s) then",
|
|
84
|
+
" redis.call('INCRBYFLOAT', k, s) code = 2 if ARGV[2] ~= '' then redis.call('SADD', ARGV[2], k) end",
|
|
85
|
+
" end",
|
|
86
|
+
" end",
|
|
87
|
+
" out[#out + 1] = code",
|
|
88
|
+
" out[#out + 1] = cur",
|
|
89
|
+
"end",
|
|
90
|
+
"return out"
|
|
91
|
+
].join('\n');
|
|
92
|
+
// Restore on miss while wipe detection runs: seed only if the sentinel exists
|
|
93
|
+
const SEED_SCRIPT = "if redis.call('EXISTS', KEYS[1]) == 0 then return -1 end " +
|
|
94
|
+
"if redis.call('SET', KEYS[2], ARGV[1], 'NX') then return 1 end " +
|
|
95
|
+
"return 0";
|
|
96
|
+
|
|
97
|
+
function nonNegativeNumber(value, defaultValue) {
|
|
98
|
+
if (value === undefined || value === null || value === '') {
|
|
99
|
+
return defaultValue;
|
|
100
|
+
}
|
|
101
|
+
const parsed = Number(value);
|
|
102
|
+
return Number.isFinite(parsed) && parsed >= 0 ? parsed : defaultValue;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function quotesEnabled() {
|
|
106
|
+
return !(process.env.QUOTES_ENABLED === 'false' || process.env.QUOTES_ENABLED === false);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* "quotes:<type>:<id_project>:<period>" -> { type, id_project, period }, null for any other key
|
|
111
|
+
* (notify flags "<key>:notify:<pct>", locks, sentinel, ...)
|
|
112
|
+
*/
|
|
113
|
+
function parseQuotaKey(key) {
|
|
114
|
+
if (typeof key !== 'string' || !key.startsWith(KEY_PREFIX)) {
|
|
115
|
+
return null;
|
|
116
|
+
}
|
|
117
|
+
const parts = key.split(':');
|
|
118
|
+
if (parts.length !== 4) {
|
|
119
|
+
return null;
|
|
120
|
+
}
|
|
121
|
+
const [, type, id_project, period] = parts;
|
|
122
|
+
if (!TYPE_REGEX.test(type) || !id_project || !PERIOD_REGEX.test(period)) {
|
|
123
|
+
return null;
|
|
124
|
+
}
|
|
125
|
+
return { type, id_project, period };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** "<counter key>:notify:<pct>" -> { counterKey, pct }, null for any other key */
|
|
129
|
+
function parseNotifyKey(key) {
|
|
130
|
+
const match = typeof key === 'string' ? NOTIFY_REGEX.exec(key) : null;
|
|
131
|
+
if (!match || !parseQuotaKey(match[1])) {
|
|
132
|
+
return null;
|
|
133
|
+
}
|
|
134
|
+
return { counterKey: match[1], pct: match[2] };
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function buildQuotaKey(id_project, type, period) {
|
|
138
|
+
return KEY_PREFIX + type + ':' + id_project + ':' + period;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function recentCutoff(now) {
|
|
142
|
+
return new Date((now || Date.now()) - RECENT_PERIOD_DAYS * 24 * 3600 * 1000);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function isRecentPeriod(period, cutoff) {
|
|
146
|
+
const date = QuotaUsage.periodToDate(period);
|
|
147
|
+
return date !== null && date >= cutoff;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function redisClient(tdCache) {
|
|
151
|
+
if (!tdCache || typeof tdCache.getClient !== 'function') {
|
|
152
|
+
return null;
|
|
153
|
+
}
|
|
154
|
+
const client = tdCache.getClient();
|
|
155
|
+
return client && typeof client.send_command === 'function' ? client : null;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function redisCommand(client, command, args) {
|
|
159
|
+
return new Promise((resolve, reject) => {
|
|
160
|
+
client.send_command(command, args, (err, reply) => {
|
|
161
|
+
if (err) {
|
|
162
|
+
return reject(err);
|
|
163
|
+
}
|
|
164
|
+
resolve(reply);
|
|
165
|
+
});
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function withTimeout(promise, ms, label) {
|
|
170
|
+
let timer;
|
|
171
|
+
const timeout = new Promise((resolve, reject) => {
|
|
172
|
+
timer = setTimeout(() => reject(new Error(label + ' timed out after ' + ms + 'ms')), ms);
|
|
173
|
+
});
|
|
174
|
+
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
async function acquireLock(client, lockKey, token, ttlSeconds) {
|
|
178
|
+
return (await redisCommand(client, 'set', [lockKey, token, 'NX', 'EX', ttlSeconds])) === 'OK';
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
async function releaseLock(client, lockKey, token) {
|
|
182
|
+
try {
|
|
183
|
+
await redisCommand(client, 'eval', [RELEASE_LOCK_SCRIPT, 1, lockKey, token]);
|
|
184
|
+
} catch (err) {
|
|
185
|
+
winston.error("[QuotaSnapshot] Unable to release " + lockKey + ": ", err);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Re-creates the notify flags of a counter that are not expired yet, with their remaining TTL (SET NX).
|
|
191
|
+
* @returns {Promise<number>} flags restored
|
|
192
|
+
*/
|
|
193
|
+
async function restoreNotifyFlags(client, counterKey, notified) {
|
|
194
|
+
if (!notified || typeof notified !== 'object') {
|
|
195
|
+
return 0;
|
|
196
|
+
}
|
|
197
|
+
const now = Date.now();
|
|
198
|
+
const commands = [];
|
|
199
|
+
Object.keys(notified).forEach((pct) => {
|
|
200
|
+
const ttlSeconds = Math.ceil((new Date(notified[pct]).getTime() - now) / 1000);
|
|
201
|
+
if (Number.isFinite(ttlSeconds) && ttlSeconds > 1) {
|
|
202
|
+
commands.push(redisCommand(client, 'set', [counterKey + ':notify:' + pct, 'true', 'NX', 'EX', ttlSeconds]));
|
|
203
|
+
}
|
|
204
|
+
});
|
|
205
|
+
const replies = await Promise.all(commands);
|
|
206
|
+
return replies.filter((r) => r === 'OK').length;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
// RESTORE ON MISS + WIPE DETECTION - START
|
|
211
|
+
|
|
212
|
+
const memoTtlMs = nonNegativeNumber(process.env.QUOTA_RESTORE_MEMO_TTL_SECONDS, 600) * 1000;
|
|
213
|
+
const verifiedKeys = new Map(); // key -> expiresAt, insertion order = age (oldest first)
|
|
214
|
+
const inflight = new Map(); // key -> pending check, so a burst on a new key does one lookup
|
|
215
|
+
const restoreStats = { existsChecks: 0, lookups: 0, restored: 0, errors: 0, sentinelChecks: 0, memoCleared: 0, wipeRestores: 0, dirtyMarks: 0 };
|
|
216
|
+
// counters already added to the dirty set since this process last saw a new snapshot generation (only used while
|
|
217
|
+
// the watcher runs: it reads the generation with the sentinel)
|
|
218
|
+
const dirtyMemo = new Set();
|
|
219
|
+
let lastDirtyGen;
|
|
220
|
+
let lastGenReadAt = 0; // time of the last successful generation read by the watcher
|
|
221
|
+
let watcherIntervalMs = 0;
|
|
222
|
+
// SADD to the dirty set only when a snapshot can drain it (quotes enabled, snapshot interval > 0)
|
|
223
|
+
let dirtyTracking = computeDirtyTracking();
|
|
224
|
+
|
|
225
|
+
function computeDirtyTracking() {
|
|
226
|
+
return quotesEnabled() && nonNegativeNumber(process.env.QUOTA_SNAPSHOT_INTERVAL_MINUTES, 60) > 0;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/** Re-reads QUOTES_ENABLED / QUOTA_SNAPSHOT_INTERVAL_MINUTES (start() calls it). */
|
|
230
|
+
function configureDirtyTracking() {
|
|
231
|
+
dirtyTracking = computeDirtyTracking();
|
|
232
|
+
return dirtyTracking;
|
|
233
|
+
}
|
|
234
|
+
let lookupFailureLogged = false;
|
|
235
|
+
let mongoDownUntil = 0;
|
|
236
|
+
let restorePromise = null;
|
|
237
|
+
let wipeDetectionTimer = null;
|
|
238
|
+
|
|
239
|
+
function markVerified(key) {
|
|
240
|
+
if (memoTtlMs <= 0) {
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
243
|
+
verifiedKeys.delete(key);
|
|
244
|
+
verifiedKeys.set(key, Date.now() + memoTtlMs);
|
|
245
|
+
if (verifiedKeys.size > MEMO_MAX_SIZE) {
|
|
246
|
+
verifiedKeys.delete(verifiedKeys.keys().next().value);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* Makes sure the Redis counter "key" exists before it is read or incremented: if it is missing and
|
|
252
|
+
* quota_usages has a snapshot for it, the key is seeded with SET NX so the next INCR lands on top.
|
|
253
|
+
* Cost: a Map lookup; one EXISTS (plus one indexed Mongo lookup when missing) per key per process per
|
|
254
|
+
* memo TTL. Never throws: on Redis/Mongo errors the caller proceeds as if there were no backup, and
|
|
255
|
+
* after a Mongo failure no lookup is tried for MONGO_FAILURE_BACKOFF_MS.
|
|
256
|
+
*/
|
|
257
|
+
function ensureQuotaKey(tdCache, key) {
|
|
258
|
+
const expiresAt = verifiedKeys.get(key);
|
|
259
|
+
if (expiresAt !== undefined) {
|
|
260
|
+
if (expiresAt > Date.now()) {
|
|
261
|
+
return Promise.resolve();
|
|
262
|
+
}
|
|
263
|
+
verifiedKeys.delete(key);
|
|
264
|
+
}
|
|
265
|
+
let pending = inflight.get(key);
|
|
266
|
+
if (!pending) {
|
|
267
|
+
pending = checkAndRestore(tdCache, key).finally(() => inflight.delete(key));
|
|
268
|
+
inflight.set(key, pending);
|
|
269
|
+
}
|
|
270
|
+
return pending;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
async function checkAndRestore(tdCache, key) {
|
|
274
|
+
const client = redisClient(tdCache);
|
|
275
|
+
const parsed = parseQuotaKey(key);
|
|
276
|
+
if (!client || !parsed || NOT_IN_REDIS_TYPES.indexOf(parsed.type) !== -1) {
|
|
277
|
+
return;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
let exists;
|
|
281
|
+
try {
|
|
282
|
+
restoreStats.existsChecks++;
|
|
283
|
+
exists = await redisCommand(client, 'exists', [key]);
|
|
284
|
+
} catch (err) {
|
|
285
|
+
restoreStats.errors++;
|
|
286
|
+
winston.error("[QuotaSnapshot] EXISTS failed for key " + key + ": ", err);
|
|
287
|
+
return;
|
|
288
|
+
}
|
|
289
|
+
if (exists) {
|
|
290
|
+
markVerified(key);
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
if (Date.now() < mongoDownUntil) {
|
|
294
|
+
return;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
let doc;
|
|
298
|
+
try {
|
|
299
|
+
restoreStats.lookups++;
|
|
300
|
+
doc = await withTimeout(
|
|
301
|
+
QuotaUsage.findOne(parsed).select('value notified').maxTimeMS(LOOKUP_TIMEOUT_MS).lean().exec(),
|
|
302
|
+
LOOKUP_TIMEOUT_MS,
|
|
303
|
+
'quota_usages lookup'
|
|
304
|
+
);
|
|
305
|
+
lookupFailureLogged = false;
|
|
306
|
+
} catch (err) {
|
|
307
|
+
restoreStats.errors++;
|
|
308
|
+
mongoDownUntil = Date.now() + MONGO_FAILURE_BACKOFF_MS;
|
|
309
|
+
if (!lookupFailureLogged) {
|
|
310
|
+
lookupFailureLogged = true;
|
|
311
|
+
winston.error("[QuotaSnapshot] Unable to read quota_usages, counters missing in Redis start from zero until it is reachable again: ", err);
|
|
312
|
+
}
|
|
313
|
+
return;
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
if (doc && Number(doc.value) > 0) {
|
|
317
|
+
try {
|
|
318
|
+
let reply;
|
|
319
|
+
if (wipeDetectionTimer) {
|
|
320
|
+
reply = await redisCommand(client, 'eval', [SEED_SCRIPT, 2, SENTINEL_KEY, key, String(doc.value)]);
|
|
321
|
+
} else {
|
|
322
|
+
reply = (await redisCommand(client, 'set', [key, String(doc.value), 'NX'])) === 'OK' ? 1 : 0;
|
|
323
|
+
}
|
|
324
|
+
if (reply === -1) {
|
|
325
|
+
// Redis was wiped: the merge restore started by the sentinel timer adds the snapshot
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
if (reply === 1) {
|
|
329
|
+
restoreStats.restored++;
|
|
330
|
+
await restoreNotifyFlags(client, key, doc.notified);
|
|
331
|
+
winston.info("[QuotaSnapshot] Restored missing quota key " + key + " from snapshot: " + doc.value);
|
|
332
|
+
}
|
|
333
|
+
} catch (err) {
|
|
334
|
+
restoreStats.errors++;
|
|
335
|
+
winston.error("[QuotaSnapshot] Unable to restore quota key " + key + ": ", err);
|
|
336
|
+
return;
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
markVerified(key);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
/**
|
|
343
|
+
* Checks the sentinel. When it is missing Redis was wiped: clears the memo and starts (or joins) the
|
|
344
|
+
* restore of the current period. Never throws.
|
|
345
|
+
* @param {Object} options { wait: await the restore }
|
|
346
|
+
* @returns {Promise<{wiped: boolean, restore?: Object}>}
|
|
347
|
+
*/
|
|
348
|
+
async function checkSentinel(tdCache, options) {
|
|
349
|
+
options = options || {};
|
|
350
|
+
const client = redisClient(tdCache);
|
|
351
|
+
if (!client || !quotesEnabled()) {
|
|
352
|
+
return { wiped: false, skipped: true };
|
|
353
|
+
}
|
|
354
|
+
let present;
|
|
355
|
+
try {
|
|
356
|
+
restoreStats.sentinelChecks++;
|
|
357
|
+
const reply = await redisCommand(client, 'mget', [SENTINEL_KEY, DIRTY_GEN_KEY]);
|
|
358
|
+
present = reply[0];
|
|
359
|
+
if (reply[1] !== lastDirtyGen) {
|
|
360
|
+
lastDirtyGen = reply[1];
|
|
361
|
+
dirtyMemo.clear();
|
|
362
|
+
}
|
|
363
|
+
lastGenReadAt = Date.now();
|
|
364
|
+
} catch (err) {
|
|
365
|
+
// unknown generation: stop trusting the memo
|
|
366
|
+
dirtyMemo.clear();
|
|
367
|
+
lastGenReadAt = 0;
|
|
368
|
+
restoreStats.errors++;
|
|
369
|
+
winston.error("[QuotaSnapshot] Unable to read " + SENTINEL_KEY + ": ", err);
|
|
370
|
+
return { wiped: false, error: err.message };
|
|
371
|
+
}
|
|
372
|
+
if (present !== null && present !== undefined) {
|
|
373
|
+
return { wiped: false };
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
verifiedKeys.clear();
|
|
377
|
+
dirtyMemo.clear();
|
|
378
|
+
restoreStats.memoCleared++;
|
|
379
|
+
const initiator = !restorePromise;
|
|
380
|
+
if (initiator) {
|
|
381
|
+
winston.warn("[QuotaSnapshot] " + SENTINEL_KEY + " is missing: Redis was wiped, restoring the quota counters from quota_usages");
|
|
382
|
+
restorePromise = restoreAfterWipe(client, tdCache).finally(() => { restorePromise = null; });
|
|
383
|
+
}
|
|
384
|
+
const pending = restorePromise;
|
|
385
|
+
if (!options.wait) {
|
|
386
|
+
return { wiped: true };
|
|
387
|
+
}
|
|
388
|
+
const result = await pending;
|
|
389
|
+
return { wiped: true, restore: initiator ? result : { restored: false, reason: 'in_progress' } };
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
async function restoreAfterWipe(client, tdCache) {
|
|
393
|
+
const started = Date.now();
|
|
394
|
+
let locked = false;
|
|
395
|
+
try {
|
|
396
|
+
locked = await acquireLock(client, RESTORE_LOCK_KEY, INSTANCE_TOKEN, Math.ceil(RESTORE_LOCK_TTL_MS / 1000));
|
|
397
|
+
if (!locked) {
|
|
398
|
+
winston.info("[QuotaSnapshot] Another replica is restoring the quota counters, waiting for " + SENTINEL_KEY);
|
|
399
|
+
return { restored: false, reason: 'locked' };
|
|
400
|
+
}
|
|
401
|
+
if ((await redisCommand(client, 'get', [SENTINEL_KEY])) !== null) {
|
|
402
|
+
return { restored: false, reason: 'already_restored' };
|
|
403
|
+
}
|
|
404
|
+
const extendLock = async () => {
|
|
405
|
+
const extended = await redisCommand(client, 'eval', [EXTEND_LOCK_SCRIPT, 1, RESTORE_LOCK_KEY, INSTANCE_TOKEN, RESTORE_LOCK_TTL_MS]);
|
|
406
|
+
if (extended !== 1) {
|
|
407
|
+
// the merge is atomic per key and idempotent, a concurrent restore cannot double count
|
|
408
|
+
winston.warn("[QuotaSnapshot] " + RESTORE_LOCK_KEY + " lost while restoring");
|
|
409
|
+
}
|
|
410
|
+
};
|
|
411
|
+
const { summary } = await restoreFromSnapshots({
|
|
412
|
+
tdCache: tdCache, period: 'current', mode: 'merge', collectResults: false, onBatch: extendLock
|
|
413
|
+
});
|
|
414
|
+
await redisCommand(client, 'set', [SENTINEL_KEY, INSTANCE_TOKEN]);
|
|
415
|
+
restoreStats.wipeRestores++;
|
|
416
|
+
winston.warn("[QuotaSnapshot] Quota counters restored after a Redis wipe in " + (Date.now() - started) + "ms: " + JSON.stringify(summary));
|
|
417
|
+
return { restored: true, summary: summary };
|
|
418
|
+
} catch (err) {
|
|
419
|
+
restoreStats.errors++;
|
|
420
|
+
winston.error("[QuotaSnapshot] Restore after Redis wipe failed, retrying at the next sentinel check: ", err);
|
|
421
|
+
return { restored: false, reason: 'error', error: err.message };
|
|
422
|
+
} finally {
|
|
423
|
+
if (locked) {
|
|
424
|
+
await releaseLock(client, RESTORE_LOCK_KEY, INSTANCE_TOKEN);
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
/**
|
|
430
|
+
* Starts the sentinel timer: first check now (a wipe that happened while the server was down is restored
|
|
431
|
+
* too), then every intervalSeconds. While it runs, restore on miss seeds only if the sentinel exists.
|
|
432
|
+
*/
|
|
433
|
+
function startWipeDetection(tdCache, options) {
|
|
434
|
+
stopWipeDetection();
|
|
435
|
+
const intervalMs = (options && options.intervalSeconds) * 1000;
|
|
436
|
+
watcherIntervalMs = intervalMs;
|
|
437
|
+
checkSentinel(tdCache);
|
|
438
|
+
wipeDetectionTimer = setInterval(() => checkSentinel(tdCache), intervalMs);
|
|
439
|
+
wipeDetectionTimer.unref();
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
function stopWipeDetection() {
|
|
443
|
+
if (wipeDetectionTimer) {
|
|
444
|
+
clearInterval(wipeDetectionTimer);
|
|
445
|
+
wipeDetectionTimer = null;
|
|
446
|
+
}
|
|
447
|
+
watcherIntervalMs = 0;
|
|
448
|
+
lastGenReadAt = 0;
|
|
449
|
+
dirtyMemo.clear();
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
function logRedisError(label) {
|
|
453
|
+
return (err) => {
|
|
454
|
+
if (err) {
|
|
455
|
+
winston.error("[QuotaSnapshot] " + label + " failed: ", err);
|
|
456
|
+
}
|
|
457
|
+
};
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
/**
|
|
461
|
+
* The memo is trusted only while the watcher runs and its last generation read is younger than
|
|
462
|
+
* 1.5 x its interval (the cycle waits 2 x interval + 1 s after bumping the generation).
|
|
463
|
+
*/
|
|
464
|
+
function dirtyMemoHit(key) {
|
|
465
|
+
return wipeDetectionTimer !== null &&
|
|
466
|
+
Date.now() - lastGenReadAt < 1.5 * watcherIntervalMs &&
|
|
467
|
+
dirtyMemo.has(key);
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
function rememberDirty(key) {
|
|
471
|
+
if (wipeDetectionTimer === null) {
|
|
472
|
+
return;
|
|
473
|
+
}
|
|
474
|
+
if (dirtyMemo.size >= MEMO_MAX_SIZE) {
|
|
475
|
+
dirtyMemo.clear(); // forgetting is always safe: the next increment marks again
|
|
476
|
+
}
|
|
477
|
+
dirtyMemo.add(key);
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
/** Test hook: makes the last generation read look older. */
|
|
481
|
+
function ageGenerationRead(ms) {
|
|
482
|
+
lastGenReadAt -= ms;
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
/**
|
|
486
|
+
* Increments a counter (command: incr | incrby | incrbyfloat). When the counter must be marked dirty
|
|
487
|
+
* it sends MULTI, the increment, SADD quotes:dirty, EXEC (one round trip); on a memo hit, or when the
|
|
488
|
+
* snapshot is disabled, only the plain increment. The key is memoized only after a successful SADD.
|
|
489
|
+
* Like TdCache, the returned promise resolves once the commands are queued on the connection (commands
|
|
490
|
+
* on the connection run in order) and errors are logged.
|
|
491
|
+
*/
|
|
492
|
+
function incrementCounter(tdCache, key, command, amount) {
|
|
493
|
+
const client = redisClient(tdCache);
|
|
494
|
+
if (!client) {
|
|
495
|
+
return command === 'incr' ? tdCache.incr(key) : tdCache[command](key, amount);
|
|
496
|
+
}
|
|
497
|
+
const args = command === 'incr' ? [key] : [key, amount];
|
|
498
|
+
if (!dirtyTracking || dirtyMemoHit(key)) {
|
|
499
|
+
client.send_command(command, args, logRedisError(command + " " + key));
|
|
500
|
+
return Promise.resolve();
|
|
501
|
+
}
|
|
502
|
+
restoreStats.dirtyMarks++;
|
|
503
|
+
client.multi([[command].concat(args), ['sadd', DIRTY_KEY, key]]).exec((err, replies) => {
|
|
504
|
+
if (err) {
|
|
505
|
+
winston.error("[QuotaSnapshot] " + command + " + sadd " + key + " failed: ", err);
|
|
506
|
+
return;
|
|
507
|
+
}
|
|
508
|
+
const [incrReply, saddReply] = replies || [];
|
|
509
|
+
if (incrReply instanceof Error) {
|
|
510
|
+
winston.error("[QuotaSnapshot] " + command + " " + key + " failed: ", incrReply);
|
|
511
|
+
}
|
|
512
|
+
if (saddReply instanceof Error) {
|
|
513
|
+
winston.error("[QuotaSnapshot] sadd " + DIRTY_KEY + " " + key + " failed: ", saddReply);
|
|
514
|
+
return;
|
|
515
|
+
}
|
|
516
|
+
rememberDirty(key);
|
|
517
|
+
});
|
|
518
|
+
return Promise.resolve();
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
/** Adds a counter to the dirty set (e.g. a notify flag was set), unless already marked in this generation. */
|
|
522
|
+
function markDirty(tdCache, key) {
|
|
523
|
+
const client = redisClient(tdCache);
|
|
524
|
+
if (!client || !dirtyTracking || dirtyMemoHit(key)) {
|
|
525
|
+
return Promise.resolve();
|
|
526
|
+
}
|
|
527
|
+
restoreStats.dirtyMarks++;
|
|
528
|
+
client.send_command('sadd', [DIRTY_KEY, key], (err) => {
|
|
529
|
+
if (err) {
|
|
530
|
+
winston.error("[QuotaSnapshot] sadd " + DIRTY_KEY + " " + key + " failed: ", err);
|
|
531
|
+
return;
|
|
532
|
+
}
|
|
533
|
+
rememberDirty(key);
|
|
534
|
+
});
|
|
535
|
+
return Promise.resolve();
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
/** Resolves when the restore started by this process (if any) is done. */
|
|
539
|
+
function waitForRestore() {
|
|
540
|
+
return restorePromise ? restorePromise.then(() => undefined) : Promise.resolve();
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
function resetRestoreMemo() {
|
|
544
|
+
verifiedKeys.clear();
|
|
545
|
+
dirtyMemo.clear();
|
|
546
|
+
inflight.clear();
|
|
547
|
+
lookupFailureLogged = false;
|
|
548
|
+
mongoDownUntil = 0;
|
|
549
|
+
Object.keys(restoreStats).forEach((k) => { restoreStats[k] = 0; });
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
/**
|
|
553
|
+
* The notify flags of these counters were deleted (plan change): forget them in the backup too.
|
|
554
|
+
* Never throws.
|
|
555
|
+
*/
|
|
556
|
+
async function clearNotified(keys) {
|
|
557
|
+
const filters = (keys || []).map(parseQuotaKey).filter(Boolean);
|
|
558
|
+
if (filters.length === 0) {
|
|
559
|
+
return;
|
|
560
|
+
}
|
|
561
|
+
try {
|
|
562
|
+
await QuotaUsage.updateMany({ $or: filters }, { $unset: { notified: '' } }).exec();
|
|
563
|
+
} catch (err) {
|
|
564
|
+
winston.error("[QuotaSnapshot] Unable to clear the notify flags backup: ", err);
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
// RESTORE ON MISS + WIPE DETECTION - END
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
// SNAPSHOT - START
|
|
572
|
+
|
|
573
|
+
let snapshotTimers = [];
|
|
574
|
+
let running = false;
|
|
575
|
+
|
|
576
|
+
// Moves the dirty set aside for this cycle. A saving set left by a crashed cycle is kept and the new
|
|
577
|
+
// changes are merged into it. Bumps the generation when there is something to save. Returns 1 / 0.
|
|
578
|
+
const MOVE_DIRTY_SCRIPT = [
|
|
579
|
+
"if redis.call('EXISTS', KEYS[2]) == 1 then",
|
|
580
|
+
" if redis.call('EXISTS', KEYS[1]) == 1 then",
|
|
581
|
+
" redis.call('SUNIONSTORE', KEYS[2], KEYS[2], KEYS[1])",
|
|
582
|
+
" redis.call('DEL', KEYS[1])",
|
|
583
|
+
" end",
|
|
584
|
+
"elseif redis.call('EXISTS', KEYS[1]) == 1 then",
|
|
585
|
+
" redis.call('RENAME', KEYS[1], KEYS[2])",
|
|
586
|
+
"else",
|
|
587
|
+
" return 0",
|
|
588
|
+
"end",
|
|
589
|
+
"redis.call('INCR', KEYS[3])",
|
|
590
|
+
"return 1"
|
|
591
|
+
].join('\n');
|
|
592
|
+
// Per counter: value, then the PTTL of each notify flag (ARGV = checkpoints)
|
|
593
|
+
const READ_COUNTERS_SCRIPT = [
|
|
594
|
+
"local out = {}",
|
|
595
|
+
"for _, k in ipairs(KEYS) do",
|
|
596
|
+
" out[#out + 1] = redis.call('GET', k)",
|
|
597
|
+
" for _, pct in ipairs(ARGV) do",
|
|
598
|
+
" out[#out + 1] = redis.call('PTTL', k .. ':notify:' .. pct)",
|
|
599
|
+
" end",
|
|
600
|
+
"end",
|
|
601
|
+
"return out"
|
|
602
|
+
].join('\n');
|
|
603
|
+
|
|
604
|
+
function yieldToEventLoop() {
|
|
605
|
+
return new Promise((resolve) => setImmediate(resolve));
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
function sleep(ms) {
|
|
609
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
/**
|
|
613
|
+
* How long the cycle waits after moving the dirty set aside: every process must have seen the new
|
|
614
|
+
* generation (watcher tick) before the values are read, so that the increments it did without SADD
|
|
615
|
+
* (memoized) are included.
|
|
616
|
+
*/
|
|
617
|
+
function defaultSettleMs() {
|
|
618
|
+
// all replicas must share QUOTA_SENTINEL_CHECK_SECONDS; when it is 0 here use the default as a floor,
|
|
619
|
+
// in case other replicas still memoize
|
|
620
|
+
const checkSeconds = nonNegativeNumber(process.env.QUOTA_SENTINEL_CHECK_SECONDS, 15) || 15;
|
|
621
|
+
return 2 * checkSeconds * 1000 + 1000;
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
/**
|
|
625
|
+
* Saves up to BATCH_SIZE counters: one Lua call reads the values and the notify flag TTLs, one bulk upsert
|
|
626
|
+
* with $max (never lowers the backup; no read back: an unchanged doc is a no-op in mongo).
|
|
627
|
+
*/
|
|
628
|
+
async function saveCounters(client, keys, now, stats) {
|
|
629
|
+
const recent = [];
|
|
630
|
+
keys.forEach((k) => {
|
|
631
|
+
const parsed = parseQuotaKey(k);
|
|
632
|
+
if (!parsed) {
|
|
633
|
+
return;
|
|
634
|
+
}
|
|
635
|
+
if (!isRecentPeriod(parsed.period, recentCutoff(now.getTime()))) {
|
|
636
|
+
stats.skippedOld++;
|
|
637
|
+
return;
|
|
638
|
+
}
|
|
639
|
+
recent.push({ key: k, filter: parsed });
|
|
640
|
+
});
|
|
641
|
+
if (recent.length === 0) {
|
|
642
|
+
return;
|
|
643
|
+
}
|
|
644
|
+
const reply = await redisCommand(client, 'eval', [READ_COUNTERS_SCRIPT, recent.length]
|
|
645
|
+
.concat(recent.map((r) => r.key)).concat(NOTIFY_PCTS));
|
|
646
|
+
const stride = 1 + NOTIFY_PCTS.length;
|
|
647
|
+
const ops = [];
|
|
648
|
+
recent.forEach((r, idx) => {
|
|
649
|
+
const raw = reply[idx * stride];
|
|
650
|
+
const value = Number(raw);
|
|
651
|
+
if (raw === null || raw === undefined || !Number.isFinite(value)) {
|
|
652
|
+
return;
|
|
653
|
+
}
|
|
654
|
+
// update pipeline (MongoDB 4.2+): $max on the value and on each flag expiry, updatedAt only when one of
|
|
655
|
+
// them grows, so an unchanged counter is a no-op (no write, no oplog entry)
|
|
656
|
+
const changed = [{ $lt: [{ $ifNull: ['$value', -1] }, value] }];
|
|
657
|
+
const flags = {};
|
|
658
|
+
NOTIFY_PCTS.forEach((pct, p) => {
|
|
659
|
+
const ttl = Number(reply[idx * stride + 1 + p]);
|
|
660
|
+
if (ttl === -2) {
|
|
661
|
+
return; // no flag
|
|
662
|
+
}
|
|
663
|
+
const stored = '$notified.' + pct;
|
|
664
|
+
stats.notifyFlags++;
|
|
665
|
+
if (ttl === -1) {
|
|
666
|
+
// flag without expiry (set by hand): keep the stored expiry, else assume the default EX
|
|
667
|
+
flags[pct] = { $ifNull: [stored, new Date(now.getTime() + NOTIFY_DEFAULT_TTL_MS)] };
|
|
668
|
+
changed.push({ $eq: [{ $ifNull: [stored, null] }, null] });
|
|
669
|
+
return;
|
|
670
|
+
}
|
|
671
|
+
const expiresAt = new Date(now.getTime() + ttl);
|
|
672
|
+
const later = { $lt: [{ $ifNull: [stored, new Date(0)] }, new Date(expiresAt.getTime() - NOTIFY_EXPIRY_TOLERANCE_MS)] };
|
|
673
|
+
flags[pct] = { $cond: [later, expiresAt, stored] };
|
|
674
|
+
changed.push(later);
|
|
675
|
+
});
|
|
676
|
+
const set = {
|
|
677
|
+
value: { $max: ['$value', value] },
|
|
678
|
+
periodDate: { $ifNull: ['$periodDate', QuotaUsage.periodToDate(r.filter.period)] },
|
|
679
|
+
updatedAt: { $cond: [{ $or: changed }, now, '$updatedAt'] }
|
|
680
|
+
};
|
|
681
|
+
if (Object.keys(flags).length > 0) {
|
|
682
|
+
set.notified = { $mergeObjects: [{ $ifNull: ['$notified', {}] }, flags] };
|
|
683
|
+
}
|
|
684
|
+
ops.push({ updateOne: { filter: r.filter, update: [{ $set: set }], upsert: true } });
|
|
685
|
+
});
|
|
686
|
+
if (ops.length > 0) {
|
|
687
|
+
const result = await QuotaUsage.bulkWrite(ops, { ordered: false });
|
|
688
|
+
stats.counters += ops.length;
|
|
689
|
+
stats.upserted += result.upsertedCount || 0;
|
|
690
|
+
stats.modified += result.modifiedCount || 0;
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
async function saveInBatches(client, keys, now, stats) {
|
|
695
|
+
for (let i = 0; i < keys.length; i += BATCH_SIZE) {
|
|
696
|
+
stats.batches++;
|
|
697
|
+
try {
|
|
698
|
+
await saveCounters(client, keys.slice(i, i + BATCH_SIZE), now, stats);
|
|
699
|
+
} catch (err) {
|
|
700
|
+
stats.failedBatches++;
|
|
701
|
+
winston.error("[QuotaSnapshot] Snapshot batch " + stats.batches + " failed, continuing: ", err);
|
|
702
|
+
}
|
|
703
|
+
await yieldToEventLoop();
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
/** Saves the counters of the saving set (SSCAN in batches). Returns the keys seen. */
|
|
708
|
+
async function saveDirtyCounters(client, now, stats) {
|
|
709
|
+
const seen = new Set();
|
|
710
|
+
let cursor = '0';
|
|
711
|
+
do {
|
|
712
|
+
const reply = await redisCommand(client, 'sscan', [DIRTY_SAVING_KEY, cursor, 'COUNT', BATCH_SIZE]);
|
|
713
|
+
cursor = reply[0];
|
|
714
|
+
const keys = (reply[1] || []).filter((k) => !seen.has(k));
|
|
715
|
+
keys.forEach((k) => seen.add(k));
|
|
716
|
+
stats.dirty += keys.length;
|
|
717
|
+
await saveInBatches(client, keys, now, stats);
|
|
718
|
+
} while (cursor !== '0');
|
|
719
|
+
return seen;
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
/** Full reconciliation: SCAN the keyspace for the counters not already saved in this cycle. */
|
|
723
|
+
async function saveAllCounters(client, now, stats, alreadySaved) {
|
|
724
|
+
let cursor = '0';
|
|
725
|
+
let pending = [];
|
|
726
|
+
do {
|
|
727
|
+
const reply = await redisCommand(client, 'scan', [cursor, 'MATCH', KEY_PREFIX + '*', 'COUNT', SCAN_COUNT]);
|
|
728
|
+
cursor = reply[0];
|
|
729
|
+
const keys = reply[1] || [];
|
|
730
|
+
stats.scanned += keys.length;
|
|
731
|
+
keys.forEach((k) => {
|
|
732
|
+
if (!alreadySaved.has(k) && parseQuotaKey(k)) {
|
|
733
|
+
alreadySaved.add(k);
|
|
734
|
+
pending.push(k);
|
|
735
|
+
}
|
|
736
|
+
});
|
|
737
|
+
if (pending.length >= BATCH_SIZE || cursor === '0') {
|
|
738
|
+
await saveInBatches(client, pending, now, stats);
|
|
739
|
+
pending = [];
|
|
740
|
+
} else {
|
|
741
|
+
await yieldToEventLoop();
|
|
742
|
+
}
|
|
743
|
+
} while (cursor !== '0');
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
async function fullScanDue(client, now) {
|
|
747
|
+
const hours = nonNegativeNumber(process.env.QUOTA_FULL_SCAN_INTERVAL_HOURS, 168);
|
|
748
|
+
if (hours === 0) {
|
|
749
|
+
return false;
|
|
750
|
+
}
|
|
751
|
+
const last = Number(await redisCommand(client, 'get', [FULL_SCAN_KEY]));
|
|
752
|
+
return !(last > 0) || now - last >= hours * 3600 * 1000;
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
/**
|
|
756
|
+
* One snapshot cycle. Saves the counters of the dirty set (no keyspace SCAN, no read back of the backup);
|
|
757
|
+
* every QUOTA_FULL_SCAN_INTERVAL_HOURS (and on the first cycle) also reconciles the whole keyspace.
|
|
758
|
+
* Returns stats, or { skipped: reason }. Never throws: a failing batch is logged, the next batches are
|
|
759
|
+
* written and the saving set is kept for the next cycle.
|
|
760
|
+
* @param {Object} options { tdCache, lockTtlSeconds, settleMs, full }
|
|
761
|
+
*/
|
|
762
|
+
async function runSnapshotCycle(options) {
|
|
763
|
+
options = options || {};
|
|
764
|
+
if (!quotesEnabled()) {
|
|
765
|
+
return { skipped: 'quotes_disabled' };
|
|
766
|
+
}
|
|
767
|
+
const client = redisClient(options.tdCache);
|
|
768
|
+
if (!client) {
|
|
769
|
+
winston.warn("[QuotaSnapshot] Redis client not available, snapshot skipped");
|
|
770
|
+
return { skipped: 'no_redis' };
|
|
771
|
+
}
|
|
772
|
+
if (running) {
|
|
773
|
+
return { skipped: 'already_running' };
|
|
774
|
+
}
|
|
775
|
+
running = true;
|
|
776
|
+
|
|
777
|
+
const started = Date.now();
|
|
778
|
+
const lockTtl = Math.max(1, Math.round(options.lockTtlSeconds || 1800));
|
|
779
|
+
const lockToken = INSTANCE_TOKEN + ':' + crypto.randomBytes(4).toString('hex');
|
|
780
|
+
const settleMs = options.settleMs !== undefined ? options.settleMs : defaultSettleMs();
|
|
781
|
+
let locked = false;
|
|
782
|
+
const stats = { full: false, dirty: 0, scanned: 0, counters: 0, upserted: 0, modified: 0, skippedOld: 0, notifyFlags: 0, batches: 0, failedBatches: 0, settleMs: 0, workMs: 0, durationMs: 0 };
|
|
783
|
+
|
|
784
|
+
try {
|
|
785
|
+
locked = await acquireLock(client, LOCK_KEY, lockToken, lockTtl);
|
|
786
|
+
if (!locked) {
|
|
787
|
+
winston.verbose("[QuotaSnapshot] Another replica holds " + LOCK_KEY + ", snapshot skipped");
|
|
788
|
+
return { skipped: 'locked' };
|
|
789
|
+
}
|
|
790
|
+
stats.full = options.full === true || await fullScanDue(client, started);
|
|
791
|
+
|
|
792
|
+
const moved = await redisCommand(client, 'eval', [MOVE_DIRTY_SCRIPT, 3, DIRTY_KEY, DIRTY_SAVING_KEY, DIRTY_GEN_KEY]);
|
|
793
|
+
if (moved === 1 && settleMs > 0) {
|
|
794
|
+
const settleStarted = Date.now();
|
|
795
|
+
await sleep(settleMs);
|
|
796
|
+
stats.settleMs = Date.now() - settleStarted;
|
|
797
|
+
}
|
|
798
|
+
const now = new Date();
|
|
799
|
+
const saved = moved === 1 ? await saveDirtyCounters(client, now, stats) : new Set();
|
|
800
|
+
const dirtyFailed = stats.failedBatches;
|
|
801
|
+
if (moved === 1 && dirtyFailed === 0) {
|
|
802
|
+
await redisCommand(client, 'del', [DIRTY_SAVING_KEY]);
|
|
803
|
+
}
|
|
804
|
+
if (stats.full) {
|
|
805
|
+
await saveAllCounters(client, now, stats, saved);
|
|
806
|
+
if (stats.failedBatches === 0) {
|
|
807
|
+
await redisCommand(client, 'set', [FULL_SCAN_KEY, String(Date.now())]);
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
stats.durationMs = Date.now() - started;
|
|
812
|
+
stats.workMs = stats.durationMs - stats.settleMs;
|
|
813
|
+
const log = stats.failedBatches > 0 ? winston.warn.bind(winston) : winston.info.bind(winston);
|
|
814
|
+
log("[QuotaSnapshot] Snapshot done" + (stats.full ? " (full scan, " + stats.scanned + " scanned keys)" : "") + ": " +
|
|
815
|
+
stats.counters + " counters saved (" + stats.upserted + " new, " + stats.modified + " changed) from " + stats.dirty + " dirty, " +
|
|
816
|
+
stats.skippedOld + " old periods skipped, " + stats.notifyFlags + " notify flags, " + stats.failedBatches + "/" +
|
|
817
|
+
stats.batches + " batches failed" + (dirtyFailed > 0 ? " (" + DIRTY_SAVING_KEY + " kept for the next cycle)" : "") +
|
|
818
|
+
" in " + stats.workMs + "ms of work + " + stats.settleMs + "ms of settle");
|
|
819
|
+
return stats;
|
|
820
|
+
} catch (err) {
|
|
821
|
+
stats.durationMs = Date.now() - started;
|
|
822
|
+
winston.error("[QuotaSnapshot] Snapshot failed after " + stats.durationMs + "ms (" + stats.counters + " counters written): ", err);
|
|
823
|
+
return Object.assign({ error: err.message || String(err) }, stats);
|
|
824
|
+
} finally {
|
|
825
|
+
running = false;
|
|
826
|
+
if (locked) {
|
|
827
|
+
await releaseLock(client, LOCK_KEY, lockToken);
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
/**
|
|
833
|
+
* Server bootstrap (not called in tests): wipe detection (QUOTA_SENTINEL_CHECK_SECONDS, default 15,
|
|
834
|
+
* 0 disables) and the periodic snapshot (QUOTA_SNAPSHOT_INTERVAL_MINUTES, default 60, 0 disables;
|
|
835
|
+
* first cycle after min(5 minutes, interval)).
|
|
836
|
+
*/
|
|
837
|
+
function start(options) {
|
|
838
|
+
options = options || {};
|
|
839
|
+
const tdCache = options.tdCache;
|
|
840
|
+
configureDirtyTracking();
|
|
841
|
+
const dropDirtySets = () => {
|
|
842
|
+
const client = redisClient(tdCache);
|
|
843
|
+
if (client) {
|
|
844
|
+
client.send_command('del', [DIRTY_KEY, DIRTY_SAVING_KEY], logRedisError("del " + DIRTY_KEY));
|
|
845
|
+
}
|
|
846
|
+
};
|
|
847
|
+
if (!quotesEnabled()) {
|
|
848
|
+
dropDirtySets();
|
|
849
|
+
winston.info("[QuotaSnapshot] Quotes disabled, quota snapshot not started");
|
|
850
|
+
return false;
|
|
851
|
+
}
|
|
852
|
+
stop();
|
|
853
|
+
|
|
854
|
+
// The indexes must exist even when MONGOOSE_AUTOINDEX is false
|
|
855
|
+
QuotaUsage.createIndexes().catch((err) => {
|
|
856
|
+
winston.error("[QuotaSnapshot] Error creating quota_usages indexes: ", err);
|
|
857
|
+
});
|
|
858
|
+
|
|
859
|
+
const sentinelCheckSeconds = nonNegativeNumber(process.env.QUOTA_SENTINEL_CHECK_SECONDS, 15);
|
|
860
|
+
if (sentinelCheckSeconds > 0) {
|
|
861
|
+
startWipeDetection(tdCache, { intervalSeconds: sentinelCheckSeconds });
|
|
862
|
+
winston.info("[QuotaSnapshot] Wipe detection started, sentinel check every (s): " + sentinelCheckSeconds);
|
|
863
|
+
} else {
|
|
864
|
+
winston.info("[QuotaSnapshot] QUOTA_SENTINEL_CHECK_SECONDS=0, wipe detection disabled");
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
const intervalMinutes = nonNegativeNumber(process.env.QUOTA_SNAPSHOT_INTERVAL_MINUTES, 60);
|
|
868
|
+
if (intervalMinutes === 0) {
|
|
869
|
+
dropDirtySets(); // nobody would drain them
|
|
870
|
+
winston.info("[QuotaSnapshot] QUOTA_SNAPSHOT_INTERVAL_MINUTES=0, quota snapshot disabled, counters not tracked");
|
|
871
|
+
return true;
|
|
872
|
+
}
|
|
873
|
+
const intervalMs = intervalMinutes * 60 * 1000;
|
|
874
|
+
// the cycle waits defaultSettleMs() before reading: keep the lock well above it
|
|
875
|
+
const lockTtlSeconds = Math.max(Math.round(defaultSettleMs() / 1000) + 120, Math.round(intervalMs / 2000));
|
|
876
|
+
const cycle = () => runSnapshotCycle({ tdCache: tdCache, lockTtlSeconds: lockTtlSeconds });
|
|
877
|
+
const first = setTimeout(cycle, Math.min(5 * 60 * 1000, intervalMs));
|
|
878
|
+
const every = setInterval(cycle, intervalMs);
|
|
879
|
+
first.unref();
|
|
880
|
+
every.unref();
|
|
881
|
+
snapshotTimers = [first, every];
|
|
882
|
+
winston.info("[QuotaSnapshot] Quota snapshot started, interval (minutes): " + intervalMinutes + ", lock ttl (s): " + lockTtlSeconds);
|
|
883
|
+
return true;
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
function stop() {
|
|
887
|
+
snapshotTimers.forEach((t) => clearTimeout(t));
|
|
888
|
+
snapshotTimers = [];
|
|
889
|
+
stopWipeDetection();
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
// SNAPSHOT - END
|
|
893
|
+
|
|
894
|
+
|
|
895
|
+
// RESTORE / RESET - START
|
|
896
|
+
|
|
897
|
+
/** id_project -> current period string, null when the project does not exist */
|
|
898
|
+
async function currentPeriods(projectIds) {
|
|
899
|
+
// required lazily: QuoteManager requires this module
|
|
900
|
+
const { QuoteManager } = require('./QuoteManager');
|
|
901
|
+
const Project = require('../models/project');
|
|
902
|
+
const qm = new QuoteManager({ tdCache: {} });
|
|
903
|
+
const now = { createdAt: new Date() };
|
|
904
|
+
const periods = new Map(projectIds.map((id) => [id, null]));
|
|
905
|
+
const validIds = projectIds.filter((id) => mongoose.Types.ObjectId.isValid(id));
|
|
906
|
+
for (let i = 0; i < validIds.length; i += BATCH_SIZE) {
|
|
907
|
+
const projects = await Project.find({ _id: { $in: validIds.slice(i, i + BATCH_SIZE) } }).exec();
|
|
908
|
+
for (const project of projects) {
|
|
909
|
+
qm.project = project;
|
|
910
|
+
const key = await qm.generateKey(now, 'requests');
|
|
911
|
+
periods.set(project._id.toString(), key.substring(key.lastIndexOf(':') + 1));
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
return periods;
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
function restoreAction(mode, dryRun, code, previous, snapshot) {
|
|
918
|
+
const verb = mode === 'merge' ? 'merged' : 'raised';
|
|
919
|
+
if (dryRun) {
|
|
920
|
+
if (previous === null) return 'would-set';
|
|
921
|
+
if (mode === 'nx') return 'skip-exists';
|
|
922
|
+
return previous < snapshot ? 'would-' + (mode === 'merge' ? 'merge' : 'raise') : 'skip-higher';
|
|
923
|
+
}
|
|
924
|
+
if (code === 1) return 'set';
|
|
925
|
+
if (code === 2) return verb;
|
|
926
|
+
return mode === 'nx' ? 'skip-exists' : 'skip-higher';
|
|
927
|
+
}
|
|
928
|
+
|
|
929
|
+
/**
|
|
930
|
+
* Seeds Redis from quota_usages (counters and the notify flags not expired yet), streaming the docs
|
|
931
|
+
* and doing the Redis work in one Lua call per batch.
|
|
932
|
+
* @param {Object} options
|
|
933
|
+
* tdCache connected TdCache
|
|
934
|
+
* project optional id_project filter
|
|
935
|
+
* period 'current' (default: the period that contains now, per project) or 'all'
|
|
936
|
+
* dryRun only report
|
|
937
|
+
* mode 'nx' (default): create missing keys only
|
|
938
|
+
* 'max' (or force: true): set to max(redis value, snapshot)
|
|
939
|
+
* 'merge': missing -> snapshot, lower than snapshot -> snapshot + redis value (after a clean wipe)
|
|
940
|
+
* collectResults false to return only the summary (default true)
|
|
941
|
+
* onBatch async callback after every batch (e.g. to extend a lock)
|
|
942
|
+
* @returns {Promise<{results: Array, summary: Object}>}
|
|
943
|
+
*/
|
|
944
|
+
async function restoreFromSnapshots(options) {
|
|
945
|
+
options = options || {};
|
|
946
|
+
const client = redisClient(options.tdCache);
|
|
947
|
+
if (!client) {
|
|
948
|
+
throw new Error('Redis client not available');
|
|
949
|
+
}
|
|
950
|
+
const period = options.period || 'current';
|
|
951
|
+
if (period !== 'current' && period !== 'all') {
|
|
952
|
+
throw new Error("period must be 'current' or 'all'");
|
|
953
|
+
}
|
|
954
|
+
const mode = options.mode || (options.force ? 'max' : 'nx');
|
|
955
|
+
if (['nx', 'max', 'merge'].indexOf(mode) === -1) {
|
|
956
|
+
throw new Error("mode must be 'nx', 'max' or 'merge'");
|
|
957
|
+
}
|
|
958
|
+
const collect = options.collectResults !== false;
|
|
959
|
+
|
|
960
|
+
const query = { type: { $in: COUNTED_TYPES } };
|
|
961
|
+
if (options.project) {
|
|
962
|
+
query.id_project = String(options.project);
|
|
963
|
+
}
|
|
964
|
+
if (period === 'current') {
|
|
965
|
+
// the current period started at most a month ago
|
|
966
|
+
query.periodDate = { $gte: recentCutoff() };
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
const results = [];
|
|
970
|
+
const summary = {};
|
|
971
|
+
const periods = new Map();
|
|
972
|
+
const record = (key, snapshot, redisValue, action) => {
|
|
973
|
+
summary[action] = (summary[action] || 0) + 1;
|
|
974
|
+
if (collect) {
|
|
975
|
+
results.push({ key: key, snapshot: snapshot, redis: redisValue, action: action });
|
|
976
|
+
}
|
|
977
|
+
};
|
|
978
|
+
|
|
979
|
+
const processBatch = async (docs) => {
|
|
980
|
+
if (period === 'current') {
|
|
981
|
+
const unknown = [...new Set(docs.map((d) => d.id_project))].filter((id) => !periods.has(id));
|
|
982
|
+
if (unknown.length > 0) {
|
|
983
|
+
(await currentPeriods(unknown)).forEach((value, id) => periods.set(id, value));
|
|
984
|
+
}
|
|
985
|
+
docs = docs.filter((d) => {
|
|
986
|
+
const current = periods.get(d.id_project);
|
|
987
|
+
if (current === null) {
|
|
988
|
+
record(buildQuotaKey(d.id_project, d.type, d.period), d.value, undefined, 'skip-no-project');
|
|
989
|
+
return false;
|
|
990
|
+
}
|
|
991
|
+
return d.period === current;
|
|
992
|
+
});
|
|
993
|
+
}
|
|
994
|
+
const work = [];
|
|
995
|
+
docs.forEach((d) => {
|
|
996
|
+
const key = buildQuotaKey(d.id_project, d.type, d.period);
|
|
997
|
+
if (!(Number(d.value) > 0)) {
|
|
998
|
+
record(key, Number(d.value), undefined, 'skip-zero');
|
|
999
|
+
} else {
|
|
1000
|
+
work.push({ key: key, snapshot: Number(d.value), notified: d.notified });
|
|
1001
|
+
}
|
|
1002
|
+
});
|
|
1003
|
+
if (work.length === 0) {
|
|
1004
|
+
return;
|
|
1005
|
+
}
|
|
1006
|
+
const reply = await redisCommand(client, 'eval', [RESTORE_BATCH_SCRIPT, work.length]
|
|
1007
|
+
.concat(work.map((w) => w.key))
|
|
1008
|
+
.concat([options.dryRun ? 'read' : mode, dirtyTracking ? DIRTY_KEY : ''])
|
|
1009
|
+
.concat(work.map((w) => String(w.snapshot))));
|
|
1010
|
+
let flags = 0;
|
|
1011
|
+
if (!options.dryRun) {
|
|
1012
|
+
const restored = await Promise.all(work.map((w) => restoreNotifyFlags(client, w.key, w.notified)));
|
|
1013
|
+
flags = restored.reduce((a, b) => a + b, 0);
|
|
1014
|
+
}
|
|
1015
|
+
work.forEach((w, idx) => {
|
|
1016
|
+
const code = reply[idx * 2];
|
|
1017
|
+
const previous = reply[idx * 2 + 1] === null || reply[idx * 2 + 1] === undefined ? null : Number(reply[idx * 2 + 1]);
|
|
1018
|
+
record(w.key, w.snapshot, previous, restoreAction(mode, options.dryRun, code, previous, w.snapshot));
|
|
1019
|
+
});
|
|
1020
|
+
if (flags > 0) {
|
|
1021
|
+
summary['notify-restored'] = (summary['notify-restored'] || 0) + flags;
|
|
1022
|
+
}
|
|
1023
|
+
};
|
|
1024
|
+
|
|
1025
|
+
const cursor = QuotaUsage.find(query).sort({ id_project: 1, type: 1, period: 1 }).lean().cursor();
|
|
1026
|
+
let batch = [];
|
|
1027
|
+
for (let doc = await cursor.next(); doc != null; doc = await cursor.next()) {
|
|
1028
|
+
batch.push(doc);
|
|
1029
|
+
if (batch.length >= BATCH_SIZE) {
|
|
1030
|
+
await processBatch(batch);
|
|
1031
|
+
batch = [];
|
|
1032
|
+
if (options.onBatch) await options.onBatch();
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
if (batch.length > 0) {
|
|
1036
|
+
await processBatch(batch);
|
|
1037
|
+
if (options.onBatch) await options.onBatch();
|
|
1038
|
+
}
|
|
1039
|
+
return { results: results, summary: summary };
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
/**
|
|
1043
|
+
* Sets a counter to a value (default 0) in Redis AND in quota_usages (bypassing $max) and clears its
|
|
1044
|
+
* notify flags in both. Runs under the snapshot lock so a running snapshot cannot raise it back.
|
|
1045
|
+
* @param {Object} options { tdCache, project, type, period: 'current' (default) | "M/D/YYYY", value, dryRun }
|
|
1046
|
+
*/
|
|
1047
|
+
async function resetCounter(options) {
|
|
1048
|
+
options = options || {};
|
|
1049
|
+
const client = redisClient(options.tdCache);
|
|
1050
|
+
if (!client) {
|
|
1051
|
+
throw new Error('Redis client not available');
|
|
1052
|
+
}
|
|
1053
|
+
if (!options.project) {
|
|
1054
|
+
throw new Error('project is mandatory');
|
|
1055
|
+
}
|
|
1056
|
+
if (COUNTED_TYPES.indexOf(options.type) === -1) {
|
|
1057
|
+
throw new Error('type must be one of ' + COUNTED_TYPES.join(', '));
|
|
1058
|
+
}
|
|
1059
|
+
const value = options.value === undefined ? 0 : Number(options.value);
|
|
1060
|
+
if (!Number.isFinite(value) || value < 0) {
|
|
1061
|
+
throw new Error('value must be a number >= 0');
|
|
1062
|
+
}
|
|
1063
|
+
const id_project = String(options.project);
|
|
1064
|
+
let period = options.period || 'current';
|
|
1065
|
+
if (period === 'current') {
|
|
1066
|
+
period = (await currentPeriods([id_project])).get(id_project);
|
|
1067
|
+
if (!period) {
|
|
1068
|
+
throw new Error('project ' + id_project + ' not found');
|
|
1069
|
+
}
|
|
1070
|
+
} else if (!PERIOD_REGEX.test(period)) {
|
|
1071
|
+
throw new Error('period must be current or M/D/YYYY');
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
const key = buildQuotaKey(id_project, options.type, period);
|
|
1075
|
+
const raw = await redisCommand(client, 'get', [key]);
|
|
1076
|
+
const doc = await QuotaUsage.findOne({ id_project: id_project, type: options.type, period: period }).lean().exec();
|
|
1077
|
+
const notifyKeys = [];
|
|
1078
|
+
let cursor = '0';
|
|
1079
|
+
do {
|
|
1080
|
+
const reply = await redisCommand(client, 'scan', [cursor, 'MATCH', key + ':notify:*', 'COUNT', SCAN_COUNT]);
|
|
1081
|
+
cursor = reply[0];
|
|
1082
|
+
notifyKeys.push(...(reply[1] || []));
|
|
1083
|
+
} while (cursor !== '0');
|
|
1084
|
+
const result = {
|
|
1085
|
+
key: key,
|
|
1086
|
+
redis: raw === null ? null : Number(raw),
|
|
1087
|
+
snapshot: doc ? doc.value : null,
|
|
1088
|
+
notifyFlags: notifyKeys.length + (doc && doc.notified ? Object.keys(doc.notified).length : 0),
|
|
1089
|
+
value: value,
|
|
1090
|
+
dryRun: !!options.dryRun
|
|
1091
|
+
};
|
|
1092
|
+
if (options.dryRun) {
|
|
1093
|
+
return result;
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
// wait for a running snapshot (it could raise the counter back with $max) up to its lock TTL
|
|
1097
|
+
const token = INSTANCE_TOKEN + ':reset';
|
|
1098
|
+
const onWait = typeof options.onWait === 'function' ? options.onWait : () => {};
|
|
1099
|
+
const progressEveryMs = options.progressEveryMs || 5000;
|
|
1100
|
+
let deadline = null;
|
|
1101
|
+
let lastProgress = 0;
|
|
1102
|
+
while (!(await acquireLock(client, LOCK_KEY, token, 30))) {
|
|
1103
|
+
const pttl = Number(await redisCommand(client, 'pttl', [LOCK_KEY]));
|
|
1104
|
+
if (deadline === null) {
|
|
1105
|
+
deadline = Date.now() + Math.max(pttl, 0) + 5000;
|
|
1106
|
+
}
|
|
1107
|
+
if (Date.now() > deadline) {
|
|
1108
|
+
throw new Error(LOCK_KEY + ' is still held after its TTL, retry later');
|
|
1109
|
+
}
|
|
1110
|
+
if (Date.now() - lastProgress >= progressEveryMs) {
|
|
1111
|
+
lastProgress = Date.now();
|
|
1112
|
+
onWait("A quota snapshot is running (" + LOCK_KEY + " expires in " + Math.max(0, Math.ceil(pttl / 1000)) + " s), waiting for it to finish...");
|
|
1113
|
+
}
|
|
1114
|
+
await sleep(200);
|
|
1115
|
+
}
|
|
1116
|
+
try {
|
|
1117
|
+
await redisCommand(client, 'set', [key, String(value)]);
|
|
1118
|
+
if (dirtyTracking) {
|
|
1119
|
+
await redisCommand(client, 'sadd', [DIRTY_KEY, key]);
|
|
1120
|
+
}
|
|
1121
|
+
if (notifyKeys.length > 0) {
|
|
1122
|
+
await redisCommand(client, 'del', notifyKeys);
|
|
1123
|
+
}
|
|
1124
|
+
await QuotaUsage.updateOne(
|
|
1125
|
+
{ id_project: id_project, type: options.type, period: period },
|
|
1126
|
+
{ $set: { value: value, updatedAt: new Date() }, $unset: { notified: '' }, $setOnInsert: { periodDate: QuotaUsage.periodToDate(period) } },
|
|
1127
|
+
{ upsert: true }
|
|
1128
|
+
).exec();
|
|
1129
|
+
} finally {
|
|
1130
|
+
await releaseLock(client, LOCK_KEY, token);
|
|
1131
|
+
}
|
|
1132
|
+
winston.info("[QuotaSnapshot] Counter " + key + " reset to " + value);
|
|
1133
|
+
return result;
|
|
1134
|
+
}
|
|
1135
|
+
|
|
1136
|
+
// RESTORE / RESET - END
|
|
1137
|
+
|
|
1138
|
+
module.exports = {
|
|
1139
|
+
start,
|
|
1140
|
+
stop,
|
|
1141
|
+
runSnapshotCycle,
|
|
1142
|
+
ensureQuotaKey,
|
|
1143
|
+
checkSentinel,
|
|
1144
|
+
startWipeDetection,
|
|
1145
|
+
stopWipeDetection,
|
|
1146
|
+
waitForRestore,
|
|
1147
|
+
clearNotified,
|
|
1148
|
+
restoreFromSnapshots,
|
|
1149
|
+
resetCounter,
|
|
1150
|
+
incrementCounter,
|
|
1151
|
+
markDirty,
|
|
1152
|
+
configureDirtyTracking,
|
|
1153
|
+
ageGenerationRead,
|
|
1154
|
+
parseQuotaKey,
|
|
1155
|
+
parseNotifyKey,
|
|
1156
|
+
buildQuotaKey,
|
|
1157
|
+
resetRestoreMemo,
|
|
1158
|
+
restoreStats,
|
|
1159
|
+
COUNTED_TYPES,
|
|
1160
|
+
LOCK_KEY,
|
|
1161
|
+
RESTORE_LOCK_KEY,
|
|
1162
|
+
SENTINEL_KEY,
|
|
1163
|
+
DIRTY_KEY,
|
|
1164
|
+
DIRTY_SAVING_KEY,
|
|
1165
|
+
DIRTY_GEN_KEY,
|
|
1166
|
+
FULL_SCAN_KEY
|
|
1167
|
+
};
|