@vercel/board 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -27
- package/dist/{a2-C41hD8SF.mjs → a2-ChRLRupa.mjs} +969 -25
- package/dist/access.d.mts +1 -1
- package/dist/access.mjs +3 -3
- package/dist/{actors-DjMeZpnU.mjs → actors-BOamkEuz.mjs} +3 -73
- package/dist/attempts-CLG840Oj.mjs +20 -0
- package/dist/{auth-COzc09sn.mjs → auth-Drg9FiC2.mjs} +225 -2
- package/dist/backend-B-SJ7p6p.d.mts +680 -0
- package/dist/backend-a2/contract.d.mts +873 -2
- package/dist/backend-a2/contract.mjs +2 -2
- package/dist/backend-a2/scheduler-local.d.mts +1 -1
- package/dist/backend-a2/scheduler-local.mjs +1 -1
- package/dist/backend-a2/scheduler-qstash.d.mts +78 -1
- package/dist/backend-a2/scheduler-qstash.mjs +521 -1
- package/dist/backend-a2/scheduler-vercel.d.mts +37 -1
- package/dist/backend-a2/scheduler-vercel.mjs +203 -1
- package/dist/backend-a2/store-memory.d.mts +2 -2
- package/dist/backend-a2/store-memory.mjs +3 -3
- package/dist/backend-a2/store-postgres.d.mts +1 -1
- package/dist/backend-a2/store-postgres.mjs +2850 -2
- package/dist/backend-a2/store-redis-http.d.mts +1 -1
- package/dist/backend-a2/store-redis-http.mjs +490 -2
- package/dist/backend-a2/store-redis.d.mts +10 -2
- package/dist/backend-a2/store-redis.mjs +51 -2
- package/dist/backend-a2/store-sqlite.d.mts +1 -1
- package/dist/backend-a2/store-sqlite.mjs +3 -3
- package/dist/backend-a2.d.mts +2706 -3
- package/dist/backend-a2.mjs +1 -1
- package/dist/{backend-http-Bid7RIpC.d.mts → backend-http-7cNb8nJI.d.mts} +10 -3
- package/dist/{backend-http-0roEmjl0.mjs → backend-http-CMlL3yq3.mjs} +2 -2
- package/dist/{backend-http-BVRno-ca.mjs → backend-http-Ddzd5kTS.mjs} +32 -3
- package/dist/backend-http.d.mts +1 -1
- package/dist/backend-http.mjs +1 -1
- package/dist/backend-http.node.d.mts +1 -1
- package/dist/backend-http.node.mjs +1 -1
- package/dist/{chat-provider-XCnQlxq6.d.mts → chat-provider-DNJQyajS.d.mts} +16 -4
- package/dist/{chat-provider-jFSeyNmB.mjs → chat-provider-YHy-WbrB.mjs} +15 -9
- package/dist/cli.mjs +30 -10
- package/dist/{client-DGDPaN7U.mjs → client-BA-J7usH.mjs} +10 -2
- package/dist/{conformance-DRR23YNy.mjs → conformance-CT5EeHeC.mjs} +5 -2
- package/dist/conformance.d.mts +1 -1
- package/dist/conformance.mjs +1 -1
- package/dist/{engine-Bir5s8ts.mjs → engine-D0oBUBEK.mjs} +249 -163
- package/dist/events-CewD4kw7.mjs +75 -0
- package/dist/{exports-Dvc1o52I.mjs → exports-CgjfMrQ-.mjs} +35 -32
- package/dist/exports-ZsgjzEnM.d.mts +4 -0
- package/dist/external-BV_cUncs.mjs +151 -0
- package/dist/folds-CRYY14kc.d.mts +534 -0
- package/dist/{reducers-mUPFuFJV.mjs → folds-Dyjput8M.mjs} +6 -73
- package/dist/ids-C6AYo1bu.mjs +31 -0
- package/dist/index.d.mts +4 -4
- package/dist/{index.default-Bkg77Pxn.mjs → index.default-C5Cpz3sa.mjs} +1 -1
- package/dist/index.mjs +5 -5
- package/dist/index.node.d.mts +4 -4
- package/dist/index.node.mjs +3 -3
- package/dist/inspect/app.js +31 -31
- package/dist/inspect/engine.js +5 -5
- package/dist/internal-CvsrprnU-CRY89I-b.mjs +203 -0
- package/dist/{local-Cr_dSjvs.mjs → local-DhkrAqPP.mjs} +1 -1
- package/dist/mentions-CB6ShY4R.mjs +72 -0
- package/dist/{targets-yM5JBUgr.mjs → middleware-CqaCwucP.mjs} +1 -33
- package/dist/platform-B4TnJtWu-CwcTyKg2.mjs +32 -0
- package/dist/{project-CFiArraX.mjs → project-DMDST9kl.mjs} +1 -1
- package/dist/{provider-BGjz-mDa.d.mts → provider-B0b9Qxw4.d.mts} +170 -40
- package/dist/provider-B3DpYSDq.mjs +220 -0
- package/dist/redis-tables-DN9PfIDi.mjs +3897 -0
- package/dist/reducer-CYPHsSmd-BQIfozGi.d.mts +170 -0
- package/dist/reducer-CjmvZSAb-D_OSISmj.mjs +74 -0
- package/dist/{contract-CuJmlrhw.mjs → reducers-BFmIXilu.mjs} +260 -287
- package/dist/retryable-lazy-DZWmHpii-Bye-W8q7.mjs +19 -0
- package/dist/scheduler-task-X1SToDjP-BVoWuInI.mjs +69 -0
- package/dist/{serve-BsOOc-6_.mjs → serve-BGYw0uNL.mjs} +5 -5
- package/dist/{server-jwPY7_wi.mjs → server-DPxBSogx.mjs} +4 -4
- package/dist/server-DtFrWxi3-Boa6gdWO.mjs +3602 -0
- package/dist/server-qgez0Xny-D5mLfqFQ.d.mts +385 -0
- package/dist/{serving-BNbzc96L.mjs → serving-CNkgUPW1.mjs} +1 -1
- package/dist/session-clear-wHd5GCpZ-BnKSxReK.mjs +51 -0
- package/dist/{sql-DbiEnqM0.mjs → sql-U9WTxrwJ.mjs} +2 -2
- package/dist/{sqlite-db-Cbj03vmp.mjs → sqlite-db-Ds4majtT.mjs} +1 -1
- package/dist/store-D0MhBvg6-4Z2L0Kqd.d.mts +530 -0
- package/dist/store-N8PXxDAS-BWBgzBHp.mjs +5 -0
- package/dist/store-codec-DwrWlgQx-CYxGW0aD.mjs +61 -0
- package/dist/store-memory-Bas9Uq5C.mjs +872 -0
- package/dist/store-memory-ZLiZbH3R.mjs +2 -0
- package/dist/store-polling-BZt2Rf3D-BiEM6_HB.mjs +148 -0
- package/dist/store-presence-polling-JuAy2Ck2-CYpuK75X.mjs +109 -0
- package/dist/store-sqlite-BmwL3pUR.mjs +2 -0
- package/dist/store-sqlite-CxxPCcCT.mjs +1053 -0
- package/dist/stream-buffer-DrTyiIuo-BKy-vI14.mjs +53 -0
- package/dist/sync-github.d.mts +24 -17
- package/dist/sync-github.mjs +89 -22
- package/dist/sync-kapso.d.mts +15 -13
- package/dist/sync-kapso.mjs +17 -14
- package/dist/sync-linear.d.mts +219 -0
- package/dist/sync-linear.mjs +311 -0
- package/dist/sync-slack.d.mts +12 -17
- package/dist/sync-slack.mjs +54 -42
- package/dist/sync.chat.d.mts +1 -1
- package/dist/sync.chat.mjs +1 -1
- package/dist/sync.d.mts +3 -3
- package/dist/sync.mjs +5 -4
- package/dist/targets-C-wdR5wP.mjs +33 -0
- package/dist/{tokens-DKLn5P9B.mjs → tokens-DhuzKzAl.mjs} +1 -1
- package/dist/{types-BHCjTZdB.d.mts → types-8yq2VnKI.d.mts} +2 -2
- package/dist/{users-Dls7_ANV.mjs → users-yryHKFMo.mjs} +2 -2
- package/dist/{watch-spec-W0c7ypHJ.mjs → watchers-hash-RLtnSF-t.mjs} +233 -28
- package/docs/01-board/01-mounting.md +1 -1
- package/docs/02-sync/01-overview.md +72 -48
- package/docs/02-sync/02-functions.md +398 -0
- package/docs/02-sync/03-slack.md +30 -25
- package/docs/02-sync/04-github.md +62 -26
- package/docs/02-sync/05-whatsapp.md +23 -19
- package/docs/02-sync/06-providers.md +60 -27
- package/docs/02-sync/07-linear.md +129 -0
- package/docs/03-guides/04-triage.md +1 -1
- package/docs/04-reference/01-cli.md +26 -6
- package/docs/04-reference/02-board.md +18 -5
- package/docs/04-reference/03-http.md +5 -2
- package/docs/04-reference/05-conformance.md +1 -1
- package/docs/index.md +5 -10
- package/package.json +31 -6
- package/dist/a2-D24cvwpN.d.mts +0 -3359
- package/dist/access-CJU2-vaa.mjs +0 -226
- package/dist/attempts-Bxp9KgPf.mjs +0 -919
- package/dist/exports-DLiE7fLY.d.mts +0 -4
- package/dist/provider-CuewcDNZ.mjs +0 -205
- package/dist/redis-tables-B35vv4CZ.mjs +0 -842
- package/dist/reducers-B_vUGQRp.d.mts +0 -1172
- package/dist/watchers-hash-CLjsa6od.mjs +0 -225
- package/docs/02-sync/02-callbacks.md +0 -265
|
@@ -0,0 +1,3897 @@
|
|
|
1
|
+
import { c as externalMessageKey } from "./external-BV_cUncs.mjs";
|
|
2
|
+
import { n as setOwn } from "./own-OwgPWdmh.mjs";
|
|
3
|
+
import { n as CLEAR_EVENT_TYPE, o as sessionClearedError, s as supersededAttemptError, t as A2Error } from "./session-clear-wHd5GCpZ-BnKSxReK.mjs";
|
|
4
|
+
import { m as nullProtoRecord, n as NOTIFY_TIMINGS, v as stateTailReaders } from "./internal-CvsrprnU-CRY89I-b.mjs";
|
|
5
|
+
import { i as presenceTombstones, n as isSerializedEvent, r as parseSerializedEvent, t as bufferedEvents } from "./stream-buffer-DrTyiIuo-BKy-vI14.mjs";
|
|
6
|
+
import { i as streamActivities, n as defaultSleep, r as pollingStream, t as createStreamActivity } from "./store-polling-BZt2Rf3D-BiEM6_HB.mjs";
|
|
7
|
+
import { t as retryableLazy } from "./retryable-lazy-DZWmHpii-Bye-W8q7.mjs";
|
|
8
|
+
import { i as syncReads, n as project, r as projectSync, t as concernedMessageId } from "./project-DMDST9kl.mjs";
|
|
9
|
+
import { i as idempotentReplay, n as decodeCause } from "./store-codec-DwrWlgQx-CYxGW0aD.mjs";
|
|
10
|
+
import { t as pollPresence } from "./store-presence-polling-JuAy2Ck2-CYpuK75X.mjs";
|
|
11
|
+
//#region ../../node_modules/.pnpm/experimental-a2@0.22.0_@upstash+qstash@2.12.0_@vercel+queue@0.7.0_ai@7.0.130_zod@4.6.5__52060f00b21aecfab049c27fa3abd2f9/node_modules/experimental-a2/dist/store-redis-notify-CsJw_hfD.js
|
|
12
|
+
const key = ({ participant, field }) => JSON.stringify([participant, field]);
|
|
13
|
+
function presenceRecovery({ read, deliver, isActive }) {
|
|
14
|
+
const fields = /* @__PURE__ */ new Map();
|
|
15
|
+
const deleted = presenceTombstones();
|
|
16
|
+
const reading = /* @__PURE__ */ new Set();
|
|
17
|
+
let generation = 0;
|
|
18
|
+
let latest;
|
|
19
|
+
const remember = ({ participant, field, seen, at }) => {
|
|
20
|
+
const id = key({
|
|
21
|
+
participant,
|
|
22
|
+
field
|
|
23
|
+
});
|
|
24
|
+
const previous = fields.get(id);
|
|
25
|
+
if (!previous || previous.at.getTime() < at.getTime() || previous.at.getTime() === at.getTime() && previous.seen !== seen) fields.set(id, {
|
|
26
|
+
participant,
|
|
27
|
+
field,
|
|
28
|
+
seen,
|
|
29
|
+
at
|
|
30
|
+
});
|
|
31
|
+
};
|
|
32
|
+
const snapshot = async ({ reconcile }) => {
|
|
33
|
+
const run = generation;
|
|
34
|
+
const before = new Map(fields);
|
|
35
|
+
const changed = /* @__PURE__ */ new Set();
|
|
36
|
+
reading.add(changed);
|
|
37
|
+
try {
|
|
38
|
+
const rows = await read();
|
|
39
|
+
if (!isActive()) return rows;
|
|
40
|
+
if (run !== generation) return latest;
|
|
41
|
+
const current = rows.filter((row) => !deleted.outranks(row));
|
|
42
|
+
const live = /* @__PURE__ */ new Set();
|
|
43
|
+
for (const row of current) {
|
|
44
|
+
const id = key(row);
|
|
45
|
+
live.add(id);
|
|
46
|
+
if (changed.has(id)) continue;
|
|
47
|
+
deleted.drop(row);
|
|
48
|
+
remember(row);
|
|
49
|
+
if (reconcile) deliver({
|
|
50
|
+
participant: row.participant,
|
|
51
|
+
values: { [row.field]: row.value },
|
|
52
|
+
seen: row.seen,
|
|
53
|
+
at: row.at
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
if (reconcile) for (const [id, prior] of before) {
|
|
57
|
+
if (live.has(id) || changed.has(id) || fields.get(id) !== prior) continue;
|
|
58
|
+
fields.delete(id);
|
|
59
|
+
deleted.keep(prior);
|
|
60
|
+
deliver({
|
|
61
|
+
participant: prior.participant,
|
|
62
|
+
values: { [prior.field]: null },
|
|
63
|
+
seen: prior.seen,
|
|
64
|
+
at: prior.at
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
return current;
|
|
68
|
+
} finally {
|
|
69
|
+
reading.delete(changed);
|
|
70
|
+
if (reading.size === 0) latest = void 0;
|
|
71
|
+
}
|
|
72
|
+
};
|
|
73
|
+
return {
|
|
74
|
+
async read() {
|
|
75
|
+
let pending = latest;
|
|
76
|
+
while (pending) {
|
|
77
|
+
await pending;
|
|
78
|
+
if (latest === pending) break;
|
|
79
|
+
pending = latest;
|
|
80
|
+
}
|
|
81
|
+
return snapshot({ reconcile: false });
|
|
82
|
+
},
|
|
83
|
+
async reconnect() {
|
|
84
|
+
generation += 1;
|
|
85
|
+
latest = snapshot({ reconcile: true });
|
|
86
|
+
await latest;
|
|
87
|
+
},
|
|
88
|
+
update(patch) {
|
|
89
|
+
for (const [field, value] of Object.entries(patch.values)) {
|
|
90
|
+
if (value === void 0) continue;
|
|
91
|
+
const write = {
|
|
92
|
+
participant: patch.participant,
|
|
93
|
+
field,
|
|
94
|
+
at: patch.at
|
|
95
|
+
};
|
|
96
|
+
const id = key(write);
|
|
97
|
+
const previous = fields.get(id);
|
|
98
|
+
if (previous && previous.at.getTime() > patch.at.getTime()) continue;
|
|
99
|
+
if (deleted.outranks(write)) continue;
|
|
100
|
+
for (const changed of reading) changed.add(id);
|
|
101
|
+
if (value === null) {
|
|
102
|
+
fields.delete(id);
|
|
103
|
+
deleted.keep(write);
|
|
104
|
+
} else {
|
|
105
|
+
deleted.drop(write);
|
|
106
|
+
remember({
|
|
107
|
+
...write,
|
|
108
|
+
seen: patch.seen
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
const FEED_PAGE_SIZE = 1e3;
|
|
116
|
+
const READ_CONCURRENCY = 16;
|
|
117
|
+
/** The key layout this core writes; any other is rejected. */
|
|
118
|
+
const REDIS_LAYOUT = "tagged-1";
|
|
119
|
+
/**
|
|
120
|
+
* Upstash runs a script under a database-wide lock unless it opts into
|
|
121
|
+
* locking only its keys' hash tags. Redis and Valkey reject the flag, and
|
|
122
|
+
* have no such lock to opt out of: one thread runs every command.
|
|
123
|
+
*/
|
|
124
|
+
const KEY_LOCKING$1 = "#!lua flags=allow-key-locking";
|
|
125
|
+
const READ_ONLY_KEY_LOCKING$1 = "#!lua flags=no-writes,allow-key-locking";
|
|
126
|
+
const SHEBANG_REJECTED$1 = /shebang|unexpected symbol near '#'/i;
|
|
127
|
+
const META_CHUNK_SIZE = 1e3;
|
|
128
|
+
/** A head row's dispatch columns, in the order scripts return them. */
|
|
129
|
+
const HEAD_COLUMNS = `{ 'd', 'fc', 'lc', 'ch', 'ce', 'a', 'lf', 'lfa', 'e', 'f' }`;
|
|
130
|
+
/**
|
|
131
|
+
* Issues a variadic command in calls of at most 1000 arguments, since Lua's
|
|
132
|
+
* `unpack` is bounded by its stack, and concatenates table replies. An even
|
|
133
|
+
* slice size keeps HSET's field/value pairs together.
|
|
134
|
+
*/
|
|
135
|
+
const SLICED_LUA = `
|
|
136
|
+
local function sliced(command, key, args)
|
|
137
|
+
local results = {}
|
|
138
|
+
for first = 1, #args, 1000 do
|
|
139
|
+
local last = math.min(first + 999, #args)
|
|
140
|
+
local reply
|
|
141
|
+
if key then
|
|
142
|
+
reply = redis.call(command, key, unpack(args, first, last))
|
|
143
|
+
else
|
|
144
|
+
reply = redis.call(command, unpack(args, first, last))
|
|
145
|
+
end
|
|
146
|
+
if type(reply) == 'table' then
|
|
147
|
+
for _, value in ipairs(reply) do results[#results + 1] = value end
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
return results
|
|
151
|
+
end
|
|
152
|
+
`;
|
|
153
|
+
/**
|
|
154
|
+
* Upstash bills every call a script makes, so scripts read the counter,
|
|
155
|
+
* clear marker, typed-type index, and event-id owners with one MGET
|
|
156
|
+
* (`read_bounds`), read rows with one HMGET per chunk
|
|
157
|
+
* (`read_rows`), and buffer hash writes (`put`, `drop`) into one HSET and
|
|
158
|
+
* one HDEL per hash (`flush`). Buffered writes are invisible to reads until
|
|
159
|
+
* flushed, so a script must not read a field it has buffered. Every script
|
|
160
|
+
* binds KEYS[1] to the session's counter and derives the session's other
|
|
161
|
+
* keys, which share its hash tag.
|
|
162
|
+
*/
|
|
163
|
+
const BASE_LUA = `${SLICED_LUA}
|
|
164
|
+
local COUNTER = KEYS[1]
|
|
165
|
+
local SESSION = string.sub(COUNTER, 1, -6)
|
|
166
|
+
local LOG, CLEAR = SESSION .. 'log', SESSION .. 'clear'
|
|
167
|
+
local META, READY, CURSORS = SESSION .. 'heads', SESSION .. 'runnable', SESSION .. 'cursors'
|
|
168
|
+
local PIN_COUNTS, TYPED = SESSION .. 'pins', SESSION .. 'typed'
|
|
169
|
+
local CLEAR_TYPE = '${CLEAR_EVENT_TYPE}'
|
|
170
|
+
local total, boundary = 0, 0
|
|
171
|
+
-- The event types typed log cursors read: type → { from, last }. The type's
|
|
172
|
+
-- index holds every event of it above \`from\`, and no event of it lies above
|
|
173
|
+
-- \`last\`.
|
|
174
|
+
local typed, typed_dirty = {}, false
|
|
175
|
+
-- One key per event id holds the index it names in this session. The
|
|
176
|
+
-- \`id:\` namespace keeps a raw id from spelling another session key.
|
|
177
|
+
local function id_key(id) return SESSION .. 'id:' .. id end
|
|
178
|
+
-- Reads the counter, the latest clear marker, the typed types, and the
|
|
179
|
+
-- owners of \`ids\` in one MGET. Returns each id's index, or false.
|
|
180
|
+
local function read_bounds(ids)
|
|
181
|
+
local keys = { COUNTER, CLEAR, TYPED }
|
|
182
|
+
for _, id in ipairs(ids or {}) do keys[#keys + 1] = id_key(id) end
|
|
183
|
+
local values = sliced('MGET', nil, keys)
|
|
184
|
+
total = tonumber(values[1] or values[2] or '0')
|
|
185
|
+
boundary = tonumber(values[2] or '0')
|
|
186
|
+
if values[3] then typed = cjson.decode(values[3]) end
|
|
187
|
+
local owners = {}
|
|
188
|
+
for i = 4, #keys do owners[i - 3] = values[i] end
|
|
189
|
+
return owners
|
|
190
|
+
end
|
|
191
|
+
-- Event-id owners from read_bounds: an id recorded below the latest clear
|
|
192
|
+
-- is free.
|
|
193
|
+
local function owners_of(values)
|
|
194
|
+
local owners = {}
|
|
195
|
+
for i = 1, #values do
|
|
196
|
+
local owner = values[i]
|
|
197
|
+
if owner and tonumber(owner) < boundary then owner = false end
|
|
198
|
+
owners[i] = owner
|
|
199
|
+
end
|
|
200
|
+
return owners
|
|
201
|
+
end
|
|
202
|
+
local function entry_index(id) return tonumber(string.match(id, '^(%d+)')) end
|
|
203
|
+
local function cid_of(consumer, tag)
|
|
204
|
+
return tostring(string.len(consumer)) .. ':' .. consumer .. tag
|
|
205
|
+
end
|
|
206
|
+
local function parse_cid(cid)
|
|
207
|
+
local colon = string.find(cid, ':', 1, true)
|
|
208
|
+
local n = tonumber(string.sub(cid, 1, colon - 1))
|
|
209
|
+
return string.sub(cid, colon + 1, colon + n), string.sub(cid, colon + n + 1)
|
|
210
|
+
end
|
|
211
|
+
-- A cursor's rows: its consumer, then '#' for an event cursor or 'L' .. lane
|
|
212
|
+
-- for a lane or log cursor, as requests name it.
|
|
213
|
+
local function row_of(consumer, tag)
|
|
214
|
+
local kind = string.sub(tag, 1, 1)
|
|
215
|
+
if kind == '#' then return cid_of(consumer, '#') end
|
|
216
|
+
if kind == 'G' then return cid_of(consumer, 'L' .. string.sub(tag, 2)) end
|
|
217
|
+
return cid_of(consumer, tag)
|
|
218
|
+
end
|
|
219
|
+
local function chunk_key(idx)
|
|
220
|
+
return META .. ':' .. tostring(math.floor((idx - 1) / ${META_CHUNK_SIZE}))
|
|
221
|
+
end
|
|
222
|
+
local function row_name(idx, row, suffix)
|
|
223
|
+
return tostring(idx) .. ':' .. row .. ':' .. suffix
|
|
224
|
+
end
|
|
225
|
+
-- Rows of (index, row) refs, aligned with refs; one HMGET per chunk.
|
|
226
|
+
local function read_rows(refs, suffixes)
|
|
227
|
+
local groups, order = {}, {}
|
|
228
|
+
for position, ref in ipairs(refs) do
|
|
229
|
+
local key = chunk_key(ref[1])
|
|
230
|
+
local group = groups[key]
|
|
231
|
+
if not group then
|
|
232
|
+
group = { positions = {}, names = {} }
|
|
233
|
+
groups[key] = group
|
|
234
|
+
order[#order + 1] = key
|
|
235
|
+
end
|
|
236
|
+
group.positions[#group.positions + 1] = position
|
|
237
|
+
for _, suffix in ipairs(suffixes) do
|
|
238
|
+
group.names[#group.names + 1] = row_name(ref[1], ref[2], suffix)
|
|
239
|
+
end
|
|
240
|
+
end
|
|
241
|
+
local rows = {}
|
|
242
|
+
for _, key in ipairs(order) do
|
|
243
|
+
local group = groups[key]
|
|
244
|
+
local values = sliced('HMGET', key, group.names)
|
|
245
|
+
for n, position in ipairs(group.positions) do
|
|
246
|
+
local row = {}
|
|
247
|
+
for column = 1, #suffixes do row[column] = values[(n - 1) * #suffixes + column] end
|
|
248
|
+
rows[position] = row
|
|
249
|
+
end
|
|
250
|
+
end
|
|
251
|
+
return rows
|
|
252
|
+
end
|
|
253
|
+
local writes, write_order, deletes, delete_order = {}, {}, {}, {}
|
|
254
|
+
local function put(key, field, value)
|
|
255
|
+
local fields = writes[key]
|
|
256
|
+
if not fields then
|
|
257
|
+
fields = {}
|
|
258
|
+
writes[key] = fields
|
|
259
|
+
write_order[#write_order + 1] = key
|
|
260
|
+
end
|
|
261
|
+
fields[#fields + 1] = field
|
|
262
|
+
fields[#fields + 1] = value
|
|
263
|
+
end
|
|
264
|
+
local function drop(key, field)
|
|
265
|
+
local fields = deletes[key]
|
|
266
|
+
if not fields then
|
|
267
|
+
fields = {}
|
|
268
|
+
deletes[key] = fields
|
|
269
|
+
delete_order[#delete_order + 1] = key
|
|
270
|
+
end
|
|
271
|
+
fields[#fields + 1] = field
|
|
272
|
+
end
|
|
273
|
+
local function flush()
|
|
274
|
+
for _, key in ipairs(write_order) do sliced('HSET', key, writes[key]) end
|
|
275
|
+
for _, key in ipairs(delete_order) do sliced('HDEL', key, deletes[key]) end
|
|
276
|
+
writes, write_order, deletes, delete_order = {}, {}, {}, {}
|
|
277
|
+
end
|
|
278
|
+
local function put_row(idx, row, suffix, value)
|
|
279
|
+
put(chunk_key(idx), row_name(idx, row, suffix), value)
|
|
280
|
+
end
|
|
281
|
+
local function drop_row(idx, row, suffix)
|
|
282
|
+
drop(chunk_key(idx), row_name(idx, row, suffix))
|
|
283
|
+
end
|
|
284
|
+
-- A request names a cursor by tag: '#' for an event cursor, 'L' .. lane for a
|
|
285
|
+
-- lane or log cursor. A claimed row records its kind: '#', or the cursor's tag.
|
|
286
|
+
local function owns(kind, tag)
|
|
287
|
+
if not kind then return false end
|
|
288
|
+
if tag == '#' then return kind == '#' end
|
|
289
|
+
return kind == tag or kind == 'G' .. string.sub(tag, 2)
|
|
290
|
+
end
|
|
291
|
+
local function cid_for(consumer, kind, idx)
|
|
292
|
+
if kind == '#' then return cid_of(consumer, '#' .. tostring(idx)) end
|
|
293
|
+
return cid_of(consumer, kind)
|
|
294
|
+
end
|
|
295
|
+
-- One type's index, named by the type's bytes in hex so no type can spell
|
|
296
|
+
-- another key of the session.
|
|
297
|
+
local function type_key(name)
|
|
298
|
+
return SESSION .. 'type:' .. (string.gsub(name, '.', function(c)
|
|
299
|
+
return string.format('%02x', string.byte(c))
|
|
300
|
+
end))
|
|
301
|
+
end
|
|
302
|
+
local ready_changes, ready_order = {}, {}
|
|
303
|
+
local function ready_set(cid, score)
|
|
304
|
+
if ready_changes[cid] == nil then ready_order[#ready_order + 1] = cid end
|
|
305
|
+
ready_changes[cid] = score
|
|
306
|
+
end
|
|
307
|
+
local function flush_ready()
|
|
308
|
+
local adds, removes = {}, {}
|
|
309
|
+
for _, cid in ipairs(ready_order) do
|
|
310
|
+
local score = ready_changes[cid]
|
|
311
|
+
if score then
|
|
312
|
+
adds[#adds + 1] = score
|
|
313
|
+
adds[#adds + 1] = cid
|
|
314
|
+
else
|
|
315
|
+
removes[#removes + 1] = cid
|
|
316
|
+
end
|
|
317
|
+
end
|
|
318
|
+
sliced('ZADD', READY, adds)
|
|
319
|
+
sliced('ZREM', READY, removes)
|
|
320
|
+
ready_changes, ready_order = {}, {}
|
|
321
|
+
end
|
|
322
|
+
local function release_pins(idx, row, pins)
|
|
323
|
+
if not pins or pins == '' then return end
|
|
324
|
+
for _, ref in ipairs(cjson.decode(pins)) do
|
|
325
|
+
local count = redis.call('HINCRBY', PIN_COUNTS, ref, -1)
|
|
326
|
+
if count <= 0 then
|
|
327
|
+
redis.call('HDEL', PIN_COUNTS, ref)
|
|
328
|
+
local checkpoint = cjson.decode(ref)
|
|
329
|
+
redis.call('ZREMRANGEBYSCORE', checkpoint[1], checkpoint[2], checkpoint[2])
|
|
330
|
+
end
|
|
331
|
+
end
|
|
332
|
+
put_row(idx, row, 'pins', '')
|
|
333
|
+
end
|
|
334
|
+
`;
|
|
335
|
+
/**
|
|
336
|
+
* Cursor movement for scripts that add events or move cursors. Lane and log
|
|
337
|
+
* cursor records are read once per script (`load_states`) and written back
|
|
338
|
+
* once (`flush_states`). A record whose head lies at or below the latest clear
|
|
339
|
+
* marker is idle, and a log cursor counts from the marker.
|
|
340
|
+
*
|
|
341
|
+
* A typed log cursor finds its next head in its types' indexes, one bounded
|
|
342
|
+
* lookup per type, skipped when `typed` shows no newer event of it. When an
|
|
343
|
+
* index does not reach back far enough yet (a typed backfill, or a type a
|
|
344
|
+
* cursor newly reads), the lookup returns the type, and the script returns
|
|
345
|
+
* `{ 'index', type, after }` before writing anything: the caller indexes
|
|
346
|
+
* that type down to `after` (INDEX_TYPE_LUA, one page per call) and runs the
|
|
347
|
+
* operation again.
|
|
348
|
+
*/
|
|
349
|
+
const CURSOR_LUA = `
|
|
350
|
+
local served_all, served_set, served_logs = true, {}, {}
|
|
351
|
+
local pending_fast = false
|
|
352
|
+
local function types_of(raw)
|
|
353
|
+
if not raw or raw == '*' then return nil end
|
|
354
|
+
local set = {}
|
|
355
|
+
for _, name in ipairs(cjson.decode(raw)) do set[name] = true end
|
|
356
|
+
return set
|
|
357
|
+
end
|
|
358
|
+
local function takes(types, kind)
|
|
359
|
+
return kind ~= CLEAR_TYPE and (types == nil or types[kind] == true)
|
|
360
|
+
end
|
|
361
|
+
-- The served consumers: '' for every consumer, else [[name] or
|
|
362
|
+
-- [name, lane, typesJson or '*', backfill]].
|
|
363
|
+
local function use_served(raw)
|
|
364
|
+
if raw == '' then return end
|
|
365
|
+
served_all = false
|
|
366
|
+
for _, entry in ipairs(cjson.decode(raw)) do
|
|
367
|
+
served_set[entry[1]] = true
|
|
368
|
+
if entry[2] then
|
|
369
|
+
served_logs[#served_logs + 1] = {
|
|
370
|
+
name = entry[1], lane = entry[2], ty = entry[3],
|
|
371
|
+
types = types_of(entry[3]), backfill = entry[4] == 1,
|
|
372
|
+
}
|
|
373
|
+
end
|
|
374
|
+
end
|
|
375
|
+
end
|
|
376
|
+
local function served_has(name) return served_all or served_set[name] == true end
|
|
377
|
+
-- Whether a served consumer has a live head. Newest heads come first: a
|
|
378
|
+
-- removed consumer's backlog is older than the work a deployment serves.
|
|
379
|
+
local function has_pending()
|
|
380
|
+
if pending_fast then return '1' end
|
|
381
|
+
if served_all then return redis.call('ZCOUNT', READY, 1, '+inf') > 0 and '1' or '0' end
|
|
382
|
+
if next(served_set) == nil then return '0' end
|
|
383
|
+
local page = redis.call('ZREVRANGEBYSCORE', READY, '+inf', 1, 'LIMIT', 0, 32)
|
|
384
|
+
for _, cid in ipairs(page) do
|
|
385
|
+
if served_set[(parse_cid(cid))] then return '1' end
|
|
386
|
+
end
|
|
387
|
+
if #page < 32 then return '0' end
|
|
388
|
+
for _, cid in ipairs(redis.call('ZREVRANGEBYSCORE', READY, '+inf', 1, 'LIMIT', 32, -1)) do
|
|
389
|
+
if served_set[(parse_cid(cid))] then return '1' end
|
|
390
|
+
end
|
|
391
|
+
return '0'
|
|
392
|
+
end
|
|
393
|
+
local function targets_of(raw)
|
|
394
|
+
if raw == '' then return {} end
|
|
395
|
+
return cjson.decode(raw)
|
|
396
|
+
end
|
|
397
|
+
-- Starts indexing a type: every later event of it.
|
|
398
|
+
local function register(name, from)
|
|
399
|
+
typed[name] = { from, from }
|
|
400
|
+
typed_dirty = true
|
|
401
|
+
end
|
|
402
|
+
-- The first stored event in (after, total] of a type in \`types\` (nil: any
|
|
403
|
+
-- type), or false. Nil and a type when that type's index can't answer yet.
|
|
404
|
+
local function stored_head(after, types)
|
|
405
|
+
if types == nil then
|
|
406
|
+
if after + 1 <= total then return after + 1 end
|
|
407
|
+
return false
|
|
408
|
+
end
|
|
409
|
+
local best = false
|
|
410
|
+
for name in pairs(types) do
|
|
411
|
+
local entry = typed[name]
|
|
412
|
+
if entry == nil then
|
|
413
|
+
if after < total then return nil, name end
|
|
414
|
+
register(name, total)
|
|
415
|
+
elseif entry[2] > after then
|
|
416
|
+
if entry[1] > after then return nil, name end
|
|
417
|
+
local found = redis.call('ZRANGEBYSCORE', type_key(name),
|
|
418
|
+
'(' .. tostring(after), tostring(total), 'LIMIT', 0, 1)
|
|
419
|
+
if #found > 0 then
|
|
420
|
+
local idx = tonumber(found[1])
|
|
421
|
+
if not best or idx < best then best = idx end
|
|
422
|
+
end
|
|
423
|
+
end
|
|
424
|
+
end
|
|
425
|
+
return best
|
|
426
|
+
end
|
|
427
|
+
-- The first event after \`after\` a log cursor takes: stored, then in the
|
|
428
|
+
-- batch being appended.
|
|
429
|
+
local function next_log_head(after, types, batch)
|
|
430
|
+
local head, missing = stored_head(after, types)
|
|
431
|
+
if head == nil then return nil, missing end
|
|
432
|
+
if head then return head end
|
|
433
|
+
for _, event in ipairs(batch) do
|
|
434
|
+
if event.idx > after and takes(types, event.type) then return event.idx end
|
|
435
|
+
end
|
|
436
|
+
return false
|
|
437
|
+
end
|
|
438
|
+
-- Indexes a batch's events of the types typed log cursors read.
|
|
439
|
+
local function index_batch(batch)
|
|
440
|
+
local adds, order = {}, {}
|
|
441
|
+
for _, event in ipairs(batch) do
|
|
442
|
+
local entry = typed[event.type]
|
|
443
|
+
if entry then
|
|
444
|
+
local list = adds[event.type]
|
|
445
|
+
if not list then
|
|
446
|
+
list = {}
|
|
447
|
+
adds[event.type] = list
|
|
448
|
+
order[#order + 1] = event.type
|
|
449
|
+
end
|
|
450
|
+
list[#list + 1] = event.idx
|
|
451
|
+
list[#list + 1] = tostring(event.idx)
|
|
452
|
+
entry[2] = event.idx
|
|
453
|
+
typed_dirty = true
|
|
454
|
+
end
|
|
455
|
+
end
|
|
456
|
+
for _, name in ipairs(order) do sliced('ZADD', type_key(name), adds[name]) end
|
|
457
|
+
end
|
|
458
|
+
local states, state_order = {}, {}
|
|
459
|
+
local function load_states(cids)
|
|
460
|
+
local need, names = {}, {}
|
|
461
|
+
for _, cid in ipairs(cids) do
|
|
462
|
+
local state = states[cid]
|
|
463
|
+
if (state == nil or state.partial) and not need[cid] then
|
|
464
|
+
need[cid] = true
|
|
465
|
+
need[#need + 1] = cid
|
|
466
|
+
names[#names + 1] = cid .. ':h'
|
|
467
|
+
names[#names + 1] = cid .. ':p'
|
|
468
|
+
names[#names + 1] = cid .. ':t'
|
|
469
|
+
names[#names + 1] = cid .. ':ty'
|
|
470
|
+
end
|
|
471
|
+
end
|
|
472
|
+
if #need == 0 then return end
|
|
473
|
+
local values = sliced('HMGET', CURSORS, names)
|
|
474
|
+
for n, cid in ipairs(need) do
|
|
475
|
+
local h, p, t, ty = values[n * 4 - 3], values[n * 4 - 2], values[n * 4 - 1], values[n * 4]
|
|
476
|
+
local tail = (t and t ~= '') and tonumber(t) or false
|
|
477
|
+
local state = states[cid]
|
|
478
|
+
if state then
|
|
479
|
+
state.t = tail
|
|
480
|
+
state.partial = nil
|
|
481
|
+
else
|
|
482
|
+
local _, tag = parse_cid(cid)
|
|
483
|
+
state = {
|
|
484
|
+
kind = string.sub(tag, 1, 1), exists = p ~= false,
|
|
485
|
+
h = (h and h ~= '') and tonumber(h) or false,
|
|
486
|
+
p = tonumber(p or '0'), t = tail, ty = ty or false,
|
|
487
|
+
}
|
|
488
|
+
if state.h and state.h <= boundary then state.h = false end
|
|
489
|
+
if state.kind == 'G' and state.exists and state.p < boundary then state.p = boundary end
|
|
490
|
+
states[cid] = state
|
|
491
|
+
state_order[#state_order + 1] = cid
|
|
492
|
+
end
|
|
493
|
+
end
|
|
494
|
+
end
|
|
495
|
+
local function flush_states()
|
|
496
|
+
for _, cid in ipairs(state_order) do
|
|
497
|
+
local state = states[cid]
|
|
498
|
+
if state.dirty then
|
|
499
|
+
put(CURSORS, cid .. ':h', state.h and tostring(state.h) or '')
|
|
500
|
+
put(CURSORS, cid .. ':p', tostring(state.p))
|
|
501
|
+
if state.kind == 'L' then
|
|
502
|
+
if not state.partial then put(CURSORS, cid .. ':t', state.t and tostring(state.t) or '') end
|
|
503
|
+
elseif state.ty then
|
|
504
|
+
put(CURSORS, cid .. ':ty', state.ty)
|
|
505
|
+
end
|
|
506
|
+
end
|
|
507
|
+
end
|
|
508
|
+
end
|
|
509
|
+
-- Moves a cursor past its head \`idx\`. \`kind\` is the head row's kind;
|
|
510
|
+
-- \`batch\` is the returned events, appended after the stored log. Writes
|
|
511
|
+
-- nothing; returns nil and a type when that type's index must reach back
|
|
512
|
+
-- to \`idx\` first.
|
|
513
|
+
local function advance(consumer, kind, idx, row_next, batch)
|
|
514
|
+
if kind == '#' then
|
|
515
|
+
ready_set(cid_of(consumer, '#' .. tostring(idx)), false)
|
|
516
|
+
return true
|
|
517
|
+
end
|
|
518
|
+
local cid = cid_of(consumer, kind)
|
|
519
|
+
local state = states[cid]
|
|
520
|
+
local next_head
|
|
521
|
+
if string.sub(kind, 1, 1) == 'L' then
|
|
522
|
+
next_head = row_next and tonumber(row_next) or false
|
|
523
|
+
if not state then
|
|
524
|
+
state = { kind = 'L', exists = true, p = idx, h = false, t = false, ty = false, partial = true }
|
|
525
|
+
states[cid] = state
|
|
526
|
+
state_order[#state_order + 1] = cid
|
|
527
|
+
end
|
|
528
|
+
if not next_head then
|
|
529
|
+
state.t = false
|
|
530
|
+
state.partial = nil
|
|
531
|
+
end
|
|
532
|
+
else
|
|
533
|
+
local lane, raw = string.sub(kind, 2), nil
|
|
534
|
+
for _, spec in ipairs(served_logs) do
|
|
535
|
+
if spec.name == consumer and spec.lane == lane then raw = spec.ty end
|
|
536
|
+
end
|
|
537
|
+
if not raw then
|
|
538
|
+
load_states({ cid })
|
|
539
|
+
raw = states[cid].ty or '*'
|
|
540
|
+
end
|
|
541
|
+
local head, missing = next_log_head(idx, types_of(raw), batch)
|
|
542
|
+
if head == nil then return nil, missing end
|
|
543
|
+
next_head = head
|
|
544
|
+
state = states[cid]
|
|
545
|
+
if not state then
|
|
546
|
+
state = { kind = 'G', exists = true, p = idx, h = false, t = false, ty = raw }
|
|
547
|
+
states[cid] = state
|
|
548
|
+
state_order[#state_order + 1] = cid
|
|
549
|
+
end
|
|
550
|
+
state.ty = raw
|
|
551
|
+
end
|
|
552
|
+
state.h = next_head
|
|
553
|
+
state.p = idx
|
|
554
|
+
state.exists = true
|
|
555
|
+
state.dirty = true
|
|
556
|
+
ready_set(cid, next_head)
|
|
557
|
+
return true
|
|
558
|
+
end
|
|
559
|
+
-- Reads what an appended batch's cursor work needs: its lane targets'
|
|
560
|
+
-- records, and the head of each served log consumer whose types it carries.
|
|
561
|
+
-- Writes nothing; returns nil, a type, and an index when that type's index
|
|
562
|
+
-- must reach back to the index first.
|
|
563
|
+
local log_plan = {}
|
|
564
|
+
local function plan(batch, base)
|
|
565
|
+
local need = {}
|
|
566
|
+
for _, event in ipairs(batch) do
|
|
567
|
+
for _, target in ipairs(event.targets) do
|
|
568
|
+
if target[2] ~= '#' then need[#need + 1] = cid_of(target[1], target[2]) end
|
|
569
|
+
end
|
|
570
|
+
end
|
|
571
|
+
for _, spec in ipairs(served_logs) do
|
|
572
|
+
for _, event in ipairs(batch) do
|
|
573
|
+
if takes(spec.types, event.type) then
|
|
574
|
+
local cid = cid_of(spec.name, 'G' .. spec.lane)
|
|
575
|
+
log_plan[#log_plan + 1] = { spec = spec, first = event.idx, cid = cid }
|
|
576
|
+
need[#need + 1] = cid
|
|
577
|
+
break
|
|
578
|
+
end
|
|
579
|
+
end
|
|
580
|
+
end
|
|
581
|
+
load_states(need)
|
|
582
|
+
for _, log in ipairs(log_plan) do
|
|
583
|
+
local state, spec = states[log.cid], log.spec
|
|
584
|
+
if not state.exists then
|
|
585
|
+
if spec.backfill then
|
|
586
|
+
local head, missing = stored_head(boundary, spec.types)
|
|
587
|
+
if head == nil then return nil, missing, boundary end
|
|
588
|
+
log.p, log.h = boundary, head or log.first
|
|
589
|
+
else
|
|
590
|
+
-- Nothing it takes is stored after its start: index its types from there.
|
|
591
|
+
for name in pairs(spec.types or {}) do
|
|
592
|
+
if typed[name] == nil then register(name, base) end
|
|
593
|
+
end
|
|
594
|
+
log.p, log.h = base, log.first
|
|
595
|
+
end
|
|
596
|
+
elseif not state.h then
|
|
597
|
+
-- Not the batch's first: a deployment that doesn't serve this consumer
|
|
598
|
+
-- may have appended events it takes since it went idle.
|
|
599
|
+
local head, missing = stored_head(state.p, spec.types)
|
|
600
|
+
if head == nil then return nil, missing, state.p end
|
|
601
|
+
log.h = head or log.first
|
|
602
|
+
end
|
|
603
|
+
end
|
|
604
|
+
return true
|
|
605
|
+
end
|
|
606
|
+
-- Starts the cursor work \`plan\` read: each event's targets, and each
|
|
607
|
+
-- planned log cursor.
|
|
608
|
+
local function enqueue(batch)
|
|
609
|
+
for _, event in ipairs(batch) do
|
|
610
|
+
for _, target in ipairs(event.targets) do
|
|
611
|
+
local consumer, tag = target[1], target[2]
|
|
612
|
+
if served_has(consumer) then pending_fast = true end
|
|
613
|
+
if tag == '#' then
|
|
614
|
+
ready_set(cid_of(consumer, '#' .. tostring(event.idx)), event.idx)
|
|
615
|
+
else
|
|
616
|
+
local cid = cid_of(consumer, tag)
|
|
617
|
+
local state = states[cid]
|
|
618
|
+
if state.h and state.t then
|
|
619
|
+
put_row(state.t, row_of(consumer, tag), 'next', tostring(event.idx))
|
|
620
|
+
else
|
|
621
|
+
state.h = event.idx
|
|
622
|
+
if state.p < boundary then state.p = boundary end
|
|
623
|
+
ready_set(cid, event.idx)
|
|
624
|
+
end
|
|
625
|
+
state.t = event.idx
|
|
626
|
+
state.exists = true
|
|
627
|
+
state.dirty = true
|
|
628
|
+
end
|
|
629
|
+
end
|
|
630
|
+
end
|
|
631
|
+
for _, log in ipairs(log_plan) do
|
|
632
|
+
pending_fast = true
|
|
633
|
+
local state = states[log.cid]
|
|
634
|
+
if not state.exists then
|
|
635
|
+
state.exists, state.p, state.h = true, log.p, log.h
|
|
636
|
+
ready_set(log.cid, log.h)
|
|
637
|
+
elseif not state.h then
|
|
638
|
+
state.h = log.h
|
|
639
|
+
ready_set(log.cid, log.h)
|
|
640
|
+
end
|
|
641
|
+
state.ty = log.spec.ty
|
|
642
|
+
state.dirty = true
|
|
643
|
+
end
|
|
644
|
+
end
|
|
645
|
+
`;
|
|
646
|
+
/**
|
|
647
|
+
* KEYS: counter.
|
|
648
|
+
* ARGV: sessionId, nowMs, n, served consumers, origin index or '', then per
|
|
649
|
+
* event: id, type, payloadJson, causeJson or '', targets.
|
|
650
|
+
* Returns ['empty', hasPending] | ['ok', firstIndex, hasPending] |
|
|
651
|
+
* ['dup', hasPending, ...indexes] | ['partial', count] |
|
|
652
|
+
* ['superseded', position] | ['cleared', originIndex] |
|
|
653
|
+
* ['index', type, after].
|
|
654
|
+
*/
|
|
655
|
+
const APPEND_LUA = `${BASE_LUA}${CURSOR_LUA}
|
|
656
|
+
local now, n = ARGV[2], tonumber(ARGV[3])
|
|
657
|
+
use_served(ARGV[4])
|
|
658
|
+
if n == 0 then return { 'empty', has_pending() } end
|
|
659
|
+
local ids = {}
|
|
660
|
+
for i = 1, n do ids[i] = ARGV[5 + (i - 1) * 5 + 1] end
|
|
661
|
+
local owners = owners_of(read_bounds(ids))
|
|
662
|
+
local base = total
|
|
663
|
+
local dups = {}
|
|
664
|
+
for i = 1, n do
|
|
665
|
+
if owners[i] then dups[#dups + 1] = owners[i] end
|
|
666
|
+
end
|
|
667
|
+
if #dups == n then
|
|
668
|
+
local res = { 'dup', has_pending() }
|
|
669
|
+
for i = 1, #dups do res[#res + 1] = dups[i] end
|
|
670
|
+
return res
|
|
671
|
+
end
|
|
672
|
+
if #dups > 0 then return { 'partial', tostring(#dups) } end
|
|
673
|
+
local origin = ARGV[5]
|
|
674
|
+
if origin ~= '' and tonumber(origin) <= boundary then return { 'cleared', origin } end
|
|
675
|
+
-- The attempt-currency fence: a cause's head row must still be its cursor's
|
|
676
|
+
-- head, at that attempt, not dead-lettered.
|
|
677
|
+
local refs, fenced = {}, {}
|
|
678
|
+
for i = 1, n do
|
|
679
|
+
local raw = ARGV[5 + (i - 1) * 5 + 4]
|
|
680
|
+
if raw ~= '' then
|
|
681
|
+
local cause = cjson.decode(raw)
|
|
682
|
+
local index = tonumber(cause.index)
|
|
683
|
+
if index <= boundary then return { 'superseded', tostring(i) } end
|
|
684
|
+
local tag = type(cause.lane) == 'string' and ('L' .. cause.lane) or '#'
|
|
685
|
+
refs[#refs + 1] = { index, row_of(cause.consumer, tag) }
|
|
686
|
+
fenced[#fenced + 1] = {
|
|
687
|
+
position = i, consumer = cause.consumer, attempt = tonumber(cause.attempt), tag = tag,
|
|
688
|
+
}
|
|
689
|
+
end
|
|
690
|
+
end
|
|
691
|
+
local rows = read_rows(refs, { 'd', 'f', 'p', 'l' })
|
|
692
|
+
for k, cause in ipairs(fenced) do
|
|
693
|
+
local row = rows[k]
|
|
694
|
+
if tonumber(row[1] or '0') ~= cause.attempt or row[2] or row[3] or not owns(row[4], cause.tag) then
|
|
695
|
+
return { 'superseded', tostring(cause.position) }
|
|
696
|
+
end
|
|
697
|
+
if served_has(cause.consumer) then pending_fast = true end
|
|
698
|
+
end
|
|
699
|
+
local batch = {}
|
|
700
|
+
for i = 1, n do
|
|
701
|
+
local o = 5 + (i - 1) * 5
|
|
702
|
+
batch[i] = { idx = base + i, type = ARGV[o + 2], targets = targets_of(ARGV[o + 5]) }
|
|
703
|
+
end
|
|
704
|
+
local planned, missing, after = plan(batch, base)
|
|
705
|
+
if not planned then return { 'index', missing, tostring(after) } end
|
|
706
|
+
local marks = {}
|
|
707
|
+
for i = 1, n do
|
|
708
|
+
local o = 5 + (i - 1) * 5
|
|
709
|
+
local field = tostring(base + i)
|
|
710
|
+
redis.call('XADD', LOG, field .. '-0',
|
|
711
|
+
'id', ARGV[o + 1], 'type', ARGV[o + 2], 'payload', ARGV[o + 3],
|
|
712
|
+
'created_at', now, 'cause', ARGV[o + 4])
|
|
713
|
+
marks[#marks + 1] = id_key(ARGV[o + 1])
|
|
714
|
+
marks[#marks + 1] = field
|
|
715
|
+
end
|
|
716
|
+
index_batch(batch)
|
|
717
|
+
enqueue(batch)
|
|
718
|
+
marks[#marks + 1] = COUNTER
|
|
719
|
+
marks[#marks + 1] = tostring(base + n)
|
|
720
|
+
if typed_dirty then
|
|
721
|
+
marks[#marks + 1] = TYPED
|
|
722
|
+
marks[#marks + 1] = cjson.encode(typed)
|
|
723
|
+
end
|
|
724
|
+
sliced('MSET', nil, marks)
|
|
725
|
+
flush_states()
|
|
726
|
+
flush()
|
|
727
|
+
flush_ready()
|
|
728
|
+
return { 'ok', tostring(base + 1), has_pending() }
|
|
729
|
+
`;
|
|
730
|
+
/**
|
|
731
|
+
* KEYS: counter.
|
|
732
|
+
* ARGV: type, after. Indexes one page of the stored log below the type's
|
|
733
|
+
* index, toward `after`, registering the type first if no cursor read it
|
|
734
|
+
* yet. Returns ['1'] once the index reaches `after`, else ['0'].
|
|
735
|
+
*/
|
|
736
|
+
const INDEX_TYPE_LUA = `${BASE_LUA}
|
|
737
|
+
read_bounds()
|
|
738
|
+
local name, after = ARGV[1], tonumber(ARGV[2])
|
|
739
|
+
local entry = typed[name]
|
|
740
|
+
if not entry then
|
|
741
|
+
entry = { total, total }
|
|
742
|
+
typed[name] = entry
|
|
743
|
+
end
|
|
744
|
+
-- Entries below the latest clear belong to an old lifetime: never indexed.
|
|
745
|
+
local floor = math.max(after, boundary - 1)
|
|
746
|
+
if entry[1] > floor then
|
|
747
|
+
local entries = redis.call('XREVRANGE', LOG, tostring(entry[1]) .. '-0',
|
|
748
|
+
tostring(floor + 1) .. '-0', 'COUNT', 256)
|
|
749
|
+
local adds = {}
|
|
750
|
+
for _, stored in ipairs(entries) do
|
|
751
|
+
local fields = stored[2]
|
|
752
|
+
for j = 1, #fields, 2 do
|
|
753
|
+
if fields[j] == 'type' then
|
|
754
|
+
if fields[j + 1] == name then
|
|
755
|
+
local idx = entry_index(stored[1])
|
|
756
|
+
adds[#adds + 1] = idx
|
|
757
|
+
adds[#adds + 1] = tostring(idx)
|
|
758
|
+
end
|
|
759
|
+
break
|
|
760
|
+
end
|
|
761
|
+
end
|
|
762
|
+
end
|
|
763
|
+
sliced('ZADD', type_key(name), adds)
|
|
764
|
+
if #entries < 256 then
|
|
765
|
+
entry[1] = after
|
|
766
|
+
else
|
|
767
|
+
entry[1] = entry_index(entries[#entries][1]) - 1
|
|
768
|
+
end
|
|
769
|
+
elseif entry[1] > after then
|
|
770
|
+
entry[1] = after
|
|
771
|
+
end
|
|
772
|
+
redis.call('SET', TYPED, cjson.encode(typed))
|
|
773
|
+
return { entry[1] <= after and '1' or '0' }
|
|
774
|
+
`;
|
|
775
|
+
/**
|
|
776
|
+
* KEYS: counter.
|
|
777
|
+
* ARGV: holder, nowMs, expiresAtMs, maxFailures or '', served consumers,
|
|
778
|
+
* the highest head to claim or '', excluded count, then per excluded head:
|
|
779
|
+
* consumer, tag.
|
|
780
|
+
* Live heads are the ready set's positive scores, so an empty one is enough
|
|
781
|
+
* to report 'settled'. A claimed result ends with the clear marker.
|
|
782
|
+
*/
|
|
783
|
+
const CLAIM_AVAILABLE_LUA = `${BASE_LUA}${CURSOR_LUA}
|
|
784
|
+
local ready = redis.call('ZRANGEBYSCORE', READY, 1, ARGV[6] == '' and '+inf' or ARGV[6], 'WITHSCORES')
|
|
785
|
+
if #ready == 0 then return { 'settled' } end
|
|
786
|
+
read_bounds()
|
|
787
|
+
local holder, now, expires_at = ARGV[1], tonumber(ARGV[2]), ARGV[3]
|
|
788
|
+
local max_failures = tonumber(ARGV[4])
|
|
789
|
+
use_served(ARGV[5])
|
|
790
|
+
local excluded = {}
|
|
791
|
+
for i = 1, tonumber(ARGV[7]) do
|
|
792
|
+
excluded[ARGV[7 + (i - 1) * 2 + 1] .. '\\0' .. ARGV[7 + (i - 1) * 2 + 2]] = true
|
|
793
|
+
end
|
|
794
|
+
local candidates = {}
|
|
795
|
+
for i = 1, #ready, 2 do
|
|
796
|
+
local cid, idx = ready[i], tonumber(ready[i + 1])
|
|
797
|
+
local consumer, tag = parse_cid(cid)
|
|
798
|
+
local kind = string.sub(tag, 1, 1)
|
|
799
|
+
local key = consumer .. '\\0' .. (kind == 'G' and ('L' .. string.sub(tag, 2)) or tag)
|
|
800
|
+
if served_has(consumer) and not excluded[key] then
|
|
801
|
+
candidates[#candidates + 1] = {
|
|
802
|
+
idx, row_of(consumer, tag), consumer = consumer, cid = cid,
|
|
803
|
+
kind = kind == '#' and '#' or tag,
|
|
804
|
+
}
|
|
805
|
+
end
|
|
806
|
+
end
|
|
807
|
+
local rows = read_rows(candidates, { 'd', 'fc', 'lc', 'ch', 'ce', 'a', 'lf', 'lfa', 'e', 'f', 'pins' })
|
|
808
|
+
local takes_over, retry_at, wanted = {}, nil, {}
|
|
809
|
+
for position, candidate in ipairs(candidates) do
|
|
810
|
+
local row = rows[position]
|
|
811
|
+
local expires = tonumber(row[5] or '0')
|
|
812
|
+
if expires > now then
|
|
813
|
+
if not retry_at or expires < retry_at then retry_at = expires end
|
|
814
|
+
else
|
|
815
|
+
takes_over[#takes_over + 1] = { candidate = candidate, row = row }
|
|
816
|
+
wanted[candidate[1]] = true
|
|
817
|
+
end
|
|
818
|
+
end
|
|
819
|
+
if #takes_over == 0 then
|
|
820
|
+
if retry_at then return { 'busy', tostring(retry_at) } end
|
|
821
|
+
return { 'settled' }
|
|
822
|
+
end
|
|
823
|
+
-- One XRANGE per run of consecutive claimed indexes.
|
|
824
|
+
local indexes, entries = {}, {}
|
|
825
|
+
for idx in pairs(wanted) do indexes[#indexes + 1] = idx end
|
|
826
|
+
table.sort(indexes)
|
|
827
|
+
local first = 1
|
|
828
|
+
while first <= #indexes do
|
|
829
|
+
local last = first
|
|
830
|
+
while last < #indexes and indexes[last + 1] == indexes[last] + 1 do last = last + 1 end
|
|
831
|
+
local range = redis.call('XRANGE', LOG, tostring(indexes[first]) .. '-0', tostring(indexes[last]) .. '-0')
|
|
832
|
+
for _, entry in ipairs(range) do entries[entry_index(entry[1])] = entry end
|
|
833
|
+
first = last + 1
|
|
834
|
+
end
|
|
835
|
+
for _, idx in ipairs(indexes) do
|
|
836
|
+
if not entries[idx] then return { 'missing', tostring(idx) } end
|
|
837
|
+
end
|
|
838
|
+
local result = { 'claimed', tostring(#takes_over) }
|
|
839
|
+
for _, take in ipairs(takes_over) do
|
|
840
|
+
local candidate, row = take.candidate, take.row
|
|
841
|
+
local idx, at = candidate[1], candidate[2]
|
|
842
|
+
local attempt = tostring(tonumber(row[1] or '0') + 1)
|
|
843
|
+
-- A claim still held here lapsed: its attempt never finished.
|
|
844
|
+
local lapsed = row[4] ~= false
|
|
845
|
+
local failures = tonumber(row[6] or '0')
|
|
846
|
+
if lapsed then
|
|
847
|
+
failures = failures + 1
|
|
848
|
+
put_row(idx, at, 'a', tostring(failures))
|
|
849
|
+
end
|
|
850
|
+
put_row(idx, at, 'd', attempt)
|
|
851
|
+
local columns
|
|
852
|
+
if lapsed and max_failures and failures >= max_failures then
|
|
853
|
+
-- That takeover spent the last failed attempt: dead-letter, unclaimed.
|
|
854
|
+
-- Advancing the attempt fences the lapsed one; nothing runs.
|
|
855
|
+
put_row(idx, at, 'f', ARGV[2])
|
|
856
|
+
drop_row(idx, at, 'ch')
|
|
857
|
+
drop_row(idx, at, 'ce')
|
|
858
|
+
ready_set(candidate.cid, -idx)
|
|
859
|
+
release_pins(idx, at, row[11])
|
|
860
|
+
columns = { attempt, row[2], row[3], false, false, tostring(failures), row[7], row[8], row[9], ARGV[2] }
|
|
861
|
+
else
|
|
862
|
+
local first_claimed = row[2] or ARGV[2]
|
|
863
|
+
put_row(idx, at, 'fc', first_claimed)
|
|
864
|
+
put_row(idx, at, 'lc', ARGV[2])
|
|
865
|
+
put_row(idx, at, 'ch', holder)
|
|
866
|
+
put_row(idx, at, 'ce', expires_at)
|
|
867
|
+
put_row(idx, at, 'l', candidate.kind)
|
|
868
|
+
columns = { attempt, first_claimed, ARGV[2], holder, expires_at, tostring(failures), row[7], row[8], row[9], false }
|
|
869
|
+
end
|
|
870
|
+
result[#result + 1] = { candidate.consumer, candidate.kind, tostring(idx), columns, entries[idx] }
|
|
871
|
+
end
|
|
872
|
+
flush()
|
|
873
|
+
flush_ready()
|
|
874
|
+
result[#result + 1] = tostring(boundary)
|
|
875
|
+
result[#result + 1] = tostring(total)
|
|
876
|
+
return result
|
|
877
|
+
`;
|
|
878
|
+
/**
|
|
879
|
+
* KEYS: counter.
|
|
880
|
+
* ARGV: holder, nowMs, expiresAtMs, n, then per claim: consumer, tag, index,
|
|
881
|
+
* attempt. Returns [renewedCount, ...renewed, ...superseded] as 1-based
|
|
882
|
+
* positions in the request.
|
|
883
|
+
*/
|
|
884
|
+
const RENEW_CLAIMS_LUA = `${BASE_LUA}
|
|
885
|
+
read_bounds()
|
|
886
|
+
local holder, now, expires_at = ARGV[1], tonumber(ARGV[2]), ARGV[3]
|
|
887
|
+
local refs, items, superseded = {}, {}, {}
|
|
888
|
+
for i = 1, tonumber(ARGV[4]) do
|
|
889
|
+
local o = 4 + (i - 1) * 4
|
|
890
|
+
local idx = tonumber(ARGV[o + 3])
|
|
891
|
+
if idx <= boundary then
|
|
892
|
+
superseded[#superseded + 1] = tostring(i)
|
|
893
|
+
else
|
|
894
|
+
local row = row_of(ARGV[o + 1], ARGV[o + 2])
|
|
895
|
+
refs[#refs + 1] = { idx, row }
|
|
896
|
+
items[#items + 1] = { position = i, idx = idx, row = row, tag = ARGV[o + 2], attempt = tonumber(ARGV[o + 4]) }
|
|
897
|
+
end
|
|
898
|
+
end
|
|
899
|
+
local rows = read_rows(refs, { 'd', 'ce', 'p', 'f', 'ch', 'l' })
|
|
900
|
+
local renewed = {}
|
|
901
|
+
for k, item in ipairs(items) do
|
|
902
|
+
local row = rows[k]
|
|
903
|
+
local attempt = tonumber(row[1] or '0')
|
|
904
|
+
if row[6] and not owns(row[6], item.tag) then
|
|
905
|
+
-- Another cursor's head: this one never claimed it.
|
|
906
|
+
elseif row[3] or attempt > item.attempt then
|
|
907
|
+
superseded[#superseded + 1] = tostring(item.position)
|
|
908
|
+
elseif attempt == item.attempt and tonumber(expires_at) > now and
|
|
909
|
+
not row[4] and row[5] == holder and tonumber(row[2] or '0') > now then
|
|
910
|
+
put_row(item.idx, item.row, 'ce', expires_at)
|
|
911
|
+
renewed[#renewed + 1] = tostring(item.position)
|
|
912
|
+
end
|
|
913
|
+
end
|
|
914
|
+
flush()
|
|
915
|
+
local result = { tostring(#renewed) }
|
|
916
|
+
for i = 1, #renewed do result[#result + 1] = renewed[i] end
|
|
917
|
+
for i = 1, #superseded do result[#result + 1] = superseded[i] end
|
|
918
|
+
return result
|
|
919
|
+
`;
|
|
920
|
+
/**
|
|
921
|
+
* KEYS: counter.
|
|
922
|
+
* ARGV: sessionId, nowMs, consumer, tag, index, attempt, child causeJson,
|
|
923
|
+
* served consumers, returned idsJson, n, then per child: id, type,
|
|
924
|
+
* payloadJson, targets.
|
|
925
|
+
*/
|
|
926
|
+
const COMPLETE_ATTEMPT_LUA = `${BASE_LUA}${CURSOR_LUA}
|
|
927
|
+
local now = ARGV[2]
|
|
928
|
+
local consumer, tag, idx, attempt = ARGV[3], ARGV[4], tonumber(ARGV[5]), tonumber(ARGV[6])
|
|
929
|
+
local at = row_of(consumer, tag)
|
|
930
|
+
local n = tonumber(ARGV[10])
|
|
931
|
+
local ids = {}
|
|
932
|
+
for i = 1, n do ids[i] = ARGV[10 + (i - 1) * 4 + 1] end
|
|
933
|
+
local owners = owners_of(read_bounds(ids))
|
|
934
|
+
if idx <= boundary then return { 'superseded' } end
|
|
935
|
+
if idx < 1 or idx > total then return { 'missing' } end
|
|
936
|
+
local row = read_rows({ { idx, at } }, { 'd', 'p', 'pa', 'ri', 'ch', 'f', 'l', 'next', 'pins' })[1]
|
|
937
|
+
if row[2] then
|
|
938
|
+
-- The cursor moved past this head: a retry of its completion replays.
|
|
939
|
+
if not owns(row[7], tag) then return { 'superseded' } end
|
|
940
|
+
if not row[3] or tonumber(row[3]) ~= attempt then return { 'superseded' } end
|
|
941
|
+
local recorded = cjson.decode(row[4] or '[]')
|
|
942
|
+
if #recorded ~= n then return { 'partial' } end
|
|
943
|
+
for i = 1, n do
|
|
944
|
+
if recorded[i] ~= ARGV[10 + (i - 1) * 4 + 1] then return { 'partial' } end
|
|
945
|
+
end
|
|
946
|
+
local result = { 'duplicate' }
|
|
947
|
+
for i = 1, n do
|
|
948
|
+
if not owners[i] then return { 'partial' } end
|
|
949
|
+
result[#result + 1] = owners[i]
|
|
950
|
+
end
|
|
951
|
+
return result
|
|
952
|
+
end
|
|
953
|
+
if tonumber(row[1] or '0') ~= attempt or not row[5] or row[6] or not owns(row[7], tag) then
|
|
954
|
+
return { 'superseded' }
|
|
955
|
+
end
|
|
956
|
+
for i = 1, n do
|
|
957
|
+
if owners[i] then return { 'partial' } end
|
|
958
|
+
end
|
|
959
|
+
use_served(ARGV[8])
|
|
960
|
+
local batch = {}
|
|
961
|
+
for i = 1, n do
|
|
962
|
+
local o = 10 + (i - 1) * 4
|
|
963
|
+
batch[i] = { idx = total + i, type = ARGV[o + 2], targets = targets_of(ARGV[o + 4]) }
|
|
964
|
+
end
|
|
965
|
+
local moved, missing = advance(consumer, row[7], idx, row[8], batch)
|
|
966
|
+
if not moved then return { 'index', missing, tostring(idx) } end
|
|
967
|
+
local planned, unindexed, after = plan(batch, total)
|
|
968
|
+
if not planned then return { 'index', unindexed, tostring(after) } end
|
|
969
|
+
put_row(idx, at, 'p', now)
|
|
970
|
+
put_row(idx, at, 'pa', ARGV[6])
|
|
971
|
+
put_row(idx, at, 'ri', ARGV[9])
|
|
972
|
+
release_pins(idx, at, row[9])
|
|
973
|
+
local marks = {}
|
|
974
|
+
for i = 1, n do
|
|
975
|
+
local o = 10 + (i - 1) * 4
|
|
976
|
+
local field = tostring(total + i)
|
|
977
|
+
redis.call('XADD', LOG, field .. '-0',
|
|
978
|
+
'id', ARGV[o + 1], 'type', ARGV[o + 2], 'payload', ARGV[o + 3],
|
|
979
|
+
'created_at', now, 'cause', ARGV[7])
|
|
980
|
+
marks[#marks + 1] = id_key(ARGV[o + 1])
|
|
981
|
+
marks[#marks + 1] = field
|
|
982
|
+
end
|
|
983
|
+
index_batch(batch)
|
|
984
|
+
enqueue(batch)
|
|
985
|
+
if n > 0 then
|
|
986
|
+
marks[#marks + 1] = COUNTER
|
|
987
|
+
marks[#marks + 1] = tostring(total + n)
|
|
988
|
+
end
|
|
989
|
+
if typed_dirty then
|
|
990
|
+
marks[#marks + 1] = TYPED
|
|
991
|
+
marks[#marks + 1] = cjson.encode(typed)
|
|
992
|
+
end
|
|
993
|
+
sliced('MSET', nil, marks)
|
|
994
|
+
flush_states()
|
|
995
|
+
flush()
|
|
996
|
+
flush_ready()
|
|
997
|
+
return { 'ok', tostring(total + 1) }
|
|
998
|
+
`;
|
|
999
|
+
/**
|
|
1000
|
+
* KEYS: counter.
|
|
1001
|
+
* ARGV: consumer, tag, index, attempt, error, maxFailures, nowMs, retryAtMs
|
|
1002
|
+
* or ''.
|
|
1003
|
+
*/
|
|
1004
|
+
const FAIL_ATTEMPT_LUA = `${BASE_LUA}
|
|
1005
|
+
read_bounds()
|
|
1006
|
+
local consumer, tag, idx, attempt = ARGV[1], ARGV[2], tonumber(ARGV[3]), tonumber(ARGV[4])
|
|
1007
|
+
local at = row_of(consumer, tag)
|
|
1008
|
+
if idx <= boundary then return { 'superseded', '0' } end
|
|
1009
|
+
if idx < 1 or idx > total then return { 'missing', '0' } end
|
|
1010
|
+
local row = read_rows({ { idx, at } }, { 'a', 'd', 'p', 'f', 'lfa', 'ch', 'l', 'pins' })[1]
|
|
1011
|
+
if row[3] or (row[7] and not owns(row[7], tag)) then return { 'superseded', '0' } end
|
|
1012
|
+
local failures = tonumber(row[1] or '0')
|
|
1013
|
+
if tonumber(row[2] or '0') ~= attempt then return { 'superseded', tostring(failures) } end
|
|
1014
|
+
if not row[6] and row[5] and tonumber(row[5]) == attempt then
|
|
1015
|
+
return { row[4] and 'dead_lettered' or 'failed', tostring(failures) }
|
|
1016
|
+
end
|
|
1017
|
+
if row[4] then return { 'dead_lettered', tostring(failures) } end
|
|
1018
|
+
if not row[6] then return { 'superseded', tostring(failures) } end
|
|
1019
|
+
failures = failures + 1
|
|
1020
|
+
local dead = failures >= tonumber(ARGV[6])
|
|
1021
|
+
put_row(idx, at, 'a', tostring(failures))
|
|
1022
|
+
put_row(idx, at, 'e', ARGV[5])
|
|
1023
|
+
put_row(idx, at, 'lf', ARGV[7])
|
|
1024
|
+
put_row(idx, at, 'lfa', ARGV[4])
|
|
1025
|
+
drop_row(idx, at, 'ch')
|
|
1026
|
+
if dead then
|
|
1027
|
+
put_row(idx, at, 'f', ARGV[7])
|
|
1028
|
+
drop_row(idx, at, 'ce')
|
|
1029
|
+
ready_set(cid_for(consumer, row[7], idx), -idx)
|
|
1030
|
+
release_pins(idx, at, row[8])
|
|
1031
|
+
elseif ARGV[8] ~= '' then
|
|
1032
|
+
put_row(idx, at, 'ce', ARGV[8])
|
|
1033
|
+
else
|
|
1034
|
+
drop_row(idx, at, 'ce')
|
|
1035
|
+
end
|
|
1036
|
+
flush()
|
|
1037
|
+
flush_ready()
|
|
1038
|
+
return { dead and 'dead_lettered' or 'failed', tostring(failures) }
|
|
1039
|
+
`;
|
|
1040
|
+
/**
|
|
1041
|
+
* KEYS: counter. ARGV: consumer, index, action, nowMs, served
|
|
1042
|
+
* consumers. Returns [redriven, hasPending] | ['index', type, after]. Acts
|
|
1043
|
+
* only on the consumer's dead-lettered head at the index, above the clear
|
|
1044
|
+
* marker.
|
|
1045
|
+
*/
|
|
1046
|
+
const REDRIVE_LUA = `${BASE_LUA}${CURSOR_LUA}
|
|
1047
|
+
read_bounds()
|
|
1048
|
+
use_served(ARGV[5])
|
|
1049
|
+
local consumer, idx = ARGV[1], tonumber(ARGV[2])
|
|
1050
|
+
local function result(redriven) return { redriven, has_pending() } end
|
|
1051
|
+
if idx <= boundary or idx < 1 or idx > total then return result('0') end
|
|
1052
|
+
local cid, tag = nil, nil
|
|
1053
|
+
for _, member in ipairs(redis.call('ZRANGEBYSCORE', READY, -idx, -idx)) do
|
|
1054
|
+
local owner, member_tag = parse_cid(member)
|
|
1055
|
+
if owner == consumer then
|
|
1056
|
+
cid, tag = member, member_tag
|
|
1057
|
+
break
|
|
1058
|
+
end
|
|
1059
|
+
end
|
|
1060
|
+
if not cid then return result('0') end
|
|
1061
|
+
local kind = string.sub(tag, 1, 1) == '#' and '#' or tag
|
|
1062
|
+
local at = row_of(consumer, tag)
|
|
1063
|
+
if ARGV[3] == 'skip' then
|
|
1064
|
+
local row = read_rows({ { idx, at } }, { 'next' })[1]
|
|
1065
|
+
local moved, missing = advance(consumer, kind, idx, row[1], {})
|
|
1066
|
+
if not moved then return { 'index', missing, tostring(idx) } end
|
|
1067
|
+
put_row(idx, at, 'p', ARGV[4])
|
|
1068
|
+
flush_states()
|
|
1069
|
+
if typed_dirty then redis.call('SET', TYPED, cjson.encode(typed)) end
|
|
1070
|
+
else
|
|
1071
|
+
drop_row(idx, at, 'f')
|
|
1072
|
+
drop_row(idx, at, 'a')
|
|
1073
|
+
ready_set(cid, idx)
|
|
1074
|
+
end
|
|
1075
|
+
flush()
|
|
1076
|
+
flush_ready()
|
|
1077
|
+
return result('1')
|
|
1078
|
+
`;
|
|
1079
|
+
/**
|
|
1080
|
+
* KEYS: counter. ARGV: consumer, lane, types JSON or '*', backfill
|
|
1081
|
+
* ('1' or '0'). Brings one log cursor up to date without an append, as
|
|
1082
|
+
* `plan` and `enqueue` would for an append carrying its types: a missing
|
|
1083
|
+
* cursor starts at the clear marker with backfill, else at the head with
|
|
1084
|
+
* nothing to run; an idle one takes the next stored event of its types; one
|
|
1085
|
+
* with a head, claimed or not, keeps it. Records the types, and writes
|
|
1086
|
+
* nothing when nothing changed. Returns ['ok', position, head or nil, head
|
|
1087
|
+
* columns, session head] | ['index', type, after].
|
|
1088
|
+
*/
|
|
1089
|
+
const WAKE_LOG_LUA = `${BASE_LUA}${CURSOR_LUA}
|
|
1090
|
+
read_bounds()
|
|
1091
|
+
local consumer, lane, ty = ARGV[1], ARGV[2], ARGV[3]
|
|
1092
|
+
local types = types_of(ty)
|
|
1093
|
+
local cid = cid_of(consumer, 'G' .. lane)
|
|
1094
|
+
load_states({ cid })
|
|
1095
|
+
local state = states[cid]
|
|
1096
|
+
local had_head = state.h
|
|
1097
|
+
-- Nothing after the latest clear: a missing cursor stays missing.
|
|
1098
|
+
if not state.exists and total == boundary then
|
|
1099
|
+
return { 'ok', tostring(boundary), false, {}, tostring(total) }
|
|
1100
|
+
end
|
|
1101
|
+
if not state.exists then
|
|
1102
|
+
state.exists, state.p = true, total
|
|
1103
|
+
if ARGV[4] == '1' then
|
|
1104
|
+
local head, missing = stored_head(boundary, types)
|
|
1105
|
+
if head == nil then return { 'index', missing, tostring(boundary) } end
|
|
1106
|
+
state.p, state.h = boundary, head
|
|
1107
|
+
else
|
|
1108
|
+
-- Nothing it takes is stored after its start: index its types from there.
|
|
1109
|
+
for name in pairs(types or {}) do
|
|
1110
|
+
if typed[name] == nil then register(name, total) end
|
|
1111
|
+
end
|
|
1112
|
+
end
|
|
1113
|
+
state.dirty = true
|
|
1114
|
+
elseif not state.h then
|
|
1115
|
+
-- A deployment that doesn't serve this consumer may have appended events
|
|
1116
|
+
-- it takes since it went idle.
|
|
1117
|
+
local head, missing = stored_head(state.p, types)
|
|
1118
|
+
if head == nil then return { 'index', missing, tostring(state.p) } end
|
|
1119
|
+
if head then state.h, state.dirty = head, true end
|
|
1120
|
+
end
|
|
1121
|
+
if state.ty ~= ty then state.ty, state.dirty = ty, true end
|
|
1122
|
+
local columns = {}
|
|
1123
|
+
if had_head then
|
|
1124
|
+
columns = read_rows({ { state.h, row_of(consumer, 'G' .. lane) } }, ${HEAD_COLUMNS})[1]
|
|
1125
|
+
elseif state.h then
|
|
1126
|
+
ready_set(cid, state.h)
|
|
1127
|
+
end
|
|
1128
|
+
flush_states()
|
|
1129
|
+
if typed_dirty then redis.call('SET', TYPED, cjson.encode(typed)) end
|
|
1130
|
+
flush()
|
|
1131
|
+
flush_ready()
|
|
1132
|
+
return { 'ok', tostring(state.p), state.h and tostring(state.h) or false, columns, tostring(total) }
|
|
1133
|
+
`;
|
|
1134
|
+
/**
|
|
1135
|
+
* KEYS: counter. Returns the dead-lettered heads above the clear
|
|
1136
|
+
* marker as [consumer, tag, index, columns, entry] tuples.
|
|
1137
|
+
*/
|
|
1138
|
+
const DEAD_LETTERS_LUA = `${BASE_LUA}
|
|
1139
|
+
local dead = redis.call('ZRANGEBYSCORE', READY, '-inf', '(0', 'WITHSCORES')
|
|
1140
|
+
if #dead == 0 then return {} end
|
|
1141
|
+
read_bounds()
|
|
1142
|
+
local refs, items = {}, {}
|
|
1143
|
+
for i = 1, #dead, 2 do
|
|
1144
|
+
local consumer, tag = parse_cid(dead[i])
|
|
1145
|
+
local idx = -tonumber(dead[i + 1])
|
|
1146
|
+
if idx > boundary then
|
|
1147
|
+
refs[#refs + 1] = { idx, row_of(consumer, tag) }
|
|
1148
|
+
items[#items + 1] = { consumer, tag, idx }
|
|
1149
|
+
end
|
|
1150
|
+
end
|
|
1151
|
+
local rows = read_rows(refs, ${HEAD_COLUMNS})
|
|
1152
|
+
local result = {}
|
|
1153
|
+
for k, item in ipairs(items) do
|
|
1154
|
+
local entries = redis.call('XRANGE', LOG, tostring(item[3]) .. '-0', tostring(item[3]) .. '-0')
|
|
1155
|
+
if #entries > 0 then
|
|
1156
|
+
result[#result + 1] = { item[1], item[2], tostring(item[3]), rows[k], entries[1] }
|
|
1157
|
+
end
|
|
1158
|
+
end
|
|
1159
|
+
return result
|
|
1160
|
+
`;
|
|
1161
|
+
/**
|
|
1162
|
+
* KEYS: counter. Returns every listed cursor as [consumer, tag,
|
|
1163
|
+
* position, head or nil, columns]: lane and log cursors from the cursor
|
|
1164
|
+
* hash, read against the clear marker, then the event cursors with a head.
|
|
1165
|
+
*/
|
|
1166
|
+
const CURSORS_LUA = `${BASE_LUA}
|
|
1167
|
+
read_bounds()
|
|
1168
|
+
local listed = {}
|
|
1169
|
+
local fields = redis.call('HGETALL', CURSORS)
|
|
1170
|
+
local records, order = {}, {}
|
|
1171
|
+
for i = 1, #fields, 2 do
|
|
1172
|
+
local at = string.find(fields[i], ':[^:]*$')
|
|
1173
|
+
local cid, suffix = string.sub(fields[i], 1, at - 1), string.sub(fields[i], at + 1)
|
|
1174
|
+
if not records[cid] then
|
|
1175
|
+
records[cid] = {}
|
|
1176
|
+
order[#order + 1] = cid
|
|
1177
|
+
end
|
|
1178
|
+
records[cid][suffix] = fields[i + 1]
|
|
1179
|
+
end
|
|
1180
|
+
for _, cid in ipairs(order) do
|
|
1181
|
+
local record = records[cid]
|
|
1182
|
+
local consumer, tag = parse_cid(cid)
|
|
1183
|
+
local head = (record.h and record.h ~= '') and tonumber(record.h) or false
|
|
1184
|
+
local position = tonumber(record.p or '0')
|
|
1185
|
+
if head and head <= boundary then head = false end
|
|
1186
|
+
local kind = string.sub(tag, 1, 1)
|
|
1187
|
+
if kind == 'G' and position < boundary then position = boundary end
|
|
1188
|
+
if kind == 'G' or head or position > boundary then
|
|
1189
|
+
listed[#listed + 1] = { consumer, tag, position, head }
|
|
1190
|
+
end
|
|
1191
|
+
end
|
|
1192
|
+
local ready = redis.call('ZRANGE', READY, 0, -1, 'WITHSCORES')
|
|
1193
|
+
for i = 1, #ready, 2 do
|
|
1194
|
+
local consumer, tag = parse_cid(ready[i])
|
|
1195
|
+
if string.sub(tag, 1, 1) == '#' then
|
|
1196
|
+
local head = math.abs(tonumber(ready[i + 1]))
|
|
1197
|
+
listed[#listed + 1] = { consumer, '#', head - 1, head }
|
|
1198
|
+
end
|
|
1199
|
+
end
|
|
1200
|
+
local refs, positions = {}, {}
|
|
1201
|
+
for position, cursor in ipairs(listed) do
|
|
1202
|
+
if cursor[4] then
|
|
1203
|
+
refs[#refs + 1] = { cursor[4], row_of(cursor[1], cursor[2]) }
|
|
1204
|
+
positions[#refs] = position
|
|
1205
|
+
end
|
|
1206
|
+
end
|
|
1207
|
+
local rows = read_rows(refs, ${HEAD_COLUMNS})
|
|
1208
|
+
local columns = {}
|
|
1209
|
+
for k, position in ipairs(positions) do columns[position] = rows[k] end
|
|
1210
|
+
local result = {}
|
|
1211
|
+
for position, cursor in ipairs(listed) do
|
|
1212
|
+
result[#result + 1] = {
|
|
1213
|
+
cursor[1], cursor[2], tostring(cursor[3]),
|
|
1214
|
+
cursor[4] and tostring(cursor[4]) or false, columns[position] or {},
|
|
1215
|
+
}
|
|
1216
|
+
end
|
|
1217
|
+
return result
|
|
1218
|
+
`;
|
|
1219
|
+
/**
|
|
1220
|
+
* KEYS: counter, head, history.
|
|
1221
|
+
* ARGV: write count, then per write: index, snapshot JSON, pin count, then
|
|
1222
|
+
* per pin: consumer, tag, index; finally the reducer name. Writes from
|
|
1223
|
+
* before the latest clear are dropped, and a head from before it is
|
|
1224
|
+
* replaced as if absent. A pin holds only on an unfinished, live head: a
|
|
1225
|
+
* claimed row says so itself, an unclaimed head by its cursor's ready score.
|
|
1226
|
+
*/
|
|
1227
|
+
const PUT_SNAPSHOTS_LUA = `${BASE_LUA}
|
|
1228
|
+
read_bounds()
|
|
1229
|
+
local HEAD, HISTORY = KEYS[2], KEYS[3]
|
|
1230
|
+
local entries, refs, pins = {}, {}, {}
|
|
1231
|
+
local offset = 2
|
|
1232
|
+
for w = 1, tonumber(ARGV[1]) do
|
|
1233
|
+
local index = tonumber(ARGV[offset])
|
|
1234
|
+
local pin_count = tonumber(ARGV[offset + 2])
|
|
1235
|
+
local entry = { index = index, snapshot = ARGV[offset + 1], pins = {} }
|
|
1236
|
+
for p = 1, pin_count do
|
|
1237
|
+
local o = offset + 2 + (p - 1) * 3
|
|
1238
|
+
local pin = { consumer = ARGV[o + 1], tag = ARGV[o + 2], idx = tonumber(ARGV[o + 3]) }
|
|
1239
|
+
if index >= boundary and pin.idx > boundary and pin.idx <= total then
|
|
1240
|
+
pin.row = row_of(pin.consumer, pin.tag)
|
|
1241
|
+
refs[#refs + 1] = { pin.idx, pin.row }
|
|
1242
|
+
pins[#pins + 1] = pin
|
|
1243
|
+
entry.pins[#entry.pins + 1] = pin
|
|
1244
|
+
end
|
|
1245
|
+
end
|
|
1246
|
+
entries[w] = entry
|
|
1247
|
+
offset = offset + 3 + pin_count * 3
|
|
1248
|
+
end
|
|
1249
|
+
local function live(consumer, tag, idx)
|
|
1250
|
+
return tonumber(redis.call('ZSCORE', READY, cid_of(consumer, tag)) or '0') == idx
|
|
1251
|
+
end
|
|
1252
|
+
local rows = read_rows(refs, { 'pins', 'p', 'f', 'l' })
|
|
1253
|
+
local held, held_order = {}, {}
|
|
1254
|
+
for k, pin in ipairs(pins) do
|
|
1255
|
+
local row = rows[k]
|
|
1256
|
+
local valid
|
|
1257
|
+
if row[2] or row[3] then
|
|
1258
|
+
valid = false
|
|
1259
|
+
elseif row[4] then
|
|
1260
|
+
valid = owns(row[4], pin.tag)
|
|
1261
|
+
elseif pin.tag == '#' then
|
|
1262
|
+
valid = live(pin.consumer, '#' .. tostring(pin.idx), pin.idx)
|
|
1263
|
+
else
|
|
1264
|
+
valid = live(pin.consumer, pin.tag, pin.idx) or
|
|
1265
|
+
live(pin.consumer, 'G' .. string.sub(pin.tag, 2), pin.idx)
|
|
1266
|
+
end
|
|
1267
|
+
if valid then
|
|
1268
|
+
local key = tostring(pin.idx) .. ':' .. pin.row
|
|
1269
|
+
if not held[key] then
|
|
1270
|
+
local list = (row[1] and row[1] ~= '') and cjson.decode(row[1]) or {}
|
|
1271
|
+
local set = {}
|
|
1272
|
+
for _, ref in ipairs(list) do set[ref] = true end
|
|
1273
|
+
held[key] = { list = list, set = set, idx = pin.idx, row = pin.row }
|
|
1274
|
+
held_order[#held_order + 1] = key
|
|
1275
|
+
end
|
|
1276
|
+
pin.held = held[key]
|
|
1277
|
+
end
|
|
1278
|
+
end
|
|
1279
|
+
local snapshots = {}
|
|
1280
|
+
for _, entry in ipairs(entries) do
|
|
1281
|
+
local ref = cjson.encode({ HISTORY, tostring(entry.index) })
|
|
1282
|
+
for _, pin in ipairs(entry.pins) do
|
|
1283
|
+
if pin.held and not pin.held.set[ref] then
|
|
1284
|
+
pin.held.set[ref] = true
|
|
1285
|
+
pin.held.list[#pin.held.list + 1] = ref
|
|
1286
|
+
pin.held.dirty = true
|
|
1287
|
+
redis.call('HINCRBY', PIN_COUNTS, ref, 1)
|
|
1288
|
+
end
|
|
1289
|
+
end
|
|
1290
|
+
if entry.index >= boundary then
|
|
1291
|
+
snapshots[#snapshots + 1] = { index = entry.index, snapshot = entry.snapshot, ref = ref }
|
|
1292
|
+
end
|
|
1293
|
+
end
|
|
1294
|
+
for _, key in ipairs(held_order) do
|
|
1295
|
+
local row = held[key]
|
|
1296
|
+
if row.dirty then put_row(row.idx, row.row, 'pins', cjson.encode(row.list)) end
|
|
1297
|
+
end
|
|
1298
|
+
flush()
|
|
1299
|
+
if #snapshots == 0 then return 1 end
|
|
1300
|
+
|
|
1301
|
+
local head = redis.call('GET', HEAD)
|
|
1302
|
+
local head_index = nil
|
|
1303
|
+
local registered = false
|
|
1304
|
+
if head then
|
|
1305
|
+
local record = cjson.decode(head)
|
|
1306
|
+
head_index = tonumber(record['index'])
|
|
1307
|
+
registered = record['r'] == 1
|
|
1308
|
+
end
|
|
1309
|
+
if head_index and head_index < boundary then
|
|
1310
|
+
head, head_index, registered = nil, nil, false
|
|
1311
|
+
redis.call('ZREMRANGEBYSCORE', HISTORY, '-inf', '(' .. boundary)
|
|
1312
|
+
end
|
|
1313
|
+
-- Deletion finds each reducer's keys through this set instead of a scan.
|
|
1314
|
+
-- Heads written since carry the mark, so a reducer registers once.
|
|
1315
|
+
if not registered then
|
|
1316
|
+
redis.call('SADD', SESSION .. 'snap-reducers', ARGV[#ARGV])
|
|
1317
|
+
end
|
|
1318
|
+
local latest = snapshots[#snapshots]
|
|
1319
|
+
local writes_head = not head_index or latest.index >= head_index
|
|
1320
|
+
local advances = not head_index or latest.index > head_index
|
|
1321
|
+
local final_head_index = advances and latest.index or head_index
|
|
1322
|
+
|
|
1323
|
+
if advances and head and tonumber(redis.call('HGET', PIN_COUNTS, cjson.encode({ HISTORY, tostring(head_index) })) or '0') > 0 then
|
|
1324
|
+
redis.call('ZREMRANGEBYSCORE', HISTORY, head_index, head_index)
|
|
1325
|
+
redis.call('ZADD', HISTORY, head_index, head)
|
|
1326
|
+
end
|
|
1327
|
+
|
|
1328
|
+
for _, write in ipairs(snapshots) do
|
|
1329
|
+
if write.index < final_head_index then
|
|
1330
|
+
redis.call('ZREMRANGEBYSCORE', HISTORY, write.index, write.index)
|
|
1331
|
+
if tonumber(redis.call('HGET', PIN_COUNTS, write.ref) or '0') > 0 then
|
|
1332
|
+
redis.call('ZADD', HISTORY, write.index, write.snapshot)
|
|
1333
|
+
end
|
|
1334
|
+
end
|
|
1335
|
+
end
|
|
1336
|
+
|
|
1337
|
+
if writes_head then
|
|
1338
|
+
redis.call('SET', HEAD, latest.snapshot)
|
|
1339
|
+
redis.call('ZREMRANGEBYSCORE', HISTORY, latest.index, latest.index)
|
|
1340
|
+
end
|
|
1341
|
+
return 1
|
|
1342
|
+
`;
|
|
1343
|
+
/**
|
|
1344
|
+
* Presence entries are `{"a":at,"e":expiresAt,"s":seen,"v":value}`, stamps
|
|
1345
|
+
* first and both integers, so scripts read them with a pattern instead of
|
|
1346
|
+
* decoding the value.
|
|
1347
|
+
*/
|
|
1348
|
+
const PRESENCE_STAMPS_LUA = `
|
|
1349
|
+
local function stamps(entry)
|
|
1350
|
+
local a, e = string.match(entry, '^{"a":(%-?%d+),"e":(%-?%d+),')
|
|
1351
|
+
return tonumber(a), tonumber(e)
|
|
1352
|
+
end
|
|
1353
|
+
`;
|
|
1354
|
+
/**
|
|
1355
|
+
* KEYS: presence hash. ARGV: atMs, ttlMs, nowMs, n, then per entry:
|
|
1356
|
+
* hashField, entryJson (a null value is a tombstone). Field-wise LWW by the
|
|
1357
|
+
* sender's `at`: an unexpired stored entry with a strictly newer stamp
|
|
1358
|
+
* survives; ties go to the incoming write (memory-tier parity). Returns the applied 1-based entry
|
|
1359
|
+
* positions. The whole-key PEXPIRE is the coarse bound, sound because core
|
|
1360
|
+
* sends one uniform ttlMs; the per-entry `e` stamp is the fine bound (the
|
|
1361
|
+
* injected storage clock at the applied set plus ttlMs — never the sender
|
|
1362
|
+
* stamp), filtered on read against the same clock.
|
|
1363
|
+
*/
|
|
1364
|
+
const PRESENCE_SET_LUA = `${SLICED_LUA}${PRESENCE_STAMPS_LUA}
|
|
1365
|
+
local at, now, n = tonumber(ARGV[1]), tonumber(ARGV[3]), tonumber(ARGV[4])
|
|
1366
|
+
local fields = {}
|
|
1367
|
+
for i = 1, n do fields[i] = ARGV[4 + (i - 1) * 2 + 1] end
|
|
1368
|
+
local stored = sliced('HMGET', KEYS[1], fields)
|
|
1369
|
+
local applied, sets = {}, {}
|
|
1370
|
+
for i = 1, n do
|
|
1371
|
+
local a, e = false, false
|
|
1372
|
+
if stored[i] then a, e = stamps(stored[i]) end
|
|
1373
|
+
if not stored[i] or a <= at or e <= now then
|
|
1374
|
+
sets[#sets + 1] = fields[i]
|
|
1375
|
+
sets[#sets + 1] = ARGV[4 + (i - 1) * 2 + 2]
|
|
1376
|
+
applied[#applied + 1] = tostring(i)
|
|
1377
|
+
end
|
|
1378
|
+
end
|
|
1379
|
+
sliced('HSET', KEYS[1], sets)
|
|
1380
|
+
if #applied > 0 then redis.call('PEXPIRE', KEYS[1], ARGV[2]) end
|
|
1381
|
+
return applied
|
|
1382
|
+
`;
|
|
1383
|
+
/**
|
|
1384
|
+
* KEYS: presence hash. ARGV: nowMs, then the fields a read saw expired.
|
|
1385
|
+
* Deletes those still at or past their expiry, re-checked in this atomic
|
|
1386
|
+
* step so a stale reader never sweeps a concurrently refreshed field.
|
|
1387
|
+
*/
|
|
1388
|
+
const PRESENCE_PRUNE_LUA = `${SLICED_LUA}${PRESENCE_STAMPS_LUA}
|
|
1389
|
+
local now = tonumber(ARGV[1])
|
|
1390
|
+
local fields = {}
|
|
1391
|
+
for i = 2, #ARGV do fields[#fields + 1] = ARGV[i] end
|
|
1392
|
+
local stored = sliced('HMGET', KEYS[1], fields)
|
|
1393
|
+
local stale = {}
|
|
1394
|
+
for i = 1, #fields do
|
|
1395
|
+
if stored[i] then
|
|
1396
|
+
local _, e = stamps(stored[i])
|
|
1397
|
+
if e <= now then stale[#stale + 1] = fields[i] end
|
|
1398
|
+
end
|
|
1399
|
+
end
|
|
1400
|
+
sliced('HDEL', KEYS[1], stale)
|
|
1401
|
+
return 1
|
|
1402
|
+
`;
|
|
1403
|
+
/**
|
|
1404
|
+
* KEYS: counter, head, history. ARGV: snapshot frontier or ''.
|
|
1405
|
+
* Returns one atomic multiversion-cache-plus-log-frontier read, with the
|
|
1406
|
+
* clear boundary; checkpoints from before it are ignored.
|
|
1407
|
+
*/
|
|
1408
|
+
const READ_STATE_LUA = `
|
|
1409
|
+
local session = string.sub(KEYS[1], 1, -6)
|
|
1410
|
+
local counts = redis.call('MGET', KEYS[1], session .. 'clear')
|
|
1411
|
+
local boundary = tonumber(counts[2] or '0')
|
|
1412
|
+
local head = redis.call('GET', KEYS[2])
|
|
1413
|
+
local head_index = head and cjson.decode(head)['index'] or nil
|
|
1414
|
+
if head and (type(head_index) ~= 'number' or head_index < 0 or head_index > 9007199254740991 or head_index ~= math.floor(head_index)) then
|
|
1415
|
+
return redis.error_reply('invalid snapshot head index')
|
|
1416
|
+
end
|
|
1417
|
+
if head_index and head_index < boundary then
|
|
1418
|
+
head = nil
|
|
1419
|
+
head_index = nil
|
|
1420
|
+
end
|
|
1421
|
+
local cap = ARGV[1]
|
|
1422
|
+
local snapshot = nil
|
|
1423
|
+
if head_index and (cap == '' or head_index <= tonumber(cap)) then
|
|
1424
|
+
snapshot = head
|
|
1425
|
+
else
|
|
1426
|
+
local upper = cap == '' and '+inf' or cap
|
|
1427
|
+
local historical = redis.call('ZREVRANGEBYSCORE', KEYS[3], upper, boundary, 'LIMIT', 0, 1)
|
|
1428
|
+
if #historical > 0 then snapshot = historical[1] end
|
|
1429
|
+
end
|
|
1430
|
+
return { head_index or '', snapshot or '', counts[1] or counts[2] or '0', tostring(boundary) }
|
|
1431
|
+
`;
|
|
1432
|
+
/**
|
|
1433
|
+
* What reclaim removes once no entry is left below the latest clear: every
|
|
1434
|
+
* registered reducer's checkpoints below the latest clear and the pin counts
|
|
1435
|
+
* naming them (snapshot writes register each reducer in
|
|
1436
|
+
* {prefix}:{session}:snap-reducers, so no key scan is needed), every head
|
|
1437
|
+
* row below it (whole chunks, then the boundary chunk's older rows), the
|
|
1438
|
+
* type indexes' older entries, and the lane cursors it left idle.
|
|
1439
|
+
*
|
|
1440
|
+
* A lifetime that ended empty then collapses to the clear marker, the log
|
|
1441
|
+
* holding it (readers resuming before the marker still read it), and the log
|
|
1442
|
+
* cursors the sweep reset to the marker. The counter reads from the marker
|
|
1443
|
+
* when absent, type indexes rebuild lazily, and the marker's own id key goes
|
|
1444
|
+
* too.
|
|
1445
|
+
*/
|
|
1446
|
+
const SWEEP_LUA = `
|
|
1447
|
+
local function sweep(boundary)
|
|
1448
|
+
for _, reducer in ipairs(redis.call('SMEMBERS', SESSION .. 'snap-reducers')) do
|
|
1449
|
+
local head_key = SESSION .. 'snap:' .. reducer
|
|
1450
|
+
local head = redis.call('GET', head_key)
|
|
1451
|
+
if head and tonumber(cjson.decode(head)['index']) < boundary then
|
|
1452
|
+
redis.call('DEL', head_key)
|
|
1453
|
+
end
|
|
1454
|
+
redis.call('ZREMRANGEBYSCORE', SESSION .. 'snap-history:' .. reducer, '-inf', '(' .. boundary)
|
|
1455
|
+
end
|
|
1456
|
+
local counts = redis.call('HGETALL', PIN_COUNTS)
|
|
1457
|
+
local stale = {}
|
|
1458
|
+
for i = 1, #counts, 2 do
|
|
1459
|
+
if tonumber(cjson.decode(counts[i])[2]) < boundary then stale[#stale + 1] = counts[i] end
|
|
1460
|
+
end
|
|
1461
|
+
sliced('HDEL', PIN_COUNTS, stale)
|
|
1462
|
+
local raw = redis.call('GET', TYPED)
|
|
1463
|
+
if raw then
|
|
1464
|
+
for name in pairs(cjson.decode(raw)) do
|
|
1465
|
+
redis.call('ZREMRANGEBYSCORE', type_key(name), '-inf', '(' .. boundary)
|
|
1466
|
+
end
|
|
1467
|
+
end
|
|
1468
|
+
local last = math.floor((boundary - 1) / ${META_CHUNK_SIZE})
|
|
1469
|
+
local chunks = {}
|
|
1470
|
+
for chunk = 0, last - 1 do chunks[#chunks + 1] = META .. ':' .. tostring(chunk) end
|
|
1471
|
+
sliced('UNLINK', nil, chunks)
|
|
1472
|
+
local older = {}
|
|
1473
|
+
for _, field in ipairs(redis.call('HKEYS', META .. ':' .. tostring(last))) do
|
|
1474
|
+
local idx = tonumber(string.match(field, '^(%d+):'))
|
|
1475
|
+
if idx and idx < boundary then older[#older + 1] = field end
|
|
1476
|
+
end
|
|
1477
|
+
sliced('HDEL', META .. ':' .. tostring(last), older)
|
|
1478
|
+
local fields = redis.call('HGETALL', CURSORS)
|
|
1479
|
+
local records, order = {}, {}
|
|
1480
|
+
for i = 1, #fields, 2 do
|
|
1481
|
+
local at = string.find(fields[i], ':[^:]*$')
|
|
1482
|
+
local cid, suffix = string.sub(fields[i], 1, at - 1), string.sub(fields[i], at + 1)
|
|
1483
|
+
if not records[cid] then
|
|
1484
|
+
records[cid] = {}
|
|
1485
|
+
order[#order + 1] = cid
|
|
1486
|
+
end
|
|
1487
|
+
records[cid][suffix] = fields[i + 1]
|
|
1488
|
+
end
|
|
1489
|
+
local idle, moved = {}, {}
|
|
1490
|
+
for _, cid in ipairs(order) do
|
|
1491
|
+
local record = records[cid]
|
|
1492
|
+
local _, tag = parse_cid(cid)
|
|
1493
|
+
local head = (record.h and record.h ~= '') and tonumber(record.h) or false
|
|
1494
|
+
local position = tonumber(record.p or '0')
|
|
1495
|
+
if head and head <= boundary then head = false end
|
|
1496
|
+
if string.sub(tag, 1, 1) == 'G' then
|
|
1497
|
+
if position < boundary then position = boundary end
|
|
1498
|
+
moved[#moved + 1] = cid .. ':h'
|
|
1499
|
+
moved[#moved + 1] = head and tostring(head) or ''
|
|
1500
|
+
moved[#moved + 1] = cid .. ':p'
|
|
1501
|
+
moved[#moved + 1] = tostring(position)
|
|
1502
|
+
elseif not head and position <= boundary then
|
|
1503
|
+
for suffix in pairs(record) do idle[#idle + 1] = cid .. ':' .. suffix end
|
|
1504
|
+
elseif not head and record.h ~= '' then
|
|
1505
|
+
moved[#moved + 1] = cid .. ':h'
|
|
1506
|
+
moved[#moved + 1] = ''
|
|
1507
|
+
end
|
|
1508
|
+
end
|
|
1509
|
+
sliced('HDEL', CURSORS, idle)
|
|
1510
|
+
sliced('HSET', CURSORS, moved)
|
|
1511
|
+
local count = redis.call('GET', COUNTER)
|
|
1512
|
+
if not count or tonumber(count) ~= boundary or redis.call('ZCARD', READY) > 0 then
|
|
1513
|
+
return
|
|
1514
|
+
end
|
|
1515
|
+
local stale = { COUNTER, TYPED, PIN_COUNTS, chunk_key(boundary) }
|
|
1516
|
+
local reducers = SESSION .. 'snap-reducers'
|
|
1517
|
+
for _, reducer in ipairs(redis.call('SMEMBERS', reducers)) do
|
|
1518
|
+
stale[#stale + 1] = SESSION .. 'snap:' .. reducer
|
|
1519
|
+
stale[#stale + 1] = SESSION .. 'snap-history:' .. reducer
|
|
1520
|
+
end
|
|
1521
|
+
stale[#stale + 1] = reducers
|
|
1522
|
+
local types = redis.call('GET', TYPED)
|
|
1523
|
+
if types then
|
|
1524
|
+
for name in pairs(cjson.decode(types)) do stale[#stale + 1] = type_key(name) end
|
|
1525
|
+
end
|
|
1526
|
+
local marker = redis.call('XRANGE', LOG, boundary .. '-0', boundary .. '-0')[1]
|
|
1527
|
+
if marker then
|
|
1528
|
+
for i = 1, #marker[2], 2 do
|
|
1529
|
+
if marker[2][i] == 'id' then stale[#stale + 1] = id_key(marker[2][i + 1]) end
|
|
1530
|
+
end
|
|
1531
|
+
end
|
|
1532
|
+
sliced('UNLINK', nil, stale)
|
|
1533
|
+
end
|
|
1534
|
+
`;
|
|
1535
|
+
/**
|
|
1536
|
+
* KEYS: counter.
|
|
1537
|
+
* ARGV: sessionId, nowMs, ifLastIndex or '', cause consumer or '', cause tag,
|
|
1538
|
+
* cause index, cause attempt, marker id.
|
|
1539
|
+
* Returns ['ok', cleared, head, boundary, appended?] | ['superseded'] |
|
|
1540
|
+
* ['foreign']; `appended` is '1' when this call appended the marker.
|
|
1541
|
+
* The marker joins the same log: the old lifetime's entries stay below it
|
|
1542
|
+
* until reclaim deletes them, and every read of the log starts at the
|
|
1543
|
+
* marker. (A stream can't be renamed aside: Upstash refuses RENAME on one.)
|
|
1544
|
+
* The ready set is unlinked, so claims and pending checks see only the new
|
|
1545
|
+
* lifetime. Type indexes keep their older entries until reclaim too:
|
|
1546
|
+
* lookups start above the marker.
|
|
1547
|
+
*/
|
|
1548
|
+
const CLEAR_LUA = `${BASE_LUA}
|
|
1549
|
+
local marker_owner = owners_of(read_bounds({ ARGV[8] }))[1]
|
|
1550
|
+
local head = total
|
|
1551
|
+
if ARGV[4] ~= '' then
|
|
1552
|
+
local idx = tonumber(ARGV[6])
|
|
1553
|
+
if idx <= boundary then return { 'superseded' } end
|
|
1554
|
+
local row = read_rows({ { idx, row_of(ARGV[4], ARGV[5]) } }, { 'd', 'f', 'p', 'l' })[1]
|
|
1555
|
+
if tonumber(row[1] or '0') ~= tonumber(ARGV[7]) or row[2] or row[3] or not owns(row[4], ARGV[5]) then
|
|
1556
|
+
return { 'superseded' }
|
|
1557
|
+
end
|
|
1558
|
+
end
|
|
1559
|
+
local expected = ARGV[3]
|
|
1560
|
+
if expected ~= '' and boundary == tonumber(expected) + 1 then
|
|
1561
|
+
return { 'ok', '1', tostring(head), tostring(boundary) }
|
|
1562
|
+
end
|
|
1563
|
+
if expected ~= '' and tonumber(expected) ~= head then
|
|
1564
|
+
return { 'ok', '0', tostring(head), tostring(boundary) }
|
|
1565
|
+
end
|
|
1566
|
+
if head == 0 or boundary == head then
|
|
1567
|
+
return { 'ok', '1', tostring(head), tostring(boundary) }
|
|
1568
|
+
end
|
|
1569
|
+
if marker_owner then return { 'foreign' } end
|
|
1570
|
+
local field = tostring(head + 1)
|
|
1571
|
+
redis.call('UNLINK', READY)
|
|
1572
|
+
redis.call('XADD', LOG, field .. '-0',
|
|
1573
|
+
'id', ARGV[8], 'type', CLEAR_TYPE, 'payload', 'null',
|
|
1574
|
+
'created_at', ARGV[2], 'cause', '')
|
|
1575
|
+
redis.call('MSET', id_key(ARGV[8]), field, COUNTER, field, CLEAR, field)
|
|
1576
|
+
return { 'ok', '1', field, field, '1' }
|
|
1577
|
+
`;
|
|
1578
|
+
/**
|
|
1579
|
+
* KEYS: counter. ARGV: page size.
|
|
1580
|
+
* The next deletion batch: up to a page of [index, id] pairs of the log's
|
|
1581
|
+
* entries below the latest clear. With none left it sweeps what the old
|
|
1582
|
+
* lifetimes left and returns nothing, so a retry finishes an interrupted
|
|
1583
|
+
* deletion.
|
|
1584
|
+
*/
|
|
1585
|
+
const RECLAIM_BATCH_LUA = `${BASE_LUA}${SWEEP_LUA}
|
|
1586
|
+
local boundary = tonumber(redis.call('GET', CLEAR) or '0')
|
|
1587
|
+
if boundary == 0 then return {} end
|
|
1588
|
+
local entries = redis.call('XRANGE', LOG, '-', tostring(boundary - 1) .. '-0', 'COUNT', ARGV[1])
|
|
1589
|
+
if #entries == 0 then
|
|
1590
|
+
sweep(boundary)
|
|
1591
|
+
return {}
|
|
1592
|
+
end
|
|
1593
|
+
local batch = {}
|
|
1594
|
+
for _, entry in ipairs(entries) do
|
|
1595
|
+
local fields = entry[2]
|
|
1596
|
+
for i = 1, #fields, 2 do
|
|
1597
|
+
if fields[i] == 'id' then
|
|
1598
|
+
batch[#batch + 1] = string.match(entry[1], '^(%d+)')
|
|
1599
|
+
batch[#batch + 1] = fields[i + 1]
|
|
1600
|
+
break
|
|
1601
|
+
end
|
|
1602
|
+
end
|
|
1603
|
+
end
|
|
1604
|
+
return batch
|
|
1605
|
+
`;
|
|
1606
|
+
/**
|
|
1607
|
+
* KEYS: counter.
|
|
1608
|
+
* ARGV: sessionId, n, then per entry: index, id.
|
|
1609
|
+
* Deletes the listed entries below the latest clear and their id records.
|
|
1610
|
+
* An id record survives when it names another event, so an id reused in the
|
|
1611
|
+
* new lifetime keeps its owner. Returns [deleted, done].
|
|
1612
|
+
*/
|
|
1613
|
+
const RECLAIM_LUA = `${BASE_LUA}${SWEEP_LUA}
|
|
1614
|
+
local boundary = tonumber(redis.call('GET', CLEAR) or '0')
|
|
1615
|
+
local ids, indexes, stream_ids = {}, {}, {}
|
|
1616
|
+
-- A concurrent reclaim may list the same batch; only XDEL says who removed it.
|
|
1617
|
+
local removed = 0
|
|
1618
|
+
for i = 1, tonumber(ARGV[2]) do
|
|
1619
|
+
local field = ARGV[2 + (i - 1) * 2 + 1]
|
|
1620
|
+
if tonumber(field) < boundary then
|
|
1621
|
+
ids[#ids + 1] = ARGV[2 + (i - 1) * 2 + 2]
|
|
1622
|
+
indexes[#ids] = tonumber(field)
|
|
1623
|
+
stream_ids[#ids] = field .. '-0'
|
|
1624
|
+
end
|
|
1625
|
+
end
|
|
1626
|
+
if #ids > 0 then
|
|
1627
|
+
local keys = {}
|
|
1628
|
+
for i = 1, #ids do keys[i] = id_key(ids[i]) end
|
|
1629
|
+
local owners, stale = sliced('MGET', nil, keys), {}
|
|
1630
|
+
for i = 1, #ids do
|
|
1631
|
+
if owners[i] and tonumber(owners[i]) == indexes[i] then
|
|
1632
|
+
stale[#stale + 1] = keys[i]
|
|
1633
|
+
end
|
|
1634
|
+
end
|
|
1635
|
+
sliced('DEL', nil, stale)
|
|
1636
|
+
removed = redis.call('XDEL', LOG, unpack(stream_ids))
|
|
1637
|
+
end
|
|
1638
|
+
local done = 0
|
|
1639
|
+
if #redis.call('XRANGE', LOG, '-', tostring(boundary - 1) .. '-0', 'COUNT', 1) == 0 then
|
|
1640
|
+
sweep(boundary)
|
|
1641
|
+
done = 1
|
|
1642
|
+
end
|
|
1643
|
+
return { removed, done }
|
|
1644
|
+
`;
|
|
1645
|
+
const WAKE_PAYLOAD_LIMIT = 32768;
|
|
1646
|
+
const encoder = new TextEncoder();
|
|
1647
|
+
/**
|
|
1648
|
+
* A wake-up announcing events through `frontier`. It carries them, so a
|
|
1649
|
+
* caught-up reader applies them without reading the stream:
|
|
1650
|
+
* `{"f":frontier,"t":createdAt,"e":[[index,id,type,payload],...]}`, with
|
|
1651
|
+
* payloads inlined as their stored JSON text. One over WAKE_PAYLOAD_LIMIT
|
|
1652
|
+
* bytes, or one without entries, announces only `{"f":frontier}`, and
|
|
1653
|
+
* readers catch up with XRANGE.
|
|
1654
|
+
*/
|
|
1655
|
+
const wakeMessage = (frontier, createdAt, entries) => {
|
|
1656
|
+
const bare = `{"f":${frontier}}`;
|
|
1657
|
+
if (entries.length === 0) return bare;
|
|
1658
|
+
const carried = `{"f":${frontier},"t":${createdAt},"e":[${entries.map(([index, id, type, payload]) => `[${index},${JSON.stringify(id)},${JSON.stringify(type)},${payload}]`).join(",")}]}`;
|
|
1659
|
+
return encoder.encode(carried).byteLength > WAKE_PAYLOAD_LIMIT ? bare : carried;
|
|
1660
|
+
};
|
|
1661
|
+
const fieldMap = (fields) => {
|
|
1662
|
+
const map = {};
|
|
1663
|
+
for (let i = 0; i + 1 < fields.length; i += 2) map[fields[i]] = fields[i + 1];
|
|
1664
|
+
return map;
|
|
1665
|
+
};
|
|
1666
|
+
const entryIndex = (entryId) => Number(entryId.slice(0, entryId.indexOf("-")));
|
|
1667
|
+
const toEvent = (sessionId, entry) => {
|
|
1668
|
+
const fields = fieldMap(entry[1]);
|
|
1669
|
+
return {
|
|
1670
|
+
id: fields["id"],
|
|
1671
|
+
type: fields["type"],
|
|
1672
|
+
payload: JSON.parse(fields["payload"]),
|
|
1673
|
+
index: entryIndex(entry[0]),
|
|
1674
|
+
sessionId,
|
|
1675
|
+
createdAt: new Date(Number(fields["created_at"]))
|
|
1676
|
+
};
|
|
1677
|
+
};
|
|
1678
|
+
/**
|
|
1679
|
+
* A stream entry's causal edge. Releases before consumers (0.17 to 0.20)
|
|
1680
|
+
* recorded only the parent's index and attempt: their edge reads as the
|
|
1681
|
+
* handlers table's, `a2.handlers`, as Postgres's upgrade names it, with no
|
|
1682
|
+
* lane, since nothing reads an earlier release's lane.
|
|
1683
|
+
*/
|
|
1684
|
+
const decodeStreamCause = (raw) => {
|
|
1685
|
+
const parsed = JSON.parse(raw);
|
|
1686
|
+
return decodeCause(parsed.consumer === void 0 ? {
|
|
1687
|
+
consumer: "a2.handlers",
|
|
1688
|
+
lane: null,
|
|
1689
|
+
...parsed
|
|
1690
|
+
} : parsed);
|
|
1691
|
+
};
|
|
1692
|
+
/** A stream entry with its causal edge. */
|
|
1693
|
+
const toStoredEvent = (sessionId, entry) => {
|
|
1694
|
+
const cause = fieldMap(entry[1])["cause"];
|
|
1695
|
+
return Object.assign(toEvent(sessionId, entry), { cause: cause === void 0 || cause === "" ? null : decodeStreamCause(cause) });
|
|
1696
|
+
};
|
|
1697
|
+
const fieldBytes = (name) => new TextEncoder().encode(name);
|
|
1698
|
+
const ID_FIELD = fieldBytes("id");
|
|
1699
|
+
const TYPE_FIELD = fieldBytes("type");
|
|
1700
|
+
const PAYLOAD_FIELD = fieldBytes("payload");
|
|
1701
|
+
const CREATED_AT_FIELD = fieldBytes("created_at");
|
|
1702
|
+
const utf8 = new TextDecoder("utf-8", { ignoreBOM: true });
|
|
1703
|
+
const isField = (name, field) => {
|
|
1704
|
+
if (name.length !== field.length) return false;
|
|
1705
|
+
for (let at = 0; at < field.length; at++) if (name[at] !== field[at]) return false;
|
|
1706
|
+
return true;
|
|
1707
|
+
};
|
|
1708
|
+
const bytesToSerializedEvent = (sessionId, entry) => {
|
|
1709
|
+
const values = entry[1];
|
|
1710
|
+
let id;
|
|
1711
|
+
let type;
|
|
1712
|
+
let payloadJson;
|
|
1713
|
+
let createdAt;
|
|
1714
|
+
for (let at = 0; at + 1 < values.length; at += 2) {
|
|
1715
|
+
const name = values[at];
|
|
1716
|
+
if (isField(name, ID_FIELD)) id = utf8.decode(values[at + 1]);
|
|
1717
|
+
else if (isField(name, TYPE_FIELD)) type = utf8.decode(values[at + 1]);
|
|
1718
|
+
else if (isField(name, PAYLOAD_FIELD)) payloadJson = utf8.decode(values[at + 1]);
|
|
1719
|
+
else if (isField(name, CREATED_AT_FIELD)) createdAt = utf8.decode(values[at + 1]);
|
|
1720
|
+
}
|
|
1721
|
+
return {
|
|
1722
|
+
id,
|
|
1723
|
+
type,
|
|
1724
|
+
payloadJson,
|
|
1725
|
+
index: entryIndex(utf8.decode(entry[0])),
|
|
1726
|
+
sessionId,
|
|
1727
|
+
createdAtMs: Number(createdAt)
|
|
1728
|
+
};
|
|
1729
|
+
};
|
|
1730
|
+
const toSerializedEvent = (sessionId, entry) => {
|
|
1731
|
+
const fields = fieldMap(entry[1]);
|
|
1732
|
+
return {
|
|
1733
|
+
id: fields["id"],
|
|
1734
|
+
type: fields["type"],
|
|
1735
|
+
payloadJson: fields["payload"],
|
|
1736
|
+
index: entryIndex(entry[0]),
|
|
1737
|
+
sessionId,
|
|
1738
|
+
createdAtMs: Number(fields["created_at"])
|
|
1739
|
+
};
|
|
1740
|
+
};
|
|
1741
|
+
const json = (value) => JSON.stringify(value) ?? "null";
|
|
1742
|
+
/** How scripts name a cursor within its consumer: `#`, or `L` and its lane. */
|
|
1743
|
+
const tagOf = (lane) => lane === null ? "#" : `L${lane}`;
|
|
1744
|
+
/** The lane a cursor tag names: null for an event cursor. */
|
|
1745
|
+
const laneOf = (tag) => tag.startsWith("#") ? null : tag.slice(1);
|
|
1746
|
+
const targetsArg = (targets) => targets === void 0 || targets.length === 0 ? "" : JSON.stringify(targets.map((target) => [target.consumer, tagOf(target.lane)]));
|
|
1747
|
+
/** A log consumer's types, as scripts read them; '*' takes every type. */
|
|
1748
|
+
const logTypesArg = (types) => types === null ? "*" : JSON.stringify(types);
|
|
1749
|
+
/** The served consumers, as scripts read them; '' serves every consumer. */
|
|
1750
|
+
const servedArg = (consumers) => consumers === void 0 ? "" : JSON.stringify(consumers.map((consumer) => consumer.log ? [
|
|
1751
|
+
consumer.name,
|
|
1752
|
+
consumer.log.lane,
|
|
1753
|
+
logTypesArg(consumer.log.types),
|
|
1754
|
+
consumer.log.backfill ? 1 : 0
|
|
1755
|
+
] : [consumer.name]));
|
|
1756
|
+
/** A head's dispatch state from a row's HEAD_COLUMNS. */
|
|
1757
|
+
const dispatchOf = (consumer, lane, columns) => {
|
|
1758
|
+
const column = (at) => {
|
|
1759
|
+
const value = columns?.[at];
|
|
1760
|
+
return value === null || value === void 0 ? null : String(value);
|
|
1761
|
+
};
|
|
1762
|
+
const date = (at) => {
|
|
1763
|
+
const value = column(at);
|
|
1764
|
+
return value === null ? null : new Date(Number(value));
|
|
1765
|
+
};
|
|
1766
|
+
const count = (at) => {
|
|
1767
|
+
const value = column(at);
|
|
1768
|
+
return value === null ? null : Number(value);
|
|
1769
|
+
};
|
|
1770
|
+
return {
|
|
1771
|
+
consumer,
|
|
1772
|
+
lane,
|
|
1773
|
+
firstClaimedAt: date(1),
|
|
1774
|
+
lastClaimedAt: date(2),
|
|
1775
|
+
attemptCount: count(0) ?? 0,
|
|
1776
|
+
claimHolder: column(3),
|
|
1777
|
+
claimExpiresAt: date(4),
|
|
1778
|
+
failureCount: count(5) ?? 0,
|
|
1779
|
+
lastFailedAt: date(6),
|
|
1780
|
+
lastFailedAttempt: count(7),
|
|
1781
|
+
lastError: column(8),
|
|
1782
|
+
failedAt: date(9)
|
|
1783
|
+
};
|
|
1784
|
+
};
|
|
1785
|
+
const toHeadEvent = (sessionId, head) => ({
|
|
1786
|
+
...toStoredEvent(sessionId, head[4]),
|
|
1787
|
+
...dispatchOf(head[0], laneOf(head[1]), head[3])
|
|
1788
|
+
});
|
|
1789
|
+
const compareText = (a, b) => a < b ? -1 : a > b ? 1 : 0;
|
|
1790
|
+
const byHead = (a, b) => a.index - b.index || compareText(a.consumer, b.consumer);
|
|
1791
|
+
/** Consumer, then lane, with concurrent cursors last by head. */
|
|
1792
|
+
const byCursor = (a, b) => compareText(a.consumer, b.consumer) || (a.lane === null ? b.lane === null ? (a.head ?? 0) - (b.head ?? 0) : 1 : b.lane === null ? -1 : compareText(a.lane, b.lane));
|
|
1793
|
+
const toClaimAvailableResult = (sessionId, reply) => {
|
|
1794
|
+
const outcome = String(reply[0]);
|
|
1795
|
+
if (outcome === "claimed") {
|
|
1796
|
+
const count = Number(reply[1]);
|
|
1797
|
+
const events = reply.slice(2, 2 + count).map((head) => toHeadEvent(sessionId, head)).toSorted(byHead);
|
|
1798
|
+
const clearIndex = Number(reply[2 + count] ?? 0);
|
|
1799
|
+
if (clearIndex > 0) for (const event of events) event.clearIndex = clearIndex;
|
|
1800
|
+
return {
|
|
1801
|
+
outcome: "claimed",
|
|
1802
|
+
events,
|
|
1803
|
+
head: Number(reply[3 + count])
|
|
1804
|
+
};
|
|
1805
|
+
}
|
|
1806
|
+
if (outcome === "busy") return {
|
|
1807
|
+
outcome,
|
|
1808
|
+
retryAt: new Date(Number(reply[1]))
|
|
1809
|
+
};
|
|
1810
|
+
if (outcome === "settled") return { outcome };
|
|
1811
|
+
throw new TypeError(`claim referenced a missing event in session '${sessionId}'`);
|
|
1812
|
+
};
|
|
1813
|
+
const toIndex = ({ raw, sessionId, name }) => {
|
|
1814
|
+
const index = Number(raw ?? 0);
|
|
1815
|
+
if (!Number.isSafeInteger(index) || index < 0) throw new A2Error("STORE_UNAVAILABLE", `redis ${name} for session '${sessionId}' is invalid`);
|
|
1816
|
+
return index;
|
|
1817
|
+
};
|
|
1818
|
+
/**
|
|
1819
|
+
* A range read that met a gap. Reclaim deletes the entries below a clear, so
|
|
1820
|
+
* a read whose bounds predate the clear finds nothing where it expected
|
|
1821
|
+
* events; anywhere else a gap is corruption.
|
|
1822
|
+
*/
|
|
1823
|
+
const rangeGap = (sessionId, message) => Object.assign(new A2Error("STORE_UNAVAILABLE", `redis event range for session '${sessionId}' ${message}`), { rangeGap: true });
|
|
1824
|
+
const isRangeGap = (error) => error instanceof A2Error && error.rangeGap === true;
|
|
1825
|
+
/**
|
|
1826
|
+
* Run a whole read, bounds included. A gap means a clear raced it, so the
|
|
1827
|
+
* read runs once more from fresh bounds. A second gap, from another clear or
|
|
1828
|
+
* from corruption, surfaces as the retryable STORE_UNAVAILABLE.
|
|
1829
|
+
*/
|
|
1830
|
+
const retryAfterGap = async (read) => {
|
|
1831
|
+
try {
|
|
1832
|
+
return await read();
|
|
1833
|
+
} catch (error) {
|
|
1834
|
+
if (!isRangeGap(error)) throw error;
|
|
1835
|
+
return read();
|
|
1836
|
+
}
|
|
1837
|
+
};
|
|
1838
|
+
const wrap = async (fn) => {
|
|
1839
|
+
try {
|
|
1840
|
+
return await fn();
|
|
1841
|
+
} catch (err) {
|
|
1842
|
+
if (err instanceof A2Error || err instanceof TypeError) throw err;
|
|
1843
|
+
throw new A2Error("STORE_UNAVAILABLE", "redis store operation failed", { cause: err });
|
|
1844
|
+
}
|
|
1845
|
+
};
|
|
1846
|
+
/**
|
|
1847
|
+
* A session's key. Every key of a session shares one hash tag, so a script
|
|
1848
|
+
* locks only its session on Upstash and stays in one slot on a cluster. The
|
|
1849
|
+
* tag is the session id as JSON with its braces escaped: it can't end early,
|
|
1850
|
+
* and no session's tag spells another's.
|
|
1851
|
+
*/
|
|
1852
|
+
const redisSessionKey = ({ prefix, sessionId, suffix }) => `${prefix}:{${JSON.stringify(sessionId).replaceAll("{", "\\u007b").replaceAll("}", "\\u007d")}}:${suffix}`;
|
|
1853
|
+
function createRedisStoreCore(options) {
|
|
1854
|
+
const { call, callBytes, clock } = options;
|
|
1855
|
+
const generateId = options.ids;
|
|
1856
|
+
const prefix = options.keyPrefix;
|
|
1857
|
+
if (prefix.includes("{") || prefix.includes("}")) throw new TypeError("a redis keyPrefix cannot contain braces: they would set the hash tag of every key");
|
|
1858
|
+
const sessionKey = (sessionId, suffix) => redisSessionKey({
|
|
1859
|
+
prefix,
|
|
1860
|
+
sessionId,
|
|
1861
|
+
suffix
|
|
1862
|
+
});
|
|
1863
|
+
const storeKey = (s) => sessionKey(s, "log");
|
|
1864
|
+
const countKey = (s) => sessionKey(s, "count");
|
|
1865
|
+
const clearKey = (s) => sessionKey(s, "clear");
|
|
1866
|
+
const schemaKey = `${prefix}:schema`;
|
|
1867
|
+
const snapKey = (s, r) => sessionKey(s, `snap:${r}`);
|
|
1868
|
+
const snapHistoryKey = (s, r) => sessionKey(s, `snap-history:${r}`);
|
|
1869
|
+
const presenceKey = (s) => sessionKey(s, "presence");
|
|
1870
|
+
const notifyChannel = (s) => `${prefix}:${s}:notify`;
|
|
1871
|
+
const presenceChannel = (s) => `${prefix}:${s}:presence`;
|
|
1872
|
+
const layoutError = () => new A2Error("STORE_NOT_CONFIGURED", `the redis keys under prefix '${prefix}' hold an A2 layout other than '${REDIS_LAYOUT}', from another release; point the store at a fresh keyPrefix, or give the contract a new name so it starts new sessions`);
|
|
1873
|
+
/**
|
|
1874
|
+
* Once per store, before its first script: claim or check the prefix's
|
|
1875
|
+
* layout, and ask the server whether scripts may lock only their keys.
|
|
1876
|
+
* Only the server's own rejection of the flag means no; a transport error
|
|
1877
|
+
* fails this attempt, and the next operation asks again.
|
|
1878
|
+
*/
|
|
1879
|
+
const prepared = retryableLazy(async () => {
|
|
1880
|
+
const [, keyLocking] = await Promise.all([(async () => {
|
|
1881
|
+
await call("SET", schemaKey, REDIS_LAYOUT, "NX");
|
|
1882
|
+
if (await call("GET", schemaKey) !== "tagged-1") throw layoutError();
|
|
1883
|
+
})(), call("EVAL", `${KEY_LOCKING$1}\nreturn 1`, 1, schemaKey).then(() => true, (error) => {
|
|
1884
|
+
if (SHEBANG_REJECTED$1.test(String(error?.message))) return false;
|
|
1885
|
+
throw error;
|
|
1886
|
+
})]);
|
|
1887
|
+
return { keyLocking };
|
|
1888
|
+
});
|
|
1889
|
+
/** Runs a script; a `readOnly` one takes Upstash's shared lock. */
|
|
1890
|
+
const runScript = async ({ script, keys, args, readOnly }) => {
|
|
1891
|
+
const { keyLocking } = await prepared.get();
|
|
1892
|
+
let source = script;
|
|
1893
|
+
if (keyLocking) source = `${readOnly ? READ_ONLY_KEY_LOCKING$1 : KEY_LOCKING$1}\n${script}`;
|
|
1894
|
+
return call("EVAL", source, keys.length, ...keys, ...args);
|
|
1895
|
+
};
|
|
1896
|
+
/** Runs a writing script over a session: KEYS start with its counter. */
|
|
1897
|
+
const evalScript = (script, sessionId, keys, args) => runScript({
|
|
1898
|
+
script,
|
|
1899
|
+
keys: [countKey(sessionId), ...keys],
|
|
1900
|
+
args,
|
|
1901
|
+
readOnly: false
|
|
1902
|
+
});
|
|
1903
|
+
/**
|
|
1904
|
+
* Publishes a write's wake-up after its script, never inside it: a script
|
|
1905
|
+
* holds its lock while PUBLISH waits on subscribers. Wake-ups are hints (a
|
|
1906
|
+
* lost one costs subscribers a catch-up read, never an event), so a failed
|
|
1907
|
+
* publish is dropped and the write waits for it at most
|
|
1908
|
+
* NOTIFY_TIMINGS.publishMs.
|
|
1909
|
+
*/
|
|
1910
|
+
const publish = async (channel, message) => {
|
|
1911
|
+
let timer;
|
|
1912
|
+
await Promise.race([call("PUBLISH", channel, message).catch(() => void 0), new Promise((resolve) => {
|
|
1913
|
+
timer = setTimeout(resolve, NOTIFY_TIMINGS.publishMs);
|
|
1914
|
+
timer.unref?.();
|
|
1915
|
+
})]);
|
|
1916
|
+
clearTimeout(timer);
|
|
1917
|
+
};
|
|
1918
|
+
/**
|
|
1919
|
+
* Runs a write that may need a type's index to reach further back first.
|
|
1920
|
+
* Such a write returns ['index', type, after] having written nothing, and
|
|
1921
|
+
* runs again once INDEX_TYPE_LUA has indexed that history, a page per call.
|
|
1922
|
+
*/
|
|
1923
|
+
const indexing = async (sessionId, run) => {
|
|
1924
|
+
for (;;) {
|
|
1925
|
+
const reply = await run();
|
|
1926
|
+
if (reply[0] !== "index") return reply;
|
|
1927
|
+
for (;;) {
|
|
1928
|
+
const [done] = await evalScript(INDEX_TYPE_LUA, sessionId, [], [String(reply[1]), Number(reply[2])]);
|
|
1929
|
+
if (done === "1") break;
|
|
1930
|
+
}
|
|
1931
|
+
}
|
|
1932
|
+
};
|
|
1933
|
+
/** The frontier and clear marker, in one MGET. */
|
|
1934
|
+
const readBounds = async (sessionId) => {
|
|
1935
|
+
await prepared.get();
|
|
1936
|
+
const [count, cleared] = await call("MGET", countKey(sessionId), clearKey(sessionId));
|
|
1937
|
+
return {
|
|
1938
|
+
frontier: toIndex({
|
|
1939
|
+
raw: count ?? cleared,
|
|
1940
|
+
sessionId,
|
|
1941
|
+
name: "event count"
|
|
1942
|
+
}),
|
|
1943
|
+
clearIndex: toIndex({
|
|
1944
|
+
raw: cleared,
|
|
1945
|
+
sessionId,
|
|
1946
|
+
name: "clear boundary"
|
|
1947
|
+
})
|
|
1948
|
+
};
|
|
1949
|
+
};
|
|
1950
|
+
const readClearIndex = async (sessionId) => {
|
|
1951
|
+
await prepared.get();
|
|
1952
|
+
return toIndex({
|
|
1953
|
+
raw: await call("GET", clearKey(sessionId)),
|
|
1954
|
+
sessionId,
|
|
1955
|
+
name: "clear boundary"
|
|
1956
|
+
});
|
|
1957
|
+
};
|
|
1958
|
+
const readEventPages = async (sessionId, afterIndex, throughIndex) => {
|
|
1959
|
+
if (throughIndex <= afterIndex) return [];
|
|
1960
|
+
const events = [];
|
|
1961
|
+
let nextIndex = afterIndex + 1;
|
|
1962
|
+
while (nextIndex <= throughIndex) {
|
|
1963
|
+
const entries = await call("XRANGE", storeKey(sessionId), `${nextIndex}-0`, `${throughIndex}-0`, "COUNT", 256);
|
|
1964
|
+
if (entries.length === 0) throw rangeGap(sessionId, `ended before index ${throughIndex}`);
|
|
1965
|
+
for (let position = 0; position < entries.length; position += 1) {
|
|
1966
|
+
const index = entryIndex(entries[position][0]);
|
|
1967
|
+
if (index !== nextIndex + position || index > throughIndex) throw rangeGap(sessionId, "is not contiguous");
|
|
1968
|
+
}
|
|
1969
|
+
events.push(...entries.map((entry) => toEvent(sessionId, entry)));
|
|
1970
|
+
nextIndex = events.at(-1).index + 1;
|
|
1971
|
+
}
|
|
1972
|
+
return events;
|
|
1973
|
+
};
|
|
1974
|
+
const readEventRange = async (sessionId, afterIndex, throughIndex) => {
|
|
1975
|
+
const events = [];
|
|
1976
|
+
for (let start = afterIndex; start < throughIndex; start += 256 * READ_CONCURRENCY) {
|
|
1977
|
+
const ranges = Array.from({ length: Math.min(READ_CONCURRENCY, Math.ceil((throughIndex - start) / 256)) }, (_, position) => {
|
|
1978
|
+
const after = start + position * 256;
|
|
1979
|
+
return readEventPages(sessionId, after, Math.min(after + 256, throughIndex));
|
|
1980
|
+
});
|
|
1981
|
+
const results = await Promise.allSettled(ranges);
|
|
1982
|
+
for (const result of results) {
|
|
1983
|
+
if (result.status === "rejected") throw result.reason;
|
|
1984
|
+
events.push(...result.value);
|
|
1985
|
+
}
|
|
1986
|
+
}
|
|
1987
|
+
return events;
|
|
1988
|
+
};
|
|
1989
|
+
/** Bounded XRANGE pages restore the full-read contract on capped providers. */
|
|
1990
|
+
const readRange = async (sessionId, afterIndex, throughIndex) => (await readBoundedRange(sessionId, afterIndex, throughIndex)).rows;
|
|
1991
|
+
/** `readRange`, with the clear boundary its rows start at. */
|
|
1992
|
+
const readBoundedRange = async (sessionId, afterIndex, throughIndex) => {
|
|
1993
|
+
if (throughIndex !== void 0 && throughIndex <= afterIndex) return {
|
|
1994
|
+
rows: [],
|
|
1995
|
+
clearIndex: 0
|
|
1996
|
+
};
|
|
1997
|
+
const { frontier: currentFrontier, clearIndex } = await readBounds(sessionId);
|
|
1998
|
+
const frontier = Math.min(throughIndex ?? currentFrontier, currentFrontier);
|
|
1999
|
+
const after = Math.max(afterIndex, clearIndex - 1);
|
|
2000
|
+
if (frontier <= after) return {
|
|
2001
|
+
rows: [],
|
|
2002
|
+
clearIndex
|
|
2003
|
+
};
|
|
2004
|
+
const rows = [];
|
|
2005
|
+
let nextIndex = after + 1;
|
|
2006
|
+
while (nextIndex <= frontier) {
|
|
2007
|
+
const entries = await call("XRANGE", storeKey(sessionId), `${nextIndex}-0`, `${frontier}-0`, "COUNT", 256);
|
|
2008
|
+
if (entries.length === 0) throw rangeGap(sessionId, `ended before index ${frontier}`);
|
|
2009
|
+
const page = entries.map((entry) => toStoredEvent(sessionId, entry));
|
|
2010
|
+
for (let position = 0; position < page.length; position += 1) if (page[position].index !== nextIndex + position) throw rangeGap(sessionId, "is not contiguous");
|
|
2011
|
+
const lastIndex = page.at(-1).index;
|
|
2012
|
+
if (lastIndex < nextIndex || lastIndex > frontier) throw new A2Error("STORE_UNAVAILABLE", `redis event range for session '${sessionId}' did not advance`);
|
|
2013
|
+
rows.push(...page);
|
|
2014
|
+
nextIndex = lastIndex + 1;
|
|
2015
|
+
}
|
|
2016
|
+
return {
|
|
2017
|
+
rows,
|
|
2018
|
+
clearIndex
|
|
2019
|
+
};
|
|
2020
|
+
};
|
|
2021
|
+
const readStream = options.readStream;
|
|
2022
|
+
return {
|
|
2023
|
+
...readStream ? { streamEvents: async function* ({ sessionId, afterIndex, signal }) {
|
|
2024
|
+
try {
|
|
2025
|
+
for await (const entries of readStream({
|
|
2026
|
+
key: storeKey(sessionId),
|
|
2027
|
+
from: `${afterIndex + 1}-0`,
|
|
2028
|
+
signal
|
|
2029
|
+
})) {
|
|
2030
|
+
signal.throwIfAborted();
|
|
2031
|
+
yield entries.map((entry) => toSerializedEvent(sessionId, entry));
|
|
2032
|
+
}
|
|
2033
|
+
} catch (cause) {
|
|
2034
|
+
if (cause instanceof A2Error) throw cause;
|
|
2035
|
+
throw new A2Error("STORE_UNAVAILABLE", "redis stream read failed", { cause });
|
|
2036
|
+
}
|
|
2037
|
+
} } : {},
|
|
2038
|
+
async append(sessionId, events, appendOptions) {
|
|
2039
|
+
return wrap(async () => {
|
|
2040
|
+
const suppliedIds = events.filter((e) => e.id !== void 0).map((e) => e.id);
|
|
2041
|
+
if (new Set(suppliedIds).size !== suppliedIds.length) throw new A2Error("PARTIAL_DUPLICATE_BATCH", "batch contains the same event id more than once");
|
|
2042
|
+
const now = clock.now().getTime();
|
|
2043
|
+
const withIds = events.map((e) => ({
|
|
2044
|
+
id: e.id ?? generateId(),
|
|
2045
|
+
type: e.type,
|
|
2046
|
+
payload: e.payload,
|
|
2047
|
+
cause: e.cause ? { ...e.cause } : null,
|
|
2048
|
+
targets: e.targets
|
|
2049
|
+
}));
|
|
2050
|
+
const payloads = withIds.map((e) => json(e.payload));
|
|
2051
|
+
const reply = await indexing(sessionId, () => evalScript(APPEND_LUA, sessionId, [], [
|
|
2052
|
+
sessionId,
|
|
2053
|
+
now,
|
|
2054
|
+
withIds.length,
|
|
2055
|
+
servedArg(appendOptions?.consumers),
|
|
2056
|
+
appendOptions?.originIndex ?? "",
|
|
2057
|
+
...withIds.flatMap((e, i) => [
|
|
2058
|
+
e.id,
|
|
2059
|
+
e.type,
|
|
2060
|
+
payloads[i],
|
|
2061
|
+
e.cause ? json(e.cause) : "",
|
|
2062
|
+
targetsArg(e.targets)
|
|
2063
|
+
])
|
|
2064
|
+
]));
|
|
2065
|
+
const [verdict] = reply;
|
|
2066
|
+
if (verdict === "empty") return {
|
|
2067
|
+
events: [],
|
|
2068
|
+
hasPending: reply[1] === "1"
|
|
2069
|
+
};
|
|
2070
|
+
if (verdict === "cleared") throw sessionClearedError(`scheduled work from event ${reply[1]} was cleared in session '${sessionId}'`);
|
|
2071
|
+
if (verdict === "superseded") throw supersededAttemptError(sessionId, withIds[Number(reply[1]) - 1].cause);
|
|
2072
|
+
if (verdict === "partial") throw new A2Error("PARTIAL_DUPLICATE_BATCH", `batch mixes ${reply[1]} already-appended and ${events.length - Number(reply[1])} fresh events`);
|
|
2073
|
+
if (verdict === "dup") {
|
|
2074
|
+
const hasPending = reply[1] === "1";
|
|
2075
|
+
const firstIndex = Math.min(...reply.slice(2).map(Number));
|
|
2076
|
+
const { rows: existing, clearIndex } = await retryAfterGap(() => readBoundedRange(sessionId, firstIndex - 1, firstIndex + events.length - 1));
|
|
2077
|
+
if (firstIndex < clearIndex) throw new A2Error("STORE_UNAVAILABLE", `session '${sessionId}' was cleared while an append retry replayed; retry it`);
|
|
2078
|
+
const replayed = idempotentReplay(events, existing);
|
|
2079
|
+
await publish(notifyChannel(sessionId), wakeMessage(firstIndex + events.length - 1, now, []));
|
|
2080
|
+
return {
|
|
2081
|
+
events: replayed,
|
|
2082
|
+
hasPending
|
|
2083
|
+
};
|
|
2084
|
+
}
|
|
2085
|
+
const base = Number(reply[1]) - 1;
|
|
2086
|
+
await publish(notifyChannel(sessionId), wakeMessage(base + withIds.length, now, withIds.map((e, i) => [
|
|
2087
|
+
base + 1 + i,
|
|
2088
|
+
e.id,
|
|
2089
|
+
e.type,
|
|
2090
|
+
payloads[i]
|
|
2091
|
+
])));
|
|
2092
|
+
return {
|
|
2093
|
+
events: withIds.map((e, i) => ({
|
|
2094
|
+
id: e.id,
|
|
2095
|
+
type: e.type,
|
|
2096
|
+
payload: structuredClone(e.payload),
|
|
2097
|
+
index: base + 1 + i,
|
|
2098
|
+
sessionId,
|
|
2099
|
+
createdAt: new Date(now),
|
|
2100
|
+
cause: e.cause
|
|
2101
|
+
})),
|
|
2102
|
+
hasPending: reply[2] === "1"
|
|
2103
|
+
};
|
|
2104
|
+
});
|
|
2105
|
+
},
|
|
2106
|
+
async cursors(sessionId) {
|
|
2107
|
+
return wrap(async () => {
|
|
2108
|
+
return (await runScript({
|
|
2109
|
+
script: CURSORS_LUA,
|
|
2110
|
+
keys: [countKey(sessionId)],
|
|
2111
|
+
args: [],
|
|
2112
|
+
readOnly: true
|
|
2113
|
+
})).map(([consumer, tag, position, head, columns]) => Object.assign(dispatchOf(consumer, laneOf(tag), columns), {
|
|
2114
|
+
position: Number(position),
|
|
2115
|
+
head: head === null ? null : Number(head)
|
|
2116
|
+
})).toSorted(byCursor);
|
|
2117
|
+
});
|
|
2118
|
+
},
|
|
2119
|
+
async read(sessionId, opts) {
|
|
2120
|
+
return wrap(() => retryAfterGap(() => readRange(sessionId, opts?.afterIndex ?? 0, opts?.throughIndex)));
|
|
2121
|
+
},
|
|
2122
|
+
async readHistory(sessionId, opts) {
|
|
2123
|
+
const after = opts?.afterIndex ?? 0;
|
|
2124
|
+
if (opts?.throughIndex !== void 0 && opts.throughIndex <= after) return [];
|
|
2125
|
+
return wrap(() => retryAfterGap(async () => {
|
|
2126
|
+
const { frontier, clearIndex } = await readBounds(sessionId);
|
|
2127
|
+
return readEventRange(sessionId, Math.max(after, clearIndex - 1), Math.min(opts?.throughIndex ?? frontier, frontier));
|
|
2128
|
+
}));
|
|
2129
|
+
},
|
|
2130
|
+
async claimAvailable({ sessionId, holder, ttlMs, expiresAtMs, exclude, consumers, maxFailures, through }) {
|
|
2131
|
+
return wrap(async () => {
|
|
2132
|
+
const now = clock.now().getTime();
|
|
2133
|
+
const expiresAt = expiresAtMs ?? now + ttlMs;
|
|
2134
|
+
const excluded = exclude ?? [];
|
|
2135
|
+
const reply = await evalScript(CLAIM_AVAILABLE_LUA, sessionId, [], [
|
|
2136
|
+
holder,
|
|
2137
|
+
now,
|
|
2138
|
+
expiresAt,
|
|
2139
|
+
maxFailures ?? "",
|
|
2140
|
+
servedArg(consumers),
|
|
2141
|
+
through ?? "",
|
|
2142
|
+
excluded.length,
|
|
2143
|
+
...excluded.flatMap((ref) => [ref.consumer, ref.lane === null ? `#${ref.index}` : `L${ref.lane}`])
|
|
2144
|
+
]);
|
|
2145
|
+
return toClaimAvailableResult(sessionId, reply);
|
|
2146
|
+
});
|
|
2147
|
+
},
|
|
2148
|
+
async renewClaims({ sessionId, holder, claims, ttlMs, expiresAtMs }) {
|
|
2149
|
+
if (claims.length === 0) return {
|
|
2150
|
+
renewed: [],
|
|
2151
|
+
superseded: []
|
|
2152
|
+
};
|
|
2153
|
+
return wrap(async () => {
|
|
2154
|
+
const now = clock.now().getTime();
|
|
2155
|
+
const expiresAt = expiresAtMs ?? now + ttlMs;
|
|
2156
|
+
const reply = await evalScript(RENEW_CLAIMS_LUA, sessionId, [], [
|
|
2157
|
+
holder,
|
|
2158
|
+
now,
|
|
2159
|
+
expiresAt,
|
|
2160
|
+
claims.length,
|
|
2161
|
+
...claims.flatMap((claim) => [
|
|
2162
|
+
claim.consumer,
|
|
2163
|
+
tagOf(claim.lane),
|
|
2164
|
+
claim.index,
|
|
2165
|
+
claim.attempt
|
|
2166
|
+
])
|
|
2167
|
+
]);
|
|
2168
|
+
const renewedCount = Number(reply[0]);
|
|
2169
|
+
const refs = reply.slice(1).map((position) => {
|
|
2170
|
+
const { consumer, lane, index } = claims[Number(position) - 1];
|
|
2171
|
+
return {
|
|
2172
|
+
consumer,
|
|
2173
|
+
lane,
|
|
2174
|
+
index
|
|
2175
|
+
};
|
|
2176
|
+
});
|
|
2177
|
+
return {
|
|
2178
|
+
renewed: refs.slice(0, renewedCount).toSorted(byHead),
|
|
2179
|
+
superseded: refs.slice(renewedCount).toSorted(byHead)
|
|
2180
|
+
};
|
|
2181
|
+
});
|
|
2182
|
+
},
|
|
2183
|
+
async completeAttempt({ sessionId, consumer, lane, index, attempt, events, consumers }) {
|
|
2184
|
+
return wrap(async () => {
|
|
2185
|
+
const ids = events.map((event) => event.id);
|
|
2186
|
+
if (new Set(ids).size !== ids.length) throw new A2Error("PARTIAL_DUPLICATE_BATCH", "batch contains the same event id more than once");
|
|
2187
|
+
const now = clock.now().getTime();
|
|
2188
|
+
const cause = {
|
|
2189
|
+
consumer,
|
|
2190
|
+
lane,
|
|
2191
|
+
index,
|
|
2192
|
+
attempt
|
|
2193
|
+
};
|
|
2194
|
+
const payloads = events.map((event) => json(event.payload));
|
|
2195
|
+
const reply = await indexing(sessionId, () => evalScript(COMPLETE_ATTEMPT_LUA, sessionId, [], [
|
|
2196
|
+
sessionId,
|
|
2197
|
+
now,
|
|
2198
|
+
consumer,
|
|
2199
|
+
tagOf(lane),
|
|
2200
|
+
index,
|
|
2201
|
+
attempt,
|
|
2202
|
+
json(cause),
|
|
2203
|
+
servedArg(consumers),
|
|
2204
|
+
json(ids),
|
|
2205
|
+
events.length,
|
|
2206
|
+
...events.flatMap((event, i) => [
|
|
2207
|
+
event.id,
|
|
2208
|
+
event.type,
|
|
2209
|
+
payloads[i],
|
|
2210
|
+
targetsArg(event.targets)
|
|
2211
|
+
])
|
|
2212
|
+
]));
|
|
2213
|
+
const outcome = reply[0];
|
|
2214
|
+
if (outcome === "missing") throw new TypeError(`no event at index ${index} in session '${sessionId}'`);
|
|
2215
|
+
if (outcome === "partial") throw new A2Error("PARTIAL_DUPLICATE_BATCH", "returned event ids do not match the completed attempt");
|
|
2216
|
+
if (outcome === "superseded") return { outcome };
|
|
2217
|
+
if (outcome === "ok") {
|
|
2218
|
+
const base = Number(reply[1]) - 1;
|
|
2219
|
+
if (events.length > 0) await publish(notifyChannel(sessionId), wakeMessage(base + events.length, now, events.map((event, i) => [
|
|
2220
|
+
base + 1 + i,
|
|
2221
|
+
event.id,
|
|
2222
|
+
event.type,
|
|
2223
|
+
payloads[i]
|
|
2224
|
+
])));
|
|
2225
|
+
return {
|
|
2226
|
+
outcome: "completed",
|
|
2227
|
+
events: events.map((event, i) => ({
|
|
2228
|
+
id: event.id,
|
|
2229
|
+
type: event.type,
|
|
2230
|
+
payload: structuredClone(event.payload),
|
|
2231
|
+
index: base + 1 + i,
|
|
2232
|
+
sessionId,
|
|
2233
|
+
createdAt: new Date(now),
|
|
2234
|
+
cause: { ...cause }
|
|
2235
|
+
}))
|
|
2236
|
+
};
|
|
2237
|
+
}
|
|
2238
|
+
if (outcome === "duplicate") {
|
|
2239
|
+
if (events.length === 0) return {
|
|
2240
|
+
outcome: "completed",
|
|
2241
|
+
events: []
|
|
2242
|
+
};
|
|
2243
|
+
const firstIndex = Math.min(...reply.slice(1).map(Number));
|
|
2244
|
+
const { rows: existing, clearIndex } = await retryAfterGap(() => readBoundedRange(sessionId, firstIndex - 1, firstIndex + events.length - 1));
|
|
2245
|
+
if (index <= clearIndex) return { outcome: "superseded" };
|
|
2246
|
+
if (existing.some((row) => row.cause?.consumer !== consumer || row.cause.lane !== lane || row.cause.index !== index || row.cause.attempt !== attempt)) throw new A2Error("PARTIAL_DUPLICATE_BATCH", "returned event retry does not match the committed batch");
|
|
2247
|
+
const replayed = idempotentReplay(events, existing);
|
|
2248
|
+
await publish(notifyChannel(sessionId), wakeMessage(firstIndex + events.length - 1, now, []));
|
|
2249
|
+
return {
|
|
2250
|
+
outcome: "completed",
|
|
2251
|
+
events: replayed
|
|
2252
|
+
};
|
|
2253
|
+
}
|
|
2254
|
+
throw new TypeError(`unexpected completion outcome '${outcome}'`);
|
|
2255
|
+
});
|
|
2256
|
+
},
|
|
2257
|
+
async failAttempt({ sessionId, consumer, lane, index, attempt, error, maxFailures, retryAfterMs }) {
|
|
2258
|
+
return wrap(async () => {
|
|
2259
|
+
const now = clock.now().getTime();
|
|
2260
|
+
const reply = await evalScript(FAIL_ATTEMPT_LUA, sessionId, [], [
|
|
2261
|
+
consumer,
|
|
2262
|
+
tagOf(lane),
|
|
2263
|
+
index,
|
|
2264
|
+
attempt,
|
|
2265
|
+
error,
|
|
2266
|
+
maxFailures,
|
|
2267
|
+
now,
|
|
2268
|
+
retryAfterMs === void 0 ? "" : now + retryAfterMs
|
|
2269
|
+
]);
|
|
2270
|
+
const outcome = reply[0];
|
|
2271
|
+
if (outcome === "missing") throw new TypeError(`no event at index ${index} in session '${sessionId}'`);
|
|
2272
|
+
return {
|
|
2273
|
+
outcome,
|
|
2274
|
+
failureCount: Number(reply[1])
|
|
2275
|
+
};
|
|
2276
|
+
});
|
|
2277
|
+
},
|
|
2278
|
+
async deadLetters(sessionId) {
|
|
2279
|
+
return wrap(async () => {
|
|
2280
|
+
return (await runScript({
|
|
2281
|
+
script: DEAD_LETTERS_LUA,
|
|
2282
|
+
keys: [countKey(sessionId)],
|
|
2283
|
+
args: [],
|
|
2284
|
+
readOnly: true
|
|
2285
|
+
})).map((head) => toHeadEvent(sessionId, head)).toSorted(byHead);
|
|
2286
|
+
});
|
|
2287
|
+
},
|
|
2288
|
+
async redrive({ sessionId, consumer, index, action, consumers }) {
|
|
2289
|
+
return wrap(async () => {
|
|
2290
|
+
const [redriven, hasPending] = await indexing(sessionId, () => evalScript(REDRIVE_LUA, sessionId, [], [
|
|
2291
|
+
consumer,
|
|
2292
|
+
index,
|
|
2293
|
+
action,
|
|
2294
|
+
clock.now().getTime(),
|
|
2295
|
+
servedArg(consumers)
|
|
2296
|
+
]));
|
|
2297
|
+
return {
|
|
2298
|
+
redriven: redriven === "1",
|
|
2299
|
+
hasPending: hasPending === "1"
|
|
2300
|
+
};
|
|
2301
|
+
});
|
|
2302
|
+
},
|
|
2303
|
+
async wakeLog({ sessionId, consumer }) {
|
|
2304
|
+
return wrap(async () => {
|
|
2305
|
+
const { name, log } = consumer;
|
|
2306
|
+
const [, position, head, columns, sessionHead] = await indexing(sessionId, () => evalScript(WAKE_LOG_LUA, sessionId, [], [
|
|
2307
|
+
name,
|
|
2308
|
+
log.lane,
|
|
2309
|
+
logTypesArg(log.types),
|
|
2310
|
+
log.backfill ? 1 : 0
|
|
2311
|
+
]));
|
|
2312
|
+
return {
|
|
2313
|
+
cursor: Object.assign(dispatchOf(name, log.lane, columns), {
|
|
2314
|
+
position: Number(position),
|
|
2315
|
+
head: head === null ? null : Number(head)
|
|
2316
|
+
}),
|
|
2317
|
+
head: Number(sessionHead)
|
|
2318
|
+
};
|
|
2319
|
+
});
|
|
2320
|
+
},
|
|
2321
|
+
async readState(sessionId, reducerName, stateOptions) {
|
|
2322
|
+
return wrap(() => retryAfterGap(async () => {
|
|
2323
|
+
const reply = await runScript({
|
|
2324
|
+
script: READ_STATE_LUA,
|
|
2325
|
+
keys: [
|
|
2326
|
+
countKey(sessionId),
|
|
2327
|
+
snapKey(sessionId, reducerName),
|
|
2328
|
+
snapHistoryKey(sessionId, reducerName)
|
|
2329
|
+
],
|
|
2330
|
+
args: [stateOptions?.snapshotThroughIndex ?? stateOptions?.throughIndex ?? ""],
|
|
2331
|
+
readOnly: true
|
|
2332
|
+
});
|
|
2333
|
+
const rawHead = reply[0];
|
|
2334
|
+
const raw = reply[1];
|
|
2335
|
+
const headIndex = rawHead === "" ? null : Number(rawHead);
|
|
2336
|
+
const parsed = raw === "" ? null : JSON.parse(raw);
|
|
2337
|
+
const currentFrontier = Number(reply[2]);
|
|
2338
|
+
if (!Number.isSafeInteger(currentFrontier) || currentFrontier < 0 || headIndex !== null && (!Number.isSafeInteger(headIndex) || headIndex < 0) || parsed !== null && (!Number.isSafeInteger(parsed.index) || parsed.index < 0)) throw new A2Error("STORE_UNAVAILABLE", `redis state frontier for session '${sessionId}' is invalid`);
|
|
2339
|
+
const frontier = Math.min(currentFrontier, stateOptions?.throughIndex ?? currentFrontier);
|
|
2340
|
+
const snapshot = parsed;
|
|
2341
|
+
const clearIndex = Number(reply[3] ?? 0);
|
|
2342
|
+
const events = await readEventRange(sessionId, Math.max(snapshot?.index ?? 0, clearIndex - 1), frontier);
|
|
2343
|
+
return {
|
|
2344
|
+
...clearIndex > 0 ? { clearIndex } : {},
|
|
2345
|
+
headIndex,
|
|
2346
|
+
snapshot: snapshot ? {
|
|
2347
|
+
index: snapshot.index,
|
|
2348
|
+
state: snapshot.state
|
|
2349
|
+
} : null,
|
|
2350
|
+
events
|
|
2351
|
+
};
|
|
2352
|
+
}));
|
|
2353
|
+
},
|
|
2354
|
+
async putSnapshots(sessionId, reducerName, writes) {
|
|
2355
|
+
const byIndex = /* @__PURE__ */ new Map();
|
|
2356
|
+
for (const write of writes) {
|
|
2357
|
+
const existing = byIndex.get(write.index);
|
|
2358
|
+
const pins = existing?.pins ?? /* @__PURE__ */ new Map();
|
|
2359
|
+
for (const pin of write.pins ?? []) pins.set(JSON.stringify([
|
|
2360
|
+
pin.consumer,
|
|
2361
|
+
pin.lane,
|
|
2362
|
+
pin.index
|
|
2363
|
+
]), pin);
|
|
2364
|
+
if (existing) existing.state = write.state;
|
|
2365
|
+
else byIndex.set(write.index, {
|
|
2366
|
+
index: write.index,
|
|
2367
|
+
state: write.state,
|
|
2368
|
+
pins
|
|
2369
|
+
});
|
|
2370
|
+
}
|
|
2371
|
+
const normalized = [...byIndex.values()].toSorted((a, b) => a.index - b.index);
|
|
2372
|
+
if (normalized.length === 0) return;
|
|
2373
|
+
await wrap(async () => {
|
|
2374
|
+
const args = [normalized.length];
|
|
2375
|
+
for (const write of normalized) args.push(write.index, json({
|
|
2376
|
+
index: write.index,
|
|
2377
|
+
state: write.state,
|
|
2378
|
+
updatedAt: clock.now().getTime(),
|
|
2379
|
+
r: 1
|
|
2380
|
+
}), write.pins.size, ...[...write.pins.values()].flatMap((pin) => [
|
|
2381
|
+
pin.consumer,
|
|
2382
|
+
tagOf(pin.lane),
|
|
2383
|
+
pin.index
|
|
2384
|
+
]));
|
|
2385
|
+
await evalScript(PUT_SNAPSHOTS_LUA, sessionId, [snapKey(sessionId, reducerName), snapHistoryKey(sessionId, reducerName)], [...args, reducerName]);
|
|
2386
|
+
});
|
|
2387
|
+
},
|
|
2388
|
+
presence: {
|
|
2389
|
+
async set(ns, participant, values, meta) {
|
|
2390
|
+
return wrap(async () => {
|
|
2391
|
+
const entries = Object.entries(values);
|
|
2392
|
+
if (entries.length === 0) return null;
|
|
2393
|
+
const atMs = meta.at.getTime();
|
|
2394
|
+
const nowMs = clock.now().getTime();
|
|
2395
|
+
const expiresAtMs = nowMs + meta.ttlMs;
|
|
2396
|
+
const applied = await runScript({
|
|
2397
|
+
script: PRESENCE_SET_LUA,
|
|
2398
|
+
keys: [presenceKey(ns)],
|
|
2399
|
+
args: [
|
|
2400
|
+
atMs,
|
|
2401
|
+
meta.ttlMs,
|
|
2402
|
+
nowMs,
|
|
2403
|
+
entries.length,
|
|
2404
|
+
...entries.flatMap(([field, value]) => [JSON.stringify([participant, field]), json({
|
|
2405
|
+
a: atMs,
|
|
2406
|
+
e: expiresAtMs,
|
|
2407
|
+
s: meta.seen,
|
|
2408
|
+
v: value
|
|
2409
|
+
})])
|
|
2410
|
+
],
|
|
2411
|
+
readOnly: false
|
|
2412
|
+
});
|
|
2413
|
+
if (applied.length === 0) return null;
|
|
2414
|
+
const appliedValues = nullProtoRecord();
|
|
2415
|
+
const published = [];
|
|
2416
|
+
for (const position of applied) {
|
|
2417
|
+
const [field, value] = entries[Number(position) - 1];
|
|
2418
|
+
appliedValues[field] = structuredClone(value);
|
|
2419
|
+
published.push(`${json(field)}:${json(value)}`);
|
|
2420
|
+
}
|
|
2421
|
+
return {
|
|
2422
|
+
patch: {
|
|
2423
|
+
participant,
|
|
2424
|
+
values: appliedValues,
|
|
2425
|
+
seen: meta.seen,
|
|
2426
|
+
at: new Date(atMs)
|
|
2427
|
+
},
|
|
2428
|
+
published: publish(presenceChannel(ns), `${json({
|
|
2429
|
+
src: options.presenceSource,
|
|
2430
|
+
participant,
|
|
2431
|
+
seen: meta.seen,
|
|
2432
|
+
at: atMs
|
|
2433
|
+
}).slice(0, -1)},"values":{${published.join(",")}}}`)
|
|
2434
|
+
};
|
|
2435
|
+
});
|
|
2436
|
+
},
|
|
2437
|
+
async read(ns) {
|
|
2438
|
+
return wrap(async () => {
|
|
2439
|
+
const key = presenceKey(ns);
|
|
2440
|
+
const reply = await call("HGETALL", key);
|
|
2441
|
+
const nowMs = clock.now().getTime();
|
|
2442
|
+
const rows = [];
|
|
2443
|
+
const expired = [];
|
|
2444
|
+
for (let i = 0; i + 1 < reply.length; i += 2) {
|
|
2445
|
+
const entry = JSON.parse(reply[i + 1]);
|
|
2446
|
+
if (entry.e <= nowMs) {
|
|
2447
|
+
expired.push(reply[i]);
|
|
2448
|
+
continue;
|
|
2449
|
+
}
|
|
2450
|
+
if (entry.v === null) continue;
|
|
2451
|
+
const [participant, field] = JSON.parse(reply[i]);
|
|
2452
|
+
rows.push({
|
|
2453
|
+
participant,
|
|
2454
|
+
field,
|
|
2455
|
+
value: entry.v,
|
|
2456
|
+
seen: entry.s,
|
|
2457
|
+
at: new Date(entry.a),
|
|
2458
|
+
expiresAt: new Date(entry.e)
|
|
2459
|
+
});
|
|
2460
|
+
}
|
|
2461
|
+
if (expired.length > 0) await runScript({
|
|
2462
|
+
script: PRESENCE_PRUNE_LUA,
|
|
2463
|
+
keys: [key],
|
|
2464
|
+
args: [nowMs, ...expired],
|
|
2465
|
+
readOnly: false
|
|
2466
|
+
}).catch(() => void 0);
|
|
2467
|
+
return rows;
|
|
2468
|
+
});
|
|
2469
|
+
}
|
|
2470
|
+
},
|
|
2471
|
+
async readRange(sessionId, afterIndex, throughIndex) {
|
|
2472
|
+
return wrap(() => readRange(sessionId, afterIndex, throughIndex));
|
|
2473
|
+
},
|
|
2474
|
+
async readTail(sessionId, fromIndex, throughIndex) {
|
|
2475
|
+
return wrap(async () => {
|
|
2476
|
+
const [newest, clearIndex] = await Promise.all([call("XREVRANGE", storeKey(sessionId), throughIndex === void 0 ? "+" : `${throughIndex}-0`, `${fromIndex}-0`, "COUNT", 256), readClearIndex(sessionId)]);
|
|
2477
|
+
if (clearIndex > fromIndex) return null;
|
|
2478
|
+
const page = newest.toReversed().map((entry) => toEvent(sessionId, entry));
|
|
2479
|
+
for (let position = 1; position < page.length; position += 1) if (page[position].index !== page[position - 1].index + 1) throw new A2Error("STORE_UNAVAILABLE", `redis event range for session '${sessionId}' is not contiguous`);
|
|
2480
|
+
const oldest = page[0]?.index;
|
|
2481
|
+
return oldest === void 0 || oldest <= fromIndex ? page : null;
|
|
2482
|
+
});
|
|
2483
|
+
},
|
|
2484
|
+
async clear({ sessionId, ifLastIndex, cause }) {
|
|
2485
|
+
return wrap(async () => {
|
|
2486
|
+
const markerId = generateId();
|
|
2487
|
+
const now = clock.now().getTime();
|
|
2488
|
+
const reply = await evalScript(CLEAR_LUA, sessionId, [], [
|
|
2489
|
+
sessionId,
|
|
2490
|
+
now,
|
|
2491
|
+
ifLastIndex ?? "",
|
|
2492
|
+
cause?.consumer ?? "",
|
|
2493
|
+
cause ? tagOf(cause.lane) : "",
|
|
2494
|
+
cause?.index ?? "",
|
|
2495
|
+
cause?.attempt ?? "",
|
|
2496
|
+
markerId
|
|
2497
|
+
]);
|
|
2498
|
+
if (reply[0] === "superseded") throw supersededAttemptError(sessionId, cause);
|
|
2499
|
+
if (reply[0] === "foreign") throw new A2Error("PARTIAL_DUPLICATE_BATCH", `the clear marker id '${markerId}' already exists`);
|
|
2500
|
+
const head = Number(reply[2]);
|
|
2501
|
+
if (reply[4] === "1") await publish(notifyChannel(sessionId), wakeMessage(head, now, [[
|
|
2502
|
+
head,
|
|
2503
|
+
markerId,
|
|
2504
|
+
CLEAR_EVENT_TYPE,
|
|
2505
|
+
"null"
|
|
2506
|
+
]]));
|
|
2507
|
+
else if (reply[1] === "1" && head > 0) await publish(notifyChannel(sessionId), wakeMessage(head, now, []));
|
|
2508
|
+
return {
|
|
2509
|
+
cleared: reply[1] === "1",
|
|
2510
|
+
index: Number(reply[2]),
|
|
2511
|
+
clearIndex: Number(reply[3])
|
|
2512
|
+
};
|
|
2513
|
+
});
|
|
2514
|
+
},
|
|
2515
|
+
async reclaim(sessionId) {
|
|
2516
|
+
return wrap(async () => {
|
|
2517
|
+
const pairs = await evalScript(RECLAIM_BATCH_LUA, sessionId, [], [256]);
|
|
2518
|
+
if (pairs.length === 0) return {
|
|
2519
|
+
deleted: 0,
|
|
2520
|
+
done: true
|
|
2521
|
+
};
|
|
2522
|
+
const [deleted, done] = await evalScript(RECLAIM_LUA, sessionId, [], [
|
|
2523
|
+
sessionId,
|
|
2524
|
+
pairs.length / 2,
|
|
2525
|
+
...pairs
|
|
2526
|
+
]);
|
|
2527
|
+
return {
|
|
2528
|
+
deleted: Number(deleted),
|
|
2529
|
+
done: Number(done) === 1
|
|
2530
|
+
};
|
|
2531
|
+
});
|
|
2532
|
+
},
|
|
2533
|
+
clearIndex: (sessionId) => wrap(() => readClearIndex(sessionId)),
|
|
2534
|
+
async readEvents(sessionId, afterIndex) {
|
|
2535
|
+
return wrap(async () => {
|
|
2536
|
+
return (await call("XRANGE", storeKey(sessionId), `${afterIndex + 1}-0`, "+", "COUNT", 256)).map((entry) => toEvent(sessionId, entry));
|
|
2537
|
+
});
|
|
2538
|
+
},
|
|
2539
|
+
async readFeedPage(sessionId, afterIndex) {
|
|
2540
|
+
return wrap(async () => {
|
|
2541
|
+
const range = [
|
|
2542
|
+
storeKey(sessionId),
|
|
2543
|
+
`${afterIndex + 1}-0`,
|
|
2544
|
+
"+",
|
|
2545
|
+
"COUNT",
|
|
2546
|
+
FEED_PAGE_SIZE
|
|
2547
|
+
];
|
|
2548
|
+
if (callBytes) return (await callBytes("XRANGE", ...range)).map((entry) => bytesToSerializedEvent(sessionId, entry));
|
|
2549
|
+
return (await call("XRANGE", ...range)).map((entry) => toSerializedEvent(sessionId, entry));
|
|
2550
|
+
});
|
|
2551
|
+
}
|
|
2552
|
+
};
|
|
2553
|
+
}
|
|
2554
|
+
/** A feed that falls this far behind drops its inbox and catches up. */
|
|
2555
|
+
const INBOX_BYTES = 524288;
|
|
2556
|
+
/**
|
|
2557
|
+
* Parses a wake-up message: `{"f":frontier}` or a bare frontier number
|
|
2558
|
+
* announces events without carrying them; `{"f","t","e"}` carries them.
|
|
2559
|
+
* Returns null for anything else, which callers treat as a hint to catch up.
|
|
2560
|
+
*/
|
|
2561
|
+
const parseWake = (message) => {
|
|
2562
|
+
let parsed;
|
|
2563
|
+
try {
|
|
2564
|
+
parsed = JSON.parse(message);
|
|
2565
|
+
} catch {
|
|
2566
|
+
return null;
|
|
2567
|
+
}
|
|
2568
|
+
if (typeof parsed === "number") return Number.isSafeInteger(parsed) ? {
|
|
2569
|
+
frontier: parsed,
|
|
2570
|
+
createdAt: 0,
|
|
2571
|
+
entries: []
|
|
2572
|
+
} : null;
|
|
2573
|
+
if (typeof parsed !== "object" || parsed === null) return null;
|
|
2574
|
+
const { f, t, e } = parsed;
|
|
2575
|
+
if (typeof f !== "number" || !Number.isSafeInteger(f)) return null;
|
|
2576
|
+
if (e === void 0) return {
|
|
2577
|
+
frontier: f,
|
|
2578
|
+
createdAt: 0,
|
|
2579
|
+
entries: []
|
|
2580
|
+
};
|
|
2581
|
+
if (!Array.isArray(e) || typeof t !== "number" || !Number.isFinite(t)) return null;
|
|
2582
|
+
for (const entry of e) if (!Array.isArray(entry) || entry.length !== 4 || !Number.isSafeInteger(entry[0]) || typeof entry[1] !== "string" || typeof entry[2] !== "string") return null;
|
|
2583
|
+
return {
|
|
2584
|
+
frontier: f,
|
|
2585
|
+
createdAt: t,
|
|
2586
|
+
entries: e
|
|
2587
|
+
};
|
|
2588
|
+
};
|
|
2589
|
+
function createRedisNotifyStore(options) {
|
|
2590
|
+
const { client, clock, ids, keyPrefix: prefix } = options;
|
|
2591
|
+
const notifyChannel = (sessionId) => `${prefix}:${sessionId}:notify`;
|
|
2592
|
+
const presenceChannel = (ns) => `${prefix}:${ns}:presence`;
|
|
2593
|
+
const instanceId = crypto.randomUUID();
|
|
2594
|
+
const core = createRedisStoreCore({
|
|
2595
|
+
call: async (command, ...args) => {
|
|
2596
|
+
return (await client()).call(command, ...args);
|
|
2597
|
+
},
|
|
2598
|
+
clock,
|
|
2599
|
+
ids,
|
|
2600
|
+
keyPrefix: prefix,
|
|
2601
|
+
presenceSource: instanceId,
|
|
2602
|
+
...options.callBytes ? { callBytes: options.callBytes } : {},
|
|
2603
|
+
...options.readStream ? { readStream: options.readStream } : {}
|
|
2604
|
+
});
|
|
2605
|
+
let severed = false;
|
|
2606
|
+
let subscriber = null;
|
|
2607
|
+
/** channel → live event feeds, registered before they subscribe. */
|
|
2608
|
+
const feeds = /* @__PURE__ */ new Map();
|
|
2609
|
+
/** channel → refcounted subscription held for each live iterator. */
|
|
2610
|
+
const subscriptions = /* @__PURE__ */ new Map();
|
|
2611
|
+
const presenceSubscriptions = /* @__PURE__ */ new Map();
|
|
2612
|
+
const pollingStreams = /* @__PURE__ */ new Set();
|
|
2613
|
+
options.onReconnect?.((channel) => {
|
|
2614
|
+
for (const feed of feeds.get(channel) ?? []) {
|
|
2615
|
+
feed.dirty = true;
|
|
2616
|
+
feed.wake?.();
|
|
2617
|
+
}
|
|
2618
|
+
presenceSubscriptions.get(channel)?.recovery?.reconnect().catch(() => {});
|
|
2619
|
+
});
|
|
2620
|
+
const deliverRemotePresence = ({ subscription, message }) => {
|
|
2621
|
+
let parsed;
|
|
2622
|
+
try {
|
|
2623
|
+
parsed = JSON.parse(message);
|
|
2624
|
+
} catch {
|
|
2625
|
+
return;
|
|
2626
|
+
}
|
|
2627
|
+
if (parsed.src === instanceId) return;
|
|
2628
|
+
const patch = {
|
|
2629
|
+
participant: parsed.participant,
|
|
2630
|
+
values: parsed.values,
|
|
2631
|
+
seen: parsed.seen,
|
|
2632
|
+
at: new Date(parsed.at)
|
|
2633
|
+
};
|
|
2634
|
+
subscription.recovery?.update(patch);
|
|
2635
|
+
for (const listener of subscription.listeners) listener(patch);
|
|
2636
|
+
};
|
|
2637
|
+
const getSubscriber = () => {
|
|
2638
|
+
subscriber ??= (async () => {
|
|
2639
|
+
const conn = (await client()).duplicate();
|
|
2640
|
+
conn.on("message", (channel, message) => {
|
|
2641
|
+
for (const feed of feeds.get(channel) ?? []) {
|
|
2642
|
+
if (feed.bytes + message.length <= INBOX_BYTES) {
|
|
2643
|
+
feed.inbox.push(message);
|
|
2644
|
+
feed.bytes += message.length;
|
|
2645
|
+
} else {
|
|
2646
|
+
feed.inbox.length = 0;
|
|
2647
|
+
feed.held.length = 0;
|
|
2648
|
+
feed.gapSince = void 0;
|
|
2649
|
+
feed.bytes = 0;
|
|
2650
|
+
feed.dirty = true;
|
|
2651
|
+
}
|
|
2652
|
+
feed.wake?.();
|
|
2653
|
+
}
|
|
2654
|
+
const subscription = presenceSubscriptions.get(channel);
|
|
2655
|
+
if (subscription) deliverRemotePresence({
|
|
2656
|
+
subscription,
|
|
2657
|
+
message
|
|
2658
|
+
});
|
|
2659
|
+
});
|
|
2660
|
+
return conn;
|
|
2661
|
+
})();
|
|
2662
|
+
return subscriber;
|
|
2663
|
+
};
|
|
2664
|
+
/**
|
|
2665
|
+
* The subscriber, gated on connection readiness — for channel
|
|
2666
|
+
* commands only (`close` uses the ungated promise, so it can never
|
|
2667
|
+
* hang on an unreachable server). SUBSCRIBE carries redis's
|
|
2668
|
+
* ok-loading flag, so ioredis writes it mid-handshake, ahead of the
|
|
2669
|
+
* ready check's INFO — which then finds the connection already in
|
|
2670
|
+
* subscriber mode and fails the check (a spurious error event plus
|
|
2671
|
+
* a reconnect). ECHO has no such flag: it resolves only once the
|
|
2672
|
+
* connection is ready, so every channel command chained behind it
|
|
2673
|
+
* lands after the check.
|
|
2674
|
+
*/
|
|
2675
|
+
let subscriberGate = null;
|
|
2676
|
+
const gatedSubscriber = () => {
|
|
2677
|
+
subscriberGate ??= getSubscriber().then(async (conn) => {
|
|
2678
|
+
await conn.call("echo", "ready");
|
|
2679
|
+
return conn;
|
|
2680
|
+
});
|
|
2681
|
+
return subscriberGate;
|
|
2682
|
+
};
|
|
2683
|
+
const acquireChannel = async (channel) => {
|
|
2684
|
+
const existing = subscriptions.get(channel);
|
|
2685
|
+
if (existing) {
|
|
2686
|
+
existing.refs += 1;
|
|
2687
|
+
try {
|
|
2688
|
+
await existing.ready;
|
|
2689
|
+
} catch (err) {
|
|
2690
|
+
existing.refs -= 1;
|
|
2691
|
+
throw err;
|
|
2692
|
+
}
|
|
2693
|
+
return;
|
|
2694
|
+
}
|
|
2695
|
+
const lease = {
|
|
2696
|
+
refs: 1,
|
|
2697
|
+
ready: gatedSubscriber().then(async (conn) => {
|
|
2698
|
+
await conn.call("subscribe", channel);
|
|
2699
|
+
if (subscriptions.get(channel) !== lease || severed) return;
|
|
2700
|
+
const presence = presenceSubscriptions.get(channel);
|
|
2701
|
+
if (options.polling?.() && presence) {
|
|
2702
|
+
presence.recovery = void 0;
|
|
2703
|
+
const ns = channel.slice(prefix.length + 1, -9);
|
|
2704
|
+
presence.poll = pollPresence({
|
|
2705
|
+
read: () => core.presence.read(ns),
|
|
2706
|
+
deliver: (patch) => {
|
|
2707
|
+
for (const listener of presence.listeners) listener(patch);
|
|
2708
|
+
}
|
|
2709
|
+
});
|
|
2710
|
+
await presence.poll.ready;
|
|
2711
|
+
}
|
|
2712
|
+
})
|
|
2713
|
+
};
|
|
2714
|
+
subscriptions.set(channel, lease);
|
|
2715
|
+
try {
|
|
2716
|
+
await lease.ready;
|
|
2717
|
+
} catch (err) {
|
|
2718
|
+
if (subscriptions.get(channel) === lease) {
|
|
2719
|
+
subscriptions.delete(channel);
|
|
2720
|
+
const presence = presenceSubscriptions.get(channel);
|
|
2721
|
+
presence?.poll?.stop();
|
|
2722
|
+
if (presence) presence.poll = void 0;
|
|
2723
|
+
}
|
|
2724
|
+
throw err;
|
|
2725
|
+
}
|
|
2726
|
+
};
|
|
2727
|
+
const releaseChannel = (channel) => {
|
|
2728
|
+
const lease = subscriptions.get(channel);
|
|
2729
|
+
if (!lease) return;
|
|
2730
|
+
lease.refs -= 1;
|
|
2731
|
+
if (lease.refs > 0) return;
|
|
2732
|
+
subscriptions.delete(channel);
|
|
2733
|
+
gatedSubscriber().then((conn) => conn.call("unsubscribe", channel)).catch(() => {});
|
|
2734
|
+
};
|
|
2735
|
+
const store = {
|
|
2736
|
+
append: core.append,
|
|
2737
|
+
cursors: core.cursors,
|
|
2738
|
+
read: core.read,
|
|
2739
|
+
readHistory: core.readHistory,
|
|
2740
|
+
claimAvailable: core.claimAvailable,
|
|
2741
|
+
renewClaims: core.renewClaims,
|
|
2742
|
+
completeAttempt: core.completeAttempt,
|
|
2743
|
+
failAttempt: core.failAttempt,
|
|
2744
|
+
readState: core.readState,
|
|
2745
|
+
putSnapshots: core.putSnapshots,
|
|
2746
|
+
clear: core.clear,
|
|
2747
|
+
reclaim: core.reclaim,
|
|
2748
|
+
deadLetters: core.deadLetters,
|
|
2749
|
+
redrive: core.redrive,
|
|
2750
|
+
wakeLog: core.wakeLog,
|
|
2751
|
+
presence: {
|
|
2752
|
+
async set(ns, participant, values, meta) {
|
|
2753
|
+
const applied = await core.presence.set(ns, participant, values, meta);
|
|
2754
|
+
if (!applied) return;
|
|
2755
|
+
const { patch } = applied;
|
|
2756
|
+
const channel = presenceChannel(ns);
|
|
2757
|
+
const subscription = presenceSubscriptions.get(channel);
|
|
2758
|
+
if (subscription) {
|
|
2759
|
+
subscription.poll?.update(patch);
|
|
2760
|
+
subscription.recovery?.update(patch);
|
|
2761
|
+
for (const listener of subscription.listeners) listener(patch);
|
|
2762
|
+
}
|
|
2763
|
+
await applied.published;
|
|
2764
|
+
},
|
|
2765
|
+
async read(ns) {
|
|
2766
|
+
const channel = presenceChannel(ns);
|
|
2767
|
+
await subscriptions.get(channel)?.ready;
|
|
2768
|
+
const subscription = presenceSubscriptions.get(channel);
|
|
2769
|
+
return subscription?.recovery?.read() ?? subscription?.poll?.snapshot(() => core.presence.read(ns)) ?? core.presence.read(ns);
|
|
2770
|
+
},
|
|
2771
|
+
subscribe(ns, onPatch) {
|
|
2772
|
+
const channel = presenceChannel(ns);
|
|
2773
|
+
let subscription = presenceSubscriptions.get(channel);
|
|
2774
|
+
if (!subscription) {
|
|
2775
|
+
subscription = { listeners: /* @__PURE__ */ new Set() };
|
|
2776
|
+
presenceSubscriptions.set(channel, subscription);
|
|
2777
|
+
if (options.onReconnect && !options.polling?.()) {
|
|
2778
|
+
const current = subscription;
|
|
2779
|
+
subscription.recovery = presenceRecovery({
|
|
2780
|
+
read: () => core.presence.read(ns),
|
|
2781
|
+
deliver: (patch) => {
|
|
2782
|
+
for (const listener of current.listeners) listener(patch);
|
|
2783
|
+
},
|
|
2784
|
+
isActive: () => !severed && presenceSubscriptions.get(channel) === current
|
|
2785
|
+
});
|
|
2786
|
+
}
|
|
2787
|
+
}
|
|
2788
|
+
subscription.listeners.add(onPatch);
|
|
2789
|
+
acquireChannel(channel).catch(() => {});
|
|
2790
|
+
let stopped = false;
|
|
2791
|
+
return () => {
|
|
2792
|
+
if (stopped) return;
|
|
2793
|
+
stopped = true;
|
|
2794
|
+
subscription.listeners.delete(onPatch);
|
|
2795
|
+
if (subscription.listeners.size === 0) {
|
|
2796
|
+
subscription.poll?.stop();
|
|
2797
|
+
presenceSubscriptions.delete(channel);
|
|
2798
|
+
}
|
|
2799
|
+
releaseChannel(channel);
|
|
2800
|
+
};
|
|
2801
|
+
}
|
|
2802
|
+
},
|
|
2803
|
+
stream(sessionId, opts) {
|
|
2804
|
+
const startAfter = opts?.startAfter ?? 0;
|
|
2805
|
+
const channel = notifyChannel(sessionId);
|
|
2806
|
+
return { [Symbol.asyncIterator]() {
|
|
2807
|
+
let last = startAfter;
|
|
2808
|
+
let buffer = [];
|
|
2809
|
+
let closed = false;
|
|
2810
|
+
let subscribed = false;
|
|
2811
|
+
let pending = null;
|
|
2812
|
+
let poller;
|
|
2813
|
+
let stopActivity;
|
|
2814
|
+
let catchup;
|
|
2815
|
+
let stoppingRead;
|
|
2816
|
+
const stopRead = () => {
|
|
2817
|
+
const reading = catchup;
|
|
2818
|
+
catchup = void 0;
|
|
2819
|
+
if (reading) {
|
|
2820
|
+
reading.controller.abort();
|
|
2821
|
+
stoppingRead = reading.iterator.return(void 0).then(() => void 0).catch(() => {});
|
|
2822
|
+
}
|
|
2823
|
+
return stoppingRead ?? Promise.resolve();
|
|
2824
|
+
};
|
|
2825
|
+
const nextRead = async (after) => {
|
|
2826
|
+
if (!catchup) {
|
|
2827
|
+
const controller = new AbortController();
|
|
2828
|
+
catchup = {
|
|
2829
|
+
controller,
|
|
2830
|
+
delivered: false,
|
|
2831
|
+
iterator: core.streamEvents({
|
|
2832
|
+
sessionId,
|
|
2833
|
+
afterIndex: after,
|
|
2834
|
+
signal: controller.signal
|
|
2835
|
+
})[Symbol.asyncIterator]()
|
|
2836
|
+
};
|
|
2837
|
+
}
|
|
2838
|
+
const reading = catchup;
|
|
2839
|
+
const result = await reading.iterator.next();
|
|
2840
|
+
if (result.done && catchup === reading) {
|
|
2841
|
+
catchup = void 0;
|
|
2842
|
+
if (reading.delivered) feed.dirty = true;
|
|
2843
|
+
} else if (!result.done) reading.delivered = true;
|
|
2844
|
+
return result;
|
|
2845
|
+
};
|
|
2846
|
+
const feed = {
|
|
2847
|
+
inbox: [],
|
|
2848
|
+
bytes: 0,
|
|
2849
|
+
dirty: true,
|
|
2850
|
+
held: [],
|
|
2851
|
+
gapSince: void 0,
|
|
2852
|
+
wake: null,
|
|
2853
|
+
stopRead
|
|
2854
|
+
};
|
|
2855
|
+
const activity = createStreamActivity(() => {
|
|
2856
|
+
if (poller) streamActivities.get(poller)?.activate();
|
|
2857
|
+
});
|
|
2858
|
+
const release = () => {
|
|
2859
|
+
stopRead();
|
|
2860
|
+
stopActivity?.();
|
|
2861
|
+
if (poller) {
|
|
2862
|
+
poller.return?.();
|
|
2863
|
+
pollingStreams.delete(poller);
|
|
2864
|
+
}
|
|
2865
|
+
if (!subscribed) return;
|
|
2866
|
+
subscribed = false;
|
|
2867
|
+
const set = feeds.get(channel);
|
|
2868
|
+
set?.delete(feed);
|
|
2869
|
+
if (set?.size === 0) feeds.delete(channel);
|
|
2870
|
+
releaseChannel(channel);
|
|
2871
|
+
};
|
|
2872
|
+
/**
|
|
2873
|
+
* Moves carried events that continue this feed's sequence into
|
|
2874
|
+
* the buffer. Writers publish after their scripts, so concurrent
|
|
2875
|
+
* writers' wake-ups can arrive out of order: they apply in index
|
|
2876
|
+
* order, and one past a gap is held until the gap fills or
|
|
2877
|
+
* NOTIFY_TIMINGS.reorderMs passes. A message without events past
|
|
2878
|
+
* the buffer, or an unreadable one, marks the feed dirty so it
|
|
2879
|
+
* catches up with XRANGE.
|
|
2880
|
+
*/
|
|
2881
|
+
const applyInbox = () => {
|
|
2882
|
+
const carried = feed.held.splice(0);
|
|
2883
|
+
let announced = 0;
|
|
2884
|
+
for (const message of feed.inbox.splice(0)) {
|
|
2885
|
+
const wake = parseWake(message);
|
|
2886
|
+
if (!wake) feed.dirty = true;
|
|
2887
|
+
else if (wake.entries.length === 0) announced = Math.max(announced, wake.frontier);
|
|
2888
|
+
else carried.push({
|
|
2889
|
+
wake,
|
|
2890
|
+
bytes: message.length
|
|
2891
|
+
});
|
|
2892
|
+
}
|
|
2893
|
+
carried.sort((a, b) => a.wake.entries[0][0] - b.wake.entries[0][0]);
|
|
2894
|
+
feed.bytes = 0;
|
|
2895
|
+
for (const held of carried) {
|
|
2896
|
+
if (held.wake.entries[0][0] > (buffer.at(-1)?.index ?? last) + 1) {
|
|
2897
|
+
feed.held.push(held);
|
|
2898
|
+
feed.bytes += held.bytes;
|
|
2899
|
+
continue;
|
|
2900
|
+
}
|
|
2901
|
+
for (const [index, id, type, payload] of held.wake.entries) {
|
|
2902
|
+
if (index <= (buffer.at(-1)?.index ?? last)) continue;
|
|
2903
|
+
buffer.push({
|
|
2904
|
+
id,
|
|
2905
|
+
type,
|
|
2906
|
+
payload,
|
|
2907
|
+
index,
|
|
2908
|
+
sessionId,
|
|
2909
|
+
createdAt: new Date(held.wake.createdAt)
|
|
2910
|
+
});
|
|
2911
|
+
}
|
|
2912
|
+
}
|
|
2913
|
+
if (announced > (buffer.at(-1)?.index ?? last)) feed.dirty = true;
|
|
2914
|
+
if (feed.held.length === 0) feed.gapSince = void 0;
|
|
2915
|
+
else feed.gapSince ??= Date.now();
|
|
2916
|
+
};
|
|
2917
|
+
const iterator = {
|
|
2918
|
+
async next() {
|
|
2919
|
+
for (;;) {
|
|
2920
|
+
if (closed) {
|
|
2921
|
+
release();
|
|
2922
|
+
return {
|
|
2923
|
+
value: void 0,
|
|
2924
|
+
done: true
|
|
2925
|
+
};
|
|
2926
|
+
}
|
|
2927
|
+
const row = buffer.shift();
|
|
2928
|
+
if (row) {
|
|
2929
|
+
let event;
|
|
2930
|
+
try {
|
|
2931
|
+
event = isSerializedEvent(row) ? parseSerializedEvent(row) : row;
|
|
2932
|
+
} catch (cause) {
|
|
2933
|
+
buffer = [];
|
|
2934
|
+
feed.dirty = true;
|
|
2935
|
+
release();
|
|
2936
|
+
throw new A2Error("STORE_UNAVAILABLE", "redis stream read failed", { cause });
|
|
2937
|
+
}
|
|
2938
|
+
last = row.index;
|
|
2939
|
+
return {
|
|
2940
|
+
value: event,
|
|
2941
|
+
done: false
|
|
2942
|
+
};
|
|
2943
|
+
}
|
|
2944
|
+
if (severed) {
|
|
2945
|
+
release();
|
|
2946
|
+
throw new A2Error("STORE_UNAVAILABLE", "the store was closed");
|
|
2947
|
+
}
|
|
2948
|
+
try {
|
|
2949
|
+
if (!subscribed) {
|
|
2950
|
+
subscribed = true;
|
|
2951
|
+
let set = feeds.get(channel);
|
|
2952
|
+
if (!set) {
|
|
2953
|
+
set = /* @__PURE__ */ new Set();
|
|
2954
|
+
feeds.set(channel, set);
|
|
2955
|
+
}
|
|
2956
|
+
set.add(feed);
|
|
2957
|
+
const [, clearIndex] = await Promise.all([acquireChannel(channel), core.clearIndex(sessionId)]);
|
|
2958
|
+
if (clearIndex - 1 > last) last = clearIndex - 1;
|
|
2959
|
+
if (closed || severed) continue;
|
|
2960
|
+
}
|
|
2961
|
+
if (options.polling?.()) {
|
|
2962
|
+
if (!poller) {
|
|
2963
|
+
poller = pollingStream((after) => core.readEvents(sessionId, after), { startAfter: last })[Symbol.asyncIterator]();
|
|
2964
|
+
pollingStreams.add(poller);
|
|
2965
|
+
stopActivity = streamActivities.get(poller)?.subscribe(activity.setIdle);
|
|
2966
|
+
}
|
|
2967
|
+
const result = await poller.next();
|
|
2968
|
+
if (!result.done) presenceSubscriptions.get(presenceChannel(sessionId))?.poll?.wake();
|
|
2969
|
+
if (severed) throw new A2Error("STORE_UNAVAILABLE", "the store was closed");
|
|
2970
|
+
return result;
|
|
2971
|
+
}
|
|
2972
|
+
if (catchup) {
|
|
2973
|
+
const result = await nextRead(last);
|
|
2974
|
+
if (closed || severed) continue;
|
|
2975
|
+
if (!result.done) buffer = result.value;
|
|
2976
|
+
continue;
|
|
2977
|
+
}
|
|
2978
|
+
applyInbox();
|
|
2979
|
+
if (buffer.length > 0) continue;
|
|
2980
|
+
if (feed.gapSince !== void 0 && Date.now() - feed.gapSince >= NOTIFY_TIMINGS.reorderMs) feed.dirty = true;
|
|
2981
|
+
if (feed.dirty) {
|
|
2982
|
+
feed.dirty = false;
|
|
2983
|
+
feed.held.length = 0;
|
|
2984
|
+
feed.gapSince = void 0;
|
|
2985
|
+
if (core.streamEvents) {
|
|
2986
|
+
const result = await nextRead(last);
|
|
2987
|
+
if (closed || severed) continue;
|
|
2988
|
+
if (!result.done) buffer = result.value;
|
|
2989
|
+
continue;
|
|
2990
|
+
}
|
|
2991
|
+
buffer = await core.readFeedPage(sessionId, last);
|
|
2992
|
+
if (buffer.length >= 1e3) feed.dirty = true;
|
|
2993
|
+
continue;
|
|
2994
|
+
}
|
|
2995
|
+
const woken = new Promise((resolve) => {
|
|
2996
|
+
feed.wake = () => resolve("wake");
|
|
2997
|
+
});
|
|
2998
|
+
const holding = feed.gapSince !== void 0;
|
|
2999
|
+
pending = defaultSleep(holding ? Math.max(0, NOTIFY_TIMINGS.reorderMs - (Date.now() - feed.gapSince)) : NOTIFY_TIMINGS.safetyReadMs);
|
|
3000
|
+
const outcome = await Promise.race([woken, pending.promise.then(() => "timer")]);
|
|
3001
|
+
pending.cancel();
|
|
3002
|
+
pending = null;
|
|
3003
|
+
if (outcome === "timer" && !holding) feed.dirty = true;
|
|
3004
|
+
} catch (err) {
|
|
3005
|
+
release();
|
|
3006
|
+
if (closed) return {
|
|
3007
|
+
value: void 0,
|
|
3008
|
+
done: true
|
|
3009
|
+
};
|
|
3010
|
+
if (err instanceof A2Error) throw err;
|
|
3011
|
+
throw new A2Error("STORE_UNAVAILABLE", "redis store operation failed", { cause: err });
|
|
3012
|
+
} finally {
|
|
3013
|
+
feed.wake = null;
|
|
3014
|
+
}
|
|
3015
|
+
}
|
|
3016
|
+
},
|
|
3017
|
+
async return() {
|
|
3018
|
+
closed = true;
|
|
3019
|
+
pending?.cancel();
|
|
3020
|
+
feed.wake?.();
|
|
3021
|
+
release();
|
|
3022
|
+
await stopRead();
|
|
3023
|
+
return {
|
|
3024
|
+
value: void 0,
|
|
3025
|
+
done: true
|
|
3026
|
+
};
|
|
3027
|
+
}
|
|
3028
|
+
};
|
|
3029
|
+
streamActivities.set(iterator, activity);
|
|
3030
|
+
bufferedEvents.set(iterator, () => {
|
|
3031
|
+
if (closed) return void 0;
|
|
3032
|
+
const row = buffer.shift();
|
|
3033
|
+
if (row) last = row.index;
|
|
3034
|
+
return row;
|
|
3035
|
+
});
|
|
3036
|
+
return iterator;
|
|
3037
|
+
} };
|
|
3038
|
+
},
|
|
3039
|
+
async close() {
|
|
3040
|
+
severed = true;
|
|
3041
|
+
for (const subscription of presenceSubscriptions.values()) subscription.poll?.stop();
|
|
3042
|
+
presenceSubscriptions.clear();
|
|
3043
|
+
for (const poller of pollingStreams) poller.return?.();
|
|
3044
|
+
pollingStreams.clear();
|
|
3045
|
+
const reads = [];
|
|
3046
|
+
for (const set of feeds.values()) for (const feed of set) {
|
|
3047
|
+
feed.wake?.();
|
|
3048
|
+
reads.push(feed.stopRead());
|
|
3049
|
+
}
|
|
3050
|
+
await Promise.allSettled(reads);
|
|
3051
|
+
if (subscriber) (await subscriber.catch(() => null))?.disconnect();
|
|
3052
|
+
await options.closeClient();
|
|
3053
|
+
}
|
|
3054
|
+
};
|
|
3055
|
+
stateTailReaders.set(store, core.readTail);
|
|
3056
|
+
return store;
|
|
3057
|
+
}
|
|
3058
|
+
//#endregion
|
|
3059
|
+
//#region src/store/redis-tables.ts
|
|
3060
|
+
/**
|
|
3061
|
+
* Applies an event's writes if it's the one to apply: with `next`, only the very
|
|
3062
|
+
* next index; otherwise any index past the position. The check and the writes are
|
|
3063
|
+
* one script, so a retried or superseded attempt changes nothing. Every key it
|
|
3064
|
+
* touches comes in KEYS: the position first, then the writes' own.
|
|
3065
|
+
*/
|
|
3066
|
+
const APPLY = `
|
|
3067
|
+
local pos = tonumber(redis.call('GET', KEYS[1]) or '-1')
|
|
3068
|
+
local index = tonumber(ARGV[1])
|
|
3069
|
+
if ARGV[2] == 'next' then
|
|
3070
|
+
if pos ~= index - 1 then return 0 end
|
|
3071
|
+
elseif pos >= index then
|
|
3072
|
+
return 0
|
|
3073
|
+
end
|
|
3074
|
+
for _, op in ipairs(cjson.decode(ARGV[3])) do
|
|
3075
|
+
redis.call(op[1], KEYS[op[2]], unpack(op, 3))
|
|
3076
|
+
end
|
|
3077
|
+
redis.call('SET', KEYS[1], ARGV[1])
|
|
3078
|
+
return 1
|
|
3079
|
+
`;
|
|
3080
|
+
/**
|
|
3081
|
+
* The scripts' shared part: which of a page of IDs match every group (one of
|
|
3082
|
+
* its sets each) and the text. Upstash bills each command a script runs, so a
|
|
3083
|
+
* page costs one ZMSCORE per set and one HMGET for the text, whatever its size.
|
|
3084
|
+
*/
|
|
3085
|
+
const MATCHES = `
|
|
3086
|
+
local function matching(spec, from, ids)
|
|
3087
|
+
local ok = {}
|
|
3088
|
+
for i = 1, #ids do ok[i] = true end
|
|
3089
|
+
for _, group in ipairs(spec.groups) do
|
|
3090
|
+
-- The set the page came from holds every ID in it.
|
|
3091
|
+
if not (#group == 1 and KEYS[group[1]] == from) then
|
|
3092
|
+
local any = {}
|
|
3093
|
+
for _, key in ipairs(group) do
|
|
3094
|
+
local scores = redis.call('ZMSCORE', KEYS[key], unpack(ids))
|
|
3095
|
+
for i = 1, #ids do if scores[i] then any[i] = true end end
|
|
3096
|
+
end
|
|
3097
|
+
for i = 1, #ids do if not any[i] then ok[i] = false end end
|
|
3098
|
+
end
|
|
3099
|
+
end
|
|
3100
|
+
local metas = nil
|
|
3101
|
+
if spec.text ~= cjson.null then
|
|
3102
|
+
metas = redis.call('HMGET', KEYS[3], unpack(ids))
|
|
3103
|
+
for i = 1, #ids do
|
|
3104
|
+
if ok[i] and not string.find(cjson.decode(metas[i]).t, spec.text, 1, true) then ok[i] = false end
|
|
3105
|
+
end
|
|
3106
|
+
end
|
|
3107
|
+
return ok, metas
|
|
3108
|
+
end
|
|
3109
|
+
local function driver(spec)
|
|
3110
|
+
local best, size = KEYS[spec.fallback], nil
|
|
3111
|
+
for _, key in ipairs(spec.drivers) do
|
|
3112
|
+
local n = redis.call('ZCARD', KEYS[key])
|
|
3113
|
+
if size == nil or n < size then best, size = KEYS[key], n end
|
|
3114
|
+
end
|
|
3115
|
+
return best
|
|
3116
|
+
end
|
|
3117
|
+
`;
|
|
3118
|
+
/**
|
|
3119
|
+
* Messages that match, walking the smallest index set newest first (oldest first
|
|
3120
|
+
* with \`asc\`) and testing the others a page at a time, until \`limit\` or until
|
|
3121
|
+
* it has read \`budget\` IDs: Redis runs nothing else while a script runs, so a
|
|
3122
|
+
* long read goes in several. Returns the position, the matches, and where to go
|
|
3123
|
+
* on from (false when the set ran out). KEYS: position, messages, metas, then
|
|
3124
|
+
* the sets.
|
|
3125
|
+
*/
|
|
3126
|
+
const MESSAGES = `${MATCHES}
|
|
3127
|
+
local spec = cjson.decode(ARGV[1])
|
|
3128
|
+
local from = driver(spec)
|
|
3129
|
+
local found, n, read = {}, 0, 0
|
|
3130
|
+
local low, high = spec.min, spec.max
|
|
3131
|
+
local more = true
|
|
3132
|
+
while n < spec.limit do
|
|
3133
|
+
if read >= spec.budget then break end
|
|
3134
|
+
local count = math.min(200, spec.budget - read)
|
|
3135
|
+
local ids
|
|
3136
|
+
if spec.asc then ids = redis.call('ZRANGEBYSCORE', from, low, high, 'LIMIT', 0, count)
|
|
3137
|
+
else ids = redis.call('ZREVRANGEBYSCORE', from, high, low, 'LIMIT', 0, count) end
|
|
3138
|
+
if #ids == 0 then more = false break end
|
|
3139
|
+
local ok = matching(spec, from, ids)
|
|
3140
|
+
for i, id in ipairs(ids) do
|
|
3141
|
+
read = read + 1
|
|
3142
|
+
if spec.asc then low = '(' .. id else high = '(' .. id end
|
|
3143
|
+
if ok[i] then
|
|
3144
|
+
n = n + 1
|
|
3145
|
+
found[n] = id
|
|
3146
|
+
if n >= spec.limit then break end
|
|
3147
|
+
end
|
|
3148
|
+
end
|
|
3149
|
+
end
|
|
3150
|
+
local rows = {}
|
|
3151
|
+
if n > 0 then rows = redis.call('HMGET', KEYS[2], unpack(found)) end
|
|
3152
|
+
local next = false
|
|
3153
|
+
if more and n < spec.limit then if spec.asc then next = low else next = high end end
|
|
3154
|
+
return { redis.call('GET', KEYS[1]) or '0', rows, next }
|
|
3155
|
+
`;
|
|
3156
|
+
/**
|
|
3157
|
+
* The latest threads in a channel, or across several, from their thread order
|
|
3158
|
+
* sets: each set's first \`limit\`, merged. KEYS: position, then the sets.
|
|
3159
|
+
*/
|
|
3160
|
+
const LATEST = `
|
|
3161
|
+
local spec = cjson.decode(ARGV[1])
|
|
3162
|
+
local roots = {}
|
|
3163
|
+
for i = 2, #KEYS do
|
|
3164
|
+
local ids
|
|
3165
|
+
if spec.asc then ids = redis.call('ZRANGEBYSCORE', KEYS[i], spec.min, '+inf', 'WITHSCORES', 'LIMIT', 0, spec.limit)
|
|
3166
|
+
else ids = redis.call('ZREVRANGEBYSCORE', KEYS[i], '+inf', spec.min, 'WITHSCORES', 'LIMIT', 0, spec.limit) end
|
|
3167
|
+
for j = 1, #ids, 2 do roots[#roots + 1] = ids[j] roots[#roots + 1] = ids[j + 1] end
|
|
3168
|
+
end
|
|
3169
|
+
return { redis.call('GET', KEYS[1]) or '0', roots }
|
|
3170
|
+
`;
|
|
3171
|
+
/**
|
|
3172
|
+
* The threads with a message that matches: every match's root, once, and its
|
|
3173
|
+
* latest reply, reading at most \`budget\` IDs. Returns the position, the roots
|
|
3174
|
+
* and their latest replies, and where to go on from. KEYS: position, messages,
|
|
3175
|
+
* metas, summaries, then the sets.
|
|
3176
|
+
*/
|
|
3177
|
+
const ROOTS = `${MATCHES}
|
|
3178
|
+
local spec = cjson.decode(ARGV[1])
|
|
3179
|
+
local from = driver(spec)
|
|
3180
|
+
local roots, seen, read, high = {}, {}, 0, spec.max
|
|
3181
|
+
while read < spec.budget do
|
|
3182
|
+
local ids = redis.call('ZREVRANGEBYSCORE', from, high, '-inf', 'LIMIT', 0, math.min(200, spec.budget - read))
|
|
3183
|
+
if #ids == 0 then return { redis.call('GET', KEYS[1]) or '0', roots, false } end
|
|
3184
|
+
local ok, metas = matching(spec, from, ids)
|
|
3185
|
+
local matched = {}
|
|
3186
|
+
for i, id in ipairs(ids) do if ok[i] then matched[#matched + 1] = id end end
|
|
3187
|
+
if #matched > 0 then
|
|
3188
|
+
if metas then
|
|
3189
|
+
local byId = {}
|
|
3190
|
+
for i, id in ipairs(ids) do byId[id] = metas[i] end
|
|
3191
|
+
metas = {}
|
|
3192
|
+
for j, id in ipairs(matched) do metas[j] = byId[id] end
|
|
3193
|
+
else
|
|
3194
|
+
metas = redis.call('HMGET', KEYS[3], unpack(matched))
|
|
3195
|
+
end
|
|
3196
|
+
local fresh = {}
|
|
3197
|
+
for j, id in ipairs(matched) do
|
|
3198
|
+
local meta = cjson.decode(metas[j])
|
|
3199
|
+
local root = id
|
|
3200
|
+
if meta.re ~= cjson.null then root = tostring(meta.re) end
|
|
3201
|
+
if not seen[root] then
|
|
3202
|
+
seen[root] = true
|
|
3203
|
+
fresh[#fresh + 1] = root
|
|
3204
|
+
end
|
|
3205
|
+
end
|
|
3206
|
+
if #fresh > 0 then
|
|
3207
|
+
local summaries = redis.call('HMGET', KEYS[4], unpack(fresh))
|
|
3208
|
+
for j, root in ipairs(fresh) do
|
|
3209
|
+
roots[#roots + 1] = root
|
|
3210
|
+
roots[#roots + 1] = cjson.decode(summaries[j]).at
|
|
3211
|
+
end
|
|
3212
|
+
end
|
|
3213
|
+
end
|
|
3214
|
+
read = read + #ids
|
|
3215
|
+
high = '(' .. ids[#ids]
|
|
3216
|
+
end
|
|
3217
|
+
return { redis.call('GET', KEYS[1]) or '0', roots, high }
|
|
3218
|
+
`;
|
|
3219
|
+
/** Each root, its reply count, and its latest reply (false when it has none). KEYS: messages, summaries. ARGV: the roots. */
|
|
3220
|
+
const THREAD_ROWS = `
|
|
3221
|
+
local summaries = redis.call('HMGET', KEYS[2], unpack(ARGV))
|
|
3222
|
+
local lasts, slot = {}, {}
|
|
3223
|
+
for i, root in ipairs(ARGV) do
|
|
3224
|
+
summaries[i] = cjson.decode(summaries[i])
|
|
3225
|
+
if tostring(summaries[i].at) ~= root then
|
|
3226
|
+
lasts[#lasts + 1] = tostring(summaries[i].at)
|
|
3227
|
+
slot[i] = #lasts
|
|
3228
|
+
end
|
|
3229
|
+
end
|
|
3230
|
+
local rows = redis.call('HMGET', KEYS[1], unpack(ARGV))
|
|
3231
|
+
local lastRows = {}
|
|
3232
|
+
if #lasts > 0 then lastRows = redis.call('HMGET', KEYS[1], unpack(lasts)) end
|
|
3233
|
+
local out = {}
|
|
3234
|
+
for i = 1, #ARGV do
|
|
3235
|
+
local last = false
|
|
3236
|
+
if slot[i] then last = lastRows[slot[i]] end
|
|
3237
|
+
out[i] = { rows[i], summaries[i].n, last }
|
|
3238
|
+
end
|
|
3239
|
+
return out
|
|
3240
|
+
`;
|
|
3241
|
+
/** Each message's latest version at or before ARGV[1]. KEYS: the version data, then each message's versions. */
|
|
3242
|
+
const AT = `
|
|
3243
|
+
local versions, slot = {}, {}
|
|
3244
|
+
for i = 2, #KEYS do
|
|
3245
|
+
local found = redis.call('ZREVRANGEBYSCORE', KEYS[i], ARGV[1], '-inf', 'LIMIT', 0, 1)
|
|
3246
|
+
if #found > 0 then
|
|
3247
|
+
versions[#versions + 1] = found[1]
|
|
3248
|
+
slot[i - 1] = #versions
|
|
3249
|
+
end
|
|
3250
|
+
end
|
|
3251
|
+
local data = {}
|
|
3252
|
+
if #versions > 0 then data = redis.call('HMGET', KEYS[1], unpack(versions)) end
|
|
3253
|
+
local out = {}
|
|
3254
|
+
for i = 1, #KEYS - 1 do
|
|
3255
|
+
if slot[i] then out[i] = data[slot[i]] else out[i] = false end
|
|
3256
|
+
end
|
|
3257
|
+
return out
|
|
3258
|
+
`;
|
|
3259
|
+
/** Each key's hash, whole. */
|
|
3260
|
+
const HASHES = `
|
|
3261
|
+
local out = {}
|
|
3262
|
+
for i = 1, #KEYS do out[i] = redis.call('HGETALL', KEYS[i]) end
|
|
3263
|
+
return out
|
|
3264
|
+
`;
|
|
3265
|
+
/** A person's roots posted at or after ARGV[1]. KEYS: messages, their roots by creation time. */
|
|
3266
|
+
const RECENT = `
|
|
3267
|
+
local ids = redis.call('ZRANGEBYSCORE', KEYS[2], ARGV[1], '+inf')
|
|
3268
|
+
if #ids == 0 then return {} end
|
|
3269
|
+
return redis.call('HMGET', KEYS[1], unpack(ids))
|
|
3270
|
+
`;
|
|
3271
|
+
/**
|
|
3272
|
+
* What an event reads before it applies, in one script so it reads the primary,
|
|
3273
|
+
* where the position check runs: a replica can lag it. Nothing, when the
|
|
3274
|
+
* position says it's not the one to apply (as APPLY checks). KEYS: position,
|
|
3275
|
+
* messages, metas, summaries, sync threads, the copy's hash. ARGV: the index,
|
|
3276
|
+
* the mode, the message, its root ('' to take the message's own), the sync
|
|
3277
|
+
* thread, the copy ('' for none of each). Returns the message, its root's ID,
|
|
3278
|
+
* the root, its summary, the sync thread, and the copy.
|
|
3279
|
+
*/
|
|
3280
|
+
const BEFORE = `
|
|
3281
|
+
local pos = tonumber(redis.call('GET', KEYS[1]) or '-1')
|
|
3282
|
+
local index = tonumber(ARGV[1])
|
|
3283
|
+
if ARGV[2] == 'next' then
|
|
3284
|
+
if pos ~= index - 1 then return false end
|
|
3285
|
+
elseif pos >= index then
|
|
3286
|
+
return false
|
|
3287
|
+
end
|
|
3288
|
+
local message, root = false, ARGV[4]
|
|
3289
|
+
if ARGV[3] ~= '' then
|
|
3290
|
+
message = redis.call('HGET', KEYS[2], ARGV[3])
|
|
3291
|
+
if message and root == '' then
|
|
3292
|
+
local re = cjson.decode(redis.call('HGET', KEYS[3], ARGV[3])).re
|
|
3293
|
+
if re ~= cjson.null then root = tostring(re) end
|
|
3294
|
+
end
|
|
3295
|
+
end
|
|
3296
|
+
local rootData, summary = false, false
|
|
3297
|
+
if root ~= '' then
|
|
3298
|
+
rootData = redis.call('HGET', KEYS[2], root)
|
|
3299
|
+
summary = redis.call('HGET', KEYS[4], root)
|
|
3300
|
+
end
|
|
3301
|
+
local thread, copy = false, false
|
|
3302
|
+
if ARGV[5] ~= '' then thread = redis.call('HGET', KEYS[5], ARGV[5]) end
|
|
3303
|
+
if ARGV[6] ~= '' then copy = redis.call('HGET', KEYS[6], ARGV[6]) end
|
|
3304
|
+
return { message, root, rootData, summary, thread, copy }
|
|
3305
|
+
`;
|
|
3306
|
+
/** The newest reply in a thread that isn't deleted, other than ARGV (false when none). KEYS: the thread, metas. */
|
|
3307
|
+
const NEWEST = `
|
|
3308
|
+
local skip = {}
|
|
3309
|
+
for _, id in ipairs(ARGV) do skip[id] = true end
|
|
3310
|
+
local high = '+inf'
|
|
3311
|
+
while true do
|
|
3312
|
+
local ids = redis.call('ZREVRANGEBYSCORE', KEYS[1], high, '-inf', 'LIMIT', 0, 100)
|
|
3313
|
+
if #ids == 0 then return false end
|
|
3314
|
+
local metas = redis.call('HMGET', KEYS[2], unpack(ids))
|
|
3315
|
+
for i, id in ipairs(ids) do
|
|
3316
|
+
if not skip[id] and not cjson.decode(metas[i]).d then return id end
|
|
3317
|
+
end
|
|
3318
|
+
high = '(' .. ids[#ids]
|
|
3319
|
+
end
|
|
3320
|
+
`;
|
|
3321
|
+
/** Every message in a set of IDs (a thread, or those carrying an attachment), a thousand to an HMGET. KEYS: the set, messages. */
|
|
3322
|
+
const THREAD = `
|
|
3323
|
+
local ids = redis.call('ZRANGE', KEYS[1], 0, -1)
|
|
3324
|
+
local rows = {}
|
|
3325
|
+
for first = 1, #ids, 1000 do
|
|
3326
|
+
local part = redis.call('HMGET', KEYS[2], unpack(ids, first, math.min(first + 999, #ids)))
|
|
3327
|
+
for i = 1, #part do rows[#rows + 1] = part[i] end
|
|
3328
|
+
end
|
|
3329
|
+
return rows
|
|
3330
|
+
`;
|
|
3331
|
+
/** The position and every channel, read together. KEYS: position, channels. */
|
|
3332
|
+
const CHANNELS = `
|
|
3333
|
+
return { redis.call('GET', KEYS[1]) or '0', redis.call('SMEMBERS', KEYS[2]) }
|
|
3334
|
+
`;
|
|
3335
|
+
/**
|
|
3336
|
+
* Upstash runs a script under a lock on the whole database unless the script
|
|
3337
|
+
* sets `allow-key-locking`, and then locks only the hash tags of its KEYS and
|
|
3338
|
+
* refuses any other key. Every key of the tables shares one tag, so a script
|
|
3339
|
+
* locks only its board's tables; a `no-writes` one takes the shared lock. Redis
|
|
3340
|
+
* and Valkey reject the flag, and have no such lock: one thread runs every
|
|
3341
|
+
* command. A2's Redis store does the same for its sessions.
|
|
3342
|
+
*/
|
|
3343
|
+
const KEY_LOCKING = "#!lua flags=allow-key-locking";
|
|
3344
|
+
const READ_ONLY_KEY_LOCKING = "#!lua flags=no-writes,allow-key-locking";
|
|
3345
|
+
/** A server that doesn't know the flag (Redis 7, Valkey), or shebangs at all (Redis 6). */
|
|
3346
|
+
const SHEBANG_REJECTED = /shebang|unexpected symbol near '#'/i;
|
|
3347
|
+
const SEP = "";
|
|
3348
|
+
/** How many IDs one query script reads before it hands back where it got to. */
|
|
3349
|
+
const SCAN_BUDGET = 4e3;
|
|
3350
|
+
/** JSON a script decodes: Redis's cjson refuses an escaped lone surrogate, so each goes out as U+FFFD. */
|
|
3351
|
+
const luaJson = (value) => JSON.stringify(value, (_key, item) => typeof item === "string" ? item.toWellFormed() : item);
|
|
3352
|
+
/** The label sets a message is in: each key, and each key with its value. */
|
|
3353
|
+
const labelTokens = (labels) => Object.entries(labels).flatMap(([key, value]) => [`l${SEP}${key}`, `lv${SEP}${key}${SEP}${value}`]);
|
|
3354
|
+
/** A message is in its own label sets and, for a reply, its root's: "on the message or its root" is one lookup. */
|
|
3355
|
+
const inLabelSets = (own, root) => /* @__PURE__ */ new Set([...labelTokens(own), ...root ? labelTokens(root) : []]);
|
|
3356
|
+
/** The tables' keys, `<prefix>:{<prefix>:board}:<name>`: one hash tag, its board's own. */
|
|
3357
|
+
const tablesKeyBase = (prefix) => `${prefix}:{${prefix}:board}`;
|
|
3358
|
+
/**
|
|
3359
|
+
* The tables on Redis: messages, versions, and sync rows in hashes and sorted
|
|
3360
|
+
* sets under `<prefix>:{<prefix>:board}`, and a sorted set per filter value,
|
|
3361
|
+
* scored by message ID. Each event is two scripts, one that reads what it
|
|
3362
|
+
* changes and one that checks the position and writes; each query is one
|
|
3363
|
+
* script, which walks the smallest set that applies.
|
|
3364
|
+
*
|
|
3365
|
+
* Every key shares the hash tag `{<prefix>:board}`: the board's alone, since a
|
|
3366
|
+
* prefix can't hold a brace, and never an A2 session's, which is a JSON string.
|
|
3367
|
+
* Each script declares every key it touches. Keys of an earlier layout,
|
|
3368
|
+
* `<prefix>:board:<name>`, have other names, so they're never read: the tables
|
|
3369
|
+
* fill from the log.
|
|
3370
|
+
*/
|
|
3371
|
+
function redisTables(send, prefix, options = {}) {
|
|
3372
|
+
if (prefix.includes("{") || prefix.includes("}")) throw new TypeError("a Redis keyPrefix can't contain braces: they'd set every key's hash tag");
|
|
3373
|
+
const call = (command, ...args) => send(command, ...args.map((arg) => typeof arg === "string" ? arg.toWellFormed() : arg));
|
|
3374
|
+
const budget = options.scanBudget ?? SCAN_BUDGET;
|
|
3375
|
+
const p = tablesKeyBase(prefix);
|
|
3376
|
+
const k = {
|
|
3377
|
+
pos: `${p}:pos`,
|
|
3378
|
+
messages: `${p}:m`,
|
|
3379
|
+
metas: `${p}:x`,
|
|
3380
|
+
summaries: `${p}:s`,
|
|
3381
|
+
events: `${p}:e`,
|
|
3382
|
+
versionData: `${p}:vd`,
|
|
3383
|
+
all: `${p}:i:all`,
|
|
3384
|
+
channels: `${p}:channels`,
|
|
3385
|
+
threads: `${p}:t`,
|
|
3386
|
+
syncThreads: `${p}:st`,
|
|
3387
|
+
externalThreads: `${p}:et`,
|
|
3388
|
+
externalMessages: `${p}:em`,
|
|
3389
|
+
versions: (id) => `${p}:v:${id}`,
|
|
3390
|
+
channel: (channel) => `${p}:i:c:${channel}`,
|
|
3391
|
+
channelThreads: (channel) => `${p}:t:c:${channel}`,
|
|
3392
|
+
/** A root's thread, it and its replies; a reply's holds just it. */
|
|
3393
|
+
thread: (id) => `${p}:i:r:${id}`,
|
|
3394
|
+
byActor: (actor) => `${p}:i:ba:${actor}`,
|
|
3395
|
+
byPrincipal: (principal) => `${p}:i:bp:${principal}`,
|
|
3396
|
+
mention: (actor) => `${p}:i:mn:${actor}`,
|
|
3397
|
+
attachment: (id) => `${p}:i:at:${id}`,
|
|
3398
|
+
from: (from) => `${p}:i:f:${from}`,
|
|
3399
|
+
/** A person's roots, by when they were posted: for the echo check. */
|
|
3400
|
+
rootsBy: (principal) => `${p}:rr:${principal}`,
|
|
3401
|
+
label: (token) => `${p}:i:${token}`,
|
|
3402
|
+
copies: (id) => `${p}:cp:${id}`
|
|
3403
|
+
};
|
|
3404
|
+
const parse = (value) => value === null || value === void 0 || value === false ? null : JSON.parse(String(value));
|
|
3405
|
+
/**
|
|
3406
|
+
* Whether the server takes `allow-key-locking`, asked once, before the first script. Only its
|
|
3407
|
+
* rejection of the flag means no: a transport error decides nothing, and the next script asks again.
|
|
3408
|
+
*/
|
|
3409
|
+
let keyLocking = null;
|
|
3410
|
+
const locking = () => keyLocking ??= call("EVAL", `${KEY_LOCKING}\nreturn 1`, 1, k.pos).then(() => true, (error) => {
|
|
3411
|
+
if (SHEBANG_REJECTED.test(String(error?.message ?? error))) return false;
|
|
3412
|
+
keyLocking = null;
|
|
3413
|
+
throw error;
|
|
3414
|
+
});
|
|
3415
|
+
const run = async (flags, script, keys, args) => call("EVAL", await locking() ? `${flags}\n${script}` : script, keys.length, ...keys, ...args);
|
|
3416
|
+
/** A query: on Upstash it takes the shared lock, beside other reads. */
|
|
3417
|
+
const evaluate = (script, keys, args) => run(READ_ONLY_KEY_LOCKING, script, keys, args);
|
|
3418
|
+
/**
|
|
3419
|
+
* An event's apply, and the reads its writes are worked out from: they run as writing scripts,
|
|
3420
|
+
* so they read where the write that checks the position runs, whatever a provider does with a
|
|
3421
|
+
* read-only one.
|
|
3422
|
+
*/
|
|
3423
|
+
const evaluateWrite = (script, keys, args) => run(KEY_LOCKING, script, keys, args);
|
|
3424
|
+
async function messagesOf(ids) {
|
|
3425
|
+
const out = /* @__PURE__ */ new Map();
|
|
3426
|
+
if (ids.length === 0) return out;
|
|
3427
|
+
const rows = await call("HMGET", k.messages, ...ids);
|
|
3428
|
+
ids.forEach((id, i) => {
|
|
3429
|
+
const message = parse(rows[i]);
|
|
3430
|
+
if (message) out.set(id, message);
|
|
3431
|
+
});
|
|
3432
|
+
return out;
|
|
3433
|
+
}
|
|
3434
|
+
/**
|
|
3435
|
+
* What the event reads before it applies: its message, its root's summary and labels, and the
|
|
3436
|
+
* sync rows it changes. Null when the position says it's not the one to apply.
|
|
3437
|
+
*/
|
|
3438
|
+
async function before(event, id, mode) {
|
|
3439
|
+
const posted = event.type === "message.posted";
|
|
3440
|
+
const reads = syncReads(event);
|
|
3441
|
+
const keys = [
|
|
3442
|
+
k.pos,
|
|
3443
|
+
k.messages,
|
|
3444
|
+
k.metas,
|
|
3445
|
+
k.summaries,
|
|
3446
|
+
k.syncThreads,
|
|
3447
|
+
reads.copy === null ? k.pos : k.copies(reads.copy[0])
|
|
3448
|
+
];
|
|
3449
|
+
const args = [
|
|
3450
|
+
event.index,
|
|
3451
|
+
mode,
|
|
3452
|
+
id === null || posted ? "" : String(id),
|
|
3453
|
+
posted && event.payload.re !== null ? String(event.payload.re) : "",
|
|
3454
|
+
reads.thread ?? "",
|
|
3455
|
+
reads.copy?.[1] ?? ""
|
|
3456
|
+
];
|
|
3457
|
+
const found = await evaluateWrite(BEFORE, keys, args);
|
|
3458
|
+
if (!found) return null;
|
|
3459
|
+
const [data, rootId, rootData, rootSummary, thread, copy] = found;
|
|
3460
|
+
const sync = {
|
|
3461
|
+
thread: parse(thread) ?? void 0,
|
|
3462
|
+
copy: parse(copy) ?? void 0
|
|
3463
|
+
};
|
|
3464
|
+
if (id === null) return {
|
|
3465
|
+
read: null,
|
|
3466
|
+
sync
|
|
3467
|
+
};
|
|
3468
|
+
const message = parse(data) ?? void 0;
|
|
3469
|
+
const summary = parse(rootSummary);
|
|
3470
|
+
const root = summary ? {
|
|
3471
|
+
replies: summary.n,
|
|
3472
|
+
at: summary.at
|
|
3473
|
+
} : void 0;
|
|
3474
|
+
const read = {
|
|
3475
|
+
message,
|
|
3476
|
+
root,
|
|
3477
|
+
rootLabels: parse(rootData)?.labels ?? null
|
|
3478
|
+
};
|
|
3479
|
+
if (event.type === "message.deleted" && message && !message.deleted && root?.at === id) {
|
|
3480
|
+
const newest = await evaluateWrite(NEWEST, [k.thread(Number(rootId)), k.metas], [String(rootId), String(id)]);
|
|
3481
|
+
read.previousLast = newest === null || newest === void 0 || newest === false ? null : Number(newest);
|
|
3482
|
+
}
|
|
3483
|
+
return {
|
|
3484
|
+
read,
|
|
3485
|
+
sync
|
|
3486
|
+
};
|
|
3487
|
+
}
|
|
3488
|
+
/** A root's replies, when its labels changed: they're in its label sets too. */
|
|
3489
|
+
async function repliesOf(read, change) {
|
|
3490
|
+
const after = change.message?.after;
|
|
3491
|
+
if (!after || after.re !== null || change.message.created || !change.message.terms) return [];
|
|
3492
|
+
if (JSON.stringify(after.labels) === JSON.stringify(read.message.labels)) return [];
|
|
3493
|
+
return (await evaluateWrite(THREAD, [k.thread(after.id), k.messages], [])).flatMap((row) => {
|
|
3494
|
+
const message = parse(row);
|
|
3495
|
+
return message && message.id !== after.id ? [message] : [];
|
|
3496
|
+
});
|
|
3497
|
+
}
|
|
3498
|
+
/** The event's writes, as commands. */
|
|
3499
|
+
function writes(event, read, replies, change, sync) {
|
|
3500
|
+
const out = [];
|
|
3501
|
+
const m = change.message?.after;
|
|
3502
|
+
if (m && read) {
|
|
3503
|
+
const id = String(m.id);
|
|
3504
|
+
const created = change.message.created;
|
|
3505
|
+
const meta = {
|
|
3506
|
+
re: m.re,
|
|
3507
|
+
c: m.channel,
|
|
3508
|
+
t: m.text.toLowerCase(),
|
|
3509
|
+
ca: Date.parse(m.createdAt),
|
|
3510
|
+
d: m.deleted
|
|
3511
|
+
};
|
|
3512
|
+
out.push([
|
|
3513
|
+
"HSET",
|
|
3514
|
+
k.messages,
|
|
3515
|
+
id,
|
|
3516
|
+
JSON.stringify(m)
|
|
3517
|
+
], [
|
|
3518
|
+
"HSET",
|
|
3519
|
+
k.metas,
|
|
3520
|
+
id,
|
|
3521
|
+
luaJson(meta)
|
|
3522
|
+
]);
|
|
3523
|
+
out.push([
|
|
3524
|
+
"ZADD",
|
|
3525
|
+
k.versions(id),
|
|
3526
|
+
String(event.index),
|
|
3527
|
+
String(event.index)
|
|
3528
|
+
], [
|
|
3529
|
+
"HSET",
|
|
3530
|
+
k.versionData,
|
|
3531
|
+
String(event.index),
|
|
3532
|
+
JSON.stringify(m)
|
|
3533
|
+
]);
|
|
3534
|
+
if (created) {
|
|
3535
|
+
out.push([
|
|
3536
|
+
"HSET",
|
|
3537
|
+
k.events,
|
|
3538
|
+
event.id,
|
|
3539
|
+
id
|
|
3540
|
+
], [
|
|
3541
|
+
"SADD",
|
|
3542
|
+
k.channels,
|
|
3543
|
+
m.channel
|
|
3544
|
+
]);
|
|
3545
|
+
for (const key of [
|
|
3546
|
+
k.all,
|
|
3547
|
+
k.channel(m.channel),
|
|
3548
|
+
k.thread(id),
|
|
3549
|
+
k.byActor(m.by.actor),
|
|
3550
|
+
k.byPrincipal(m.by.principal),
|
|
3551
|
+
k.from(m.from ?? "board")
|
|
3552
|
+
]) out.push([
|
|
3553
|
+
"ZADD",
|
|
3554
|
+
key,
|
|
3555
|
+
id,
|
|
3556
|
+
id
|
|
3557
|
+
]);
|
|
3558
|
+
if (m.re !== null) out.push([
|
|
3559
|
+
"ZADD",
|
|
3560
|
+
k.thread(m.re),
|
|
3561
|
+
id,
|
|
3562
|
+
id
|
|
3563
|
+
]);
|
|
3564
|
+
else out.push([
|
|
3565
|
+
"ZADD",
|
|
3566
|
+
k.rootsBy(m.by.principal),
|
|
3567
|
+
String(meta.ca),
|
|
3568
|
+
id
|
|
3569
|
+
]);
|
|
3570
|
+
}
|
|
3571
|
+
if (change.message.terms) {
|
|
3572
|
+
const old = read.message;
|
|
3573
|
+
const rootLabels = m.re === null ? null : read.rootLabels;
|
|
3574
|
+
const was = created ? /* @__PURE__ */ new Set() : inLabelSets(old.labels, rootLabels);
|
|
3575
|
+
const is = inLabelSets(m.labels, rootLabels);
|
|
3576
|
+
for (const token of was) if (!is.has(token)) out.push([
|
|
3577
|
+
"ZREM",
|
|
3578
|
+
k.label(token),
|
|
3579
|
+
id
|
|
3580
|
+
]);
|
|
3581
|
+
for (const token of is) if (!was.has(token)) out.push([
|
|
3582
|
+
"ZADD",
|
|
3583
|
+
k.label(token),
|
|
3584
|
+
id,
|
|
3585
|
+
id
|
|
3586
|
+
]);
|
|
3587
|
+
const mentionedBefore = new Set(created ? [] : old.mentions);
|
|
3588
|
+
const mentioned = new Set(m.mentions);
|
|
3589
|
+
for (const actor of mentionedBefore) if (!mentioned.has(actor)) out.push([
|
|
3590
|
+
"ZREM",
|
|
3591
|
+
k.mention(actor),
|
|
3592
|
+
id
|
|
3593
|
+
]);
|
|
3594
|
+
for (const actor of mentioned) if (!mentionedBefore.has(actor)) out.push([
|
|
3595
|
+
"ZADD",
|
|
3596
|
+
k.mention(actor),
|
|
3597
|
+
id,
|
|
3598
|
+
id
|
|
3599
|
+
]);
|
|
3600
|
+
const filesBefore = new Set(created ? [] : old.attachments.map((file) => file.id));
|
|
3601
|
+
const files = new Set(m.attachments.map((file) => file.id));
|
|
3602
|
+
for (const file of filesBefore) if (!files.has(file)) out.push([
|
|
3603
|
+
"ZREM",
|
|
3604
|
+
k.attachment(file),
|
|
3605
|
+
id
|
|
3606
|
+
]);
|
|
3607
|
+
for (const file of files) if (!filesBefore.has(file)) out.push([
|
|
3608
|
+
"ZADD",
|
|
3609
|
+
k.attachment(file),
|
|
3610
|
+
id,
|
|
3611
|
+
id
|
|
3612
|
+
]);
|
|
3613
|
+
for (const reply of replies) {
|
|
3614
|
+
const replyWas = inLabelSets(reply.labels, old.labels);
|
|
3615
|
+
const replyIs = inLabelSets(reply.labels, m.labels);
|
|
3616
|
+
for (const token of replyWas) if (!replyIs.has(token)) out.push([
|
|
3617
|
+
"ZREM",
|
|
3618
|
+
k.label(token),
|
|
3619
|
+
String(reply.id)
|
|
3620
|
+
]);
|
|
3621
|
+
for (const token of replyIs) if (!replyWas.has(token)) out.push([
|
|
3622
|
+
"ZADD",
|
|
3623
|
+
k.label(token),
|
|
3624
|
+
String(reply.id),
|
|
3625
|
+
String(reply.id)
|
|
3626
|
+
]);
|
|
3627
|
+
}
|
|
3628
|
+
}
|
|
3629
|
+
}
|
|
3630
|
+
if (change.summary) {
|
|
3631
|
+
const { root, replies, at } = change.summary;
|
|
3632
|
+
const channel = m?.channel ?? read?.message?.channel;
|
|
3633
|
+
out.push([
|
|
3634
|
+
"HSET",
|
|
3635
|
+
k.summaries,
|
|
3636
|
+
String(root),
|
|
3637
|
+
JSON.stringify({
|
|
3638
|
+
n: replies,
|
|
3639
|
+
at
|
|
3640
|
+
})
|
|
3641
|
+
], [
|
|
3642
|
+
"ZADD",
|
|
3643
|
+
k.threads,
|
|
3644
|
+
String(at),
|
|
3645
|
+
String(root)
|
|
3646
|
+
]);
|
|
3647
|
+
if (channel !== void 0) out.push([
|
|
3648
|
+
"ZADD",
|
|
3649
|
+
k.channelThreads(channel),
|
|
3650
|
+
String(at),
|
|
3651
|
+
String(root)
|
|
3652
|
+
]);
|
|
3653
|
+
}
|
|
3654
|
+
if (sync.thread) out.push([
|
|
3655
|
+
"HSET",
|
|
3656
|
+
k.syncThreads,
|
|
3657
|
+
sync.thread.key,
|
|
3658
|
+
JSON.stringify(sync.thread.data)
|
|
3659
|
+
]);
|
|
3660
|
+
for (const [external, thread] of sync.externalThreads) out.push([
|
|
3661
|
+
"HSET",
|
|
3662
|
+
k.externalThreads,
|
|
3663
|
+
external,
|
|
3664
|
+
thread
|
|
3665
|
+
]);
|
|
3666
|
+
for (const [message, target, copy] of sync.copies) out.push([
|
|
3667
|
+
"HSET",
|
|
3668
|
+
k.copies(message),
|
|
3669
|
+
target,
|
|
3670
|
+
JSON.stringify(copy)
|
|
3671
|
+
]);
|
|
3672
|
+
for (const [key, message] of sync.externalMessages) out.push([
|
|
3673
|
+
"HSET",
|
|
3674
|
+
k.externalMessages,
|
|
3675
|
+
key,
|
|
3676
|
+
String(message)
|
|
3677
|
+
]);
|
|
3678
|
+
return out;
|
|
3679
|
+
}
|
|
3680
|
+
async function commit(event, mode) {
|
|
3681
|
+
const found = await before(event, concernedMessageId(event), mode);
|
|
3682
|
+
if (!found) return false;
|
|
3683
|
+
const { read, sync: syncRead } = found;
|
|
3684
|
+
const change = read ? project(read, event) : {};
|
|
3685
|
+
const ops = writes(event, read, read ? await repliesOf(read, change) : [], change, projectSync(syncRead, event));
|
|
3686
|
+
const keys = [k.pos];
|
|
3687
|
+
const slots = /* @__PURE__ */ new Map();
|
|
3688
|
+
const encoded = ops.map(([command, key, ...args]) => {
|
|
3689
|
+
if (!slots.has(key)) slots.set(key, keys.push(key));
|
|
3690
|
+
return [
|
|
3691
|
+
command,
|
|
3692
|
+
slots.get(key),
|
|
3693
|
+
...args
|
|
3694
|
+
];
|
|
3695
|
+
});
|
|
3696
|
+
return Number(await evaluateWrite(APPLY, keys, [
|
|
3697
|
+
event.index,
|
|
3698
|
+
mode,
|
|
3699
|
+
luaJson(encoded)
|
|
3700
|
+
])) === 1;
|
|
3701
|
+
}
|
|
3702
|
+
/**
|
|
3703
|
+
* The channels under `path`, and the position that list reflects. A read's
|
|
3704
|
+
* position goes no further: a post in a channel newer than the list is past
|
|
3705
|
+
* it, so the feed from that position carries it.
|
|
3706
|
+
*/
|
|
3707
|
+
async function channelsUnder(path) {
|
|
3708
|
+
const [at, all] = await evaluate(CHANNELS, [k.pos, k.channels], []);
|
|
3709
|
+
return {
|
|
3710
|
+
under: all.filter((c) => c.startsWith(`${path}/`)),
|
|
3711
|
+
asOf: Number(at)
|
|
3712
|
+
};
|
|
3713
|
+
}
|
|
3714
|
+
/** A filter as groups of sets, each group needing one: the query scripts' input, and the position it holds for. */
|
|
3715
|
+
async function compile(filter, keys) {
|
|
3716
|
+
const slot = (key) => keys.push(key);
|
|
3717
|
+
const groups = [];
|
|
3718
|
+
const group = (sets) => groups.push(sets.map(slot));
|
|
3719
|
+
let asOf = Number.POSITIVE_INFINITY;
|
|
3720
|
+
if (filter.channel) {
|
|
3721
|
+
const { path, subtree } = filter.channel;
|
|
3722
|
+
let under = [];
|
|
3723
|
+
if (subtree) ({under, asOf} = await channelsUnder(path));
|
|
3724
|
+
group([path, ...under].map(k.channel));
|
|
3725
|
+
}
|
|
3726
|
+
if (filter.channels) group(filter.channels.map(k.channel));
|
|
3727
|
+
if (filter.re !== void 0) group([k.thread(filter.re)]);
|
|
3728
|
+
if (filter.by) group([...filter.by.actors.map(k.byActor), ...filter.by.principals.map(k.byPrincipal)]);
|
|
3729
|
+
if (filter.mentions) group(filter.mentions.map(k.mention));
|
|
3730
|
+
if (filter.from !== void 0) group([k.from(filter.from)]);
|
|
3731
|
+
for (const label of filter.labels ?? []) group(label.values === null ? [k.label(`l${SEP}${label.key}`)] : label.values.map((value) => k.label(`lv${SEP}${label.key}${SEP}${value}`)));
|
|
3732
|
+
return {
|
|
3733
|
+
spec: {
|
|
3734
|
+
groups,
|
|
3735
|
+
drivers: groups.filter((sets) => sets.length === 1).map((sets) => sets[0]),
|
|
3736
|
+
fallback: slot(k.all),
|
|
3737
|
+
text: filter.text ?? null
|
|
3738
|
+
},
|
|
3739
|
+
asOf
|
|
3740
|
+
};
|
|
3741
|
+
}
|
|
3742
|
+
return {
|
|
3743
|
+
async position() {
|
|
3744
|
+
return Number(await call("GET", k.pos) ?? 0);
|
|
3745
|
+
},
|
|
3746
|
+
async apply(event) {
|
|
3747
|
+
await commit(event, "any");
|
|
3748
|
+
},
|
|
3749
|
+
applyNext: (event) => commit(event, "next"),
|
|
3750
|
+
async message(id) {
|
|
3751
|
+
return (await messagesOf([id])).get(id) ?? null;
|
|
3752
|
+
},
|
|
3753
|
+
async byEventId(eventId) {
|
|
3754
|
+
const id = await call("HGET", k.events, eventId);
|
|
3755
|
+
return id === null || id === void 0 ? null : (await messagesOf([Number(id)])).get(Number(id)) ?? null;
|
|
3756
|
+
},
|
|
3757
|
+
async messagesAt(ids, index) {
|
|
3758
|
+
const out = /* @__PURE__ */ new Map();
|
|
3759
|
+
const wanted = [...new Set(ids)];
|
|
3760
|
+
if (wanted.length === 0) return out;
|
|
3761
|
+
const rows = await evaluate(AT, [k.versionData, ...wanted.map(k.versions)], [index]);
|
|
3762
|
+
wanted.forEach((id, i) => {
|
|
3763
|
+
const message = parse(rows[i]);
|
|
3764
|
+
if (message) out.set(id, message);
|
|
3765
|
+
});
|
|
3766
|
+
return out;
|
|
3767
|
+
},
|
|
3768
|
+
async thread(root) {
|
|
3769
|
+
const ids = (await call("ZRANGE", k.thread(root), 0, -1)).map(Number);
|
|
3770
|
+
const found = await messagesOf(ids);
|
|
3771
|
+
return ids.flatMap((id) => found.has(id) ? [found.get(id)] : []);
|
|
3772
|
+
},
|
|
3773
|
+
async messages(filter, page) {
|
|
3774
|
+
const keys = [
|
|
3775
|
+
k.pos,
|
|
3776
|
+
k.messages,
|
|
3777
|
+
k.metas
|
|
3778
|
+
];
|
|
3779
|
+
const asc = page.after !== void 0;
|
|
3780
|
+
const { spec: compiled, asOf } = await compile(filter, keys);
|
|
3781
|
+
const spec = {
|
|
3782
|
+
...compiled,
|
|
3783
|
+
limit: page.limit,
|
|
3784
|
+
budget,
|
|
3785
|
+
asc,
|
|
3786
|
+
min: asc ? `(${page.after}` : "-inf",
|
|
3787
|
+
max: page.before !== void 0 ? `(${page.before}` : "+inf"
|
|
3788
|
+
};
|
|
3789
|
+
let position = null;
|
|
3790
|
+
const messages = [];
|
|
3791
|
+
for (;;) {
|
|
3792
|
+
const [at, rows, next] = await evaluate(MESSAGES, keys, [luaJson(spec)]);
|
|
3793
|
+
position ??= Number(at);
|
|
3794
|
+
for (const row of rows) messages.push(parse(row));
|
|
3795
|
+
spec.limit = page.limit - messages.length;
|
|
3796
|
+
if (next === null || spec.limit === 0) break;
|
|
3797
|
+
if (asc) spec.min = next;
|
|
3798
|
+
else spec.max = next;
|
|
3799
|
+
}
|
|
3800
|
+
return {
|
|
3801
|
+
messages: asc ? messages : messages.reverse(),
|
|
3802
|
+
position: Math.min(position, asOf)
|
|
3803
|
+
};
|
|
3804
|
+
},
|
|
3805
|
+
async threads(filter, page) {
|
|
3806
|
+
const asc = page.after !== void 0;
|
|
3807
|
+
const { channel, channels, ...rest } = filter;
|
|
3808
|
+
let position = null;
|
|
3809
|
+
const roots = /* @__PURE__ */ new Map();
|
|
3810
|
+
let asOf = Number.POSITIVE_INFINITY;
|
|
3811
|
+
if (Object.keys(rest).length === 0 && !(channel && channels)) {
|
|
3812
|
+
let under = [];
|
|
3813
|
+
if (channel?.subtree) ({under, asOf} = await channelsUnder(channel.path));
|
|
3814
|
+
const sets = channels ? channels.map(k.channelThreads) : channel ? [channel.path, ...under].map(k.channelThreads) : [k.threads];
|
|
3815
|
+
const spec = {
|
|
3816
|
+
limit: page.limit,
|
|
3817
|
+
asc,
|
|
3818
|
+
min: asc ? `(${page.after}` : "-inf"
|
|
3819
|
+
};
|
|
3820
|
+
const [at, pairs] = await evaluate(LATEST, [k.pos, ...sets], [luaJson(spec)]);
|
|
3821
|
+
position = Number(at);
|
|
3822
|
+
for (let i = 0; i < pairs.length; i += 2) roots.set(pairs[i], Number(pairs[i + 1]));
|
|
3823
|
+
} else {
|
|
3824
|
+
const keys = [
|
|
3825
|
+
k.pos,
|
|
3826
|
+
k.messages,
|
|
3827
|
+
k.metas,
|
|
3828
|
+
k.summaries
|
|
3829
|
+
];
|
|
3830
|
+
const compiled = await compile(filter, keys);
|
|
3831
|
+
asOf = compiled.asOf;
|
|
3832
|
+
const spec = {
|
|
3833
|
+
...compiled.spec,
|
|
3834
|
+
budget,
|
|
3835
|
+
max: "+inf"
|
|
3836
|
+
};
|
|
3837
|
+
for (;;) {
|
|
3838
|
+
const [at, pairs, next] = await evaluate(ROOTS, keys, [luaJson(spec)]);
|
|
3839
|
+
position ??= Number(at);
|
|
3840
|
+
for (let i = 0; i < pairs.length; i += 2) roots.set(String(pairs[i]), Number(pairs[i + 1]));
|
|
3841
|
+
if (next === null) break;
|
|
3842
|
+
spec.max = next;
|
|
3843
|
+
}
|
|
3844
|
+
}
|
|
3845
|
+
const order = [...roots].filter(([, at]) => page.after === void 0 || at > page.after).sort(([, a], [, b]) => asc ? a - b : b - a).slice(0, page.limit).map(([root]) => root);
|
|
3846
|
+
const threads = (order.length === 0 ? [] : await evaluate(THREAD_ROWS, [k.messages, k.summaries], order)).map(([root, replies, last]) => ({
|
|
3847
|
+
root: parse(root),
|
|
3848
|
+
replies: Number(replies),
|
|
3849
|
+
last: parse(last)
|
|
3850
|
+
}));
|
|
3851
|
+
return {
|
|
3852
|
+
threads: asc ? threads : threads.reverse(),
|
|
3853
|
+
position: Math.min(position, asOf)
|
|
3854
|
+
};
|
|
3855
|
+
},
|
|
3856
|
+
async syncOf(thread) {
|
|
3857
|
+
return parse(await call("HGET", k.syncThreads, thread));
|
|
3858
|
+
},
|
|
3859
|
+
async threadOf(externalThread) {
|
|
3860
|
+
const thread = await call("HGET", k.externalThreads, externalThread);
|
|
3861
|
+
return thread === null || thread === void 0 ? null : String(thread);
|
|
3862
|
+
},
|
|
3863
|
+
async copiesOf(ids) {
|
|
3864
|
+
const out = /* @__PURE__ */ new Map();
|
|
3865
|
+
const wanted = [...new Set(ids)];
|
|
3866
|
+
if (wanted.length === 0) return out;
|
|
3867
|
+
const rows = await evaluate(HASHES, wanted.map(k.copies), []);
|
|
3868
|
+
wanted.forEach((id, i) => {
|
|
3869
|
+
const flat = rows[i] ?? [];
|
|
3870
|
+
if (flat.length === 0) return;
|
|
3871
|
+
const copies = {};
|
|
3872
|
+
for (let at = 0; at < flat.length; at += 2) setOwn(copies, flat[at], JSON.parse(flat[at + 1]));
|
|
3873
|
+
out.set(id, copies);
|
|
3874
|
+
});
|
|
3875
|
+
return out;
|
|
3876
|
+
},
|
|
3877
|
+
async messageOf(externalThread, platformMessage) {
|
|
3878
|
+
const id = await call("HGET", k.externalMessages, externalMessageKey(externalThread, platformMessage));
|
|
3879
|
+
return id === null || id === void 0 ? null : Number(id);
|
|
3880
|
+
},
|
|
3881
|
+
async externalThreads() {
|
|
3882
|
+
return await call("HKEYS", k.externalThreads);
|
|
3883
|
+
},
|
|
3884
|
+
async withAttachment(attachment) {
|
|
3885
|
+
return (await evaluate(THREAD, [k.attachment(attachment), k.messages], [])).flatMap((row) => {
|
|
3886
|
+
const message = parse(row);
|
|
3887
|
+
return message ? [message] : [];
|
|
3888
|
+
});
|
|
3889
|
+
},
|
|
3890
|
+
async recentRoots(principal, since) {
|
|
3891
|
+
return (await evaluate(RECENT, [k.messages, k.rootsBy(principal)], [since])).map((row) => parse(row)).sort((a, b) => a.id - b.id);
|
|
3892
|
+
},
|
|
3893
|
+
async close() {}
|
|
3894
|
+
};
|
|
3895
|
+
}
|
|
3896
|
+
//#endregion
|
|
3897
|
+
export { createRedisNotifyStore as n, redisTables as t };
|