okengine 0.23.0 → 0.23.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/site/content/docs/ai/mcp.mdx +4 -4
- package/site/content/docs/elements/channel/index.mdx +4 -4
- package/site/content/docs/elements/channel/receipts.mdx +7 -5
- package/site/content/docs/elements/channel/sms.mdx +2 -1
- package/site/content/docs/elements/flow/index.mdx +22 -18
- package/site/content/docs/elements/signal/broadcast.mdx +4 -2
- package/site/content/docs/elements/signal/index.mdx +15 -14
- package/site/content/docs/plugins/otp.mdx +3 -3
- package/site/content/docs/reference/configuration.mdx +16 -0
- package/site/content/docs/reference/errors.mdx +31 -19
- package/site/content/docs/reference/fx.mdx +12 -4
- package/src/auth/tenants.ts +17 -0
- package/src/cli/doctor-diff.test.ts +29 -1
- package/src/cli/doctor-diff.ts +35 -0
- package/src/cli/manifest-pr-diff.ts +16 -0
- package/src/compiler/aot.test.ts +3 -2
- package/src/compiler/aot.ts +45 -1
- package/src/compiler/dynamic.ts +1 -1
- package/src/compiler/effects-fetch.test.ts +77 -0
- package/src/compiler/effects-infer.ts +87 -16
- package/src/compiler/effects-join.test.ts +48 -0
- package/src/compiler/extract.test.ts +28 -0
- package/src/compiler/extract.ts +11 -1
- package/src/compiler/fx-index.ts +58 -4
- package/src/compiler/http-parse.test.ts +101 -0
- package/src/compiler/http-parse.ts +188 -9
- package/src/compiler/interpret.ts +25 -2
- package/src/console/index.ts +1 -1
- package/src/console/ui-next/dist/assets/{access-page-DDKkhT9v.js → access-page-zBMhZnFm.js} +1 -1
- package/src/console/ui-next/dist/assets/{agent-disclosure-62Xts8Xi.js → agent-disclosure-CBj9f7Ju.js} +1 -1
- package/src/console/ui-next/dist/assets/{cache-glyph-DyeoKHKA.js → cache-glyph-D3MY49h3.js} +1 -1
- package/src/console/ui-next/dist/assets/{call-pii-button-DvfpLXsU.js → call-pii-button-L9JxdBhp.js} +1 -1
- package/src/console/ui-next/dist/assets/{collapsible-BY03SeCg.js → collapsible-BvFaX-Zt.js} +1 -1
- package/src/console/ui-next/dist/assets/{decisions-page-CgkKlA56.js → decisions-page-DgB76m-X.js} +1 -1
- package/src/console/ui-next/dist/assets/{duration-tone-Y6HLjyJ5.js → duration-tone-l1DSJ2Vz.js} +1 -1
- package/src/console/ui-next/dist/assets/{flows-page-DLPfNy-_.js → flows-page-Bb9t1lu6.js} +1 -1
- package/src/console/ui-next/dist/assets/{highlighted-json-D7nNzpgB.js → highlighted-json-DMvsi3Ti.js} +1 -1
- package/src/console/ui-next/dist/assets/{http-method-DdL19zzh.js → http-method-BwhJBTcQ.js} +1 -1
- package/src/console/ui-next/dist/assets/{index-C0lc_s9d.js → index-RhV2jT_7.js} +3 -3
- package/src/console/ui-next/dist/assets/{observability-page-CRlcyLCC.js → observability-page-BRfUILBq.js} +1 -1
- package/src/console/ui-next/dist/assets/{replica-lag-Bb3FWUu9.js → replica-lag-BsceAzIG.js} +1 -1
- package/src/console/ui-next/dist/assets/{request-meta-DYBMWhX8.js → request-meta-C6lTyCXp.js} +1 -1
- package/src/console/ui-next/dist/assets/{store-page-CcE-SXC8.js → store-page-YmE806bJ.js} +1 -1
- package/src/console/ui-next/dist/assets/{trace-detail-sheet-CKMSEQ4s.js → trace-detail-sheet-Di4irS49.js} +1 -1
- package/src/console/ui-next/dist/assets/{tree-expand-toggle-QOfk0M0H.js → tree-expand-toggle-DYeL6Rye.js} +1 -1
- package/src/console/ui-next/dist/assets/{units-page-DKC1CDDd.js → units-page-07agasRA.js} +1 -1
- package/src/console/ui-next/dist/assets/{vault-page-B54v4Yit.js → vault-page-CHBck4n_.js} +1 -1
- package/src/console/ui-next/dist/index.html +1 -1
- package/src/console/ui-next/seed-parked-approval.ts +5 -1
- package/src/drivers/journal-postgres.test.ts +106 -2
- package/src/drivers/journal-postgres.ts +475 -14
- package/src/drivers/postgres.test.ts +60 -0
- package/src/drivers/postgres.ts +79 -5
- package/src/drivers/signal-compete.test.ts +238 -0
- package/src/drivers/signal-postgres.test.ts +170 -0
- package/src/drivers/signal-postgres.ts +486 -213
- package/src/drivers/signal-redis.test.ts +224 -0
- package/src/drivers/signal-redis.ts +934 -93
- package/src/drivers/signal-types.ts +12 -0
- package/src/elements/ai/approval.test.ts +15 -3
- package/src/elements/ai/approval.ts +15 -9
- package/src/elements/ai/mcp-protocol.ts +5 -6
- package/src/elements/ai/runtime.ts +20 -19
- package/src/elements/channel/runtime.ts +42 -4
- package/src/elements/channel/sql-ledger.test.ts +404 -0
- package/src/elements/channel/sql-ledger.ts +395 -0
- package/src/elements/clock/chaos-child.ts +12 -2
- package/src/elements/clock/durable.ts +11 -0
- package/src/elements/signal/runtime.ts +9 -0
- package/src/elements/signal.test.ts +1 -0
- package/src/elements/store/cache-bus.test.ts +121 -0
- package/src/elements/store/cache-bus.ts +278 -0
- package/src/elements/store/cache.test.ts +179 -15
- package/src/elements/store/cache.ts +357 -51
- package/src/elements/store/prepare-row.test.ts +6 -2
- package/src/elements/store/runtime.ts +44 -2
- package/src/elements/store/sql-errors.test.ts +131 -0
- package/src/elements/store/sql-errors.ts +39 -16
- package/src/elements/store/sql-nested-tx.test.ts +344 -0
- package/src/elements/store/sql-session.test.ts +2 -0
- package/src/elements/store/sql-session.ts +275 -9
- package/src/elements/store.ts +1 -0
- package/src/kernel/app.ts +207 -43
- package/src/kernel/auto-cache.test.ts +228 -0
- package/src/kernel/auto-registry.test.ts +5 -1
- package/src/kernel/boot-bind/channel.ts +63 -2
- package/src/kernel/boot-bind/honor-config.test.ts +40 -10
- package/src/kernel/boot-bind/signal.ts +72 -25
- package/src/kernel/boot-bind/store.ts +16 -1
- package/src/kernel/boot.test.ts +2 -2
- package/src/kernel/boot.ts +54 -10
- package/src/kernel/boundary-contract.ts +7 -0
- package/src/kernel/builtin-errors.ts +2 -0
- package/src/kernel/call.test.ts +1 -0
- package/src/kernel/errors-compiler.ts +11 -0
- package/src/kernel/errors-text.ts +16 -0
- package/src/kernel/errors.ts +8 -0
- package/src/kernel/flow.ts +33 -3
- package/src/kernel/fx-call-types.test.ts +32 -0
- package/src/kernel/fx-decide.ts +11 -7
- package/src/kernel/fx-sql-handle.ts +52 -7
- package/src/kernel/fx.test.ts +1 -3
- package/src/kernel/fx.ts +27 -16
- package/src/kernel/horizontal-child.ts +54 -1
- package/src/kernel/horizontal.integration.test.ts +131 -7
- package/src/kernel/idempotency.test.ts +4 -4
- package/src/kernel/journal-boot.test.ts +83 -6
- package/src/kernel/journal.test.ts +158 -0
- package/src/kernel/journal.ts +515 -54
- package/src/kernel/pipeline-tenant.ts +5 -1
- package/src/kernel/pipeline.ts +5 -0
- package/src/kernel/signal-tx.ts +28 -0
- package/src/kernel/store-transaction.test.ts +95 -0
- package/src/mcp/authorization.ts +5 -11
- package/src/mcp/confirmation.ts +203 -38
- package/src/mcp/docs-server.ts +26 -4
- package/src/mcp/index.ts +4 -0
- package/src/mcp/mcp.test.ts +404 -33
- package/src/mcp/protocol.ts +8 -2
- package/src/mcp/server.ts +31 -5
- package/src/mcp/session.ts +7 -0
- package/src/mcp/tools.ts +68 -19
- package/src/mcp/versions.ts +60 -0
- package/src/plugins/index.ts +1 -0
- package/src/plugins/mena.ts +56 -0
- package/src/runtime/bun.ts +5 -0
- package/src/runtime/types.ts +5 -0
|
@@ -3,90 +3,538 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Emit still enrols in a transactional outbox (semantics never regress).
|
|
5
5
|
* After commit, a relay pushes to Redis Streams (`once`) or pub/sub
|
|
6
|
-
* (`broadcast` / `live`).
|
|
7
|
-
*
|
|
6
|
+
* (`broadcast` / `live`). With `compete: true`, `once` delivery is
|
|
7
|
+
* `XREADGROUP` / `XACK`, with `XAUTOCLAIM` reclaim on the same read.
|
|
8
|
+
* Broadcast and live stay on the outbox plus pub/sub.
|
|
8
9
|
*
|
|
9
10
|
* Production bind: {@link createBunSignalRedisClient} (typed `publish` /
|
|
10
11
|
* `subscribe` / `xadd` / `xreadgroup` / `xack` / `xgroup` on Bun ≥1.4).
|
|
11
12
|
*/
|
|
12
13
|
|
|
13
14
|
import { createSignalEngine } from "./signal-engine.ts";
|
|
15
|
+
import { createLiveIterable } from "./signal-live-iter.ts";
|
|
14
16
|
import type {
|
|
17
|
+
DeadLetter,
|
|
18
|
+
LiveEvent,
|
|
15
19
|
SignalBus,
|
|
16
20
|
SignalDriver,
|
|
21
|
+
SignalFailureReason,
|
|
22
|
+
SignalHandler,
|
|
23
|
+
SignalMessage,
|
|
17
24
|
SignalOpenOptions,
|
|
18
25
|
SignalRedisClientLike,
|
|
19
26
|
} from "./signal-types.ts";
|
|
20
27
|
|
|
28
|
+
/**
|
|
29
|
+
* Minimum idle time before `XAUTOCLAIM` steals a pending `once` entry.
|
|
30
|
+
* A handler that is still running refreshes its lease about every third of this window.
|
|
31
|
+
*/
|
|
32
|
+
export const SIGNAL_REDIS_MIN_IDLE_MS = 30_000;
|
|
33
|
+
|
|
34
|
+
/** Drop group consumers idle at least this long when their PEL is empty. */
|
|
35
|
+
const CONSUMER_IDLE_RETENTION_MS = 86_400_000;
|
|
36
|
+
|
|
37
|
+
const STREAM_GROUP = "oke";
|
|
38
|
+
|
|
21
39
|
/** Bun 1.4 typed stream methods — `@types/bun` may lag the runtime. */
|
|
22
40
|
interface BunRedisStreams {
|
|
23
41
|
xadd(key: string, ...args: (string | number)[]): Promise<unknown>;
|
|
24
42
|
xreadgroup(...args: (string | number)[]): Promise<unknown>;
|
|
43
|
+
xautoclaim(
|
|
44
|
+
key: string,
|
|
45
|
+
group: string,
|
|
46
|
+
consumer: string,
|
|
47
|
+
minIdleTime: number,
|
|
48
|
+
start: string,
|
|
49
|
+
...options: (string | number)[]
|
|
50
|
+
): Promise<unknown>;
|
|
51
|
+
xclaim(
|
|
52
|
+
key: string,
|
|
53
|
+
group: string,
|
|
54
|
+
consumer: string,
|
|
55
|
+
minIdleTime: number,
|
|
56
|
+
id: string,
|
|
57
|
+
...rest: (string | number)[]
|
|
58
|
+
): Promise<unknown>;
|
|
25
59
|
xack(key: string, group: string, id: string): Promise<unknown>;
|
|
26
60
|
xgroup(subcommand: string, ...args: (string | number)[]): Promise<unknown>;
|
|
61
|
+
xrange(key: string, start: string, end: string): Promise<unknown>;
|
|
62
|
+
xinfo(subcommand: string, ...args: (string | number)[]): Promise<unknown>;
|
|
63
|
+
hincrby(key: string, field: string, increment: number): Promise<number>;
|
|
64
|
+
hdel(key: string, field: string): Promise<number>;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
interface StreamEntry {
|
|
68
|
+
id: string;
|
|
69
|
+
fields: Record<string, string>;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** One row from `XINFO CONSUMERS`. */
|
|
73
|
+
interface RedisConsumerInfo {
|
|
74
|
+
readonly name: string;
|
|
75
|
+
readonly pending: number;
|
|
76
|
+
readonly idle: number;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Extra Redis commands the signal bus uses on top of {@link SignalRedisClientLike}.
|
|
81
|
+
* Implemented by {@link createBunSignalRedisClient} and {@link createSignalRedisFake}.
|
|
82
|
+
*/
|
|
83
|
+
interface SignalRedisWire extends SignalRedisClientLike {
|
|
84
|
+
xclaim(
|
|
85
|
+
key: string,
|
|
86
|
+
group: string,
|
|
87
|
+
consumer: string,
|
|
88
|
+
minIdleMs: number,
|
|
89
|
+
ids: readonly string[],
|
|
90
|
+
): Promise<void>;
|
|
91
|
+
hincrby(key: string, field: string, increment: number): Promise<number>;
|
|
92
|
+
hdel(key: string, field: string): Promise<number>;
|
|
93
|
+
xrange(key: string): Promise<StreamEntry[]>;
|
|
94
|
+
xinfoConsumers(key: string, group: string): Promise<RedisConsumerInfo[]>;
|
|
95
|
+
xgroupDelConsumer(key: string, group: string, consumer: string): Promise<void>;
|
|
96
|
+
close(): Promise<void>;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
interface InflightBatch {
|
|
100
|
+
readonly key: string;
|
|
101
|
+
readonly group: string;
|
|
102
|
+
readonly consumer: string;
|
|
103
|
+
readonly ids: Set<string>;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
interface PendingEntry {
|
|
107
|
+
id: string;
|
|
108
|
+
fields: Record<string, string>;
|
|
109
|
+
consumer: string;
|
|
110
|
+
claimedAt: number;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** Pub/sub body published for `broadcast` / `live`. */
|
|
114
|
+
interface SignalPubEnvelope {
|
|
115
|
+
readonly __oke_env: 1;
|
|
116
|
+
readonly id: string;
|
|
117
|
+
readonly origin: string;
|
|
118
|
+
readonly payload: unknown;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const inflightByClient = new WeakMap<object, Map<string, InflightBatch>>();
|
|
122
|
+
const heartbeats = new WeakMap<object, Set<ReturnType<typeof setInterval>>>();
|
|
123
|
+
|
|
124
|
+
function bunStreams(redis: Bun.RedisClient): BunRedisStreams {
|
|
125
|
+
return redis as unknown as BunRedisStreams;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function asWire(client: SignalRedisClientLike): SignalRedisWire {
|
|
129
|
+
return client as SignalRedisWire;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function streamKey(name: string): string {
|
|
133
|
+
return `oke:signal:${name}`;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function attemptsKey(name: string): string {
|
|
137
|
+
return `oke:signal:attempts:${name}`;
|
|
27
138
|
}
|
|
28
139
|
|
|
29
|
-
function
|
|
30
|
-
return
|
|
140
|
+
function deadKey(name: string): string {
|
|
141
|
+
return `oke:signal:dead:${name}`;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function bcastChannel(name: string): string {
|
|
145
|
+
return `oke:signal:bcast:${name}`;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function liveChannel(name: string): string {
|
|
149
|
+
return `oke:signal:live:${name}`;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function inflightSlot(key: string, consumer: string): string {
|
|
153
|
+
return `${key}\0${consumer}`;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function markInflight(
|
|
157
|
+
client: object,
|
|
158
|
+
key: string,
|
|
159
|
+
group: string,
|
|
160
|
+
consumer: string,
|
|
161
|
+
id: string,
|
|
162
|
+
): void {
|
|
163
|
+
let batches = inflightByClient.get(client);
|
|
164
|
+
if (!batches) {
|
|
165
|
+
batches = new Map();
|
|
166
|
+
inflightByClient.set(client, batches);
|
|
167
|
+
}
|
|
168
|
+
const slot = inflightSlot(key, consumer);
|
|
169
|
+
let batch = batches.get(slot);
|
|
170
|
+
if (!batch) {
|
|
171
|
+
batch = { key, group, consumer, ids: new Set() };
|
|
172
|
+
batches.set(slot, batch);
|
|
173
|
+
}
|
|
174
|
+
batch.ids.add(id);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function unmarkInflight(client: object, key: string, consumer: string, id: string): void {
|
|
178
|
+
inflightByClient.get(client)?.get(inflightSlot(key, consumer))?.ids.delete(id);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function isInflight(client: object, key: string, consumer: string, id: string): boolean {
|
|
182
|
+
return inflightByClient.get(client)?.get(inflightSlot(key, consumer))?.ids.has(id) ?? false;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* `XCLAIM ... JUSTID` for every id this process is still handling.
|
|
187
|
+
*
|
|
188
|
+
* @param client - Redis client that owns the in-flight set
|
|
189
|
+
*/
|
|
190
|
+
async function refreshInflight(client: object): Promise<void> {
|
|
191
|
+
const batches = inflightByClient.get(client);
|
|
192
|
+
if (!batches) return;
|
|
193
|
+
const ops = client as SignalRedisWire;
|
|
194
|
+
for (const batch of batches.values()) {
|
|
195
|
+
if (batch.ids.size === 0) continue;
|
|
196
|
+
try {
|
|
197
|
+
await ops.xclaim(batch.key, batch.group, batch.consumer, 0, [...batch.ids]);
|
|
198
|
+
} catch {
|
|
199
|
+
/* Lease refresh is best-effort. */
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* Start the idle-reset timer for one bus. Call the returned function on shutdown.
|
|
206
|
+
*
|
|
207
|
+
* @param client - Redis client whose in-flight ids should be refreshed
|
|
208
|
+
*/
|
|
209
|
+
function retainHeartbeat(client: object): () => void {
|
|
210
|
+
const timer = setInterval(
|
|
211
|
+
() => {
|
|
212
|
+
void refreshInflight(client);
|
|
213
|
+
},
|
|
214
|
+
Math.floor(SIGNAL_REDIS_MIN_IDLE_MS / 3),
|
|
215
|
+
);
|
|
216
|
+
if (typeof timer.unref === "function") timer.unref();
|
|
217
|
+
let timers = heartbeats.get(client);
|
|
218
|
+
if (!timers) {
|
|
219
|
+
timers = new Set();
|
|
220
|
+
heartbeats.set(client, timers);
|
|
221
|
+
}
|
|
222
|
+
const bucket = timers;
|
|
223
|
+
bucket.add(timer);
|
|
224
|
+
return () => {
|
|
225
|
+
clearInterval(timer);
|
|
226
|
+
bucket.delete(timer);
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
function clearHeartbeats(client: object): void {
|
|
231
|
+
const timers = heartbeats.get(client);
|
|
232
|
+
if (!timers) return;
|
|
233
|
+
for (const timer of timers) clearInterval(timer);
|
|
234
|
+
timers.clear();
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function parseFields(flat: unknown): Record<string, string> {
|
|
238
|
+
const fields: Record<string, string> = {};
|
|
239
|
+
if (!Array.isArray(flat)) return fields;
|
|
240
|
+
for (let i = 0; i + 1 < flat.length; i += 2) {
|
|
241
|
+
const key = flat[i];
|
|
242
|
+
const value = flat[i + 1];
|
|
243
|
+
if (key === undefined || value === undefined) continue;
|
|
244
|
+
fields[String(key)] = String(value);
|
|
245
|
+
}
|
|
246
|
+
return fields;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
function parseEntry(entry: unknown): StreamEntry | undefined {
|
|
250
|
+
if (!Array.isArray(entry) || entry.length < 2) return undefined;
|
|
251
|
+
const id = entry[0];
|
|
252
|
+
if (id === undefined || typeof id === "object") return undefined;
|
|
253
|
+
return { id: String(id), fields: parseFields(entry[1]) };
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
function parseEntryList(entries: unknown): StreamEntry[] {
|
|
257
|
+
if (!Array.isArray(entries)) return [];
|
|
258
|
+
const out: StreamEntry[] = [];
|
|
259
|
+
for (const entry of entries) {
|
|
260
|
+
const parsed = parseEntry(entry);
|
|
261
|
+
if (parsed) out.push(parsed);
|
|
262
|
+
}
|
|
263
|
+
return out;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/** True when `value` is a list of stream entries (`[id, fields][]`), including empty. */
|
|
267
|
+
function isEntryList(value: unknown): boolean {
|
|
268
|
+
if (!Array.isArray(value)) return false;
|
|
269
|
+
if (value.length === 0) return true;
|
|
270
|
+
return Array.isArray(value[0]);
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
function mergeEntries(
|
|
274
|
+
groups: readonly (readonly StreamEntry[])[],
|
|
275
|
+
skip: (id: string) => boolean,
|
|
276
|
+
count: number,
|
|
277
|
+
): StreamEntry[] {
|
|
278
|
+
const out: StreamEntry[] = [];
|
|
279
|
+
const seen = new Set<string>();
|
|
280
|
+
for (const group of groups) {
|
|
281
|
+
for (const entry of group) {
|
|
282
|
+
if (seen.has(entry.id) || skip(entry.id)) continue;
|
|
283
|
+
seen.add(entry.id);
|
|
284
|
+
out.push(entry);
|
|
285
|
+
if (out.length >= count) return out;
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
return out;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/**
|
|
292
|
+
* Parse Redis `XREADGROUP` and `XAUTOCLAIM` replies into field maps.
|
|
293
|
+
*
|
|
294
|
+
* Bun returns a stream map `{ "<stream>": [[id, [field, value, ...]], ...] }`
|
|
295
|
+
* for both commands. RESP nested arrays are accepted too: `XREADGROUP`
|
|
296
|
+
* `[[stream, entries], ...]` and `XAUTOCLAIM` `[cursor, entries, deletedIds]`.
|
|
297
|
+
*
|
|
298
|
+
* @param reply - Raw command result
|
|
299
|
+
*/
|
|
300
|
+
export function parseXreadgroupReply(reply: unknown): StreamEntry[] {
|
|
301
|
+
if (reply == null) return [];
|
|
302
|
+
if (typeof reply === "object" && !Array.isArray(reply)) {
|
|
303
|
+
const out: StreamEntry[] = [];
|
|
304
|
+
for (const entries of Object.values(reply)) {
|
|
305
|
+
out.push(...parseEntryList(entries));
|
|
306
|
+
}
|
|
307
|
+
return out;
|
|
308
|
+
}
|
|
309
|
+
if (!Array.isArray(reply) || reply.length === 0) return [];
|
|
310
|
+
|
|
311
|
+
// XAUTOCLAIM: [nextCursor, [[id, fields], ...], deleted?]
|
|
312
|
+
if (typeof reply[0] === "string") return parseEntryList(reply[1]);
|
|
313
|
+
|
|
314
|
+
const head = reply[0];
|
|
315
|
+
if (!Array.isArray(head) || head.length < 2) return [];
|
|
316
|
+
// Bare [[id, [field, value, ...]], ...] — the second slot is a field list, not entries.
|
|
317
|
+
if (!isEntryList(head[1])) return parseEntryList(reply);
|
|
318
|
+
|
|
319
|
+
const out: StreamEntry[] = [];
|
|
320
|
+
for (const stream of reply) {
|
|
321
|
+
if (!Array.isArray(stream) || stream.length < 2) continue;
|
|
322
|
+
out.push(...parseEntryList(stream[1]));
|
|
323
|
+
}
|
|
324
|
+
return out;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
function parseXinfoConsumers(reply: unknown): RedisConsumerInfo[] {
|
|
328
|
+
if (!Array.isArray(reply)) return [];
|
|
329
|
+
const out: RedisConsumerInfo[] = [];
|
|
330
|
+
for (const row of reply) {
|
|
331
|
+
if (row !== null && typeof row === "object" && !Array.isArray(row)) {
|
|
332
|
+
const rec = row as Record<string, unknown>;
|
|
333
|
+
if (typeof rec.name !== "string") continue;
|
|
334
|
+
out.push({
|
|
335
|
+
name: rec.name,
|
|
336
|
+
pending: Number(rec.pending ?? 0),
|
|
337
|
+
idle: Number(rec.idle ?? 0),
|
|
338
|
+
});
|
|
339
|
+
continue;
|
|
340
|
+
}
|
|
341
|
+
if (!Array.isArray(row)) continue;
|
|
342
|
+
const fields: Record<string, string> = {};
|
|
343
|
+
for (let i = 0; i + 1 < row.length; i += 2) {
|
|
344
|
+
const key = row[i];
|
|
345
|
+
const value = row[i + 1];
|
|
346
|
+
if (key === undefined || value === undefined) continue;
|
|
347
|
+
fields[String(key)] = String(value);
|
|
348
|
+
}
|
|
349
|
+
const name = fields.name;
|
|
350
|
+
if (name === undefined) continue;
|
|
351
|
+
out.push({
|
|
352
|
+
name,
|
|
353
|
+
pending: Number(fields.pending ?? 0),
|
|
354
|
+
idle: Number(fields.idle ?? 0),
|
|
355
|
+
});
|
|
356
|
+
}
|
|
357
|
+
return out;
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
function isPubEnvelope(value: unknown): value is SignalPubEnvelope {
|
|
361
|
+
return (
|
|
362
|
+
value !== null && typeof value === "object" && !Array.isArray(value) && "__oke_env" in value
|
|
363
|
+
);
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/**
|
|
367
|
+
* Decode a pub/sub body.
|
|
368
|
+
*
|
|
369
|
+
* Own-origin envelopes are skipped (the local outbox already delivered them).
|
|
370
|
+
* A body with no `__oke_env` is a legacy raw payload.
|
|
371
|
+
*
|
|
372
|
+
* @param raw - Published string
|
|
373
|
+
* @param origin - This instance id
|
|
374
|
+
*/
|
|
375
|
+
function decodePublished(
|
|
376
|
+
raw: string,
|
|
377
|
+
origin: string,
|
|
378
|
+
): { id: string; payload: unknown } | undefined {
|
|
379
|
+
let parsed: unknown;
|
|
380
|
+
try {
|
|
381
|
+
parsed = JSON.parse(raw);
|
|
382
|
+
} catch {
|
|
383
|
+
return undefined;
|
|
384
|
+
}
|
|
385
|
+
if (isPubEnvelope(parsed)) {
|
|
386
|
+
if (parsed.origin === origin) return undefined;
|
|
387
|
+
const id =
|
|
388
|
+
typeof parsed.id === "string" && parsed.id.length > 0 ? parsed.id : crypto.randomUUID();
|
|
389
|
+
return { id, payload: parsed.payload ?? null };
|
|
390
|
+
}
|
|
391
|
+
return { id: crypto.randomUUID(), payload: parsed };
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
function pubBody(id: string, origin: string, payload: unknown): string {
|
|
395
|
+
const envelope: SignalPubEnvelope = {
|
|
396
|
+
__oke_env: 1,
|
|
397
|
+
id,
|
|
398
|
+
origin,
|
|
399
|
+
payload: payload ?? null,
|
|
400
|
+
};
|
|
401
|
+
return JSON.stringify(envelope);
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
function parseFailures(raw: string | undefined): SignalFailureReason[] {
|
|
405
|
+
if (!raw) return [];
|
|
406
|
+
try {
|
|
407
|
+
const parsed: unknown = JSON.parse(raw);
|
|
408
|
+
if (!Array.isArray(parsed)) return [];
|
|
409
|
+
const out: SignalFailureReason[] = [];
|
|
410
|
+
for (const item of parsed) {
|
|
411
|
+
if (item === null || typeof item !== "object") continue;
|
|
412
|
+
const rec = item as Record<string, unknown>;
|
|
413
|
+
if (typeof rec.code !== "string" || typeof rec.message !== "string") continue;
|
|
414
|
+
if (typeof rec.at !== "number" || typeof rec.attempt !== "number") continue;
|
|
415
|
+
out.push({ code: rec.code, message: rec.message, at: rec.at, attempt: rec.attempt });
|
|
416
|
+
}
|
|
417
|
+
return out;
|
|
418
|
+
} catch {
|
|
419
|
+
return [];
|
|
420
|
+
}
|
|
31
421
|
}
|
|
32
422
|
|
|
33
423
|
/**
|
|
34
424
|
* Bind {@link Bun.RedisClient} to {@link SignalRedisClientLike}.
|
|
35
425
|
*
|
|
36
|
-
* Streams use typed `xadd` / `xgroup` / `xreadgroup` / `
|
|
37
|
-
* Pub/sub uses typed `publish` / `subscribe`.
|
|
426
|
+
* Streams use typed `xadd` / `xgroup` / `xreadgroup` / `xautoclaim` / `xack`
|
|
427
|
+
* (Bun ≥1.4). Pub/sub uses typed `publish` / `subscribe`. `close` stops the
|
|
428
|
+
* idle-reset timer and closes connections this helper opened.
|
|
38
429
|
*
|
|
39
|
-
* @param url - Optional Redis URL
|
|
430
|
+
* @param url - Optional Redis URL. Omit to use the shared `Bun.redis` client.
|
|
40
431
|
*/
|
|
41
432
|
export function createBunSignalRedisClient(url?: string): SignalRedisClientLike {
|
|
42
|
-
const
|
|
433
|
+
const dedicated = url !== undefined;
|
|
434
|
+
const redis = dedicated ? new Bun.RedisClient(url) : Bun.redis;
|
|
43
435
|
/** Subscribe blocks a connection — keep a dedicated client. */
|
|
44
436
|
let sub: InstanceType<typeof Bun.RedisClient> | undefined;
|
|
45
437
|
|
|
46
438
|
function subClient(): InstanceType<typeof Bun.RedisClient> {
|
|
47
439
|
if (!sub) {
|
|
48
|
-
sub =
|
|
440
|
+
sub = dedicated ? new Bun.RedisClient(url) : new Bun.RedisClient();
|
|
49
441
|
}
|
|
50
442
|
return sub;
|
|
51
443
|
}
|
|
52
444
|
|
|
53
|
-
|
|
54
|
-
|
|
445
|
+
const streams = () => bunStreams(redis);
|
|
446
|
+
|
|
447
|
+
const api = {
|
|
448
|
+
async xadd(key: string, id: string, fields: Record<string, string>): Promise<string> {
|
|
55
449
|
const pairs: string[] = [];
|
|
56
450
|
for (const [k, v] of Object.entries(fields)) {
|
|
57
451
|
pairs.push(k, v);
|
|
58
452
|
}
|
|
59
|
-
return String(await
|
|
453
|
+
return String(await streams().xadd(key, id, ...pairs));
|
|
60
454
|
},
|
|
61
|
-
async xgroupCreate(
|
|
455
|
+
async xgroupCreate(
|
|
456
|
+
key: string,
|
|
457
|
+
group: string,
|
|
458
|
+
id: string,
|
|
459
|
+
opts?: { mkstream?: boolean },
|
|
460
|
+
): Promise<void> {
|
|
62
461
|
const extra = opts?.mkstream ? (["MKSTREAM"] as const) : [];
|
|
63
462
|
try {
|
|
64
|
-
await
|
|
463
|
+
await streams().xgroup("CREATE", key, group, id, ...extra);
|
|
65
464
|
} catch (err) {
|
|
66
465
|
const msg = err instanceof Error ? err.message : String(err);
|
|
67
466
|
if (!/BUSYGROUP/i.test(msg)) throw err;
|
|
68
467
|
}
|
|
69
468
|
},
|
|
70
|
-
async xreadgroup(
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
469
|
+
async xreadgroup(
|
|
470
|
+
group: string,
|
|
471
|
+
consumer: string,
|
|
472
|
+
key: string,
|
|
473
|
+
count: number,
|
|
474
|
+
): Promise<StreamEntry[]> {
|
|
475
|
+
const skip = (id: string) => isInflight(api, key, consumer, id);
|
|
476
|
+
let claimed: StreamEntry[] = [];
|
|
477
|
+
try {
|
|
478
|
+
const reply = await streams().xautoclaim(
|
|
479
|
+
key,
|
|
480
|
+
group,
|
|
481
|
+
consumer,
|
|
482
|
+
SIGNAL_REDIS_MIN_IDLE_MS,
|
|
483
|
+
"0-0",
|
|
484
|
+
"COUNT",
|
|
485
|
+
count,
|
|
486
|
+
);
|
|
487
|
+
claimed = parseXreadgroupReply(reply);
|
|
488
|
+
} catch {
|
|
489
|
+
/* Idle reclaim is best-effort; same-consumer pending still returns below. */
|
|
490
|
+
}
|
|
491
|
+
const pending = parseXreadgroupReply(
|
|
492
|
+
await streams().xreadgroup("GROUP", group, consumer, "COUNT", count, "STREAMS", key, "0"),
|
|
80
493
|
);
|
|
81
|
-
|
|
494
|
+
const owned = mergeEntries([claimed, pending], skip, count);
|
|
495
|
+
if (owned.length > 0) return owned;
|
|
496
|
+
const fresh = parseXreadgroupReply(
|
|
497
|
+
await streams().xreadgroup("GROUP", group, consumer, "COUNT", count, "STREAMS", key, ">"),
|
|
498
|
+
);
|
|
499
|
+
return mergeEntries([fresh], skip, count);
|
|
500
|
+
},
|
|
501
|
+
async xack(key: string, group: string, id: string): Promise<number> {
|
|
502
|
+
return Number(await streams().xack(key, group, id));
|
|
503
|
+
},
|
|
504
|
+
async xclaim(
|
|
505
|
+
key: string,
|
|
506
|
+
group: string,
|
|
507
|
+
consumer: string,
|
|
508
|
+
minIdleMs: number,
|
|
509
|
+
ids: readonly string[],
|
|
510
|
+
): Promise<void> {
|
|
511
|
+
const first = ids[0];
|
|
512
|
+
if (first === undefined) return;
|
|
513
|
+
await streams().xclaim(key, group, consumer, minIdleMs, first, ...ids.slice(1), "JUSTID");
|
|
514
|
+
},
|
|
515
|
+
async hincrby(key: string, field: string, increment: number): Promise<number> {
|
|
516
|
+
return Number(await streams().hincrby(key, field, increment));
|
|
517
|
+
},
|
|
518
|
+
async hdel(key: string, field: string): Promise<number> {
|
|
519
|
+
return Number(await streams().hdel(key, field));
|
|
520
|
+
},
|
|
521
|
+
async xrange(key: string): Promise<StreamEntry[]> {
|
|
522
|
+
return parseEntryList(await streams().xrange(key, "-", "+"));
|
|
523
|
+
},
|
|
524
|
+
async xinfoConsumers(key: string, group: string): Promise<RedisConsumerInfo[]> {
|
|
525
|
+
try {
|
|
526
|
+
return parseXinfoConsumers(await streams().xinfo("CONSUMERS", key, group));
|
|
527
|
+
} catch {
|
|
528
|
+
return [];
|
|
529
|
+
}
|
|
82
530
|
},
|
|
83
|
-
async
|
|
84
|
-
|
|
531
|
+
async xgroupDelConsumer(key: string, group: string, consumer: string): Promise<void> {
|
|
532
|
+
await streams().xgroup("DELCONSUMER", key, group, consumer);
|
|
85
533
|
},
|
|
86
|
-
async publish(channel, message) {
|
|
534
|
+
async publish(channel: string, message: string): Promise<number> {
|
|
87
535
|
return redis.publish(channel, message);
|
|
88
536
|
},
|
|
89
|
-
async subscribe(channel, listener) {
|
|
537
|
+
async subscribe(channel: string, listener: (message: string) => void): Promise<() => void> {
|
|
90
538
|
const client = subClient();
|
|
91
539
|
await client.subscribe(channel, (message) => {
|
|
92
540
|
listener(message);
|
|
@@ -95,57 +543,66 @@ export function createBunSignalRedisClient(url?: string): SignalRedisClientLike
|
|
|
95
543
|
void client.unsubscribe(channel);
|
|
96
544
|
};
|
|
97
545
|
},
|
|
546
|
+
async close(): Promise<void> {
|
|
547
|
+
clearHeartbeats(api);
|
|
548
|
+
sub?.close();
|
|
549
|
+
sub = undefined;
|
|
550
|
+
if (dedicated) redis.close();
|
|
551
|
+
},
|
|
98
552
|
};
|
|
99
|
-
}
|
|
100
553
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
reply: unknown,
|
|
108
|
-
): Array<{ id: string; fields: Record<string, string> }> {
|
|
109
|
-
if (reply == null || !Array.isArray(reply)) return [];
|
|
110
|
-
const out: Array<{ id: string; fields: Record<string, string> }> = [];
|
|
111
|
-
for (const stream of reply) {
|
|
112
|
-
if (!Array.isArray(stream) || stream.length < 2) continue;
|
|
113
|
-
const entries = stream[1];
|
|
114
|
-
if (!Array.isArray(entries)) continue;
|
|
115
|
-
for (const entry of entries) {
|
|
116
|
-
if (!Array.isArray(entry) || entry.length < 2) continue;
|
|
117
|
-
const id = String(entry[0]);
|
|
118
|
-
const flat = entry[1];
|
|
119
|
-
const fields: Record<string, string> = {};
|
|
120
|
-
if (Array.isArray(flat)) {
|
|
121
|
-
for (let i = 0; i + 1 < flat.length; i += 2) {
|
|
122
|
-
fields[String(flat[i])] = String(flat[i + 1]);
|
|
123
|
-
}
|
|
124
|
-
}
|
|
125
|
-
out.push({ id, fields });
|
|
126
|
-
}
|
|
554
|
+
if (dedicated) {
|
|
555
|
+
const previousOnClose = redis.onclose;
|
|
556
|
+
redis.onclose = function (this: Bun.RedisClient, error: Error): void {
|
|
557
|
+
clearHeartbeats(api);
|
|
558
|
+
previousOnClose?.call(this, error);
|
|
559
|
+
};
|
|
127
560
|
}
|
|
128
|
-
|
|
561
|
+
|
|
562
|
+
return api;
|
|
129
563
|
}
|
|
130
564
|
|
|
131
565
|
/**
|
|
132
566
|
* In-memory redis-protocol fake for signal streams + pub/sub.
|
|
567
|
+
*
|
|
568
|
+
* Pending entries carry a claim clock (`now`, default `Date.now`). Another
|
|
569
|
+
* consumer reclaims one only after {@link SIGNAL_REDIS_MIN_IDLE_MS}.
|
|
570
|
+
* `flushIdle` runs the same batched `XCLAIM ... JUSTID` the idle-reset timer uses.
|
|
571
|
+
*
|
|
572
|
+
* @param options - Optional clock for idle / reclaim tests
|
|
133
573
|
*/
|
|
134
|
-
export function createSignalRedisFake(): SignalRedisClientLike & {
|
|
574
|
+
export function createSignalRedisFake(options?: { now?: () => number }): SignalRedisClientLike & {
|
|
135
575
|
readonly streams: Map<string, Array<{ id: string; fields: Record<string, string> }>>;
|
|
136
576
|
readonly published: Array<{ channel: string; message: string }>;
|
|
577
|
+
flushIdle(): Promise<void>;
|
|
137
578
|
} {
|
|
579
|
+
const clock = options?.now ?? (() => Date.now());
|
|
138
580
|
const streams = new Map<string, Array<{ id: string; fields: Record<string, string> }>>();
|
|
139
581
|
const groups = new Map<string, Set<string>>();
|
|
140
582
|
const claimed = new Map<string, Set<string>>();
|
|
141
|
-
const
|
|
583
|
+
const pending = new Map<string, PendingEntry[]>();
|
|
584
|
+
const consumerSeen = new Map<string, Map<string, number>>();
|
|
585
|
+
const hashes = new Map<string, Map<string, number>>();
|
|
586
|
+
const subs = new Map<string, Set<(message: string) => void | Promise<void>>>();
|
|
142
587
|
const published: Array<{ channel: string; message: string }> = [];
|
|
143
588
|
let seq = 0;
|
|
144
589
|
|
|
145
|
-
|
|
590
|
+
function touchConsumer(gkey: string, consumer: string): void {
|
|
591
|
+
let names = consumerSeen.get(gkey);
|
|
592
|
+
if (!names) {
|
|
593
|
+
names = new Map();
|
|
594
|
+
consumerSeen.set(gkey, names);
|
|
595
|
+
}
|
|
596
|
+
names.set(consumer, clock());
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
const api = {
|
|
146
600
|
streams,
|
|
147
601
|
published,
|
|
148
|
-
async
|
|
602
|
+
async flushIdle(): Promise<void> {
|
|
603
|
+
await refreshInflight(api);
|
|
604
|
+
},
|
|
605
|
+
async xadd(key: string, id: string, fields: Record<string, string>): Promise<string> {
|
|
149
606
|
let list = streams.get(key);
|
|
150
607
|
if (!list) {
|
|
151
608
|
list = [];
|
|
@@ -155,42 +612,161 @@ export function createSignalRedisFake(): SignalRedisClientLike & {
|
|
|
155
612
|
list.push({ id: realId, fields: { ...fields } });
|
|
156
613
|
return realId;
|
|
157
614
|
},
|
|
158
|
-
async xgroupCreate(
|
|
615
|
+
async xgroupCreate(
|
|
616
|
+
key: string,
|
|
617
|
+
group: string,
|
|
618
|
+
_id: string,
|
|
619
|
+
opts?: { mkstream?: boolean },
|
|
620
|
+
): Promise<void> {
|
|
159
621
|
if (opts?.mkstream && !streams.has(key)) streams.set(key, []);
|
|
160
622
|
const gkey = `${key}::${group}`;
|
|
161
623
|
if (!groups.has(gkey)) groups.set(gkey, new Set());
|
|
162
624
|
if (!claimed.has(gkey)) claimed.set(gkey, new Set());
|
|
625
|
+
if (!pending.has(gkey)) pending.set(gkey, []);
|
|
163
626
|
},
|
|
164
|
-
async xreadgroup(
|
|
627
|
+
async xreadgroup(
|
|
628
|
+
group: string,
|
|
629
|
+
consumer: string,
|
|
630
|
+
key: string,
|
|
631
|
+
count: number,
|
|
632
|
+
): Promise<StreamEntry[]> {
|
|
165
633
|
const gkey = `${key}::${group}`;
|
|
166
634
|
if (!groups.has(gkey)) {
|
|
167
|
-
await
|
|
635
|
+
await api.xgroupCreate(key, group, "0", { mkstream: true });
|
|
168
636
|
}
|
|
169
|
-
|
|
637
|
+
touchConsumer(gkey, consumer);
|
|
638
|
+
const seen = claimed.get(gkey);
|
|
639
|
+
const held = pending.get(gkey);
|
|
640
|
+
if (!seen || !held) return [];
|
|
170
641
|
const list = streams.get(key) ?? [];
|
|
171
|
-
const
|
|
642
|
+
const t = clock();
|
|
643
|
+
const out: StreamEntry[] = [];
|
|
644
|
+
|
|
645
|
+
const push = (entry: PendingEntry): boolean => {
|
|
646
|
+
if (isInflight(api, key, consumer, entry.id)) return false;
|
|
647
|
+
if (out.some((row) => row.id === entry.id)) return false;
|
|
648
|
+
out.push({ id: entry.id, fields: { ...entry.fields } });
|
|
649
|
+
return out.length >= count;
|
|
650
|
+
};
|
|
651
|
+
|
|
652
|
+
let full = false;
|
|
653
|
+
for (const entry of held) {
|
|
654
|
+
if (entry.consumer === consumer) continue;
|
|
655
|
+
if (t - entry.claimedAt < SIGNAL_REDIS_MIN_IDLE_MS) continue;
|
|
656
|
+
entry.consumer = consumer;
|
|
657
|
+
entry.claimedAt = t;
|
|
658
|
+
if (push(entry)) {
|
|
659
|
+
full = true;
|
|
660
|
+
break;
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
if (!full) {
|
|
664
|
+
for (const entry of held) {
|
|
665
|
+
if (entry.consumer !== consumer) continue;
|
|
666
|
+
if (push(entry)) {
|
|
667
|
+
full = true;
|
|
668
|
+
break;
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
if (out.length > 0) return out;
|
|
673
|
+
|
|
172
674
|
for (const entry of list) {
|
|
173
675
|
if (seen.has(entry.id)) continue;
|
|
174
676
|
seen.add(entry.id);
|
|
175
|
-
|
|
677
|
+
const claimedEntry: PendingEntry = {
|
|
176
678
|
id: entry.id,
|
|
177
|
-
fields: { ...entry.fields
|
|
178
|
-
|
|
179
|
-
|
|
679
|
+
fields: { ...entry.fields },
|
|
680
|
+
consumer,
|
|
681
|
+
claimedAt: t,
|
|
682
|
+
};
|
|
683
|
+
held.push(claimedEntry);
|
|
684
|
+
if (push(claimedEntry)) break;
|
|
180
685
|
}
|
|
181
686
|
return out;
|
|
182
687
|
},
|
|
183
|
-
async xack() {
|
|
688
|
+
async xack(key: string, group: string, id: string): Promise<number> {
|
|
689
|
+
const gkey = `${key}::${group}`;
|
|
690
|
+
const held = pending.get(gkey);
|
|
691
|
+
if (!held) return 0;
|
|
692
|
+
const next = held.filter((entry) => entry.id !== id);
|
|
693
|
+
pending.set(gkey, next);
|
|
694
|
+
return next.length === held.length ? 0 : 1;
|
|
695
|
+
},
|
|
696
|
+
async xclaim(
|
|
697
|
+
key: string,
|
|
698
|
+
group: string,
|
|
699
|
+
consumer: string,
|
|
700
|
+
minIdleMs: number,
|
|
701
|
+
ids: readonly string[],
|
|
702
|
+
): Promise<void> {
|
|
703
|
+
const held = pending.get(`${key}::${group}`);
|
|
704
|
+
if (!held || ids.length === 0) return;
|
|
705
|
+
const want = new Set(ids);
|
|
706
|
+
const t = clock();
|
|
707
|
+
for (const entry of held) {
|
|
708
|
+
if (!want.has(entry.id)) continue;
|
|
709
|
+
if (t - entry.claimedAt < minIdleMs) continue;
|
|
710
|
+
entry.consumer = consumer;
|
|
711
|
+
entry.claimedAt = t;
|
|
712
|
+
}
|
|
713
|
+
},
|
|
714
|
+
async hincrby(key: string, field: string, increment: number): Promise<number> {
|
|
715
|
+
let hash = hashes.get(key);
|
|
716
|
+
if (!hash) {
|
|
717
|
+
hash = new Map();
|
|
718
|
+
hashes.set(key, hash);
|
|
719
|
+
}
|
|
720
|
+
const next = (hash.get(field) ?? 0) + increment;
|
|
721
|
+
hash.set(field, next);
|
|
722
|
+
return next;
|
|
723
|
+
},
|
|
724
|
+
async hdel(key: string, field: string): Promise<number> {
|
|
725
|
+
const hash = hashes.get(key);
|
|
726
|
+
if (!hash || !hash.has(field)) return 0;
|
|
727
|
+
hash.delete(field);
|
|
184
728
|
return 1;
|
|
185
729
|
},
|
|
186
|
-
async
|
|
730
|
+
async xrange(key: string): Promise<StreamEntry[]> {
|
|
731
|
+
const list = streams.get(key) ?? [];
|
|
732
|
+
return list.map((entry) => ({ id: entry.id, fields: { ...entry.fields } }));
|
|
733
|
+
},
|
|
734
|
+
async xinfoConsumers(key: string, group: string): Promise<RedisConsumerInfo[]> {
|
|
735
|
+
const gkey = `${key}::${group}`;
|
|
736
|
+
const names = consumerSeen.get(gkey);
|
|
737
|
+
if (!names) return [];
|
|
738
|
+
const held = pending.get(gkey) ?? [];
|
|
739
|
+
const t = clock();
|
|
740
|
+
const out: RedisConsumerInfo[] = [];
|
|
741
|
+
for (const [name, seenAt] of names) {
|
|
742
|
+
let pendingCount = 0;
|
|
743
|
+
for (const entry of held) {
|
|
744
|
+
if (entry.consumer === name) pendingCount += 1;
|
|
745
|
+
}
|
|
746
|
+
out.push({ name, pending: pendingCount, idle: Math.max(0, t - seenAt) });
|
|
747
|
+
}
|
|
748
|
+
return out;
|
|
749
|
+
},
|
|
750
|
+
async xgroupDelConsumer(key: string, group: string, consumer: string): Promise<void> {
|
|
751
|
+
const gkey = `${key}::${group}`;
|
|
752
|
+
consumerSeen.get(gkey)?.delete(consumer);
|
|
753
|
+
const held = pending.get(gkey);
|
|
754
|
+
if (!held) return;
|
|
755
|
+
pending.set(
|
|
756
|
+
gkey,
|
|
757
|
+
held.filter((entry) => entry.consumer !== consumer),
|
|
758
|
+
);
|
|
759
|
+
},
|
|
760
|
+
async publish(channel: string, message: string): Promise<number> {
|
|
187
761
|
published.push({ channel, message });
|
|
188
762
|
const set = subs.get(channel);
|
|
189
763
|
if (!set) return 0;
|
|
190
|
-
|
|
764
|
+
const listeners: Array<(message: string) => void | Promise<void>> = [];
|
|
765
|
+
for (const fn of set) listeners.push(fn);
|
|
766
|
+
for (const fn of listeners) await fn(message);
|
|
191
767
|
return set.size;
|
|
192
768
|
},
|
|
193
|
-
async subscribe(channel, listener) {
|
|
769
|
+
async subscribe(channel: string, listener: (message: string) => void): Promise<() => void> {
|
|
194
770
|
let set = subs.get(channel);
|
|
195
771
|
if (!set) {
|
|
196
772
|
set = new Set();
|
|
@@ -198,10 +774,15 @@ export function createSignalRedisFake(): SignalRedisClientLike & {
|
|
|
198
774
|
}
|
|
199
775
|
set.add(listener);
|
|
200
776
|
return () => {
|
|
201
|
-
set
|
|
777
|
+
set.delete(listener);
|
|
202
778
|
};
|
|
203
779
|
},
|
|
780
|
+
async close(): Promise<void> {
|
|
781
|
+
clearHeartbeats(api);
|
|
782
|
+
},
|
|
204
783
|
};
|
|
784
|
+
|
|
785
|
+
return api;
|
|
205
786
|
}
|
|
206
787
|
|
|
207
788
|
/**
|
|
@@ -211,29 +792,225 @@ export function createSignalRedisFake(): SignalRedisClientLike & {
|
|
|
211
792
|
*/
|
|
212
793
|
export async function openRedisSignal(options: SignalOpenOptions): Promise<SignalBus> {
|
|
213
794
|
// Same DI shape as postgres/redis/s3: inject a client in tests; production
|
|
214
|
-
// binds Bun.redis (Streams via
|
|
795
|
+
// binds Bun.redis (Streams via the typed client).
|
|
215
796
|
const redis = options.redis ?? createBunSignalRedisClient();
|
|
797
|
+
const wire = asWire(redis);
|
|
798
|
+
const ownsClient = options.redis === undefined;
|
|
216
799
|
const outbox = await createSignalEngine("redis", options);
|
|
800
|
+
const compete = options.compete === true;
|
|
801
|
+
const onceHandlers = new Map<string, SignalHandler[]>();
|
|
802
|
+
const remoteHandlers = new Map<string, SignalHandler[]>();
|
|
803
|
+
const liveEmitters = new Map<string, Set<(event: LiveEvent) => void>>();
|
|
804
|
+
const now = options.now ?? (() => Date.now());
|
|
805
|
+
const origin = options.consumerId ?? crypto.randomUUID();
|
|
806
|
+
const channelUnsubs: Array<() => void | Promise<void>> = [];
|
|
807
|
+
|
|
808
|
+
if (compete) {
|
|
809
|
+
for (const [name, decl] of options.signals) {
|
|
810
|
+
if (decl.delivery !== "once") continue;
|
|
811
|
+
await outbox.subscribe(name, "oke-relay-ack", async () => {});
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
function liveSet(name: string): Set<(event: LiveEvent) => void> {
|
|
816
|
+
let set = liveEmitters.get(name);
|
|
817
|
+
if (!set) {
|
|
818
|
+
set = new Set();
|
|
819
|
+
liveEmitters.set(name, set);
|
|
820
|
+
}
|
|
821
|
+
return set;
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
function streamMessage(
|
|
825
|
+
name: string,
|
|
826
|
+
id: string,
|
|
827
|
+
payload: unknown,
|
|
828
|
+
attempt: number,
|
|
829
|
+
): SignalMessage {
|
|
830
|
+
const at = now();
|
|
831
|
+
return {
|
|
832
|
+
id,
|
|
833
|
+
signal: name,
|
|
834
|
+
payload,
|
|
835
|
+
delivery: "once",
|
|
836
|
+
attempts: attempt,
|
|
837
|
+
failures: [],
|
|
838
|
+
createdAt: at,
|
|
839
|
+
availableAt: at,
|
|
840
|
+
status: "inflight",
|
|
841
|
+
};
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
async function deliverPub(
|
|
845
|
+
signalName: string,
|
|
846
|
+
delivery: "broadcast" | "live",
|
|
847
|
+
raw: string,
|
|
848
|
+
): Promise<void> {
|
|
849
|
+
const incoming = decodePublished(raw, origin);
|
|
850
|
+
if (!incoming) return;
|
|
851
|
+
if (delivery === "live") {
|
|
852
|
+
const emitters = liveEmitters.get(signalName);
|
|
853
|
+
if (!emitters) return;
|
|
854
|
+
for (const emit of emitters) {
|
|
855
|
+
emit({ id: incoming.id, payload: incoming.payload });
|
|
856
|
+
}
|
|
857
|
+
return;
|
|
858
|
+
}
|
|
859
|
+
const at = now();
|
|
860
|
+
const handlers = remoteHandlers.get(signalName) ?? [];
|
|
861
|
+
for (const handler of handlers) {
|
|
862
|
+
try {
|
|
863
|
+
await handler({
|
|
864
|
+
id: incoming.id,
|
|
865
|
+
signal: signalName,
|
|
866
|
+
payload: incoming.payload,
|
|
867
|
+
delivery: "broadcast",
|
|
868
|
+
attempts: 1,
|
|
869
|
+
failures: [],
|
|
870
|
+
createdAt: at,
|
|
871
|
+
availableAt: at,
|
|
872
|
+
status: "inflight",
|
|
873
|
+
});
|
|
874
|
+
} catch {
|
|
875
|
+
/* One subscriber's error must not drop the relay. */
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
for (const [name, decl] of options.signals) {
|
|
881
|
+
if (decl.delivery === "broadcast") {
|
|
882
|
+
channelUnsubs.push(
|
|
883
|
+
await redis.subscribe(bcastChannel(name), (message) => {
|
|
884
|
+
return deliverPub(name, "broadcast", message);
|
|
885
|
+
}),
|
|
886
|
+
);
|
|
887
|
+
} else if (decl.delivery === "live") {
|
|
888
|
+
channelUnsubs.push(
|
|
889
|
+
await redis.subscribe(liveChannel(name), (message) => {
|
|
890
|
+
return deliverPub(name, "live", message);
|
|
891
|
+
}),
|
|
892
|
+
);
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
const stopHeartbeat = retainHeartbeat(redis);
|
|
897
|
+
|
|
898
|
+
async function reapIdleConsumers(key: string, selfName: string): Promise<void> {
|
|
899
|
+
try {
|
|
900
|
+
const rows = await wire.xinfoConsumers(key, STREAM_GROUP);
|
|
901
|
+
for (const row of rows) {
|
|
902
|
+
if (row.name === selfName) continue;
|
|
903
|
+
if (row.pending !== 0) continue;
|
|
904
|
+
if (row.idle < CONSUMER_IDLE_RETENTION_MS) continue;
|
|
905
|
+
await wire.xgroupDelConsumer(key, STREAM_GROUP, row.name);
|
|
906
|
+
}
|
|
907
|
+
} catch {
|
|
908
|
+
/* Consumer cleanup is best-effort. */
|
|
909
|
+
}
|
|
910
|
+
}
|
|
911
|
+
|
|
912
|
+
async function readDeadLetters(name: string): Promise<DeadLetter[]> {
|
|
913
|
+
const rows = await wire.xrange(deadKey(name));
|
|
914
|
+
const out: DeadLetter[] = [];
|
|
915
|
+
for (const row of rows) {
|
|
916
|
+
let payload: unknown = null;
|
|
917
|
+
try {
|
|
918
|
+
payload = JSON.parse(row.fields.payload ?? "null");
|
|
919
|
+
} catch {
|
|
920
|
+
payload = null;
|
|
921
|
+
}
|
|
922
|
+
const at = Number(row.fields.createdAt ?? now());
|
|
923
|
+
out.push({
|
|
924
|
+
id: row.fields.id ?? row.id,
|
|
925
|
+
signal: name,
|
|
926
|
+
payload,
|
|
927
|
+
delivery: "once",
|
|
928
|
+
attempts: Number(row.fields.attempts ?? 0),
|
|
929
|
+
failures: parseFailures(row.fields.failures),
|
|
930
|
+
createdAt: at,
|
|
931
|
+
availableAt: at,
|
|
932
|
+
status: "dead",
|
|
933
|
+
});
|
|
934
|
+
}
|
|
935
|
+
return out;
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
async function drainOnceStreams(): Promise<void> {
|
|
939
|
+
const consumer = options.consumerId ?? "local";
|
|
940
|
+
for (const [name, decl] of options.signals) {
|
|
941
|
+
if (decl.delivery !== "once") continue;
|
|
942
|
+
const key = streamKey(name);
|
|
943
|
+
await wire.xgroupCreate(key, STREAM_GROUP, "0", { mkstream: true });
|
|
944
|
+
const rows = await wire.xreadgroup(STREAM_GROUP, consumer, key, 32);
|
|
945
|
+
const seen = new Set<string>();
|
|
946
|
+
for (const row of rows) {
|
|
947
|
+
if (seen.has(row.id) || isInflight(redis, key, consumer, row.id)) continue;
|
|
948
|
+
seen.add(row.id);
|
|
949
|
+
const handler = onceHandlers.get(name)?.[0];
|
|
950
|
+
if (!handler) continue;
|
|
951
|
+
let payload: unknown = null;
|
|
952
|
+
try {
|
|
953
|
+
payload = JSON.parse(row.fields.payload ?? "null");
|
|
954
|
+
} catch {
|
|
955
|
+
payload = null;
|
|
956
|
+
}
|
|
957
|
+
markInflight(redis, key, STREAM_GROUP, consumer, row.id);
|
|
958
|
+
const attempt = await wire.hincrby(attemptsKey(name), row.id, 1);
|
|
959
|
+
try {
|
|
960
|
+
await handler(streamMessage(name, row.id, payload, attempt));
|
|
961
|
+
await wire.xack(key, STREAM_GROUP, row.id);
|
|
962
|
+
await wire.hdel(attemptsKey(name), row.id);
|
|
963
|
+
} catch (err) {
|
|
964
|
+
const retries = decl.retries ?? 3;
|
|
965
|
+
if (attempt > retries) {
|
|
966
|
+
const failure: SignalFailureReason = {
|
|
967
|
+
code: "handler_error",
|
|
968
|
+
message: err instanceof Error ? err.message : String(err),
|
|
969
|
+
at: now(),
|
|
970
|
+
attempt,
|
|
971
|
+
};
|
|
972
|
+
if (decl.deadLetter) {
|
|
973
|
+
await wire.xadd(deadKey(name), "*", {
|
|
974
|
+
id: row.id,
|
|
975
|
+
signal: name,
|
|
976
|
+
payload: JSON.stringify(payload ?? null),
|
|
977
|
+
attempts: String(attempt),
|
|
978
|
+
failures: JSON.stringify([failure]),
|
|
979
|
+
createdAt: String(now()),
|
|
980
|
+
});
|
|
981
|
+
}
|
|
982
|
+
await wire.xack(key, STREAM_GROUP, row.id);
|
|
983
|
+
await wire.hdel(attemptsKey(name), row.id);
|
|
984
|
+
}
|
|
985
|
+
} finally {
|
|
986
|
+
unmarkInflight(redis, key, consumer, row.id);
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
await reapIdleConsumers(key, consumer);
|
|
990
|
+
}
|
|
991
|
+
}
|
|
217
992
|
|
|
218
993
|
async function relayToRedis(signal: string, payload: unknown): Promise<void> {
|
|
219
994
|
const decl = options.signals.get(signal);
|
|
220
995
|
if (!decl) return;
|
|
221
|
-
const body = JSON.stringify(payload ?? null);
|
|
222
996
|
if (decl.delivery === "once") {
|
|
223
|
-
const key =
|
|
224
|
-
await
|
|
225
|
-
await
|
|
226
|
-
|
|
227
|
-
|
|
997
|
+
const key = streamKey(signal);
|
|
998
|
+
await wire.xgroupCreate(key, STREAM_GROUP, "0", { mkstream: true });
|
|
999
|
+
await wire.xadd(key, "*", { payload: JSON.stringify(payload ?? null), signal });
|
|
1000
|
+
return;
|
|
1001
|
+
}
|
|
1002
|
+
const body = pubBody(crypto.randomUUID(), origin, payload);
|
|
1003
|
+
if (decl.delivery === "broadcast") {
|
|
1004
|
+
await wire.publish(bcastChannel(signal), body);
|
|
228
1005
|
} else {
|
|
229
|
-
await
|
|
1006
|
+
await wire.publish(liveChannel(signal), body);
|
|
230
1007
|
}
|
|
231
1008
|
}
|
|
232
1009
|
|
|
233
1010
|
return {
|
|
234
1011
|
driverId: "redis",
|
|
235
|
-
async emit(signal, payload,
|
|
236
|
-
await outbox.emit(signal, payload,
|
|
1012
|
+
async emit(signal, payload, emitOptions) {
|
|
1013
|
+
await outbox.emit(signal, payload, emitOptions);
|
|
237
1014
|
await relayToRedis(signal, payload);
|
|
238
1015
|
},
|
|
239
1016
|
async begin() {
|
|
@@ -241,9 +1018,9 @@ export async function openRedisSignal(options: SignalOpenOptions): Promise<Signa
|
|
|
241
1018
|
const tx = await outbox.begin();
|
|
242
1019
|
return {
|
|
243
1020
|
write: (k, v) => tx.write(k, v),
|
|
244
|
-
async emit(signal, payload,
|
|
1021
|
+
async emit(signal, payload, emitOptions) {
|
|
245
1022
|
staged.push({ signal, payload });
|
|
246
|
-
await tx.emit(signal, payload,
|
|
1023
|
+
await tx.emit(signal, payload, emitOptions);
|
|
247
1024
|
},
|
|
248
1025
|
async commit() {
|
|
249
1026
|
await tx.commit();
|
|
@@ -252,16 +1029,80 @@ export async function openRedisSignal(options: SignalOpenOptions): Promise<Signa
|
|
|
252
1029
|
rollback: () => tx.rollback(),
|
|
253
1030
|
};
|
|
254
1031
|
},
|
|
255
|
-
subscribe
|
|
256
|
-
|
|
1032
|
+
async subscribe(signal, subscriberId, handler) {
|
|
1033
|
+
if (compete && options.signals.get(signal)?.delivery === "once") {
|
|
1034
|
+
const list = onceHandlers.get(signal) ?? [];
|
|
1035
|
+
list.push(handler);
|
|
1036
|
+
onceHandlers.set(signal, list);
|
|
1037
|
+
return () => {
|
|
1038
|
+
const next = (onceHandlers.get(signal) ?? []).filter((item) => item !== handler);
|
|
1039
|
+
onceHandlers.set(signal, next);
|
|
1040
|
+
};
|
|
1041
|
+
}
|
|
1042
|
+
if (options.signals.get(signal)?.delivery === "broadcast") {
|
|
1043
|
+
const list = remoteHandlers.get(signal) ?? [];
|
|
1044
|
+
list.push(handler);
|
|
1045
|
+
remoteHandlers.set(signal, list);
|
|
1046
|
+
const unsub = await outbox.subscribe(signal, subscriberId, handler);
|
|
1047
|
+
return () => {
|
|
1048
|
+
const next = (remoteHandlers.get(signal) ?? []).filter((item) => item !== handler);
|
|
1049
|
+
if (next.length === 0) remoteHandlers.delete(signal);
|
|
1050
|
+
else remoteHandlers.set(signal, next);
|
|
1051
|
+
return unsub();
|
|
1052
|
+
};
|
|
1053
|
+
}
|
|
1054
|
+
return outbox.subscribe(signal, subscriberId, handler);
|
|
1055
|
+
},
|
|
1056
|
+
live(signal, opts) {
|
|
1057
|
+
return createLiveIterable(async (emit) => {
|
|
1058
|
+
if (opts?.afterId !== undefined && opts.afterId.length > 0) {
|
|
1059
|
+
await outbox.checkLiveResume(signal, opts.afterId);
|
|
1060
|
+
}
|
|
1061
|
+
const set = liveSet(signal);
|
|
1062
|
+
set.add(emit);
|
|
1063
|
+
const iterator = outbox.live(signal, opts)[Symbol.asyncIterator]();
|
|
1064
|
+
let stopped = false;
|
|
1065
|
+
const pump = async (): Promise<void> => {
|
|
1066
|
+
try {
|
|
1067
|
+
for (;;) {
|
|
1068
|
+
if (stopped) break;
|
|
1069
|
+
const next = await iterator.next();
|
|
1070
|
+
if (stopped || next.done) break;
|
|
1071
|
+
emit(next.value);
|
|
1072
|
+
}
|
|
1073
|
+
} catch {
|
|
1074
|
+
/* Iterator closed. */
|
|
1075
|
+
}
|
|
1076
|
+
};
|
|
1077
|
+
void pump();
|
|
1078
|
+
return () => {
|
|
1079
|
+
stopped = true;
|
|
1080
|
+
set.delete(emit);
|
|
1081
|
+
if (set.size === 0) liveEmitters.delete(signal);
|
|
1082
|
+
void iterator.return?.();
|
|
1083
|
+
};
|
|
1084
|
+
});
|
|
1085
|
+
},
|
|
257
1086
|
checkLiveResume: (signal, afterId) => outbox.checkLiveResume(signal, afterId),
|
|
258
|
-
drain
|
|
259
|
-
|
|
260
|
-
|
|
1087
|
+
async drain() {
|
|
1088
|
+
if (compete) await drainOnceStreams();
|
|
1089
|
+
await outbox.drain();
|
|
1090
|
+
},
|
|
1091
|
+
async deadLetters(signalName) {
|
|
1092
|
+
const rest = await outbox.deadLetters(signalName);
|
|
1093
|
+
const fromStream = await readDeadLetters(signalName);
|
|
1094
|
+
return [...rest, ...fromStream];
|
|
1095
|
+
},
|
|
1096
|
+
inspect: (signalName) => outbox.inspect(signalName),
|
|
261
1097
|
replay: (opts) => outbox.replay(opts),
|
|
262
1098
|
discard: (opts) => outbox.discard(opts),
|
|
263
1099
|
getWrite: (k) => outbox.getWrite(k),
|
|
264
|
-
close
|
|
1100
|
+
async close() {
|
|
1101
|
+
stopHeartbeat();
|
|
1102
|
+
for (const unsub of channelUnsubs) await unsub();
|
|
1103
|
+
if (ownsClient) await wire.close();
|
|
1104
|
+
await outbox.close();
|
|
1105
|
+
},
|
|
265
1106
|
};
|
|
266
1107
|
}
|
|
267
1108
|
|