@fedify/redis 2.0.0-pr.479.1922 → 2.0.0-pr.559.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +2 -2
- package/dist/kv.cjs +41 -0
- package/dist/kv.d.cts +8 -3
- package/dist/kv.d.ts +8 -3
- package/dist/kv.js +41 -0
- package/dist/mq.cjs +87 -42
- package/dist/mq.d.cts +7 -7
- package/dist/mq.d.ts +7 -7
- package/dist/mq.js +87 -42
- package/package.json +14 -9
package/LICENSE
CHANGED
package/README.md
CHANGED
|
@@ -38,10 +38,10 @@ const federation = createFederation({
|
|
|
38
38
|
});
|
|
39
39
|
~~~~
|
|
40
40
|
|
|
41
|
-
[JSR]: https://jsr.io/@fedify/redis
|
|
42
41
|
[JSR badge]: https://jsr.io/badges/@fedify/redis
|
|
43
|
-
[
|
|
42
|
+
[JSR]: https://jsr.io/@fedify/redis
|
|
44
43
|
[npm badge]: https://img.shields.io/npm/v/@fedify/redis?logo=npm
|
|
44
|
+
[npm]: https://www.npmjs.com/package/@fedify/redis
|
|
45
45
|
[Fedify]: https://fedify.dev/
|
|
46
46
|
[`KvStore`]: https://jsr.io/@fedify/fedify/doc/federation/~/KvStore
|
|
47
47
|
[`MessageQueue`]: https://jsr.io/@fedify/fedify/doc/federation/~/MessageQueue
|
package/dist/kv.cjs
CHANGED
|
@@ -36,6 +36,7 @@ const node_buffer = require_rolldown_runtime.__toESM(require("node:buffer"));
|
|
|
36
36
|
var RedisKvStore = class {
|
|
37
37
|
#redis;
|
|
38
38
|
#keyPrefix;
|
|
39
|
+
#keyPrefixStr;
|
|
39
40
|
#codec;
|
|
40
41
|
#textEncoder = new TextEncoder();
|
|
41
42
|
/**
|
|
@@ -46,6 +47,7 @@ var RedisKvStore = class {
|
|
|
46
47
|
constructor(redis, options = {}) {
|
|
47
48
|
this.#redis = redis;
|
|
48
49
|
this.#keyPrefix = options.keyPrefix ?? "fedify::";
|
|
50
|
+
this.#keyPrefixStr = typeof this.#keyPrefix === "string" ? this.#keyPrefix : new TextDecoder().decode(new Uint8Array(this.#keyPrefix));
|
|
49
51
|
this.#codec = options.codec ?? new require_codec.JsonCodec();
|
|
50
52
|
}
|
|
51
53
|
#serializeKey(key) {
|
|
@@ -70,6 +72,45 @@ var RedisKvStore = class {
|
|
|
70
72
|
const serializedKey = this.#serializeKey(key);
|
|
71
73
|
await this.#redis.del(serializedKey);
|
|
72
74
|
}
|
|
75
|
+
#deserializeKey(redisKey) {
|
|
76
|
+
const suffix = redisKey.slice(this.#keyPrefixStr.length);
|
|
77
|
+
return suffix.split("::").map((p) => p.replaceAll("_:", ":"));
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* {@inheritDoc KvStore.list}
|
|
81
|
+
* @since 1.10.0
|
|
82
|
+
*/
|
|
83
|
+
async *list(prefix) {
|
|
84
|
+
let pattern;
|
|
85
|
+
let exactKey = null;
|
|
86
|
+
if (prefix == null || prefix.length === 0) pattern = `${this.#keyPrefixStr}*`;
|
|
87
|
+
else {
|
|
88
|
+
const prefixKey = this.#serializeKey(prefix);
|
|
89
|
+
const prefixKeyFullStr = typeof prefixKey === "string" ? prefixKey : new TextDecoder().decode(new Uint8Array(prefixKey));
|
|
90
|
+
exactKey = prefixKey;
|
|
91
|
+
pattern = `${prefixKeyFullStr}::*`;
|
|
92
|
+
}
|
|
93
|
+
if (exactKey != null) {
|
|
94
|
+
const exactValue = await this.#redis.getBuffer(exactKey);
|
|
95
|
+
if (exactValue != null) yield {
|
|
96
|
+
key: prefix,
|
|
97
|
+
value: this.#codec.decode(exactValue)
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
let cursor = "0";
|
|
101
|
+
do {
|
|
102
|
+
const [nextCursor, keys] = await this.#redis.scan(cursor, "MATCH", pattern, "COUNT", 100);
|
|
103
|
+
cursor = nextCursor;
|
|
104
|
+
for (const key of keys) {
|
|
105
|
+
const encodedValue = await this.#redis.getBuffer(key);
|
|
106
|
+
if (encodedValue == null) continue;
|
|
107
|
+
yield {
|
|
108
|
+
key: this.#deserializeKey(key),
|
|
109
|
+
value: this.#codec.decode(encodedValue)
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
} while (cursor !== "0");
|
|
113
|
+
}
|
|
73
114
|
};
|
|
74
115
|
|
|
75
116
|
//#endregion
|
package/dist/kv.d.cts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Codec } from "./codec.cjs";
|
|
2
|
-
import { KvKey, KvStore, KvStoreSetOptions } from "@fedify/fedify";
|
|
2
|
+
import { KvKey, KvStore, KvStoreListEntry, KvStoreSetOptions } from "@fedify/fedify";
|
|
3
3
|
import { Cluster, Redis, RedisKey } from "ioredis";
|
|
4
4
|
|
|
5
5
|
//#region src/kv.d.ts
|
|
@@ -11,12 +11,12 @@ interface RedisKvStoreOptions {
|
|
|
11
11
|
* The prefix to use for all keys in the key–value store in Redis.
|
|
12
12
|
* Defaults to `"fedify::"`.
|
|
13
13
|
*/
|
|
14
|
-
keyPrefix?: RedisKey;
|
|
14
|
+
readonly keyPrefix?: RedisKey;
|
|
15
15
|
/**
|
|
16
16
|
* The codec to use for encoding and decoding values in the key–value store.
|
|
17
17
|
* Defaults to {@link JsonCodec}.
|
|
18
18
|
*/
|
|
19
|
-
codec?: Codec;
|
|
19
|
+
readonly codec?: Codec;
|
|
20
20
|
}
|
|
21
21
|
/**
|
|
22
22
|
* A key–value store that uses Redis as the underlying storage.
|
|
@@ -56,6 +56,11 @@ declare class RedisKvStore implements KvStore {
|
|
|
56
56
|
get<T = unknown>(key: KvKey): Promise<T | undefined>;
|
|
57
57
|
set(key: KvKey, value: unknown, options?: KvStoreSetOptions | undefined): Promise<void>;
|
|
58
58
|
delete(key: KvKey): Promise<void>;
|
|
59
|
+
/**
|
|
60
|
+
* {@inheritDoc KvStore.list}
|
|
61
|
+
* @since 1.10.0
|
|
62
|
+
*/
|
|
63
|
+
list(prefix?: KvKey): AsyncIterable<KvStoreListEntry>;
|
|
59
64
|
}
|
|
60
65
|
//#endregion
|
|
61
66
|
export { RedisKvStore, RedisKvStoreOptions };
|
package/dist/kv.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Temporal } from "@js-temporal/polyfill";
|
|
2
2
|
import { Codec } from "./codec.js";
|
|
3
|
-
import { KvKey, KvStore, KvStoreSetOptions } from "@fedify/fedify";
|
|
3
|
+
import { KvKey, KvStore, KvStoreListEntry, KvStoreSetOptions } from "@fedify/fedify";
|
|
4
4
|
import { Cluster, Redis, RedisKey } from "ioredis";
|
|
5
5
|
|
|
6
6
|
//#region src/kv.d.ts
|
|
@@ -12,12 +12,12 @@ interface RedisKvStoreOptions {
|
|
|
12
12
|
* The prefix to use for all keys in the key–value store in Redis.
|
|
13
13
|
* Defaults to `"fedify::"`.
|
|
14
14
|
*/
|
|
15
|
-
keyPrefix?: RedisKey;
|
|
15
|
+
readonly keyPrefix?: RedisKey;
|
|
16
16
|
/**
|
|
17
17
|
* The codec to use for encoding and decoding values in the key–value store.
|
|
18
18
|
* Defaults to {@link JsonCodec}.
|
|
19
19
|
*/
|
|
20
|
-
codec?: Codec;
|
|
20
|
+
readonly codec?: Codec;
|
|
21
21
|
}
|
|
22
22
|
/**
|
|
23
23
|
* A key–value store that uses Redis as the underlying storage.
|
|
@@ -57,6 +57,11 @@ declare class RedisKvStore implements KvStore {
|
|
|
57
57
|
get<T = unknown>(key: KvKey): Promise<T | undefined>;
|
|
58
58
|
set(key: KvKey, value: unknown, options?: KvStoreSetOptions | undefined): Promise<void>;
|
|
59
59
|
delete(key: KvKey): Promise<void>;
|
|
60
|
+
/**
|
|
61
|
+
* {@inheritDoc KvStore.list}
|
|
62
|
+
* @since 1.10.0
|
|
63
|
+
*/
|
|
64
|
+
list(prefix?: KvKey): AsyncIterable<KvStoreListEntry>;
|
|
60
65
|
}
|
|
61
66
|
//#endregion
|
|
62
67
|
export { RedisKvStore, RedisKvStoreOptions };
|
package/dist/kv.js
CHANGED
|
@@ -35,6 +35,7 @@ import { Buffer } from "node:buffer";
|
|
|
35
35
|
var RedisKvStore = class {
|
|
36
36
|
#redis;
|
|
37
37
|
#keyPrefix;
|
|
38
|
+
#keyPrefixStr;
|
|
38
39
|
#codec;
|
|
39
40
|
#textEncoder = new TextEncoder();
|
|
40
41
|
/**
|
|
@@ -45,6 +46,7 @@ var RedisKvStore = class {
|
|
|
45
46
|
constructor(redis, options = {}) {
|
|
46
47
|
this.#redis = redis;
|
|
47
48
|
this.#keyPrefix = options.keyPrefix ?? "fedify::";
|
|
49
|
+
this.#keyPrefixStr = typeof this.#keyPrefix === "string" ? this.#keyPrefix : new TextDecoder().decode(new Uint8Array(this.#keyPrefix));
|
|
48
50
|
this.#codec = options.codec ?? new JsonCodec();
|
|
49
51
|
}
|
|
50
52
|
#serializeKey(key) {
|
|
@@ -69,6 +71,45 @@ var RedisKvStore = class {
|
|
|
69
71
|
const serializedKey = this.#serializeKey(key);
|
|
70
72
|
await this.#redis.del(serializedKey);
|
|
71
73
|
}
|
|
74
|
+
#deserializeKey(redisKey) {
|
|
75
|
+
const suffix = redisKey.slice(this.#keyPrefixStr.length);
|
|
76
|
+
return suffix.split("::").map((p) => p.replaceAll("_:", ":"));
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* {@inheritDoc KvStore.list}
|
|
80
|
+
* @since 1.10.0
|
|
81
|
+
*/
|
|
82
|
+
async *list(prefix) {
|
|
83
|
+
let pattern;
|
|
84
|
+
let exactKey = null;
|
|
85
|
+
if (prefix == null || prefix.length === 0) pattern = `${this.#keyPrefixStr}*`;
|
|
86
|
+
else {
|
|
87
|
+
const prefixKey = this.#serializeKey(prefix);
|
|
88
|
+
const prefixKeyFullStr = typeof prefixKey === "string" ? prefixKey : new TextDecoder().decode(new Uint8Array(prefixKey));
|
|
89
|
+
exactKey = prefixKey;
|
|
90
|
+
pattern = `${prefixKeyFullStr}::*`;
|
|
91
|
+
}
|
|
92
|
+
if (exactKey != null) {
|
|
93
|
+
const exactValue = await this.#redis.getBuffer(exactKey);
|
|
94
|
+
if (exactValue != null) yield {
|
|
95
|
+
key: prefix,
|
|
96
|
+
value: this.#codec.decode(exactValue)
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
let cursor = "0";
|
|
100
|
+
do {
|
|
101
|
+
const [nextCursor, keys] = await this.#redis.scan(cursor, "MATCH", pattern, "COUNT", 100);
|
|
102
|
+
cursor = nextCursor;
|
|
103
|
+
for (const key of keys) {
|
|
104
|
+
const encodedValue = await this.#redis.getBuffer(key);
|
|
105
|
+
if (encodedValue == null) continue;
|
|
106
|
+
yield {
|
|
107
|
+
key: this.#deserializeKey(key),
|
|
108
|
+
value: this.#codec.decode(encodedValue)
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
} while (cursor !== "0");
|
|
112
|
+
}
|
|
72
113
|
};
|
|
73
114
|
|
|
74
115
|
//#endregion
|
package/dist/mq.cjs
CHANGED
|
@@ -47,6 +47,8 @@ var RedisMessageQueue = class {
|
|
|
47
47
|
#codec;
|
|
48
48
|
#pollIntervalMs;
|
|
49
49
|
#loopHandle;
|
|
50
|
+
#lastTimestamp = 0;
|
|
51
|
+
#sequenceInMs = 0;
|
|
50
52
|
/**
|
|
51
53
|
* Creates a new Redis message queue.
|
|
52
54
|
* @param redis The Redis client factory.
|
|
@@ -62,22 +64,53 @@ var RedisMessageQueue = class {
|
|
|
62
64
|
this.#codec = options.codec ?? new require_codec.JsonCodec();
|
|
63
65
|
this.#pollIntervalMs = Temporal.Duration.from(options.pollInterval ?? { seconds: 5 }).total("millisecond");
|
|
64
66
|
}
|
|
67
|
+
/**
|
|
68
|
+
* Returns a monotonically increasing timestamp to ensure message ordering.
|
|
69
|
+
* When multiple messages are enqueued in the same millisecond, a fractional
|
|
70
|
+
* sequence number is added to preserve insertion order.
|
|
71
|
+
*/
|
|
72
|
+
#getMonotonicTimestamp(baseTimestamp) {
|
|
73
|
+
if (baseTimestamp === this.#lastTimestamp) this.#sequenceInMs++;
|
|
74
|
+
else {
|
|
75
|
+
this.#lastTimestamp = baseTimestamp;
|
|
76
|
+
this.#sequenceInMs = 0;
|
|
77
|
+
}
|
|
78
|
+
return baseTimestamp + this.#sequenceInMs * .001;
|
|
79
|
+
}
|
|
65
80
|
async enqueue(message, options) {
|
|
66
|
-
const
|
|
67
|
-
const
|
|
81
|
+
const now = Temporal.Now.instant().epochMilliseconds;
|
|
82
|
+
const baseTs = options?.delay == null ? now : now + options.delay.total("millisecond");
|
|
83
|
+
const ts = this.#getMonotonicTimestamp(baseTs);
|
|
84
|
+
const encodedMessage = this.#codec.encode([
|
|
85
|
+
crypto.randomUUID(),
|
|
86
|
+
message,
|
|
87
|
+
options?.orderingKey
|
|
88
|
+
]);
|
|
68
89
|
await this.#redis.zadd(this.#queueKey, ts, encodedMessage);
|
|
69
|
-
if (
|
|
90
|
+
if (baseTs <= now) this.#redis.publish(this.#channelKey, "");
|
|
70
91
|
}
|
|
71
92
|
async enqueueMany(messages, options) {
|
|
72
93
|
if (messages.length === 0) return;
|
|
73
|
-
const
|
|
94
|
+
const now = Temporal.Now.instant().epochMilliseconds;
|
|
95
|
+
const baseTs = options?.delay == null ? now : now + options.delay.total("millisecond");
|
|
74
96
|
const multi = this.#redis.multi();
|
|
75
97
|
for (const message of messages) {
|
|
76
|
-
const
|
|
98
|
+
const ts = this.#getMonotonicTimestamp(baseTs);
|
|
99
|
+
const encodedMessage = this.#codec.encode([
|
|
100
|
+
crypto.randomUUID(),
|
|
101
|
+
message,
|
|
102
|
+
options?.orderingKey
|
|
103
|
+
]);
|
|
77
104
|
multi.zadd(this.#queueKey, ts, encodedMessage);
|
|
78
105
|
}
|
|
79
106
|
await multi.exec();
|
|
80
|
-
if (
|
|
107
|
+
if (baseTs <= now) this.#redis.publish(this.#channelKey, "");
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Returns the Redis key used to lock a specific ordering key.
|
|
111
|
+
*/
|
|
112
|
+
#getOrderingLockKey(orderingKey) {
|
|
113
|
+
return `${this.#lockKey}:ordering:${orderingKey}`;
|
|
81
114
|
}
|
|
82
115
|
async #poll() {
|
|
83
116
|
logger.debug("Polling for messages...");
|
|
@@ -91,10 +124,25 @@ var RedisMessageQueue = class {
|
|
|
91
124
|
logger.debug("Found {messages} messages to process.", { messages: messages.length });
|
|
92
125
|
try {
|
|
93
126
|
if (messages.length < 1) return;
|
|
94
|
-
const encodedMessage
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
127
|
+
for (const encodedMessage of messages) {
|
|
128
|
+
const decoded = this.#codec.decode(encodedMessage);
|
|
129
|
+
const orderingKey = decoded[2];
|
|
130
|
+
if (orderingKey != null) {
|
|
131
|
+
const orderingLockKey = this.#getOrderingLockKey(orderingKey);
|
|
132
|
+
const lockResult = await this.#redis.set(orderingLockKey, this.#workerId, "EX", 60, "NX");
|
|
133
|
+
if (lockResult == null) continue;
|
|
134
|
+
}
|
|
135
|
+
const removed = await this.#redis.zrem(this.#queueKey, encodedMessage);
|
|
136
|
+
if (removed === 0) {
|
|
137
|
+
if (orderingKey != null) await this.#redis.del(this.#getOrderingLockKey(orderingKey));
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
return {
|
|
141
|
+
message: decoded[1],
|
|
142
|
+
orderingKey
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
return;
|
|
98
146
|
} finally {
|
|
99
147
|
await this.#redis.del(this.#lockKey);
|
|
100
148
|
}
|
|
@@ -104,42 +152,45 @@ var RedisMessageQueue = class {
|
|
|
104
152
|
const signal = options.signal;
|
|
105
153
|
const poll = async () => {
|
|
106
154
|
while (!signal?.aborted) {
|
|
107
|
-
let
|
|
155
|
+
let result;
|
|
108
156
|
try {
|
|
109
|
-
|
|
157
|
+
result = await this.#poll();
|
|
110
158
|
} catch (error) {
|
|
111
159
|
logger.error("Error polling for messages: {error}", { error });
|
|
112
160
|
return;
|
|
113
161
|
}
|
|
114
|
-
if (
|
|
115
|
-
|
|
162
|
+
if (result === void 0) return;
|
|
163
|
+
const { message, orderingKey } = result;
|
|
164
|
+
try {
|
|
165
|
+
await handler(message);
|
|
166
|
+
} finally {
|
|
167
|
+
if (orderingKey != null) await this.#redis.del(this.#getOrderingLockKey(orderingKey));
|
|
168
|
+
}
|
|
116
169
|
}
|
|
117
170
|
};
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
subRedis.off("message", poll);
|
|
137
|
-
});
|
|
138
|
-
});
|
|
171
|
+
await this.#subRedis.subscribe(this.#channelKey);
|
|
172
|
+
/**
|
|
173
|
+
* Cast to Redis for event methods. Both Redis and Cluster extend EventEmitter
|
|
174
|
+
* and get the same methods via applyMixin at runtime, but their TypeScript
|
|
175
|
+
* interfaces are incompatible:
|
|
176
|
+
* - Redis declares specific overloads: on(event: "message", cb: (channel, message) => void)
|
|
177
|
+
* - Cluster only has generic: on(event: string | symbol, listener: Function)
|
|
178
|
+
*
|
|
179
|
+
* This makes the union type Redis | Cluster incompatible for these method calls.
|
|
180
|
+
* The cast is safe because both classes use applyMixin(Class, EventEmitter) which
|
|
181
|
+
* copies all EventEmitter prototype methods, giving them identical pub/sub functionality.
|
|
182
|
+
*
|
|
183
|
+
* @see https://github.com/redis/ioredis/blob/main/lib/Redis.ts#L863 (has specific overloads)
|
|
184
|
+
* @see https://github.com/redis/ioredis/blob/main/lib/cluster/index.ts#L1110 (empty interface)
|
|
185
|
+
*/
|
|
186
|
+
const subRedis = this.#subRedis;
|
|
187
|
+
subRedis.on("message", poll);
|
|
188
|
+
const timeouts = /* @__PURE__ */ new Set();
|
|
139
189
|
signal?.addEventListener("abort", () => {
|
|
190
|
+
subRedis.off("message", poll);
|
|
140
191
|
for (const timeout of timeouts) clearTimeout(timeout);
|
|
141
192
|
});
|
|
142
|
-
|
|
193
|
+
await poll();
|
|
143
194
|
while (!signal?.aborted) {
|
|
144
195
|
let timeout;
|
|
145
196
|
await new Promise((resolve) => {
|
|
@@ -153,12 +204,6 @@ var RedisMessageQueue = class {
|
|
|
153
204
|
if (timeout != null) timeouts.delete(timeout);
|
|
154
205
|
await poll();
|
|
155
206
|
}
|
|
156
|
-
return await new Promise((resolve) => {
|
|
157
|
-
signal?.addEventListener("abort", () => {
|
|
158
|
-
promise.catch(() => resolve()).then(() => resolve());
|
|
159
|
-
});
|
|
160
|
-
promise.catch(() => resolve()).then(() => resolve());
|
|
161
|
-
});
|
|
162
207
|
}
|
|
163
208
|
[Symbol.dispose]() {
|
|
164
209
|
clearInterval(this.#loopHandle);
|
package/dist/mq.d.cts
CHANGED
|
@@ -13,35 +13,35 @@ interface RedisMessageQueueOptions {
|
|
|
13
13
|
* This is used to prevent multiple workers from processing the same message,
|
|
14
14
|
* so it should be unique for each worker.
|
|
15
15
|
*/
|
|
16
|
-
workerId?: string;
|
|
16
|
+
readonly workerId?: string;
|
|
17
17
|
/**
|
|
18
18
|
* The Pub/Sub channel key to use for the message queue. `"fedify_channel"`
|
|
19
19
|
* by default.
|
|
20
20
|
* @default `"fedify_channel"`
|
|
21
21
|
*/
|
|
22
|
-
channelKey?: RedisKey;
|
|
22
|
+
readonly channelKey?: RedisKey;
|
|
23
23
|
/**
|
|
24
24
|
* The Sorted Set key to use for the delayed message queue. `"fedify_queue"`
|
|
25
25
|
* by default.
|
|
26
26
|
* @default `"fedify_queue"`
|
|
27
27
|
*/
|
|
28
|
-
queueKey?: RedisKey;
|
|
28
|
+
readonly queueKey?: RedisKey;
|
|
29
29
|
/**
|
|
30
30
|
* The key to use for locking the message queue. `"fedify_lock"` by default.
|
|
31
31
|
* @default `"fedify_lock"`
|
|
32
32
|
*/
|
|
33
|
-
lockKey?: RedisKey;
|
|
33
|
+
readonly lockKey?: RedisKey;
|
|
34
34
|
/**
|
|
35
35
|
* The codec to use for encoding and decoding messages in the key–value store.
|
|
36
36
|
* Defaults to {@link JsonCodec}.
|
|
37
37
|
* @default {@link JsonCodec}
|
|
38
38
|
*/
|
|
39
|
-
codec?: Codec;
|
|
39
|
+
readonly codec?: Codec;
|
|
40
40
|
/**
|
|
41
41
|
* The poll interval for the message queue. 5 seconds by default.
|
|
42
42
|
* @default `{ seconds: 5 }`
|
|
43
43
|
*/
|
|
44
|
-
pollInterval?: Temporal.Duration | Temporal.DurationLike;
|
|
44
|
+
readonly pollInterval?: Temporal.Duration | Temporal.DurationLike;
|
|
45
45
|
}
|
|
46
46
|
/**
|
|
47
47
|
* A message queue that uses Redis as the underlying storage.
|
|
@@ -78,7 +78,7 @@ declare class RedisMessageQueue implements MessageQueue, Disposable {
|
|
|
78
78
|
*/
|
|
79
79
|
constructor(redis: () => Redis | Cluster, options?: RedisMessageQueueOptions);
|
|
80
80
|
enqueue(message: any, options?: MessageQueueEnqueueOptions): Promise<void>;
|
|
81
|
-
enqueueMany(messages: any[], options?: MessageQueueEnqueueOptions): Promise<void>;
|
|
81
|
+
enqueueMany(messages: readonly any[], options?: MessageQueueEnqueueOptions): Promise<void>;
|
|
82
82
|
listen(handler: (message: any) => void | Promise<void>, options?: MessageQueueListenOptions): Promise<void>;
|
|
83
83
|
[Symbol.dispose](): void;
|
|
84
84
|
}
|
package/dist/mq.d.ts
CHANGED
|
@@ -14,35 +14,35 @@ interface RedisMessageQueueOptions {
|
|
|
14
14
|
* This is used to prevent multiple workers from processing the same message,
|
|
15
15
|
* so it should be unique for each worker.
|
|
16
16
|
*/
|
|
17
|
-
workerId?: string;
|
|
17
|
+
readonly workerId?: string;
|
|
18
18
|
/**
|
|
19
19
|
* The Pub/Sub channel key to use for the message queue. `"fedify_channel"`
|
|
20
20
|
* by default.
|
|
21
21
|
* @default `"fedify_channel"`
|
|
22
22
|
*/
|
|
23
|
-
channelKey?: RedisKey;
|
|
23
|
+
readonly channelKey?: RedisKey;
|
|
24
24
|
/**
|
|
25
25
|
* The Sorted Set key to use for the delayed message queue. `"fedify_queue"`
|
|
26
26
|
* by default.
|
|
27
27
|
* @default `"fedify_queue"`
|
|
28
28
|
*/
|
|
29
|
-
queueKey?: RedisKey;
|
|
29
|
+
readonly queueKey?: RedisKey;
|
|
30
30
|
/**
|
|
31
31
|
* The key to use for locking the message queue. `"fedify_lock"` by default.
|
|
32
32
|
* @default `"fedify_lock"`
|
|
33
33
|
*/
|
|
34
|
-
lockKey?: RedisKey;
|
|
34
|
+
readonly lockKey?: RedisKey;
|
|
35
35
|
/**
|
|
36
36
|
* The codec to use for encoding and decoding messages in the key–value store.
|
|
37
37
|
* Defaults to {@link JsonCodec}.
|
|
38
38
|
* @default {@link JsonCodec}
|
|
39
39
|
*/
|
|
40
|
-
codec?: Codec;
|
|
40
|
+
readonly codec?: Codec;
|
|
41
41
|
/**
|
|
42
42
|
* The poll interval for the message queue. 5 seconds by default.
|
|
43
43
|
* @default `{ seconds: 5 }`
|
|
44
44
|
*/
|
|
45
|
-
pollInterval?: Temporal.Duration | Temporal.DurationLike;
|
|
45
|
+
readonly pollInterval?: Temporal.Duration | Temporal.DurationLike;
|
|
46
46
|
}
|
|
47
47
|
/**
|
|
48
48
|
* A message queue that uses Redis as the underlying storage.
|
|
@@ -79,7 +79,7 @@ declare class RedisMessageQueue implements MessageQueue, Disposable {
|
|
|
79
79
|
*/
|
|
80
80
|
constructor(redis: () => Redis | Cluster, options?: RedisMessageQueueOptions);
|
|
81
81
|
enqueue(message: any, options?: MessageQueueEnqueueOptions): Promise<void>;
|
|
82
|
-
enqueueMany(messages: any[], options?: MessageQueueEnqueueOptions): Promise<void>;
|
|
82
|
+
enqueueMany(messages: readonly any[], options?: MessageQueueEnqueueOptions): Promise<void>;
|
|
83
83
|
listen(handler: (message: any) => void | Promise<void>, options?: MessageQueueListenOptions): Promise<void>;
|
|
84
84
|
[Symbol.dispose](): void;
|
|
85
85
|
}
|
package/dist/mq.js
CHANGED
|
@@ -46,6 +46,8 @@ var RedisMessageQueue = class {
|
|
|
46
46
|
#codec;
|
|
47
47
|
#pollIntervalMs;
|
|
48
48
|
#loopHandle;
|
|
49
|
+
#lastTimestamp = 0;
|
|
50
|
+
#sequenceInMs = 0;
|
|
49
51
|
/**
|
|
50
52
|
* Creates a new Redis message queue.
|
|
51
53
|
* @param redis The Redis client factory.
|
|
@@ -61,22 +63,53 @@ var RedisMessageQueue = class {
|
|
|
61
63
|
this.#codec = options.codec ?? new JsonCodec();
|
|
62
64
|
this.#pollIntervalMs = Temporal.Duration.from(options.pollInterval ?? { seconds: 5 }).total("millisecond");
|
|
63
65
|
}
|
|
66
|
+
/**
|
|
67
|
+
* Returns a monotonically increasing timestamp to ensure message ordering.
|
|
68
|
+
* When multiple messages are enqueued in the same millisecond, a fractional
|
|
69
|
+
* sequence number is added to preserve insertion order.
|
|
70
|
+
*/
|
|
71
|
+
#getMonotonicTimestamp(baseTimestamp) {
|
|
72
|
+
if (baseTimestamp === this.#lastTimestamp) this.#sequenceInMs++;
|
|
73
|
+
else {
|
|
74
|
+
this.#lastTimestamp = baseTimestamp;
|
|
75
|
+
this.#sequenceInMs = 0;
|
|
76
|
+
}
|
|
77
|
+
return baseTimestamp + this.#sequenceInMs * .001;
|
|
78
|
+
}
|
|
64
79
|
async enqueue(message, options) {
|
|
65
|
-
const
|
|
66
|
-
const
|
|
80
|
+
const now = Temporal.Now.instant().epochMilliseconds;
|
|
81
|
+
const baseTs = options?.delay == null ? now : now + options.delay.total("millisecond");
|
|
82
|
+
const ts = this.#getMonotonicTimestamp(baseTs);
|
|
83
|
+
const encodedMessage = this.#codec.encode([
|
|
84
|
+
crypto.randomUUID(),
|
|
85
|
+
message,
|
|
86
|
+
options?.orderingKey
|
|
87
|
+
]);
|
|
67
88
|
await this.#redis.zadd(this.#queueKey, ts, encodedMessage);
|
|
68
|
-
if (
|
|
89
|
+
if (baseTs <= now) this.#redis.publish(this.#channelKey, "");
|
|
69
90
|
}
|
|
70
91
|
async enqueueMany(messages, options) {
|
|
71
92
|
if (messages.length === 0) return;
|
|
72
|
-
const
|
|
93
|
+
const now = Temporal.Now.instant().epochMilliseconds;
|
|
94
|
+
const baseTs = options?.delay == null ? now : now + options.delay.total("millisecond");
|
|
73
95
|
const multi = this.#redis.multi();
|
|
74
96
|
for (const message of messages) {
|
|
75
|
-
const
|
|
97
|
+
const ts = this.#getMonotonicTimestamp(baseTs);
|
|
98
|
+
const encodedMessage = this.#codec.encode([
|
|
99
|
+
crypto.randomUUID(),
|
|
100
|
+
message,
|
|
101
|
+
options?.orderingKey
|
|
102
|
+
]);
|
|
76
103
|
multi.zadd(this.#queueKey, ts, encodedMessage);
|
|
77
104
|
}
|
|
78
105
|
await multi.exec();
|
|
79
|
-
if (
|
|
106
|
+
if (baseTs <= now) this.#redis.publish(this.#channelKey, "");
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Returns the Redis key used to lock a specific ordering key.
|
|
110
|
+
*/
|
|
111
|
+
#getOrderingLockKey(orderingKey) {
|
|
112
|
+
return `${this.#lockKey}:ordering:${orderingKey}`;
|
|
80
113
|
}
|
|
81
114
|
async #poll() {
|
|
82
115
|
logger.debug("Polling for messages...");
|
|
@@ -90,10 +123,25 @@ var RedisMessageQueue = class {
|
|
|
90
123
|
logger.debug("Found {messages} messages to process.", { messages: messages.length });
|
|
91
124
|
try {
|
|
92
125
|
if (messages.length < 1) return;
|
|
93
|
-
const encodedMessage
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
126
|
+
for (const encodedMessage of messages) {
|
|
127
|
+
const decoded = this.#codec.decode(encodedMessage);
|
|
128
|
+
const orderingKey = decoded[2];
|
|
129
|
+
if (orderingKey != null) {
|
|
130
|
+
const orderingLockKey = this.#getOrderingLockKey(orderingKey);
|
|
131
|
+
const lockResult = await this.#redis.set(orderingLockKey, this.#workerId, "EX", 60, "NX");
|
|
132
|
+
if (lockResult == null) continue;
|
|
133
|
+
}
|
|
134
|
+
const removed = await this.#redis.zrem(this.#queueKey, encodedMessage);
|
|
135
|
+
if (removed === 0) {
|
|
136
|
+
if (orderingKey != null) await this.#redis.del(this.#getOrderingLockKey(orderingKey));
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
return {
|
|
140
|
+
message: decoded[1],
|
|
141
|
+
orderingKey
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
return;
|
|
97
145
|
} finally {
|
|
98
146
|
await this.#redis.del(this.#lockKey);
|
|
99
147
|
}
|
|
@@ -103,42 +151,45 @@ var RedisMessageQueue = class {
|
|
|
103
151
|
const signal = options.signal;
|
|
104
152
|
const poll = async () => {
|
|
105
153
|
while (!signal?.aborted) {
|
|
106
|
-
let
|
|
154
|
+
let result;
|
|
107
155
|
try {
|
|
108
|
-
|
|
156
|
+
result = await this.#poll();
|
|
109
157
|
} catch (error) {
|
|
110
158
|
logger.error("Error polling for messages: {error}", { error });
|
|
111
159
|
return;
|
|
112
160
|
}
|
|
113
|
-
if (
|
|
114
|
-
|
|
161
|
+
if (result === void 0) return;
|
|
162
|
+
const { message, orderingKey } = result;
|
|
163
|
+
try {
|
|
164
|
+
await handler(message);
|
|
165
|
+
} finally {
|
|
166
|
+
if (orderingKey != null) await this.#redis.del(this.#getOrderingLockKey(orderingKey));
|
|
167
|
+
}
|
|
115
168
|
}
|
|
116
169
|
};
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
subRedis.off("message", poll);
|
|
136
|
-
});
|
|
137
|
-
});
|
|
170
|
+
await this.#subRedis.subscribe(this.#channelKey);
|
|
171
|
+
/**
|
|
172
|
+
* Cast to Redis for event methods. Both Redis and Cluster extend EventEmitter
|
|
173
|
+
* and get the same methods via applyMixin at runtime, but their TypeScript
|
|
174
|
+
* interfaces are incompatible:
|
|
175
|
+
* - Redis declares specific overloads: on(event: "message", cb: (channel, message) => void)
|
|
176
|
+
* - Cluster only has generic: on(event: string | symbol, listener: Function)
|
|
177
|
+
*
|
|
178
|
+
* This makes the union type Redis | Cluster incompatible for these method calls.
|
|
179
|
+
* The cast is safe because both classes use applyMixin(Class, EventEmitter) which
|
|
180
|
+
* copies all EventEmitter prototype methods, giving them identical pub/sub functionality.
|
|
181
|
+
*
|
|
182
|
+
* @see https://github.com/redis/ioredis/blob/main/lib/Redis.ts#L863 (has specific overloads)
|
|
183
|
+
* @see https://github.com/redis/ioredis/blob/main/lib/cluster/index.ts#L1110 (empty interface)
|
|
184
|
+
*/
|
|
185
|
+
const subRedis = this.#subRedis;
|
|
186
|
+
subRedis.on("message", poll);
|
|
187
|
+
const timeouts = /* @__PURE__ */ new Set();
|
|
138
188
|
signal?.addEventListener("abort", () => {
|
|
189
|
+
subRedis.off("message", poll);
|
|
139
190
|
for (const timeout of timeouts) clearTimeout(timeout);
|
|
140
191
|
});
|
|
141
|
-
|
|
192
|
+
await poll();
|
|
142
193
|
while (!signal?.aborted) {
|
|
143
194
|
let timeout;
|
|
144
195
|
await new Promise((resolve) => {
|
|
@@ -152,12 +203,6 @@ var RedisMessageQueue = class {
|
|
|
152
203
|
if (timeout != null) timeouts.delete(timeout);
|
|
153
204
|
await poll();
|
|
154
205
|
}
|
|
155
|
-
return await new Promise((resolve) => {
|
|
156
|
-
signal?.addEventListener("abort", () => {
|
|
157
|
-
promise.catch(() => resolve()).then(() => resolve());
|
|
158
|
-
});
|
|
159
|
-
promise.catch(() => resolve()).then(() => resolve());
|
|
160
|
-
});
|
|
161
206
|
}
|
|
162
207
|
[Symbol.dispose]() {
|
|
163
208
|
clearInterval(this.#loopHandle);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fedify/redis",
|
|
3
|
-
"version": "2.0.0-pr.
|
|
3
|
+
"version": "2.0.0-pr.559.4+6357309b",
|
|
4
4
|
"description": "Redis drivers for Fedify",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"fedify",
|
|
@@ -78,22 +78,27 @@
|
|
|
78
78
|
],
|
|
79
79
|
"dependencies": {
|
|
80
80
|
"@js-temporal/polyfill": "^0.5.1",
|
|
81
|
-
"@logtape/logtape": "^
|
|
81
|
+
"@logtape/logtape": "^2.0.0"
|
|
82
82
|
},
|
|
83
83
|
"peerDependencies": {
|
|
84
|
-
"ioredis": "^5.
|
|
85
|
-
"@fedify/fedify": "^2.0.0-pr.
|
|
84
|
+
"ioredis": "^5.8.2",
|
|
85
|
+
"@fedify/fedify": "^2.0.0-pr.559.4+6357309b"
|
|
86
86
|
},
|
|
87
87
|
"devDependencies": {
|
|
88
88
|
"@std/async": "npm:@jsr/std__async@^1.0.13",
|
|
89
89
|
"@types/node": "^22.17.0",
|
|
90
90
|
"tsdown": "^0.12.9",
|
|
91
|
-
"typescript": "^5.9.3"
|
|
91
|
+
"typescript": "^5.9.3",
|
|
92
|
+
"@fedify/fixture": "^2.0.0",
|
|
93
|
+
"@fedify/testing": "^2.0.0-pr.559.4+6357309b"
|
|
92
94
|
},
|
|
93
95
|
"scripts": {
|
|
94
|
-
"build": "tsdown",
|
|
95
|
-
"
|
|
96
|
-
"
|
|
97
|
-
"
|
|
96
|
+
"build:self": "tsdown",
|
|
97
|
+
"build": "pnpm --filter @fedify/redis... run build:self",
|
|
98
|
+
"prepublish": "pnpm build",
|
|
99
|
+
"pretest": "pnpm build",
|
|
100
|
+
"test": "node --experimental-transform-types --test",
|
|
101
|
+
"pretest:bun": "pnpm build",
|
|
102
|
+
"test:bun": "bun test --timeout=10000"
|
|
98
103
|
}
|
|
99
104
|
}
|