@valentinkolb/sync 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/publish.yml +72 -0
- package/CLAUDE.md +106 -0
- package/LICENSE +21 -0
- package/README.md +292 -0
- package/bun.lock +29 -0
- package/compose.test.yml +7 -0
- package/index.ts +18 -0
- package/package.json +21 -0
- package/src/jobs.ts +568 -0
- package/src/mutex.ts +203 -0
- package/src/ratelimit.ts +143 -0
- package/tests/jobs.test.ts +465 -0
- package/tests/mutex.test.ts +223 -0
- package/tests/preload.ts +2 -0
- package/tests/ratelimit.test.ts +119 -0
- package/tsconfig.json +31 -0
package/src/mutex.ts
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
import { redis, sleep } from "bun";
|
|
2
|
+
import { randomBytes } from "crypto";
|
|
3
|
+
|
|
4
|
+
// ==========================
|
|
5
|
+
// Configuration
|
|
6
|
+
// ==========================
|
|
7
|
+
|
|
8
|
+
const DEFAULT_RETRY_COUNT = 10;
|
|
9
|
+
const DEFAULT_RETRY_DELAY = 200;
|
|
10
|
+
const DEFAULT_TTL = 10000;
|
|
11
|
+
|
|
12
|
+
// ==========================
|
|
13
|
+
// Types
|
|
14
|
+
// ==========================
|
|
15
|
+
|
|
16
|
+
export type Lock = {
|
|
17
|
+
resource: string;
|
|
18
|
+
value: string;
|
|
19
|
+
ttl: number;
|
|
20
|
+
expiration: number;
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
export type MutexConfig = {
|
|
24
|
+
/** Key prefix for Redis keys (default: "mutex") */
|
|
25
|
+
prefix?: string;
|
|
26
|
+
/** Number of retry attempts (default: 10) */
|
|
27
|
+
retryCount?: number;
|
|
28
|
+
/** Delay between retries in ms (default: 200) */
|
|
29
|
+
retryDelay?: number;
|
|
30
|
+
/** Default lock TTL in ms (default: 10000) */
|
|
31
|
+
defaultTtl?: number;
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
export type Mutex = {
|
|
35
|
+
/** Acquire a lock on a resource */
|
|
36
|
+
acquire: (resource: string, ttl?: number) => Promise<Lock | null>;
|
|
37
|
+
/** Release a lock */
|
|
38
|
+
release: (lock: Lock) => Promise<void>;
|
|
39
|
+
/** Acquire a lock and execute a function, releasing the lock afterward */
|
|
40
|
+
withLock: <T>(resource: string, fn: (lock: Lock) => Promise<T> | T, ttl?: number) => Promise<T | null>;
|
|
41
|
+
/** Acquire a lock and execute a function, throwing if lock cannot be acquired */
|
|
42
|
+
withLockOrThrow: <T>(resource: string, fn: (lock: Lock) => Promise<T> | T, ttl?: number) => Promise<T>;
|
|
43
|
+
/** Extend a lock's TTL */
|
|
44
|
+
extend: (lock: Lock, ttl?: number) => Promise<boolean>;
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
// ==========================
|
|
48
|
+
// Lock Error
|
|
49
|
+
// ==========================
|
|
50
|
+
|
|
51
|
+
export class LockError extends Error {
|
|
52
|
+
readonly resource: string;
|
|
53
|
+
|
|
54
|
+
constructor(resource: string) {
|
|
55
|
+
super(`Failed to acquire lock on resource: ${resource}`);
|
|
56
|
+
this.name = "LockError";
|
|
57
|
+
this.resource = resource;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// ==========================
|
|
62
|
+
// Mutex Factory
|
|
63
|
+
// ==========================
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Create a mutex with the given configuration.
|
|
67
|
+
* Uses Redis SET NX for distributed locking with automatic expiry.
|
|
68
|
+
*
|
|
69
|
+
* @example
|
|
70
|
+
* ```ts
|
|
71
|
+
* const m = mutex.create({ prefix: "myapp" });
|
|
72
|
+
*
|
|
73
|
+
* // Manual acquire/release
|
|
74
|
+
* const lock = await m.acquire("resource:123");
|
|
75
|
+
* if (lock) {
|
|
76
|
+
* try {
|
|
77
|
+
* // do work
|
|
78
|
+
* } finally {
|
|
79
|
+
* await m.release(lock);
|
|
80
|
+
* }
|
|
81
|
+
* }
|
|
82
|
+
*
|
|
83
|
+
* // Or use withLock for automatic release
|
|
84
|
+
* const result = await m.withLock("resource:123", async (lock) => {
|
|
85
|
+
* return await doSomething();
|
|
86
|
+
* });
|
|
87
|
+
*
|
|
88
|
+
* // Throw if lock cannot be acquired
|
|
89
|
+
* const result = await m.withLockOrThrow("resource:123", async () => {
|
|
90
|
+
* return await doSomething();
|
|
91
|
+
* });
|
|
92
|
+
* ```
|
|
93
|
+
*/
|
|
94
|
+
const create = (config: MutexConfig = {}): Mutex => {
|
|
95
|
+
const {
|
|
96
|
+
prefix = "mutex",
|
|
97
|
+
retryCount = DEFAULT_RETRY_COUNT,
|
|
98
|
+
retryDelay = DEFAULT_RETRY_DELAY,
|
|
99
|
+
defaultTtl = DEFAULT_TTL,
|
|
100
|
+
} = config;
|
|
101
|
+
|
|
102
|
+
const acquire = async (resource: string, ttl: number = defaultTtl): Promise<Lock | null> => {
|
|
103
|
+
const key = `${prefix}:${resource}`;
|
|
104
|
+
const value = randomBytes(16).toString("hex");
|
|
105
|
+
|
|
106
|
+
for (let attempt = 0; attempt <= retryCount; attempt++) {
|
|
107
|
+
try {
|
|
108
|
+
// Use SET with NX (only set if not exists) and PX (millisecond expiry)
|
|
109
|
+
const result = await redis.send("SET", [key, value, "NX", "PX", ttl.toString()]);
|
|
110
|
+
|
|
111
|
+
if (result === "OK") {
|
|
112
|
+
return {
|
|
113
|
+
resource: key,
|
|
114
|
+
value,
|
|
115
|
+
ttl,
|
|
116
|
+
expiration: Date.now() + ttl,
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
} catch (error) {
|
|
120
|
+
console.error(`Lock acquire attempt ${attempt} failed:`, error);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// Wait before retry with jitter
|
|
124
|
+
if (attempt < retryCount) {
|
|
125
|
+
await sleep(retryDelay + Math.random() * 100);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
return null;
|
|
130
|
+
};
|
|
131
|
+
|
|
132
|
+
const release = async (lock: Lock): Promise<void> => {
|
|
133
|
+
try {
|
|
134
|
+
// Use Lua script to ensure we only delete if we own the lock
|
|
135
|
+
const releaseScript = `
|
|
136
|
+
if redis.call("get", KEYS[1]) == ARGV[1] then
|
|
137
|
+
return redis.call("del", KEYS[1])
|
|
138
|
+
else
|
|
139
|
+
return 0
|
|
140
|
+
end
|
|
141
|
+
`;
|
|
142
|
+
await redis.send("EVAL", [releaseScript, "1", lock.resource, lock.value]);
|
|
143
|
+
} catch (error) {
|
|
144
|
+
// Ignore errors during release - lock will expire anyway
|
|
145
|
+
console.error("Error releasing lock:", error);
|
|
146
|
+
}
|
|
147
|
+
};
|
|
148
|
+
|
|
149
|
+
const extend = async (lock: Lock, ttl: number = defaultTtl): Promise<boolean> => {
|
|
150
|
+
try {
|
|
151
|
+
// Use Lua script to ensure we only extend if we own the lock
|
|
152
|
+
const extendScript = `
|
|
153
|
+
if redis.call("get", KEYS[1]) == ARGV[1] then
|
|
154
|
+
return redis.call("pexpire", KEYS[1], ARGV[2])
|
|
155
|
+
else
|
|
156
|
+
return 0
|
|
157
|
+
end
|
|
158
|
+
`;
|
|
159
|
+
const result = await redis.send("EVAL", [extendScript, "1", lock.resource, lock.value, ttl.toString()]);
|
|
160
|
+
if (result === 1) {
|
|
161
|
+
lock.ttl = ttl;
|
|
162
|
+
lock.expiration = Date.now() + ttl;
|
|
163
|
+
return true;
|
|
164
|
+
}
|
|
165
|
+
return false;
|
|
166
|
+
} catch (error) {
|
|
167
|
+
console.error("Error extending lock:", error);
|
|
168
|
+
return false;
|
|
169
|
+
}
|
|
170
|
+
};
|
|
171
|
+
|
|
172
|
+
const withLock = async <T>(resource: string, fn: (lock: Lock) => Promise<T> | T, ttl?: number): Promise<T | null> => {
|
|
173
|
+
const lock = await acquire(resource, ttl);
|
|
174
|
+
if (!lock) return null;
|
|
175
|
+
|
|
176
|
+
try {
|
|
177
|
+
return await fn(lock);
|
|
178
|
+
} finally {
|
|
179
|
+
await release(lock);
|
|
180
|
+
}
|
|
181
|
+
};
|
|
182
|
+
|
|
183
|
+
const withLockOrThrow = async <T>(resource: string, fn: (lock: Lock) => Promise<T> | T, ttl?: number): Promise<T> => {
|
|
184
|
+
const lock = await acquire(resource, ttl);
|
|
185
|
+
if (!lock) {
|
|
186
|
+
throw new LockError(resource);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
try {
|
|
190
|
+
return await fn(lock);
|
|
191
|
+
} finally {
|
|
192
|
+
await release(lock);
|
|
193
|
+
}
|
|
194
|
+
};
|
|
195
|
+
|
|
196
|
+
return { acquire, release, withLock, withLockOrThrow, extend };
|
|
197
|
+
};
|
|
198
|
+
|
|
199
|
+
// ==========================
|
|
200
|
+
// Export
|
|
201
|
+
// ==========================
|
|
202
|
+
|
|
203
|
+
export const mutex = { create };
|
package/src/ratelimit.ts
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import { redis } from "bun";
|
|
2
|
+
|
|
3
|
+
// ==========================
|
|
4
|
+
// Types
|
|
5
|
+
// ==========================
|
|
6
|
+
|
|
7
|
+
export type RateLimitResult = {
|
|
8
|
+
limited: boolean;
|
|
9
|
+
remaining: number;
|
|
10
|
+
resetIn: number;
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
export type RateLimitConfig = {
|
|
14
|
+
/** Maximum number of requests allowed in the window */
|
|
15
|
+
limit: number;
|
|
16
|
+
/** Window size in seconds (default: 1) */
|
|
17
|
+
windowSecs?: number;
|
|
18
|
+
/** Key prefix for Redis keys (default: "ratelimit") */
|
|
19
|
+
prefix?: string;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
export type RateLimiter = {
|
|
23
|
+
/** Check rate limit for an identifier */
|
|
24
|
+
check: (identifier: string) => Promise<RateLimitResult>;
|
|
25
|
+
/** Check and throw if limited */
|
|
26
|
+
checkOrThrow: (identifier: string) => Promise<RateLimitResult>;
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
// ==========================
|
|
30
|
+
// Rate Limit Error
|
|
31
|
+
// ==========================
|
|
32
|
+
|
|
33
|
+
export class RateLimitError extends Error {
|
|
34
|
+
readonly remaining: number;
|
|
35
|
+
readonly resetIn: number;
|
|
36
|
+
|
|
37
|
+
constructor(result: RateLimitResult) {
|
|
38
|
+
super("Rate limit exceeded");
|
|
39
|
+
this.name = "RateLimitError";
|
|
40
|
+
this.remaining = result.remaining;
|
|
41
|
+
this.resetIn = result.resetIn;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// ==========================
|
|
46
|
+
// Lua Script for Atomic Rate Limiting
|
|
47
|
+
// ==========================
|
|
48
|
+
|
|
49
|
+
// Atomically increment counter and set expiry in one operation
|
|
50
|
+
const RATE_LIMIT_SCRIPT = `
|
|
51
|
+
local currentKey = KEYS[1]
|
|
52
|
+
local previousKey = KEYS[2]
|
|
53
|
+
local windowSecs = tonumber(ARGV[1])
|
|
54
|
+
local limit = tonumber(ARGV[2])
|
|
55
|
+
local elapsedRatio = tonumber(ARGV[3])
|
|
56
|
+
|
|
57
|
+
-- Get previous window count
|
|
58
|
+
local previousCount = tonumber(redis.call("GET", previousKey) or "0")
|
|
59
|
+
|
|
60
|
+
-- Increment current window and set expiry atomically
|
|
61
|
+
local currentCount = redis.call("INCR", currentKey)
|
|
62
|
+
if currentCount == 1 then
|
|
63
|
+
redis.call("EXPIRE", currentKey, windowSecs * 2)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
-- Calculate weighted count
|
|
67
|
+
local weightedCount = previousCount * (1 - elapsedRatio) + currentCount
|
|
68
|
+
|
|
69
|
+
return {currentCount, previousCount, weightedCount}
|
|
70
|
+
`;
|
|
71
|
+
|
|
72
|
+
// ==========================
|
|
73
|
+
// Rate Limiter Factory
|
|
74
|
+
// ==========================
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Create a rate limiter with the given configuration.
|
|
78
|
+
* Uses a sliding window algorithm for smooth rate limiting.
|
|
79
|
+
*
|
|
80
|
+
* @example
|
|
81
|
+
* ```ts
|
|
82
|
+
* const limiter = ratelimit.create({ limit: 100, windowSecs: 60 });
|
|
83
|
+
*
|
|
84
|
+
* // Check rate limit
|
|
85
|
+
* const result = await limiter.check("user:123");
|
|
86
|
+
* if (result.limited) {
|
|
87
|
+
* console.log(`Retry in ${result.resetIn}ms`);
|
|
88
|
+
* }
|
|
89
|
+
*
|
|
90
|
+
* // Or throw on limit
|
|
91
|
+
* await limiter.checkOrThrow("user:123");
|
|
92
|
+
* ```
|
|
93
|
+
*/
|
|
94
|
+
const create = (config: RateLimitConfig): RateLimiter => {
|
|
95
|
+
const { limit, windowSecs = 1, prefix = "ratelimit" } = config;
|
|
96
|
+
|
|
97
|
+
const check = async (identifier: string): Promise<RateLimitResult> => {
|
|
98
|
+
const now = Date.now();
|
|
99
|
+
const windowMs = windowSecs * 1000;
|
|
100
|
+
const currentWindow = Math.floor(now / windowMs);
|
|
101
|
+
const previousWindow = currentWindow - 1;
|
|
102
|
+
const elapsedInWindow = now % windowMs;
|
|
103
|
+
const elapsedRatio = elapsedInWindow / windowMs;
|
|
104
|
+
|
|
105
|
+
const currentKey = `${prefix}:${identifier}:${currentWindow}`;
|
|
106
|
+
const previousKey = `${prefix}:${identifier}:${previousWindow}`;
|
|
107
|
+
|
|
108
|
+
// Execute atomic Lua script
|
|
109
|
+
const result = (await redis.send("EVAL", [
|
|
110
|
+
RATE_LIMIT_SCRIPT,
|
|
111
|
+
"2",
|
|
112
|
+
currentKey,
|
|
113
|
+
previousKey,
|
|
114
|
+
windowSecs.toString(),
|
|
115
|
+
limit.toString(),
|
|
116
|
+
elapsedRatio.toString(),
|
|
117
|
+
])) as [number, number, number];
|
|
118
|
+
|
|
119
|
+
const [, , weightedCount] = result;
|
|
120
|
+
|
|
121
|
+
const limited = weightedCount > limit;
|
|
122
|
+
const remaining = Math.max(0, Math.floor(limit - weightedCount));
|
|
123
|
+
const resetIn = windowMs - elapsedInWindow;
|
|
124
|
+
|
|
125
|
+
return { limited, remaining, resetIn };
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
const checkOrThrow = async (identifier: string): Promise<RateLimitResult> => {
|
|
129
|
+
const result = await check(identifier);
|
|
130
|
+
if (result.limited) {
|
|
131
|
+
throw new RateLimitError(result);
|
|
132
|
+
}
|
|
133
|
+
return result;
|
|
134
|
+
};
|
|
135
|
+
|
|
136
|
+
return { check, checkOrThrow };
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
// ==========================
|
|
140
|
+
// Export
|
|
141
|
+
// ==========================
|
|
142
|
+
|
|
143
|
+
export const ratelimit = { create, RateLimitError };
|