@brickkit/redis-world 0.0.0-stage → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +17 -0
- package/README.md +108 -2
- package/bin/setup.js +12 -0
- package/dist/cli.d.ts +5 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +65 -0
- package/dist/cli.js.map +1 -0
- package/dist/index.d.ts +148 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +2797 -0
- package/dist/index.js.map +1 -0
- package/package.json +67 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
<http://www.apache.org/licenses/>
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
|
124
|
+
may provide additional or different license terms and conditions
|
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
128
|
+
the conditions stated in this License.
|
|
129
|
+
|
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
133
|
+
this License, without any additional terms or conditions.
|
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
135
|
+
the terms of any separate license agreement you may have executed
|
|
136
|
+
with Licensor regarding such Contributions.
|
|
137
|
+
|
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
140
|
+
except as required for reasonable and customary use in describing the
|
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
142
|
+
|
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
|
158
|
+
incidental, or consequential damages of any character arising as a
|
|
159
|
+
result of this License or out of the use or inability to use the
|
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
162
|
+
other commercial damages or losses), even if such Contributor
|
|
163
|
+
has been advised of the possibility of such damages.
|
|
164
|
+
|
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
168
|
+
or other liability obligations and/or rights consistent with this
|
|
169
|
+
License. However, in accepting such obligations, You may act only
|
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
174
|
+
of your accepting any such warranty or additional liability.
|
|
175
|
+
|
|
176
|
+
END OF TERMS AND CONDITIONS
|
|
177
|
+
|
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
179
|
+
|
|
180
|
+
To apply the Apache License to your work, attach the following
|
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
182
|
+
replaced with your own identifying information. (Don't include
|
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
184
|
+
comment syntax for the file format. We also recommend that a
|
|
185
|
+
file or class name and description of purpose be included on the
|
|
186
|
+
same "printed page" as the copyright notice for easier
|
|
187
|
+
identification within third-party archives.
|
|
188
|
+
|
|
189
|
+
Copyright 2025 Vinnymac LLC.
|
|
190
|
+
|
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
|
+
you may not use this file except in compliance with the License.
|
|
193
|
+
You may obtain a copy of the License at
|
|
194
|
+
|
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
196
|
+
|
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
|
+
See the License for the specific language governing permissions and
|
|
201
|
+
limitations under the License.
|
package/NOTICE
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
@brickkit/redis-world
|
|
2
|
+
|
|
3
|
+
The reliable list queue is derived from @fantasticfour/world-redis
|
|
4
|
+
(https://github.com/vinnymac/worlds, packages/world-redis), licensed under
|
|
5
|
+
Apache-2.0. Original project: vinnymac/worlds and its contributors.
|
|
6
|
+
|
|
7
|
+
Changes: ported the queue to Workflow SDK v5's single-queue contract; added
|
|
8
|
+
atomic reservation and persisted attempt/delivery counters, heartbeat lease
|
|
9
|
+
fencing, owner-checked dedup release, capped retries without message loss,
|
|
10
|
+
v5 routing and binary transport, and new recovery and runtime chaos tests.
|
|
11
|
+
Storage, streaming, retention, and utilities are copied from the independent
|
|
12
|
+
@brickkit/bullmq-world v5 implementation in this repository.
|
|
13
|
+
|
|
14
|
+
The copied v5 implementation is derived from @fantasticfour/world-redis-bullmq
|
|
15
|
+
(https://github.com/vinnymac/worlds, packages/world-redis-bullmq), Apache-2.0.
|
|
16
|
+
Its v5 changes include position-based event IDs, atomic writes, restart
|
|
17
|
+
recovery, independent utilities, and concurrency and chaos tests.
|
package/README.md
CHANGED
|
@@ -1,3 +1,109 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @brickkit/redis-world
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
A plain Redis World for Workflow SDK v5. Runs, events, steps, hooks, streams, and queue deliveries live in Redis. The queue uses a ready list, per-worker processing lists, a delayed sorted set, and expiring worker heartbeats. Lua scripts make enqueue, reserve, acknowledge, defer, promote, and reclaim atomic.
|
|
4
|
+
|
|
5
|
+
```sh
|
|
6
|
+
bun add @brickkit/redis-world
|
|
7
|
+
# or: npm install @brickkit/redis-world
|
|
8
|
+
```
|
|
9
|
+
|
|
10
|
+
```ts
|
|
11
|
+
import { createWorld } from "@brickkit/redis-world";
|
|
12
|
+
|
|
13
|
+
const world = createWorld({ redis: "redis://localhost:6379" });
|
|
14
|
+
await world.start();
|
|
15
|
+
// On shutdown: await world.close();
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Set `WORKFLOW_TARGET_WORLD=@brickkit/redis-world` for the Workflow runtime. Serve the runtime's generated flow route at `/.well-known/workflow/v1/flow`. Workers POST to that route unless an in-process handler or a custom route is registered. This package supplies the World, not an embedded runtime server.
|
|
19
|
+
|
|
20
|
+
## Configuration
|
|
21
|
+
|
|
22
|
+
Pass options to `createWorld`. With no arguments, Redis, key prefix, and concurrency come from the environment variables below.
|
|
23
|
+
|
|
24
|
+
| Option | Default | Meaning |
|
|
25
|
+
| ---------------------- | -------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- |
|
|
26
|
+
| `redis` | `WORKFLOW_REDIS_URL`, `REDIS_URL`, or `redis://localhost:6379` | Connection URL or ioredis `RedisOptions` |
|
|
27
|
+
| `keyPrefix` | `workflow:` | All storage and queue keys; env `WORKFLOW_REDIS_KEY_PREFIX` |
|
|
28
|
+
| `queueConcurrency` | `10` | Concurrent deliveries per process; env `WORKFLOW_REDIS_WORKER_CONCURRENCY` |
|
|
29
|
+
| `heartbeatTtlMs` | `90000` | Worker lease in ms, renewed every third of its TTL; minimum 300. Set above the longest event-loop pause |
|
|
30
|
+
| `maxAttempts` | `73` | Retry budget, minimum 49. After this budget, failed deliveries continue at `maxBackoffMs` so a transport outage cannot strand a run |
|
|
31
|
+
| `backoffType` | `exponential` | `fixed` or `exponential` |
|
|
32
|
+
| `backoffDelayMs` | `1000` | Initial retry delay in ms |
|
|
33
|
+
| `maxBackoffMs` | `30000` | Retry delay cap in ms |
|
|
34
|
+
| `baseUrl` | `WORKFLOW_BASE_URL` or `http://localhost:$PORT` with port 3000 | HTTP delivery base; registered handlers/routes take precedence |
|
|
35
|
+
| `httpTimeoutMs` | `300000` | HTTP request timeout in ms. A timeout cannot stop a handler already running on the server |
|
|
36
|
+
| `idempotencyTtlMs` | unset | Optional dedup TTL. Unset keeps the reservation until acknowledgement. A TTL shorter than the backlog/delay permits duplicates |
|
|
37
|
+
| `maxEventsPerRun` | `WORKFLOW_MAX_EVENTS` or `25000` | Event ceiling reported to the runtime |
|
|
38
|
+
| `enableAutoPipelining` | `true` | Batch Redis commands issued in the same tick |
|
|
39
|
+
| `retention` | unset | Finished-run policy described below |
|
|
40
|
+
|
|
41
|
+
`getQueueStats()` returns `waiting`, `active`, and `delayed`. `completed` and `failed` are always zero because delivery history is not retained. Exhausted failed deliveries remain delayed and retain their dedup reservation until they succeed. The Workflow runtime can record a failed run when its own delivery ceiling is exceeded.
|
|
42
|
+
|
|
43
|
+
## Reliability and recovery
|
|
44
|
+
|
|
45
|
+
One queue, `${keyPrefix}queue:flows`, carries both workflow and step work. Reservation moves a message into `${keyPrefix}queue:flows:processing:<worker>` and increments durable attempt/delivery counters before dispatch. Heartbeat expiry permits another worker to reclaim the processing list. Recovery preserves message IDs and reports attempts above one after a crash. Successful reschedules preserve the attempt number and increment delivery count. Delays and retry backoffs remain in Redis across process restarts.
|
|
46
|
+
|
|
47
|
+
Duplicate enqueues with an `idempotencyKey` return the existing message ID while that message is ready, active, or delayed. Acknowledgement releases the reservation only if it still belongs to that message. An expired lease or a reclaimed claim cannot acknowledge, defer, or release another delivery's reservation. `start()` also re-enqueues active runs with `recover:<runId>` dedup keys.
|
|
48
|
+
|
|
49
|
+
Delivery is at least once. A worker paused beyond its heartbeat TTL can overlap its replacement; queue fencing protects Redis queue state, not arbitrary external side effects. A killed step may execute again if its completion was not persisted. Use idempotent external effects. Graceful `close()` drains handlers while renewing its heartbeat.
|
|
50
|
+
|
|
51
|
+
Use Redis persistence such as AOF. Redis data loss can lose workflow history. Use a single primary, optionally with Sentinel/failover. Redis Cluster is unsupported because the storage and queue scripts use multiple keys without a shared hash slot. Workflow v4 run data is incompatible; drain old runs or use a fresh key prefix.
|
|
52
|
+
|
|
53
|
+
## Starting a run once
|
|
54
|
+
|
|
55
|
+
`start()` takes no run id, so a caller that retries (an event delivered twice, a request sent again) normally starts a second run. This world lets the caller name the run instead. Pass `runKey` and `runKeyTime` in the options to `start()`:
|
|
56
|
+
|
|
57
|
+
```ts
|
|
58
|
+
import { start } from "workflow/api";
|
|
59
|
+
|
|
60
|
+
const run = await start(myWorkflow, [input], {
|
|
61
|
+
runKey: "evt_01J...:welcome-email",
|
|
62
|
+
runKeyTime: Date.parse(event.occurredAt),
|
|
63
|
+
} as never);
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
The same `runKey` and `runKeyTime` always give the same run id, so starting again returns the run that is already there instead of a second one. This holds when the starts happen at the same moment on different workers, and the run's first message is not queued a second time. Once a run has started, starting it again does nothing: its steps do not run again.
|
|
67
|
+
|
|
68
|
+
`runKeyTime` is required with a `runKey`. A run id is a ULID, and the Workflow runtime seeds the workflow's clock (what `Date.now()` returns inside it) from the time in it. Passing the time the run is for, such as when the event happened, keeps that clock true and the same on every attempt. Pass the same time each time: a different `runKeyTime` is a different run. `runKey` is a non-empty string of at most 1024 characters. The world throws a `TypeError` for either one when it is not usable.
|
|
69
|
+
|
|
70
|
+
Without `runKey`, ids are fresh ULIDs and nothing changes. The Workflow typings do not list these two options yet, hence the cast.
|
|
71
|
+
|
|
72
|
+
## Retention
|
|
73
|
+
|
|
74
|
+
Finished runs stay forever unless you set a policy. `age` is in seconds; `count` keeps the newest runs. A status without a policy is kept.
|
|
75
|
+
|
|
76
|
+
```ts
|
|
77
|
+
createWorld({
|
|
78
|
+
redis: "redis://localhost:6379",
|
|
79
|
+
retention: {
|
|
80
|
+
completed: { age: 24 * 60 * 60, count: 1000 },
|
|
81
|
+
failed: { age: 7 * 24 * 60 * 60 },
|
|
82
|
+
// cancelled is unset, so cancelled runs stay
|
|
83
|
+
},
|
|
84
|
+
});
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
A run past either configured limit is deleted with its event log, steps, hooks, and streams. Running runs are never touched. Each worker sweeps every `retention.sweepIntervalMs` (default 60000), at most `retention.batchSize` runs per status (default 200). A Redis lock prevents simultaneous sweeps. `world.sweepRetention()` runs a sweep on demand. Queue records disappear on acknowledgement independently of run retention.
|
|
88
|
+
|
|
89
|
+
The optional in-band writer fence, `$retention: '0'` purging, hook token retention, and `createBatch` are not implemented. The v5 runtime handles their absence.
|
|
90
|
+
|
|
91
|
+
## Choosing a World
|
|
92
|
+
|
|
93
|
+
Choose `@brickkit/redis-world` when you want fewer dependencies and a hand-built list queue with durable delays, retries, and worker recovery. Choose `@brickkit/bullmq-world` when you want BullMQ's queue tooling and delivery records. Both use independent copies of the same v5 storage, streaming, and run-retention implementation. This package has no BullMQ dependency and no Bull Board integration.
|
|
94
|
+
|
|
95
|
+
BullMQ-only options `jobPrefix`, `stalledInterval`, `maxStalledCount`, `lockDuration`, and `jobRetention` are absent. `keyPrefix` isolates the queue; `heartbeatTtlMs` controls crash recovery. Promotion and orphan checks run about every 100 ms while workers are alive.
|
|
96
|
+
|
|
97
|
+
## Development
|
|
98
|
+
|
|
99
|
+
```sh
|
|
100
|
+
bun install
|
|
101
|
+
bun run typecheck
|
|
102
|
+
bun run build
|
|
103
|
+
bun run test # Docker/testcontainers unless REDIS_TEST_URL is supplied
|
|
104
|
+
bun run test:e2e # redis-server executable, or E2E_REDIS_URL for shared Redis
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
The e2e harness exercises the built package and real Workflow runtime, including SIGKILL and Redis AOF restart scenarios. Own-Redis restart tests require `redis-server` even when `E2E_REDIS_URL` is provided.
|
|
108
|
+
|
|
109
|
+
Apache-2.0. See LICENSE and NOTICE.
|
package/bin/setup.js
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// Simple wrapper to run the CLI from the bin directory
|
|
4
|
+
import("../dist/cli.js")
|
|
5
|
+
.then((module) => {
|
|
6
|
+
// Call the setupRedis function
|
|
7
|
+
return module.setupRedis();
|
|
8
|
+
})
|
|
9
|
+
.catch((err) => {
|
|
10
|
+
console.error("Failed to load CLI:", err);
|
|
11
|
+
process.exit(1);
|
|
12
|
+
});
|
package/dist/cli.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.d.ts","names":[],"sources":["../src/cli.ts"],"mappings":";iBAGe,cAAU"}
|
package/dist/cli.js
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { Redis } from "ioredis";
|
|
2
|
+
//#region src/internal/env.ts
|
|
3
|
+
/**
|
|
4
|
+
* Load an env file when present without making a `.env` file mandatory.
|
|
5
|
+
*
|
|
6
|
+
* Node 24 provides the parser natively, so published worlds do not need a
|
|
7
|
+
* runtime dependency solely to load local development configuration.
|
|
8
|
+
*/
|
|
9
|
+
function loadOptionalEnvFile(path) {
|
|
10
|
+
try {
|
|
11
|
+
if (path === void 0) process.loadEnvFile();
|
|
12
|
+
else process.loadEnvFile(path);
|
|
13
|
+
} catch (error) {
|
|
14
|
+
if (error.code !== "ENOENT") throw error;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
//#endregion
|
|
18
|
+
//#region src/cli.ts
|
|
19
|
+
async function setupRedis() {
|
|
20
|
+
loadOptionalEnvFile();
|
|
21
|
+
const connectionString = process.env.WORKFLOW_REDIS_URL || process.env.REDIS_URL || "redis://localhost:6379";
|
|
22
|
+
const keyPrefix = process.env.WORKFLOW_REDIS_KEY_PREFIX || "workflow:";
|
|
23
|
+
console.log("Setting up Redis for Workflow...");
|
|
24
|
+
console.log(`Connection: ${connectionString.replace(/^(\w+:\/\/)([^@]+)@/, "$1[redacted]@")}`);
|
|
25
|
+
console.log(`Key Prefix: ${keyPrefix}`);
|
|
26
|
+
let redis;
|
|
27
|
+
let exitCode = 0;
|
|
28
|
+
try {
|
|
29
|
+
redis = new Redis(connectionString);
|
|
30
|
+
await redis.ping();
|
|
31
|
+
console.log("Successfully connected to Redis!");
|
|
32
|
+
const version = (await redis.info("server")).match(/redis_version:([^\r\n]+)/)?.[1];
|
|
33
|
+
if (version) console.log(`Redis version: ${version}`);
|
|
34
|
+
const existingKeys = await redis.keys(`${keyPrefix}*`);
|
|
35
|
+
if (existingKeys.length > 0) {
|
|
36
|
+
console.log(`\nWarning: found ${existingKeys.length} existing workflow keys with prefix "${keyPrefix}"`);
|
|
37
|
+
console.log("\nTo flush existing workflow data, run with FLUSH=true environment variable:");
|
|
38
|
+
console.log(" FLUSH=true npx brickkit-redis-world-setup");
|
|
39
|
+
if (process.env.FLUSH === "true") {
|
|
40
|
+
console.log("\nFlushing existing workflow data...");
|
|
41
|
+
const pipeline = redis.pipeline();
|
|
42
|
+
for (const key of existingKeys) pipeline.del(key);
|
|
43
|
+
await pipeline.exec();
|
|
44
|
+
console.log(`Deleted ${existingKeys.length} keys`);
|
|
45
|
+
}
|
|
46
|
+
} else console.log(`\nNo existing workflow data found (clean slate)`);
|
|
47
|
+
console.log("\nRedis is ready for Workflow DevKit!");
|
|
48
|
+
console.log("\nThe following Redis data structures will be used:");
|
|
49
|
+
console.log(` - Hashes: ${keyPrefix}run:*, ${keyPrefix}step:*, ${keyPrefix}event:*, ${keyPrefix}hook:*`);
|
|
50
|
+
console.log(` - Sorted Sets: ${keyPrefix}runs:*, ${keyPrefix}steps:*, ${keyPrefix}events:*`);
|
|
51
|
+
console.log(` - Streams: ${keyPrefix}stream:*`);
|
|
52
|
+
console.log(` - Redis queue: ${keyPrefix}queue:flows, processing lists and delayed set`);
|
|
53
|
+
} catch (error) {
|
|
54
|
+
exitCode = 1;
|
|
55
|
+
console.error("Failed to setup Redis:", error);
|
|
56
|
+
} finally {
|
|
57
|
+
await redis?.quit().catch(() => {});
|
|
58
|
+
}
|
|
59
|
+
process.exit(exitCode);
|
|
60
|
+
}
|
|
61
|
+
if (import.meta.url === `file://${process.argv[1]}`) setupRedis();
|
|
62
|
+
//#endregion
|
|
63
|
+
export { setupRedis };
|
|
64
|
+
|
|
65
|
+
//# sourceMappingURL=cli.js.map
|
package/dist/cli.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.js","names":[],"sources":["../src/internal/env.ts","../src/cli.ts"],"sourcesContent":["/**\n * Load an env file when present without making a `.env` file mandatory.\n *\n * Node 24 provides the parser natively, so published worlds do not need a\n * runtime dependency solely to load local development configuration.\n */\nexport function loadOptionalEnvFile(path?: string): void {\n try {\n if (path === undefined) {\n process.loadEnvFile();\n } else {\n process.loadEnvFile(path);\n }\n } catch (error: unknown) {\n if ((error as NodeJS.ErrnoException).code !== \"ENOENT\") {\n throw error;\n }\n }\n}\n","import { loadOptionalEnvFile } from \"./internal/env.js\";\nimport { Redis } from \"ioredis\";\n\nasync function setupRedis() {\n // Load .env file if it exists.\n loadOptionalEnvFile();\n\n const connectionString =\n process.env.WORKFLOW_REDIS_URL ||\n process.env.REDIS_URL ||\n \"redis://localhost:6379\";\n\n const keyPrefix = process.env.WORKFLOW_REDIS_KEY_PREFIX || \"workflow:\";\n\n console.log(\"Setting up Redis for Workflow...\");\n console.log(\n `Connection: ${connectionString.replace(/^(\\w+:\\/\\/)([^@]+)@/, \"$1[redacted]@\")}`,\n );\n console.log(`Key Prefix: ${keyPrefix}`);\n\n let redis: Redis | undefined;\n let exitCode = 0;\n\n try {\n redis = new Redis(connectionString);\n\n // Test connection\n await redis.ping();\n console.log(\"Successfully connected to Redis!\");\n\n // Get Redis info\n const info = await redis.info(\"server\");\n const version = info.match(/redis_version:([^\\r\\n]+)/)?.[1];\n if (version) {\n console.log(`Redis version: ${version}`);\n }\n\n // Check if any workflow keys exist\n const existingKeys = await redis.keys(`${keyPrefix}*`);\n if (existingKeys.length > 0) {\n console.log(\n `\\nWarning: found ${existingKeys.length} existing workflow keys with prefix \"${keyPrefix}\"`,\n );\n console.log(\n \"\\nTo flush existing workflow data, run with FLUSH=true environment variable:\",\n );\n console.log(\" FLUSH=true npx brickkit-redis-world-setup\");\n\n if (process.env.FLUSH === \"true\") {\n console.log(\"\\nFlushing existing workflow data...\");\n const pipeline = redis.pipeline();\n for (const key of existingKeys) {\n pipeline.del(key);\n }\n await pipeline.exec();\n console.log(`Deleted ${existingKeys.length} keys`);\n }\n } else {\n console.log(`\\nNo existing workflow data found (clean slate)`);\n }\n\n console.log(\"\\nRedis is ready for Workflow DevKit!\");\n console.log(\"\\nThe following Redis data structures will be used:\");\n console.log(\n ` - Hashes: ${keyPrefix}run:*, ${keyPrefix}step:*, ${keyPrefix}event:*, ${keyPrefix}hook:*`,\n );\n console.log(\n ` - Sorted Sets: ${keyPrefix}runs:*, ${keyPrefix}steps:*, ${keyPrefix}events:*`,\n );\n console.log(` - Streams: ${keyPrefix}stream:*`);\n console.log(\n ` - Redis queue: ${keyPrefix}queue:flows, processing lists and delayed set`,\n );\n } catch (error) {\n exitCode = 1;\n console.error(\"Failed to setup Redis:\", error);\n } finally {\n await redis?.quit().catch(() => {});\n }\n\n process.exit(exitCode);\n}\n\n// Check if running as main module\nif (import.meta.url === `file://${process.argv[1]}`) {\n setupRedis();\n}\n\nexport { setupRedis };\n"],"mappings":";;;;;;;;AAMA,SAAgB,oBAAoB,MAAqB;CACvD,IAAI;EACF,IAAI,SAAS,KAAA,GACX,QAAQ,YAAY;OAEpB,QAAQ,YAAY,IAAI;CAE5B,SAAS,OAAgB;EACvB,IAAK,MAAgC,SAAS,UAC5C,MAAM;CAEV;AACF;;;ACfA,eAAe,aAAa;CAE1B,oBAAoB;CAEpB,MAAM,mBACJ,QAAQ,IAAI,sBACZ,QAAQ,IAAI,aACZ;CAEF,MAAM,YAAY,QAAQ,IAAI,6BAA6B;CAE3D,QAAQ,IAAI,kCAAkC;CAC9C,QAAQ,IACN,eAAe,iBAAiB,QAAQ,uBAAuB,eAAe,GAChF;CACA,QAAQ,IAAI,eAAe,WAAW;CAEtC,IAAI;CACJ,IAAI,WAAW;CAEf,IAAI;EACF,QAAQ,IAAI,MAAM,gBAAgB;EAGlC,MAAM,MAAM,KAAK;EACjB,QAAQ,IAAI,kCAAkC;EAI9C,MAAM,WAAU,MADG,MAAM,KAAK,QAAQ,EAAA,CACjB,MAAM,0BAA0B,CAAC,GAAG;EACzD,IAAI,SACF,QAAQ,IAAI,kBAAkB,SAAS;EAIzC,MAAM,eAAe,MAAM,MAAM,KAAK,GAAG,UAAU,EAAE;EACrD,IAAI,aAAa,SAAS,GAAG;GAC3B,QAAQ,IACN,oBAAoB,aAAa,OAAO,uCAAuC,UAAU,EAC3F;GACA,QAAQ,IACN,8EACF;GACA,QAAQ,IAAI,6CAA6C;GAEzD,IAAI,QAAQ,IAAI,UAAU,QAAQ;IAChC,QAAQ,IAAI,sCAAsC;IAClD,MAAM,WAAW,MAAM,SAAS;IAChC,KAAK,MAAM,OAAO,cAChB,SAAS,IAAI,GAAG;IAElB,MAAM,SAAS,KAAK;IACpB,QAAQ,IAAI,WAAW,aAAa,OAAO,MAAM;GACnD;EACF,OACE,QAAQ,IAAI,iDAAiD;EAG/D,QAAQ,IAAI,uCAAuC;EACnD,QAAQ,IAAI,qDAAqD;EACjE,QAAQ,IACN,eAAe,UAAU,SAAS,UAAU,UAAU,UAAU,WAAW,UAAU,OACvF;EACA,QAAQ,IACN,oBAAoB,UAAU,UAAU,UAAU,WAAW,UAAU,SACzE;EACA,QAAQ,IAAI,gBAAgB,UAAU,SAAS;EAC/C,QAAQ,IACN,oBAAoB,UAAU,8CAChC;CACF,SAAS,OAAO;EACd,WAAW;EACX,QAAQ,MAAM,0BAA0B,KAAK;CAC/C,UAAU;EACR,MAAM,OAAO,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC;CACpC;CAEA,QAAQ,KAAK,QAAQ;AACvB;AAGA,IAAI,YAAY,QAAQ,UAAU,QAAQ,KAAK,MAC7C,WAAW"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import { World } from "@workflow/world";
|
|
2
|
+
import { Redis, RedisOptions } from "ioredis";
|
|
3
|
+
//#region src/reaper.d.ts
|
|
4
|
+
/** How long finished runs are kept, using `age` and `count` limits. */
|
|
5
|
+
interface KeepRuns {
|
|
6
|
+
/** Delete runs that finished more than this many seconds ago. */
|
|
7
|
+
age?: number;
|
|
8
|
+
/** Keep at most this many of the newest runs; delete the rest. */
|
|
9
|
+
count?: number;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Retention for run data (the run, its event log, steps, hooks, and streams),
|
|
13
|
+
* per terminal status. A status without a policy is kept forever.
|
|
14
|
+
*/
|
|
15
|
+
interface RunRetention {
|
|
16
|
+
completed?: KeepRuns;
|
|
17
|
+
failed?: KeepRuns;
|
|
18
|
+
cancelled?: KeepRuns;
|
|
19
|
+
/** How often each worker looks for expired runs (ms). Default: 60000 */
|
|
20
|
+
sweepIntervalMs?: number;
|
|
21
|
+
/** Runs deleted per status per sweep. Default: 200 */
|
|
22
|
+
batchSize?: number;
|
|
23
|
+
}
|
|
24
|
+
//#endregion
|
|
25
|
+
//#region src/config.d.ts
|
|
26
|
+
interface RedisWorldConfig {
|
|
27
|
+
/**
|
|
28
|
+
* Redis connection URL or connection options
|
|
29
|
+
* Examples:
|
|
30
|
+
* - 'redis://localhost:6379'
|
|
31
|
+
* - 'rediss://username:password@host:6380/0'
|
|
32
|
+
* - { host: 'localhost', port: 6379 }
|
|
33
|
+
*/
|
|
34
|
+
redis: string | RedisOptions;
|
|
35
|
+
/**
|
|
36
|
+
* Number of concurrent workers processing jobs
|
|
37
|
+
* Default: 10
|
|
38
|
+
*/
|
|
39
|
+
queueConcurrency?: number;
|
|
40
|
+
/** Worker liveness TTL in milliseconds. Default 90000; minimum 300.
|
|
41
|
+
* Set above the longest event-loop pause. Renewed every TTL / 3. */
|
|
42
|
+
heartbeatTtlMs?: number;
|
|
43
|
+
/**
|
|
44
|
+
* Optional key prefix for all Redis keys
|
|
45
|
+
* Useful for multi-tenancy or namespace isolation
|
|
46
|
+
* Default: 'workflow:'
|
|
47
|
+
*/
|
|
48
|
+
keyPrefix?: string;
|
|
49
|
+
/**
|
|
50
|
+
* Delivery budget before retries continue at maxBackoffMs. Messages are never silently retired.
|
|
51
|
+
*
|
|
52
|
+
* Keep it above the runtime's own delivery ceiling (48). The runtime records a
|
|
53
|
+
* run as failed (`MAX_DELIVERIES_EXCEEDED`) on delivery 49; a job retired
|
|
54
|
+
* earlier strands its run in `running`, with nothing to redeliver it until a
|
|
55
|
+
* process calls `start()` again.
|
|
56
|
+
* Default: 73
|
|
57
|
+
*/
|
|
58
|
+
maxAttempts?: number;
|
|
59
|
+
/**
|
|
60
|
+
* Backoff strategy for retries: 'fixed' replays at the same interval,
|
|
61
|
+
* 'exponential' doubles the delay on each attempt.
|
|
62
|
+
* Default: 'exponential'
|
|
63
|
+
*/
|
|
64
|
+
backoffType?: "fixed" | "exponential";
|
|
65
|
+
/**
|
|
66
|
+
* Base delay for backoff (ms). For exponential backoff this is multiplied
|
|
67
|
+
* by 2^(attempt - 1), up to `maxBackoffMs`.
|
|
68
|
+
* Default: 1000
|
|
69
|
+
*/
|
|
70
|
+
backoffDelayMs?: number;
|
|
71
|
+
/**
|
|
72
|
+
* The longest a retry is delayed (ms). With the default number of attempts an
|
|
73
|
+
* uncapped exponential backoff would wait hours between late attempts.
|
|
74
|
+
* Default: 30000 (30s)
|
|
75
|
+
*/
|
|
76
|
+
maxBackoffMs?: number;
|
|
77
|
+
/**
|
|
78
|
+
* Base URL the worker uses to dispatch jobs back to the user's HTTP
|
|
79
|
+
* server, which must serve `world.createQueueHandler(...)` at
|
|
80
|
+
* `/.well-known/workflow/v1/flow` (steps travel on the same route). A route
|
|
81
|
+
* registered with `world.registerRoute`, or an in-process handler registered
|
|
82
|
+
* with `world.registerHandler`, takes precedence.
|
|
83
|
+
* Default: process.env.WORKFLOW_BASE_URL || `http://localhost:${process.env.PORT ?? 3000}`
|
|
84
|
+
*/
|
|
85
|
+
baseUrl?: string;
|
|
86
|
+
/**
|
|
87
|
+
* Per-job HTTP request timeout (ms). Long-running workflow steps may need a
|
|
88
|
+
* higher value.
|
|
89
|
+
* Default: 300_000 (5 minutes)
|
|
90
|
+
*/
|
|
91
|
+
httpTimeoutMs?: number;
|
|
92
|
+
/**
|
|
93
|
+
* Optional TTL (ms) for queue deduplication when `idempotencyKey`
|
|
94
|
+
* is supplied to `queue()`. By default no TTL is applied: the dedup key
|
|
95
|
+
* lives until the job is acknowledged, so duplicate enqueues are dropped
|
|
96
|
+
* for the entire lifetime of the pending/delayed job (the semantics the
|
|
97
|
+
* workflow runtime relies on when re-enqueuing pending steps on replay).
|
|
98
|
+
* Only set this if you need the dedup guard to expire on wall-clock time;
|
|
99
|
+
* it must be much larger than the longest expected queue delay/backlog,
|
|
100
|
+
* otherwise duplicate step executions become possible.
|
|
101
|
+
* Default: undefined (dedup key released on job completion)
|
|
102
|
+
*/
|
|
103
|
+
idempotencyTtlMs?: number;
|
|
104
|
+
/** Maximum events a run may accumulate, reported as `EventResult.maxEvents`.
|
|
105
|
+
* Default: `WORKFLOW_MAX_EVENTS` || 25_000 */
|
|
106
|
+
maxEventsPerRun?: number;
|
|
107
|
+
/**
|
|
108
|
+
* Batch commands issued in the same event-loop tick into one write. A win
|
|
109
|
+
* once several runs are in flight (~1.75x at 256 concurrent) and a marginal
|
|
110
|
+
* loss for a single serial caller.
|
|
111
|
+
* Default: true
|
|
112
|
+
*/
|
|
113
|
+
enableAutoPipelining?: boolean;
|
|
114
|
+
/**
|
|
115
|
+
* How long finished runs are kept, per terminal status: `age` in seconds,
|
|
116
|
+
* `count` of newest runs to keep. A reaper on each worker deletes the run, its event log, steps, hooks
|
|
117
|
+
* and streams once it passes. A status without a policy is kept forever.
|
|
118
|
+
* Default: undefined (nothing expires)
|
|
119
|
+
*
|
|
120
|
+
* @example
|
|
121
|
+
* retention: {
|
|
122
|
+
* completed: { age: 24 * 60 * 60, count: 1000 },
|
|
123
|
+
* failed: { age: 7 * 24 * 60 * 60 },
|
|
124
|
+
* }
|
|
125
|
+
*/
|
|
126
|
+
retention?: RunRetention;
|
|
127
|
+
}
|
|
128
|
+
//#endregion
|
|
129
|
+
//#region src/queue.d.ts
|
|
130
|
+
interface QueueStats {
|
|
131
|
+
waiting: number;
|
|
132
|
+
active: number;
|
|
133
|
+
delayed: number;
|
|
134
|
+
/** Completed and failed delivery records are not retained. */
|
|
135
|
+
completed: number;
|
|
136
|
+
failed: number;
|
|
137
|
+
}
|
|
138
|
+
//#endregion
|
|
139
|
+
//#region src/index.d.ts
|
|
140
|
+
export declare function createWorld(config?: RedisWorldConfig): World & {
|
|
141
|
+
start(): Promise<void>;
|
|
142
|
+
getQueueStats(): Promise<QueueStats>;
|
|
143
|
+
/** Run one retention sweep now; returns the runs deleted. 0 without `retention`. */
|
|
144
|
+
sweepRetention(): Promise<number>;
|
|
145
|
+
};
|
|
146
|
+
//#endregion
|
|
147
|
+
export type { KeepRuns, QueueStats, RedisWorldConfig, RunRetention };
|
|
148
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","names":[],"sources":["../src/reaper.ts","../src/config.ts","../src/queue.ts","../src/index.ts"],"mappings":";;;;UAIiB;;EAEf;;EAEA;;;;;;UAOe;EACf,YAAY;EACZ,SAAS;EACT,YAAY;;EAEZ;;EAEA;;;;UCnBe;;;;;;;;EAQf,gBAAgB;;;;;EAMhB;;;EAIA;;;;;;EAOA;;;;;;;;;;EAWA;;;;;;EAOA;;;;;;EAOA;;;;;;EAOA;;;;;;;;;EAUA;;;;;;EAOA;;;;;;;;;;;;EAaA;;;EAIA;;;;;;;EAQA;;;;;;;;;;;;;EAcA,YAAY;;;;UC/FG;EACf;EACA;EACA;;EAEA;EACA;;;;wBCOc,YACd,SAAQ,mBAaP;EACD,SAAS;EACT,iBAAiB,QAAQ;;EAEzB,kBAAkB"}
|