@ewhauser/world-celld 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/LICENSE +201 -0
- package/NOTICE +24 -0
- package/README.md +292 -0
- package/celld-worker/README.md +34 -0
- package/celld-worker/worker.ts +5 -0
- package/celld-worker/wrangler.jsonc +32 -0
- package/dist/index.d.ts +250 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +949 -0
- package/dist/index.js.map +1 -0
- package/dist/testing.d.ts +134 -0
- package/dist/testing.d.ts.map +1 -0
- package/dist/testing.js +1808 -0
- package/dist/testing.js.map +1 -0
- package/dist/util-BvWlkqgV.js +122 -0
- package/dist/util-BvWlkqgV.js.map +1 -0
- package/dist/worker.d.ts +406 -0
- package/dist/worker.d.ts.map +1 -0
- package/dist/worker.js +1582 -0
- package/dist/worker.js.map +1 -0
- package/package.json +90 -0
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
## 0.1.0 - 2026-08-15
|
|
6
|
+
|
|
7
|
+
- Initial `@workflow/world` implementation backed by celld Durable Objects.
|
|
8
|
+
- Durable workflow storage, streams, queues, retries, deduplication, and dead
|
|
9
|
+
letters.
|
|
10
|
+
- Authenticated fleet RPC, deployable worker bundle, local development fleet,
|
|
11
|
+
conformance tests, and the MinIO performance and loss harness.
|
package/LICENSE
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
<http://www.apache.org/licenses/>
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
|
124
|
+
may provide additional or different license terms and conditions
|
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
128
|
+
the conditions stated in this License.
|
|
129
|
+
|
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
133
|
+
this License, without any additional terms or conditions.
|
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
135
|
+
the terms of any separate license agreement you may have executed
|
|
136
|
+
with Licensor regarding such Contributions.
|
|
137
|
+
|
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
140
|
+
except as required for reasonable and customary use in describing the
|
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
142
|
+
|
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
|
158
|
+
incidental, or consequential damages of any character arising as a
|
|
159
|
+
result of this License or out of the use or inability to use the
|
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
162
|
+
other commercial damages or losses), even if such Contributor
|
|
163
|
+
has been advised of the possibility of such damages.
|
|
164
|
+
|
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
168
|
+
or other liability obligations and/or rights consistent with this
|
|
169
|
+
License. However, in accepting such obligations, You may act only
|
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
174
|
+
of your accepting any such warranty or additional liability.
|
|
175
|
+
|
|
176
|
+
END OF TERMS AND CONDITIONS
|
|
177
|
+
|
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
179
|
+
|
|
180
|
+
To apply the Apache License to your work, attach the following
|
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
182
|
+
replaced with your own identifying information. (Don't include
|
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
184
|
+
comment syntax for the file format. We also recommend that a
|
|
185
|
+
file or class name and description of purpose be included on the
|
|
186
|
+
same "printed page" as the copyright notice for easier
|
|
187
|
+
identification within third-party archives.
|
|
188
|
+
|
|
189
|
+
Copyright 2025 Vinnymac LLC.
|
|
190
|
+
|
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
|
+
you may not use this file except in compliance with the License.
|
|
193
|
+
You may obtain a copy of the License at
|
|
194
|
+
|
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
196
|
+
|
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
|
+
See the License for the specific language governing permissions and
|
|
201
|
+
limitations under the License.
|
package/NOTICE
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
@ewhauser/world-celld
|
|
2
|
+
Copyright 2026 ewhauser
|
|
3
|
+
|
|
4
|
+
This product includes software developed as part of the "worlds" project
|
|
5
|
+
(https://github.com/vinnymac/worlds), Copyright vinnymac and contributors,
|
|
6
|
+
licensed under the Apache License, Version 2.0.
|
|
7
|
+
|
|
8
|
+
The following files are vendored (verbatim or with modifications) from
|
|
9
|
+
vinnymac/worlds:
|
|
10
|
+
|
|
11
|
+
From packages/world-cloudflare/src:
|
|
12
|
+
src/apply-event.ts
|
|
13
|
+
src/storage.ts
|
|
14
|
+
src/streamer.ts
|
|
15
|
+
src/queue.ts (modified: Cloudflare Queues replaced by QueueDO)
|
|
16
|
+
src/test-mocks.ts (modified)
|
|
17
|
+
src/util.ts
|
|
18
|
+
src/worker/durable-objects/WorkflowRunDO.ts
|
|
19
|
+
src/worker/durable-objects/StreamDO.ts
|
|
20
|
+
|
|
21
|
+
From packages/shared/src:
|
|
22
|
+
src/vendor/shared/*
|
|
23
|
+
|
|
24
|
+
Portions of test/ are derived from packages/world-cloudflare/test.
|
package/README.md
ADDED
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
# world-celld
|
|
2
|
+
|
|
3
|
+
A [Workflow DevKit](https://useworkflow.dev) `World` backed by
|
|
4
|
+
[celld](https://github.com/denoland/celld).
|
|
5
|
+
|
|
6
|
+
`world-celld` stores workflow runs, hooks, and streams in celld cells. Scheduled
|
|
7
|
+
work is delivered with durable cell alarms. This gives Node applications a
|
|
8
|
+
self-hosted alternative to platform-specific Workflow backends.
|
|
9
|
+
|
|
10
|
+
> [!WARNING]
|
|
11
|
+
> `world-celld` is experimental and has not been proven in production. Its API
|
|
12
|
+
> and storage layout may change before 1.0.
|
|
13
|
+
|
|
14
|
+
## What it provides
|
|
15
|
+
|
|
16
|
+
- Workflow run, step, event, and hook persistence
|
|
17
|
+
- Durable streams
|
|
18
|
+
- Delayed work, retries, deduplication, and dead-letter storage
|
|
19
|
+
- An authenticated HTTP connection between a Node application and a celld fleet
|
|
20
|
+
- An in-process fleet for local development and conformance testing
|
|
21
|
+
|
|
22
|
+
## Try the example
|
|
23
|
+
|
|
24
|
+
You need Node.js 22 or later and pnpm 11.
|
|
25
|
+
|
|
26
|
+
```sh
|
|
27
|
+
pnpm install
|
|
28
|
+
pnpm build
|
|
29
|
+
pnpm --dir examples/demo-app build
|
|
30
|
+
pnpm --dir examples/demo-app demo
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
The example starts an in-memory fleet and runs an order workflow through steps,
|
|
34
|
+
a sleep, an approval hook, and an output stream. It uses the same worker, cell
|
|
35
|
+
classes, and HTTP protocol as a celld deployment; only persistence and cell
|
|
36
|
+
routing are emulated.
|
|
37
|
+
|
|
38
|
+
See [`examples/demo-app`](./examples/demo-app) for the application and workflow
|
|
39
|
+
source.
|
|
40
|
+
|
|
41
|
+
## Use it in an application
|
|
42
|
+
|
|
43
|
+
Install the package from npm:
|
|
44
|
+
|
|
45
|
+
```sh
|
|
46
|
+
pnpm add @ewhauser/world-celld
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Workflow DevKit can load the package from environment variables:
|
|
50
|
+
|
|
51
|
+
```sh
|
|
52
|
+
WORKFLOW_TARGET_WORLD=@ewhauser/world-celld
|
|
53
|
+
CELLD_FLEET_URL=http://fleet.internal:8080
|
|
54
|
+
CELLD_WORLD_SECRET=replace-with-a-secret
|
|
55
|
+
WORKFLOW_BASE_URL=https://workflow.example.com
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
You can also construct the World directly:
|
|
59
|
+
|
|
60
|
+
```ts
|
|
61
|
+
import { createCelldWorld } from '@ewhauser/world-celld';
|
|
62
|
+
|
|
63
|
+
const world = createCelldWorld({
|
|
64
|
+
fleetUrl: 'http://fleet.internal:8080',
|
|
65
|
+
secret: process.env.CELLD_WORLD_SECRET!,
|
|
66
|
+
baseUrl: 'https://workflow.example.com',
|
|
67
|
+
});
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
`WORKFLOW_BASE_URL` (or `baseUrl`) is where queue cells deliver flow and step
|
|
71
|
+
requests. It must be reachable from every celld node.
|
|
72
|
+
|
|
73
|
+
## Deploy the worker
|
|
74
|
+
|
|
75
|
+
Before deploying, you need a celld fleet, `esbuild` on `PATH`, and an object
|
|
76
|
+
store that meets celld's conditional-write requirements. Refer to the
|
|
77
|
+
[celld documentation](https://github.com/denoland/celld) for fleet and storage
|
|
78
|
+
setup.
|
|
79
|
+
|
|
80
|
+
From a source checkout:
|
|
81
|
+
|
|
82
|
+
```sh
|
|
83
|
+
celld deploy ./celld-worker --bucket s3://my-cells-bucket
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
From an installed package, first copy the deployable worker into your project:
|
|
87
|
+
|
|
88
|
+
```sh
|
|
89
|
+
cp -R node_modules/@ewhauser/world-celld/celld-worker ./workflow-world
|
|
90
|
+
celld deploy ./workflow-world --bucket s3://my-cells-bucket
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
The worker rejects stateful requests unless `WORLD_SECRET` is configured. Pass
|
|
94
|
+
the same secret to the fleet and the application:
|
|
95
|
+
|
|
96
|
+
```sh
|
|
97
|
+
CELLD_VAR_WORLD_SECRET="$CELLD_WORLD_SECRET" \
|
|
98
|
+
celld --bucket s3://my-cells-bucket \
|
|
99
|
+
--listen 0.0.0.0:8080 \
|
|
100
|
+
--internal-listen 10.0.0.12:8081 \
|
|
101
|
+
--advertise 10.0.0.12:8081
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Use a secret manager rather than putting the value in `wrangler.jsonc`. Keep
|
|
105
|
+
celld's internal listener on a trusted network; the World bearer token protects
|
|
106
|
+
the worker RPC routes, not celld's administrative endpoints.
|
|
107
|
+
|
|
108
|
+
More deployment detail is in [`celld-worker/README.md`](./celld-worker/README.md).
|
|
109
|
+
|
|
110
|
+
## How it works
|
|
111
|
+
|
|
112
|
+
```text
|
|
113
|
+
Workflow application
|
|
114
|
+
|
|
|
115
|
+
| authenticated HTTP RPC
|
|
116
|
+
v
|
|
117
|
+
celld worker router
|
|
118
|
+
|-- WorkflowRunDO one cell per workflow run
|
|
119
|
+
|-- StreamDO stream chunks and run/stream indexes
|
|
120
|
+
|-- IndexDO run and hook lookup indexes
|
|
121
|
+
`-- QueueDO delayed delivery, retries, and dead letters
|
|
122
|
+
|
|
|
123
|
+
| HTTP callbacks
|
|
124
|
+
v
|
|
125
|
+
Workflow application /.well-known/workflow/v1/{flow|step}
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
The application-side package implements the Workflow `World` interface and
|
|
129
|
+
translates its storage, stream, and queue operations into RPC calls. The celld
|
|
130
|
+
worker accepts only a fixed set of methods and routes each request to the named
|
|
131
|
+
cell that owns the data.
|
|
132
|
+
|
|
133
|
+
## Configuration
|
|
134
|
+
|
|
135
|
+
Application options can be passed to `createCelldWorld()` unless an environment
|
|
136
|
+
variable is shown below.
|
|
137
|
+
|
|
138
|
+
| Option | Environment variable | Default |
|
|
139
|
+
| -------------- | --------------------- | ------------------------ |
|
|
140
|
+
| `fleetUrl` | `CELLD_FLEET_URL` | required |
|
|
141
|
+
| `secret` | `CELLD_WORLD_SECRET` | required with `fleetUrl` |
|
|
142
|
+
| `baseUrl` | `WORKFLOW_BASE_URL` | `http://localhost:$PORT` |
|
|
143
|
+
| `deploymentId` | `CELLD_DEPLOYMENT_ID` | `celld-default` |
|
|
144
|
+
| `queueShards` | — | `1` |
|
|
145
|
+
| `readPollMs` | — | `250` |
|
|
146
|
+
| `rpcTimeoutMs` | — | `30000` |
|
|
147
|
+
|
|
148
|
+
The deployed worker also accepts these celld variables:
|
|
149
|
+
|
|
150
|
+
| Variable | Default | Purpose |
|
|
151
|
+
| --------------------------- | -------- | ---------------------------------------------------- |
|
|
152
|
+
| `WORLD_SECRET` | none | Required bearer secret for RPC routes |
|
|
153
|
+
| `WORKFLOW_CALLBACK_SECRET` | none | Sent with deliveries as `x-workflow-callback-secret` |
|
|
154
|
+
| `QUEUE_MAX_ATTEMPTS` | `5` | Attempts before a message is dead-lettered |
|
|
155
|
+
| `QUEUE_MAX_INFLIGHT` | `5` | Concurrent deliveries per queue cell |
|
|
156
|
+
| `QUEUE_DELIVERY_TIMEOUT_MS` | `300000` | Timeout for an application callback |
|
|
157
|
+
|
|
158
|
+
`queueShards` is part of queue placement and is pinned when a queue cell is
|
|
159
|
+
first used. Drain pending work before changing it.
|
|
160
|
+
|
|
161
|
+
## Operational notes
|
|
162
|
+
|
|
163
|
+
- Delivery is at least once. Workflow steps and other external side effects
|
|
164
|
+
must be idempotent.
|
|
165
|
+
- Queue cells expose `stats`, `listDeadLetters`, `redriveDeadLetter`,
|
|
166
|
+
`purgeDeadLetters`, and `rearmAlarm` through the authenticated RPC endpoint.
|
|
167
|
+
- A new application URL applies to newly enqueued messages. Existing messages
|
|
168
|
+
retain the callback URL with which they were created.
|
|
169
|
+
- Fleet restarts can interrupt in-flight callbacks; expired claims are
|
|
170
|
+
delivered again.
|
|
171
|
+
|
|
172
|
+
## Development
|
|
173
|
+
|
|
174
|
+
```sh
|
|
175
|
+
pnpm format
|
|
176
|
+
pnpm check
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
`pnpm check` runs formatting, linting, type checking, tests, the build, and the
|
|
180
|
+
worker and npm-package checks. The package check inspects the exact tarball and
|
|
181
|
+
installs it in a clean temporary consumer with lifecycle scripts disabled.
|
|
182
|
+
Oxlint runs its correctness, suspicious, and performance categories with
|
|
183
|
+
type-aware checks and warnings denied. The default test suite includes the
|
|
184
|
+
upstream `@workflow/world-testing` conformance suite and does not require celld.
|
|
185
|
+
To run the live fleet tests:
|
|
186
|
+
|
|
187
|
+
```sh
|
|
188
|
+
CELLD_FLEET_URL=http://fleet.internal:8080 \
|
|
189
|
+
CELLD_WORLD_SECRET=replace-with-a-secret \
|
|
190
|
+
pnpm test:integration
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
### Local MinIO performance and loss test
|
|
194
|
+
|
|
195
|
+
The opt-in performance harness starts a fresh MinIO bucket and a single celld
|
|
196
|
+
node with Docker Compose, sends concurrent queue traffic, and verifies that
|
|
197
|
+
every accepted message reaches a successful callback. A configurable portion
|
|
198
|
+
of callbacks returns one `503` first, so the same run also exercises durable
|
|
199
|
+
redelivery. The result includes enqueue and delivery throughput plus p50, p95,
|
|
200
|
+
p99, and maximum latency for all deliveries, first-attempt successes, and
|
|
201
|
+
retried messages. It is saved to `.perf-results/minio-latest.json`.
|
|
202
|
+
|
|
203
|
+
> MinIO Community is **not a supported celld production store**. It does not
|
|
204
|
+
> implement the conditional writes celld needs for ownership fencing. This
|
|
205
|
+
> deliberately single-node harness disables celld's storage probe and tests
|
|
206
|
+
> the queue's persistence, redelivery, and performance paths only; it does not
|
|
207
|
+
> validate multi-node ownership or failover correctness.
|
|
208
|
+
|
|
209
|
+
With Docker and the Compose plugin installed, run:
|
|
210
|
+
|
|
211
|
+
```sh
|
|
212
|
+
pnpm test:perf:minio
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
The defaults send 1,000 messages with concurrency 32 across two queue cells,
|
|
216
|
+
and force every twentieth message through one `503` redelivery.
|
|
217
|
+
|
|
218
|
+
#### Reference result
|
|
219
|
+
|
|
220
|
+
The following directional baseline was recorded on August 15, 2026. MinIO and
|
|
221
|
+
celld ran as native arm64 processes because Docker was unavailable on the test
|
|
222
|
+
machine; container results will differ.
|
|
223
|
+
|
|
224
|
+
| Environment | Value |
|
|
225
|
+
| ----------- | -------------------------------------------------------------------------------------------------------- |
|
|
226
|
+
| Machine | MacBook Pro (Mac15,8), Apple M3 Max, 16 cores (12 performance, 4 efficiency), 64 GB RAM |
|
|
227
|
+
| OS | macOS 26.5.2 (25F84), arm64 |
|
|
228
|
+
| Services | celld v0.2.1; MinIO RELEASE.2025-09-07T16-13-09Z |
|
|
229
|
+
| Workload | 1,000 messages; concurrency 32; 2 queue shards; 256-byte target payload; every 20th message retried once |
|
|
230
|
+
|
|
231
|
+
| Metric | Result |
|
|
232
|
+
| ----------------------- | --------------------------------------------------------------------------------: |
|
|
233
|
+
| Delivery integrity | 1,000 accepted, 1,000 delivered, 0 missing, 0 duplicate successes, 0 dead letters |
|
|
234
|
+
| Callback attempts | 1,050, including 50 forced redeliveries |
|
|
235
|
+
| Enqueue throughput | 882.55 messages/s |
|
|
236
|
+
| Delivery throughput | 808.81 messages/s |
|
|
237
|
+
| Enqueue latency | p50 24.37 ms; p95 75.60 ms; p99 176.67 ms; max 200.36 ms |
|
|
238
|
+
| All delivery latency | p50 53.95 ms; p95 167.86 ms; p99 210.84 ms; max 312.03 ms |
|
|
239
|
+
| First-attempt latency | p50 52.55 ms; p95 166.35 ms; p99 209.80 ms; max 212.01 ms |
|
|
240
|
+
| Retried-message latency | p50 82.52 ms; p95 262.63 ms; p99 312.03 ms; max 312.03 ms |
|
|
241
|
+
| Final queue state | 0 pending, 0 in flight, 0 dead letters across both shards |
|
|
242
|
+
|
|
243
|
+
Use these numbers as a smoke-test reference, not a portable performance
|
|
244
|
+
guarantee. For regression tracking, compare repeated runs on the same machine
|
|
245
|
+
and runtime configuration. Two shards gave the best latency balance for this
|
|
246
|
+
workload; benchmark your own traffic before changing the shard count. A fleet
|
|
247
|
+
pins its shard count on first use, so changing it requires draining the queues.
|
|
248
|
+
|
|
249
|
+
Override the workload or set machine-specific regression budgets with
|
|
250
|
+
environment variables:
|
|
251
|
+
|
|
252
|
+
```sh
|
|
253
|
+
PERF_MESSAGES=10000 \
|
|
254
|
+
PERF_CONCURRENCY=64 \
|
|
255
|
+
PERF_QUEUE_SHARDS=16 \
|
|
256
|
+
PERF_MIN_DELIVERY_PER_SECOND=100 \
|
|
257
|
+
PERF_MAX_DELIVERY_P99_MS=5000 \
|
|
258
|
+
pnpm test:perf:minio
|
|
259
|
+
```
|
|
260
|
+
|
|
261
|
+
Useful controls are `PERF_PAYLOAD_BYTES`, `PERF_RETRY_EVERY`,
|
|
262
|
+
`PERF_TIMEOUT_MS`, `PERF_QUEUE_MAX_INFLIGHT`,
|
|
263
|
+
`PERF_MIN_ENQUEUE_PER_SECOND`, `PERF_MIN_DELIVERY_PER_SECOND`, and
|
|
264
|
+
`PERF_MAX_DELIVERY_P99_MS`. Throughput and latency budgets default to disabled
|
|
265
|
+
because local machines vary. Message-loss, dead-letter, message-ID, queue-drain,
|
|
266
|
+
and duplicate-success checks are always enforced.
|
|
267
|
+
|
|
268
|
+
Bug reports and focused pull requests are welcome. Please include a regression
|
|
269
|
+
test for behavior changes and run the checks above before submitting a PR.
|
|
270
|
+
|
|
271
|
+
## Supply-chain security
|
|
272
|
+
|
|
273
|
+
CI and release workflows start with no permissions and grant only the access a
|
|
274
|
+
job needs. Third-party actions are pinned to verified release commits. Release
|
|
275
|
+
builds do not use caches, and the exact npm tarball is checksummed, clean-room
|
|
276
|
+
installed, and verified again before publication.
|
|
277
|
+
|
|
278
|
+
Releases publish from GitHub-hosted runners through npm trusted publishing
|
|
279
|
+
(OIDC), without a long-lived npm token. npm records provenance for public
|
|
280
|
+
releases. The Git tag and GitHub release are created only after npm accepts the
|
|
281
|
+
matching tarball; published GitHub releases are immutable. Dependency updates
|
|
282
|
+
use cooldowns, and pnpm permits lifecycle scripts only for an explicit
|
|
283
|
+
allowlist.
|
|
284
|
+
|
|
285
|
+
See [`SECURITY.md`](./SECURITY.md) to report a vulnerability and
|
|
286
|
+
[`RELEASING.md`](./RELEASING.md) for the release procedure.
|
|
287
|
+
|
|
288
|
+
## License
|
|
289
|
+
|
|
290
|
+
Apache-2.0. Parts of the implementation are adapted from
|
|
291
|
+
[`vinnymac/worlds`](https://github.com/vinnymac/worlds), also under Apache-2.0.
|
|
292
|
+
See [`NOTICE`](./NOTICE) for details.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# world-celld worker
|
|
2
|
+
|
|
3
|
+
The celld-deployable half of `@ewhauser/world-celld`: four cell classes
|
|
4
|
+
(WorkflowRunDO, StreamDO, IndexDO, QueueDO) behind an authenticated HTTP RPC
|
|
5
|
+
router.
|
|
6
|
+
|
|
7
|
+
## Deploy
|
|
8
|
+
|
|
9
|
+
Copy this directory out of `node_modules` so `celld deploy` can bundle it
|
|
10
|
+
(its `main` must live inside the project directory), then deploy against your
|
|
11
|
+
fleet bucket:
|
|
12
|
+
|
|
13
|
+
```sh
|
|
14
|
+
cp -r node_modules/@ewhauser/world-celld/celld-worker ./workflow-world
|
|
15
|
+
celld deploy ./workflow-world --bucket s3://my-cells-bucket
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Requirements:
|
|
19
|
+
|
|
20
|
+
- `esbuild` on PATH (celld shells out to it).
|
|
21
|
+
- A bucket with conditional-write support (celld's fencing requirement).
|
|
22
|
+
- `WORLD_SECRET` injected at the node level (`CELLD_VAR_WORLD_SECRET=...`) —
|
|
23
|
+
the router fails closed with 503 while it is empty.
|
|
24
|
+
|
|
25
|
+
Point the app at any node's public listener:
|
|
26
|
+
|
|
27
|
+
```ts
|
|
28
|
+
import { createCelldWorld } from '@ewhauser/world-celld';
|
|
29
|
+
|
|
30
|
+
const world = createCelldWorld({
|
|
31
|
+
fleetUrl: 'http://fleet.internal:8080',
|
|
32
|
+
secret: process.env.CELLD_WORLD_SECRET!,
|
|
33
|
+
});
|
|
34
|
+
```
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
// Deployable celld entry. `celld deploy` requires `main` to live inside the
|
|
2
|
+
// project directory; this one-liner resolves the real worker through the
|
|
3
|
+
// application's node_modules so the deployed bundle stays in step with the
|
|
4
|
+
// installed @ewhauser/world-celld version.
|
|
5
|
+
export { IndexDO, QueueDO, StreamDO, WorkflowRunDO, default } from '@ewhauser/world-celld/worker';
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
{
|
|
2
|
+
// world-celld worker: deployed to a celld fleet with `celld deploy`.
|
|
3
|
+
// celld accepts wrangler.jsonc (not .toml) and only these keys — anything
|
|
4
|
+
// else fails the deploy loudly.
|
|
5
|
+
"name": "workflow-world",
|
|
6
|
+
"main": "worker.ts",
|
|
7
|
+
"compatibility_date": "2026-01-01",
|
|
8
|
+
"durable_objects": {
|
|
9
|
+
"bindings": [
|
|
10
|
+
{ "name": "WORKFLOW_DB", "class_name": "WorkflowRunDO" },
|
|
11
|
+
{ "name": "WORKFLOW_STREAMS", "class_name": "StreamDO" },
|
|
12
|
+
{ "name": "WORKFLOW_INDEX", "class_name": "IndexDO" },
|
|
13
|
+
{ "name": "WORKFLOW_QUEUE", "class_name": "QueueDO" },
|
|
14
|
+
],
|
|
15
|
+
},
|
|
16
|
+
"migrations": [
|
|
17
|
+
{
|
|
18
|
+
"tag": "v1",
|
|
19
|
+
"new_sqlite_classes": ["WorkflowRunDO", "StreamDO", "IndexDO", "QueueDO"],
|
|
20
|
+
},
|
|
21
|
+
],
|
|
22
|
+
"vars": {
|
|
23
|
+
// REQUIRED: bearer secret for the world's RPC surface. All /v1/rpc
|
|
24
|
+
// routes fail closed with 503 while this is empty. Inject the real value
|
|
25
|
+
// at the node level with CELLD_VAR_WORLD_SECRET instead of committing it.
|
|
26
|
+
"WORLD_SECRET": "",
|
|
27
|
+
// Optional: forwarded as x-workflow-callback-secret on queue deliveries.
|
|
28
|
+
"WORKFLOW_CALLBACK_SECRET": "",
|
|
29
|
+
// Optional queue tuning (strings): QUEUE_MAX_ATTEMPTS, QUEUE_MAX_INFLIGHT,
|
|
30
|
+
// QUEUE_DELIVERY_TIMEOUT_MS.
|
|
31
|
+
},
|
|
32
|
+
}
|