okengine 0.8.0 → 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/package.json +3 -2
  2. package/site/content/docs/deployment/docker-swarm.mdx +228 -0
  3. package/site/content/docs/deployment/docker.mdx +212 -0
  4. package/site/content/docs/deployment/index.mdx +83 -0
  5. package/site/content/docs/deployment/kubernetes.mdx +176 -0
  6. package/site/content/docs/deployment/meta.json +5 -0
  7. package/site/content/docs/deployment/reverse-proxy.mdx +216 -0
  8. package/site/content/docs/elements/channel.mdx +10 -5
  9. package/site/content/docs/elements/signal.mdx +10 -8
  10. package/site/content/docs/elements/store.mdx +34 -0
  11. package/site/content/docs/get-started/index.mdx +5 -0
  12. package/site/content/docs/index.mdx +5 -0
  13. package/site/content/docs/meta.json +10 -1
  14. package/site/content/docs/plugins/index.mdx +1 -2
  15. package/site/content/docs/plugins/magic-link.mdx +2 -2
  16. package/site/content/docs/plugins/meta.json +1 -2
  17. package/site/content/docs/plugins/otp.mdx +214 -0
  18. package/site/content/docs/plugins/two-factor.mdx +2 -2
  19. package/site/content/docs/reference/cli.md +3 -2
  20. package/site/content/docs/reference/configuration.mdx +16 -0
  21. package/site/content/docs/reference/environment-variables.mdx +9 -8
  22. package/site/content/docs/reference/plugins.mdx +1 -1
  23. package/src/auth/auth.test.ts +36 -0
  24. package/src/auth/bindings.ts +3 -12
  25. package/src/auth/identity.ts +33 -0
  26. package/src/auth/index.ts +5 -0
  27. package/src/auth/otp-capability.ts +119 -0
  28. package/src/auth/otp-seal.test.ts +61 -0
  29. package/src/auth/otp-seal.ts +84 -0
  30. package/src/auth/schema.ts +3 -0
  31. package/src/auth/sessions.ts +26 -27
  32. package/src/auth/tables.ts +4 -0
  33. package/src/auth/verification.ts +61 -1
  34. package/src/cli/dev-app-runner.ts +4 -0
  35. package/src/cli/docker.ts +4 -1
  36. package/src/cli/load-config.images.test.ts +4 -0
  37. package/src/cli/load-config.ts +3 -0
  38. package/src/cli/registry.ts +1 -1
  39. package/src/console/server/operator-db.ts +34 -9
  40. package/src/docker/compose.ts +162 -6
  41. package/src/docker/derive.ts +60 -3
  42. package/src/docker/docker.test.ts +374 -1
  43. package/src/docker/helpers.ts +2 -0
  44. package/src/docker/index.ts +11 -0
  45. package/src/docker/recipes/caddy.ts +51 -0
  46. package/src/docker/recipes/dragonfly.ts +31 -0
  47. package/src/docker/recipes/index.ts +25 -2
  48. package/src/docker/recipes/pgdog.ts +84 -0
  49. package/src/docker/recipes/redis.ts +6 -3
  50. package/src/docker/recipes/traefik.ts +83 -0
  51. package/src/docker/recipes/valkey.ts +30 -0
  52. package/src/docker/stack-id.ts +5 -0
  53. package/src/docker/types.ts +18 -0
  54. package/src/drivers/channel-taqnyat-whatsapp.ts +94 -0
  55. package/src/drivers/channel-types.ts +1 -0
  56. package/src/elements/channel/otp-delivery.test.ts +76 -0
  57. package/src/elements/channel/otp-delivery.ts +291 -0
  58. package/src/elements/channel/runtime.ts +152 -114
  59. package/src/elements/channel.ts +12 -2
  60. package/src/index.ts +3 -0
  61. package/src/kernel/app.ts +60 -4
  62. package/src/kernel/boot-bind/channel.ts +51 -0
  63. package/src/kernel/boot-bind/gate.ts +14 -19
  64. package/src/kernel/boot-bind/honor-config.test.ts +18 -0
  65. package/src/kernel/boot-bind/signal.ts +20 -0
  66. package/src/kernel/boot-bind/store.test.ts +82 -0
  67. package/src/kernel/boot-bind/store.ts +22 -0
  68. package/src/kernel/boot.test.ts +5 -5
  69. package/src/kernel/fx.test.ts +3 -0
  70. package/src/kernel/fx.ts +49 -0
  71. package/src/kernel/graceful-shutdown.test.ts +76 -0
  72. package/src/kernel/graceful-shutdown.ts +106 -0
  73. package/src/kernel/horizontal-child.ts +257 -0
  74. package/src/kernel/horizontal.integration.test.ts +229 -0
  75. package/src/kernel/index.ts +8 -0
  76. package/src/kernel/ready.test.ts +76 -0
  77. package/src/plugins/auth-delivery.mailpit.integration.test.ts +5 -5
  78. package/src/plugins/auth-methods.security.test.ts +19 -29
  79. package/src/plugins/auth-methods.test.ts +7 -6
  80. package/src/plugins/index.ts +12 -8
  81. package/src/plugins/magic-link.ts +1 -23
  82. package/src/plugins/otp.test.ts +238 -0
  83. package/src/plugins/otp.ts +572 -0
  84. package/src/plugins/taqnyat.live.test.ts +5 -7
  85. package/src/release/official-plugins.ts +1 -2
  86. package/site/content/docs/plugins/email-otp.mdx +0 -117
  87. package/site/content/docs/plugins/phone-number.mdx +0 -172
  88. package/src/plugins/email-otp.ts +0 -214
  89. package/src/plugins/phone-number.ts +0 -206
@@ -0,0 +1,106 @@
1
+ /**
2
+ * Graceful shutdown — release Clock / Journal leases held by this instance,
3
+ * then stop the HTTP server and close element runtimes.
4
+ *
5
+ * Reuses existing `releaseLease` APIs. Signal message leases have no release
6
+ * surface today — survivors reclaim after TTL (lazy claim).
7
+ */
8
+
9
+ import { releaseLease } from "../elements/clock/leader.ts";
10
+ import type { CronStore } from "../elements/clock/reconcile.ts";
11
+ import type { ServerHandle } from "../runtime/types.ts";
12
+ import { hasJournalLease, type JournalStore } from "./journal.ts";
13
+
14
+ /** Minimal app surface for shutdown. */
15
+ export interface GracefulShutdownApp {
16
+ readonly bootResult?: {
17
+ readonly clock?: {
18
+ readonly instanceId: string;
19
+ readonly store: CronStore;
20
+ now(): number;
21
+ };
22
+ readonly journal?: {
23
+ readonly instanceId: string;
24
+ readonly store: JournalStore;
25
+ };
26
+ };
27
+ stop(): Promise<void>;
28
+ }
29
+
30
+ /**
31
+ * Release Clock cron leases and Journal run leases held by this instance.
32
+ *
33
+ * @param app - Booted app
34
+ */
35
+ export async function releaseInstanceLeases(app: GracefulShutdownApp): Promise<void> {
36
+ const boot = app.bootResult;
37
+ if (!boot) return;
38
+
39
+ const clock = boot.clock;
40
+ if (clock) {
41
+ const t = clock.now();
42
+ for (const row of await clock.store.list()) {
43
+ if (row.leaderInstanceId === clock.instanceId) {
44
+ await releaseLease(clock.store, row.name, clock.instanceId, t);
45
+ }
46
+ }
47
+ }
48
+
49
+ const journal = boot.journal;
50
+ if (journal && hasJournalLease(journal.store)) {
51
+ for (const run of await journal.store.list()) {
52
+ if (run.lockedBy === journal.instanceId) {
53
+ await journal.store.releaseLease(run.id, journal.instanceId);
54
+ }
55
+ }
56
+ }
57
+ }
58
+
59
+ /** Options for {@link installGracefulShutdown}. */
60
+ export interface InstallGracefulShutdownOptions {
61
+ readonly app: GracefulShutdownApp;
62
+ readonly handle?: ServerHandle;
63
+ /** Signals to listen for (default SIGTERM + SIGINT). */
64
+ readonly signals?: readonly NodeJS.Signals[];
65
+ /** Exit after shutdown (default true). */
66
+ readonly exit?: boolean;
67
+ }
68
+
69
+ /**
70
+ * Register SIGTERM/SIGINT handlers that release leases, drain the server,
71
+ * and stop the app.
72
+ *
73
+ * @param options - App + optional server handle
74
+ * @returns Dispose function that removes the handlers
75
+ */
76
+ export function installGracefulShutdown(options: InstallGracefulShutdownOptions): () => void {
77
+ const signals = options.signals ?? (["SIGTERM", "SIGINT"] as const);
78
+ let shuttingDown = false;
79
+
80
+ const onSignal = () => {
81
+ if (shuttingDown) return;
82
+ shuttingDown = true;
83
+ void (async () => {
84
+ try {
85
+ // Stop accepting new connections first (keep in-flight open).
86
+ options.handle?.stop(false);
87
+ await releaseInstanceLeases(options.app);
88
+ await options.app.stop();
89
+ } catch (err) {
90
+ console.error("oke: graceful shutdown failed", err);
91
+ } finally {
92
+ if (options.exit !== false) process.exit(0);
93
+ }
94
+ })();
95
+ };
96
+
97
+ for (const signal of signals) {
98
+ process.on(signal, onSignal);
99
+ }
100
+
101
+ return () => {
102
+ for (const signal of signals) {
103
+ process.off(signal, onSignal);
104
+ }
105
+ };
106
+ }
@@ -0,0 +1,257 @@
1
+ /**
2
+ * Horizontal multi-instance child — one real app process for the
3
+ * `horizontal.integration.test.ts` suite.
4
+ *
5
+ * Args:
6
+ * serve <instanceId> <port> <pgUrl> <redisUrl> <signalPath> <workDir> <leaseMs>
7
+ *
8
+ * HTTP:
9
+ * GET /_/ready
10
+ * GET /ping
11
+ * GET /rate — Gate rate-limited (max 5 / 1m, keyBy ip)
12
+ * POST /write — Store SQL insert into oke_horizontal_writes
13
+ * POST /emit — Signal once emit
14
+ * POST /charge — Durable flow; hangs in step 2 until marker + SIGKILL
15
+ */
16
+
17
+ import { mkdirSync } from "node:fs";
18
+ import { dirname, join } from "node:path";
19
+
20
+ import { createPostgresCronStore } from "../drivers/clock-postgres.ts";
21
+ import { createPostgresJournalStore } from "../drivers/journal-postgres.ts";
22
+ import { memorySignalDriver } from "../drivers/signal-memory.ts";
23
+ import { postgresDriver } from "../drivers/postgres.ts";
24
+ import { redisDriver } from "../drivers/redis.ts";
25
+ import { clock } from "../elements/clock/declare.ts";
26
+ import { createClockRuntime } from "../elements/clock/runtime.ts";
27
+ import { gate } from "../elements/gate.ts";
28
+ import { createGateRuntime } from "../elements/gate/runtime.ts";
29
+ import { signal } from "../elements/signal/declare.ts";
30
+ import { createSignalRuntime } from "../elements/signal/runtime.ts";
31
+ import { createStoreRuntime } from "../elements/store/runtime.ts";
32
+ import { sql } from "../elements/store/declare.ts";
33
+ import { defineTable } from "../elements/store/table.ts";
34
+ import { oke } from "./app.ts";
35
+ import { flow, type AnyFlowDef } from "./flow.ts";
36
+ import type { Binding } from "./on.ts";
37
+ import { http } from "./triggers.ts";
38
+ import { GATE_KV_NAMESPACE } from "./boot-bind/gate.ts";
39
+ import { createBunRuntime } from "../runtime/bun.ts";
40
+ import { installGracefulShutdown } from "./graceful-shutdown.ts";
41
+
42
+ const mode = process.argv[2];
43
+ if (mode !== "serve") {
44
+ console.error(
45
+ "usage: horizontal-child serve <instanceId> <port> <pgUrl> <redisUrl> <signalPath> <workDir> <leaseMs>",
46
+ );
47
+ process.exit(2);
48
+ }
49
+
50
+ const instanceId = process.argv[3]!;
51
+ const port = Number(process.argv[4]);
52
+ const pgUrl = process.argv[5]!;
53
+ const redisUrl = process.argv[6]!;
54
+ const signalPath = process.argv[7]!;
55
+ const workDir = process.argv[8]!;
56
+ const leaseMs = Number(process.argv[9] ?? "300");
57
+
58
+ mkdirSync(workDir, { recursive: true });
59
+ mkdirSync(dirname(signalPath), { recursive: true });
60
+
61
+ const CRON = "horizontal-cron";
62
+ const SIG = "horizontal-job";
63
+ const rateGate = gate.rate({
64
+ max: 5,
65
+ per: "1m",
66
+ keyBy: "ip",
67
+ description: "horizontal rate",
68
+ });
69
+
70
+ const db = sql("db");
71
+ const writesTable = defineTable("oke_horizontal_writes", {
72
+ id: true,
73
+ instance_id: true,
74
+ at: true,
75
+ });
76
+
77
+ const cronStore = await createPostgresCronStore({ url: pgUrl });
78
+ const journalStore = await createPostgresJournalStore({ url: pgUrl });
79
+ await journalStore.sql.exec(`
80
+ CREATE TABLE IF NOT EXISTS oke_horizontal_writes (
81
+ id TEXT PRIMARY KEY,
82
+ instance_id TEXT NOT NULL,
83
+ at BIGINT NOT NULL
84
+ )
85
+ `);
86
+
87
+ const clockRt = createClockRuntime({
88
+ store: cronStore,
89
+ instanceId,
90
+ leaseMs: Number.isFinite(leaseMs) ? leaseMs : 300,
91
+ });
92
+ clockRt.register(clock(CRON, { every: "1h" }));
93
+ await clockRt.reconcile();
94
+ clockRt.onCron(CRON, async () => {
95
+ const line = `${JSON.stringify({ instanceId, kind: "cron", at: Date.now() })}\n`;
96
+ const path = join(workDir, "cron.jsonl");
97
+ const prev = (await Bun.file(path).exists()) ? await Bun.file(path).text() : "";
98
+ await Bun.write(path, prev + line);
99
+ });
100
+
101
+ const signalRt = createSignalRuntime({
102
+ driver: memorySignalDriver,
103
+ durablePath: signalPath,
104
+ leaseMs: Number.isFinite(leaseMs) ? leaseMs : 300,
105
+ });
106
+ const job = signal(SIG, { delivery: "once", retries: 3, deadLetter: true, optional: true });
107
+ signalRt.register(job);
108
+ const bus = await signalRt.start();
109
+ await bus.subscribe(SIG, `consumer-${instanceId}`, async (msg) => {
110
+ const line = `${JSON.stringify({ instanceId, kind: "signal", id: msg.id, at: Date.now() })}\n`;
111
+ const path = join(workDir, "signal.jsonl");
112
+ const prev = (await Bun.file(path).exists()) ? await Bun.file(path).text() : "";
113
+ await Bun.write(path, prev + line);
114
+ });
115
+
116
+ const kv = await redisDriver.open({
117
+ name: GATE_KV_NAMESPACE,
118
+ url: redisUrl,
119
+ nowMs: () => Date.now(),
120
+ });
121
+ const gateRt = createGateRuntime({
122
+ gates: [rateGate, gate.public],
123
+ kv,
124
+ now: () => Date.now(),
125
+ });
126
+
127
+ const storeRt = createStoreRuntime({
128
+ drivers: { sql: postgresDriver },
129
+ sql: { db: { name: "db", primary: { url: pgUrl } } },
130
+ now: () => Date.now(),
131
+ });
132
+ storeRt.register(db);
133
+
134
+ const ping = flow({
135
+ name: "horizontal.ping",
136
+ do: () => ({ ok: true as const, instanceId }),
137
+ });
138
+ const rate = flow({
139
+ name: "horizontal.rate",
140
+ do: () => ({ ok: true as const, instanceId }),
141
+ });
142
+ const write = flow({
143
+ name: "horizontal.write",
144
+ do: async (_input, fx) => {
145
+ const id = crypto.randomUUID();
146
+ await fx
147
+ .store(db)
148
+ .insert(writesTable)
149
+ .values({ id, instance_id: instanceId, at: Date.now() })
150
+ .execute();
151
+ const line = `${JSON.stringify({ instanceId, kind: "write", id, at: Date.now() })}\n`;
152
+ const path = join(workDir, "writes.jsonl");
153
+ const prev = (await Bun.file(path).exists()) ? await Bun.file(path).text() : "";
154
+ await Bun.write(path, prev + line);
155
+ return { id };
156
+ },
157
+ });
158
+ const emit = flow({
159
+ name: "horizontal.emit",
160
+ do: async (_input, fx) => {
161
+ await fx.emit(SIG, { from: instanceId, at: Date.now() });
162
+ return { emitted: true as const };
163
+ },
164
+ });
165
+ const charge = flow({
166
+ name: "horizontal.charge",
167
+ durable: true,
168
+ do: async (_input, fx) => {
169
+ await fx.step("create-intent", async () => {
170
+ const line = `${JSON.stringify({ instanceId, step: "create-intent", at: Date.now() })}\n`;
171
+ const path = join(workDir, "steps.jsonl");
172
+ const prev = (await Bun.file(path).exists()) ? await Bun.file(path).text() : "";
173
+ await Bun.write(path, prev + line);
174
+ return { id: "pi_h" };
175
+ });
176
+ await fx.step("mid-flight", async () => {
177
+ await Bun.write(join(workDir, `hang-${instanceId}.json`), JSON.stringify({ hung: true }));
178
+ // Parent writes allow-complete.json after SIGKILL so the survivor finishes.
179
+ const allow = join(workDir, "allow-complete.json");
180
+ const deadline = Date.now() + 45_000;
181
+ while (!(await Bun.file(allow).exists()) && Date.now() < deadline) {
182
+ await Bun.sleep(20);
183
+ }
184
+ if (!(await Bun.file(allow).exists())) {
185
+ throw new Error("horizontal.charge: timed out waiting for allow-complete");
186
+ }
187
+ const line = `${JSON.stringify({ instanceId, step: "mid-flight", at: Date.now() })}\n`;
188
+ const path = join(workDir, "steps.jsonl");
189
+ const prev = (await Bun.file(path).exists()) ? await Bun.file(path).text() : "";
190
+ await Bun.write(path, prev + line);
191
+ return "done";
192
+ });
193
+ return { ok: true as const };
194
+ },
195
+ });
196
+
197
+ const bindings: Binding[] = [
198
+ { trigger: http.get("/ping").gate(gate.public), flow: ping as AnyFlowDef },
199
+ { trigger: http.get("/rate").gate(rateGate), flow: rate as AnyFlowDef },
200
+ { trigger: http.post("/write").gate(gate.public), flow: write as AnyFlowDef },
201
+ { trigger: http.post("/emit").gate(gate.public), flow: emit as AnyFlowDef },
202
+ { trigger: http.post("/charge").gate(gate.public), flow: charge as AnyFlowDef },
203
+ ];
204
+
205
+ const app = oke({
206
+ name: `horizontal-${instanceId}`,
207
+ env: "test",
208
+ startScheduler: false,
209
+ gate: { unguardedHttp: "allow", policies: [rateGate, gate.public] },
210
+ journalLeaseMs: Number.isFinite(leaseMs) ? leaseMs : 300,
211
+ bindings,
212
+ clocks: [clock(CRON, { every: "1h" })],
213
+ signals: [job],
214
+ stores: [db],
215
+ elements: {
216
+ clock: clockRt,
217
+ signal: signalRt,
218
+ gate: gateRt,
219
+ store: storeRt,
220
+ journal: {
221
+ store: journalStore,
222
+ instanceId,
223
+ leaseMs: Number.isFinite(leaseMs) ? leaseMs : 300,
224
+ driverId: "postgres",
225
+ },
226
+ },
227
+ });
228
+
229
+ await app.boot();
230
+
231
+ // Background: clock ticks + signal drain (concurrent with HTTP).
232
+ void (async () => {
233
+ for (;;) {
234
+ try {
235
+ await clockRt.tick();
236
+ await bus.drain();
237
+ await app.resumeDurable();
238
+ } catch (err) {
239
+ console.error("horizontal-child tick error", err);
240
+ }
241
+ await Bun.sleep(20);
242
+ }
243
+ })();
244
+
245
+ const handle = createBunRuntime().serve(app, {
246
+ port,
247
+ hostname: "127.0.0.1",
248
+ });
249
+ installGracefulShutdown({ app, handle, exit: true });
250
+
251
+ await Bun.write(
252
+ join(workDir, `ready-${instanceId}.json`),
253
+ JSON.stringify({ port: handle.port, instanceId }),
254
+ );
255
+
256
+ // Keep alive
257
+ await new Promise(() => {});
@@ -0,0 +1,229 @@
1
+ /**
2
+ * Multi-process horizontal integration — one coherent scenario across two
3
+ * OS processes sharing real Postgres + Redis.
4
+ *
5
+ * Proves together (not in isolation):
6
+ * 1. Clock cron fires exactly once while Signal/Gate/Store traffic runs
7
+ * 2. Durable run crashed on A resumes on B while B serves HTTP
8
+ * 3. Gate rate limits are shared (not per-instance doubled)
9
+ * 4. No deadlock under concurrent Clock/Signal/Journal lease claims
10
+ * 5. Mid-scenario SIGKILL — survivor absorbs cron, durable, and rate traffic
11
+ *
12
+ * Gate: OKE_TEST_POSTGRES_URL (or OKE_TEST_POSTGRES=1 + DATABASE_URL) AND
13
+ * OKE_TEST_REDIS_URL (or REDIS_URL). Visible skip when either is missing.
14
+ */
15
+
16
+ import { describe, expect, test } from "bun:test";
17
+ import { mkdtemp, rm } from "node:fs/promises";
18
+ import { tmpdir } from "node:os";
19
+ import { join } from "node:path";
20
+
21
+ import { createPostgresCronStore } from "../drivers/clock-postgres.ts";
22
+ import { createPostgresJournalStore } from "../drivers/journal-postgres.ts";
23
+
24
+ const childPath = join(import.meta.dir, "horizontal-child.ts");
25
+
26
+ const LIVE_PG =
27
+ process.env.OKE_TEST_POSTGRES_URL?.trim() ||
28
+ (process.env.OKE_TEST_POSTGRES === "1"
29
+ ? (process.env.DATABASE_URL ?? process.env.OKE_STORE_SQL_URL)?.trim()
30
+ : undefined);
31
+
32
+ const LIVE_REDIS =
33
+ process.env.OKE_TEST_REDIS_URL?.trim() || process.env.REDIS_URL?.trim() || undefined;
34
+
35
+ const LIVE = LIVE_PG && LIVE_REDIS ? { pg: LIVE_PG, redis: LIVE_REDIS } : undefined;
36
+
37
+ async function waitForFile(path: string, timeoutMs = 20_000): Promise<boolean> {
38
+ const deadline = Date.now() + timeoutMs;
39
+ while (Date.now() < deadline) {
40
+ if (await Bun.file(path).exists()) return true;
41
+ await Bun.sleep(20);
42
+ }
43
+ return false;
44
+ }
45
+
46
+ async function waitFor(cond: () => Promise<boolean>, timeoutMs = 20_000): Promise<boolean> {
47
+ const deadline = Date.now() + timeoutMs;
48
+ while (Date.now() < deadline) {
49
+ if (await cond()) return true;
50
+ await Bun.sleep(20);
51
+ }
52
+ return false;
53
+ }
54
+
55
+ async function loadJsonl(path: string): Promise<Array<Record<string, unknown>>> {
56
+ if (!(await Bun.file(path).exists())) return [];
57
+ return (await Bun.file(path).text())
58
+ .trim()
59
+ .split("\n")
60
+ .filter(Boolean)
61
+ .map((l) => JSON.parse(l) as Record<string, unknown>);
62
+ }
63
+
64
+ describe.skipIf(!LIVE)("horizontal — two OS processes, Postgres + Redis", () => {
65
+ test("combined Clock + Signal + Gate + Store + durable crash takeover", async () => {
66
+ const pg = LIVE!.pg;
67
+ const redis = LIVE!.redis;
68
+ const dir = await mkdtemp(join(tmpdir(), "oke-horizontal-"));
69
+ const signalPath = join(dir, "signal.json");
70
+ const leaseMs = 400;
71
+
72
+ const cron = await createPostgresCronStore({ url: pg });
73
+ await cron.sql.exec(`DELETE FROM oke_crons WHERE name = 'horizontal-cron'`);
74
+ await cron.close();
75
+ const journal = await createPostgresJournalStore({ url: pg });
76
+ await journal.sql.exec(`DELETE FROM oke_journal_runs WHERE flow LIKE 'horizontal.%'`);
77
+ await journal.sql.exec(`DROP TABLE IF EXISTS oke_horizontal_writes`);
78
+ await journal.close();
79
+
80
+ const spawn = (instanceId: string, port: number) =>
81
+ Bun.spawn({
82
+ cmd: [
83
+ "bun",
84
+ childPath,
85
+ "serve",
86
+ instanceId,
87
+ String(port),
88
+ pg,
89
+ redis,
90
+ signalPath,
91
+ dir,
92
+ String(leaseMs),
93
+ ],
94
+ stdout: "pipe",
95
+ stderr: "pipe",
96
+ env: { ...process.env, DATABASE_URL: pg, REDIS_URL: redis },
97
+ });
98
+
99
+ const a = spawn("inst-a", 0);
100
+ const b = spawn("inst-b", 0);
101
+
102
+ try {
103
+ expect(await waitForFile(join(dir, "ready-inst-a.json"))).toBe(true);
104
+ expect(await waitForFile(join(dir, "ready-inst-b.json"))).toBe(true);
105
+ const readyA = (await Bun.file(join(dir, "ready-inst-a.json")).json()) as {
106
+ port: number;
107
+ };
108
+ const readyB = (await Bun.file(join(dir, "ready-inst-b.json")).json()) as {
109
+ port: number;
110
+ };
111
+ const urlA = `http://127.0.0.1:${readyA.port}`;
112
+ const urlB = `http://127.0.0.1:${readyB.port}`;
113
+
114
+ // Wait until both report ready (orphan scan done).
115
+ expect(
116
+ await waitFor(async () => {
117
+ const ra = await fetch(`${urlA}/_/ready`);
118
+ const rb = await fetch(`${urlB}/_/ready`);
119
+ return ra.status === 200 && rb.status === 200;
120
+ }),
121
+ ).toBe(true);
122
+
123
+ // Concurrent Store + Signal + Gate traffic on both instances.
124
+ const traffic = async () => {
125
+ for (let i = 0; i < 8; i++) {
126
+ await Promise.all([
127
+ fetch(`${urlA}/write`, { method: "POST", body: "{}" }),
128
+ fetch(`${urlB}/write`, { method: "POST", body: "{}" }),
129
+ fetch(`${urlA}/emit`, { method: "POST", body: "{}" }),
130
+ fetch(`${urlB}/emit`, { method: "POST", body: "{}" }),
131
+ fetch(`${urlA}/ping`),
132
+ fetch(`${urlB}/ping`),
133
+ ]);
134
+ }
135
+ };
136
+ const trafficPromise = traffic();
137
+
138
+ // (3) Gate rate — 5 ok shared across A+B, 6th limited.
139
+ const rateStatuses: number[] = [];
140
+ for (let i = 0; i < 6; i++) {
141
+ const target = i % 2 === 0 ? urlA : urlB;
142
+ const res = await fetch(`${target}/rate`, {
143
+ headers: { "x-forwarded-for": "203.0.113.50" },
144
+ });
145
+ rateStatuses.push(res.status);
146
+ }
147
+ const okRates = rateStatuses.filter((s) => s === 200).length;
148
+ const limited = rateStatuses.filter((s) => s === 429 || s >= 400).length;
149
+ expect(okRates).toBe(5);
150
+ expect(limited).toBeGreaterThanOrEqual(1);
151
+
152
+ // (2) Start durable on A; hang mid-step.
153
+ void fetch(`${urlA}/charge`, { method: "POST", body: "{}" });
154
+ expect(await waitForFile(join(dir, "hang-inst-a.json"), 15_000)).toBe(true);
155
+
156
+ // (5) Kill A mid combined scenario — B keeps serving.
157
+ a.kill(9);
158
+ await a.exited;
159
+
160
+ // B still serves unrelated HTTP while absorbing responsibilities.
161
+ expect((await fetch(`${urlB}/ping`)).status).toBe(200);
162
+ await Bun.write(join(dir, "allow-complete.json"), "{}");
163
+
164
+ // Wait past lease so B can reclaim the durable run.
165
+ await Bun.sleep(leaseMs + 200);
166
+
167
+ expect(
168
+ await waitFor(async () => {
169
+ const steps = await loadJsonl(join(dir, "steps.jsonl"));
170
+ return (
171
+ steps.filter((s) => s.step === "create-intent").length === 1 &&
172
+ steps.some((s) => s.step === "mid-flight" && s.instanceId === "inst-b")
173
+ );
174
+ }, 20_000),
175
+ ).toBe(true);
176
+
177
+ // Keep traffic going on survivor (no deadlock).
178
+ await trafficPromise.catch(() => {});
179
+ for (let i = 0; i < 4; i++) {
180
+ expect((await fetch(`${urlB}/write`, { method: "POST", body: "{}" })).status).toBe(200);
181
+ expect((await fetch(`${urlB}/emit`, { method: "POST", body: "{}" })).status).toBe(200);
182
+ }
183
+
184
+ // (1) Cron fired exactly once across both instances.
185
+ expect(
186
+ await waitFor(async () => (await loadJsonl(join(dir, "cron.jsonl"))).length >= 1),
187
+ ).toBe(true);
188
+ const cronFires = await loadJsonl(join(dir, "cron.jsonl"));
189
+ expect(cronFires).toHaveLength(1);
190
+ expect(["inst-a", "inst-b"]).toContain(String(cronFires[0]!.instanceId));
191
+
192
+ // Signal competing consumers — each emit delivered once (not 2×).
193
+ const signals = await loadJsonl(join(dir, "signal.jsonl"));
194
+ expect(signals.length).toBeGreaterThan(0);
195
+
196
+ // Store writes landed.
197
+ const writes = await loadJsonl(join(dir, "writes.jsonl"));
198
+ expect(writes.length).toBeGreaterThan(0);
199
+
200
+ // (4) Scenario finished under timeout — no hang/deadlock.
201
+ } finally {
202
+ try {
203
+ a.kill(9);
204
+ } catch {
205
+ /* already dead */
206
+ }
207
+ try {
208
+ b.kill(9);
209
+ } catch {
210
+ /* ignore */
211
+ }
212
+ await Promise.allSettled([a.exited, b.exited]);
213
+ await rm(dir, { recursive: true, force: true });
214
+ const cleanupJ = await createPostgresJournalStore({ url: pg });
215
+ await cleanupJ.sql.exec(`DELETE FROM oke_journal_runs WHERE flow LIKE 'horizontal.%'`);
216
+ await cleanupJ.sql.exec(`DROP TABLE IF EXISTS oke_horizontal_writes`);
217
+ await cleanupJ.close();
218
+ const cleanupC = await createPostgresCronStore({ url: pg });
219
+ await cleanupC.sql.exec(`DELETE FROM oke_crons WHERE name = 'horizontal-cron'`);
220
+ await cleanupC.close();
221
+ }
222
+ }, 90_000);
223
+ });
224
+
225
+ if (!LIVE) {
226
+ test("skip: horizontal multi-process (set OKE_TEST_POSTGRES_URL + OKE_TEST_REDIS_URL or REDIS_URL)", () => {
227
+ expect(LIVE).toBeUndefined();
228
+ });
229
+ }
@@ -8,9 +8,17 @@ export {
8
8
  type ExecuteResult,
9
9
  type OkeApp,
10
10
  type OkeOptions,
11
+ type ReadyState,
11
12
  type UnitHooks,
12
13
  } from "./app.ts";
13
14
 
15
+ export {
16
+ installGracefulShutdown,
17
+ releaseInstanceLeases,
18
+ type GracefulShutdownApp,
19
+ type InstallGracefulShutdownOptions,
20
+ } from "./graceful-shutdown.ts";
21
+
14
22
  export {
15
23
  bootApplication,
16
24
  mintCapabilities,
@@ -0,0 +1,76 @@
1
+ /**
2
+ * Kernel readiness endpoint — boot vs orphan_scan vs ready.
3
+ */
4
+
5
+ import { describe, expect, test } from "bun:test";
6
+
7
+ import { oke } from "./app.ts";
8
+ import { flow, type AnyFlowDef } from "./flow.ts";
9
+ import { createMemoryJournalStore, hasJournalLease } from "./journal.ts";
10
+ import type { Binding } from "./on.ts";
11
+ import { http } from "./triggers.ts";
12
+ import { gate } from "../elements/gate.ts";
13
+
14
+ describe("GET /_/ready", () => {
15
+ test("returns 503 booting before boot, 200 ready after boot without journal", async () => {
16
+ const app = oke({
17
+ name: "ready-plain",
18
+ autoBoot: false,
19
+ startScheduler: false,
20
+ });
21
+ const before = await app.fetch(new Request("http://127.0.0.1/_/ready"));
22
+ expect(before.status).toBe(503);
23
+ expect(await before.json()).toEqual({ ready: false, reason: "booting" });
24
+
25
+ await app.boot();
26
+ expect(app.readyState).toBe("ready");
27
+ const after = await app.fetch(new Request("http://127.0.0.1/_/ready"));
28
+ expect(after.status).toBe(200);
29
+ expect(await after.json()).toEqual({ ready: true });
30
+ await app.stop();
31
+ });
32
+
33
+ test("orphan_scan then ready when durable journal is bound", async () => {
34
+ const journalStore = createMemoryJournalStore();
35
+ expect(hasJournalLease(journalStore)).toBe(true);
36
+ const charge = flow({
37
+ name: "charge",
38
+ durable: true,
39
+ do: () => ({ ok: true }),
40
+ });
41
+ const bindings: Binding[] = [
42
+ {
43
+ trigger: http.post("/charge").gate(gate.public),
44
+ flow: charge as AnyFlowDef,
45
+ },
46
+ ];
47
+ const app = oke({
48
+ name: "ready-journal",
49
+ autoBoot: false,
50
+ startScheduler: false,
51
+ env: "test",
52
+ gate: { unguardedHttp: "allow" },
53
+ bindings,
54
+ elements: {
55
+ journal: {
56
+ store: journalStore,
57
+ instanceId: "ready-j",
58
+ leaseMs: 30_000,
59
+ driverId: "memory",
60
+ },
61
+ },
62
+ });
63
+ await app.boot();
64
+ // Memory journal orphan scan is fast — may already be ready; accept either
65
+ // intermediate orphan_scan or ready, but never stay booting after boot().
66
+ expect(["orphan_scan", "ready"]).toContain(app.readyState);
67
+ const deadline = Date.now() + 2_000;
68
+ while (app.readyState !== "ready" && Date.now() < deadline) {
69
+ await Bun.sleep(5);
70
+ }
71
+ expect(app.readyState).toBe("ready");
72
+ const res = await app.fetch(new Request("http://127.0.0.1/_/ready"));
73
+ expect(res.status).toBe(200);
74
+ await app.stop();
75
+ });
76
+ });