okengine 0.23.1 → 0.23.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/package.json +1 -1
  2. package/site/content/docs/ai/mcp.mdx +3 -3
  3. package/site/content/docs/elements/signal/broadcast.mdx +4 -2
  4. package/site/content/docs/reference/configuration.mdx +16 -0
  5. package/site/content/docs/reference/errors.mdx +31 -19
  6. package/src/compiler/aot.test.ts +3 -2
  7. package/src/compiler/aot.ts +45 -1
  8. package/src/compiler/dynamic.ts +1 -1
  9. package/src/compiler/http-parse.test.ts +101 -0
  10. package/src/compiler/http-parse.ts +188 -9
  11. package/src/compiler/interpret.ts +25 -2
  12. package/src/console/ui-next/dist/assets/{access-page-C_N4qvai.js → access-page-zBMhZnFm.js} +1 -1
  13. package/src/console/ui-next/dist/assets/{agent-disclosure-Cq1TxniW.js → agent-disclosure-CBj9f7Ju.js} +1 -1
  14. package/src/console/ui-next/dist/assets/{cache-glyph-BaI4uxZz.js → cache-glyph-D3MY49h3.js} +1 -1
  15. package/src/console/ui-next/dist/assets/{call-pii-button-DNyK7IIY.js → call-pii-button-L9JxdBhp.js} +1 -1
  16. package/src/console/ui-next/dist/assets/{collapsible-CRhBJMZR.js → collapsible-BvFaX-Zt.js} +1 -1
  17. package/src/console/ui-next/dist/assets/{decisions-page-D9twj8Xy.js → decisions-page-DgB76m-X.js} +1 -1
  18. package/src/console/ui-next/dist/assets/{duration-tone-CsHWlVPU.js → duration-tone-l1DSJ2Vz.js} +1 -1
  19. package/src/console/ui-next/dist/assets/{flows-page-ZaTJAhR9.js → flows-page-Bb9t1lu6.js} +1 -1
  20. package/src/console/ui-next/dist/assets/{highlighted-json-DQ-EMAfz.js → highlighted-json-DMvsi3Ti.js} +1 -1
  21. package/src/console/ui-next/dist/assets/{http-method-Cn_Fl79s.js → http-method-BwhJBTcQ.js} +1 -1
  22. package/src/console/ui-next/dist/assets/{index-CmRFeWdT.js → index-RhV2jT_7.js} +3 -3
  23. package/src/console/ui-next/dist/assets/{observability-page-Cn6R_dXz.js → observability-page-BRfUILBq.js} +1 -1
  24. package/src/console/ui-next/dist/assets/{replica-lag-Bx6EZhSe.js → replica-lag-BsceAzIG.js} +1 -1
  25. package/src/console/ui-next/dist/assets/{request-meta-DmdiUrfh.js → request-meta-C6lTyCXp.js} +1 -1
  26. package/src/console/ui-next/dist/assets/{store-page-8P6vGXnn.js → store-page-YmE806bJ.js} +1 -1
  27. package/src/console/ui-next/dist/assets/{trace-detail-sheet-BfGvpFSv.js → trace-detail-sheet-Di4irS49.js} +1 -1
  28. package/src/console/ui-next/dist/assets/{tree-expand-toggle-CWdUHbc0.js → tree-expand-toggle-DYeL6Rye.js} +1 -1
  29. package/src/console/ui-next/dist/assets/{units-page-CZOFZzay.js → units-page-07agasRA.js} +1 -1
  30. package/src/console/ui-next/dist/assets/{vault-page-ki_ishmX.js → vault-page-CHBck4n_.js} +1 -1
  31. package/src/console/ui-next/dist/index.html +1 -1
  32. package/src/console/ui-next/seed-parked-approval.ts +5 -1
  33. package/src/drivers/journal-postgres.test.ts +84 -1
  34. package/src/drivers/journal-postgres.ts +282 -32
  35. package/src/drivers/postgres.test.ts +60 -0
  36. package/src/drivers/postgres.ts +79 -5
  37. package/src/drivers/signal-compete.test.ts +6 -2
  38. package/src/drivers/signal-postgres.test.ts +170 -0
  39. package/src/drivers/signal-postgres.ts +449 -196
  40. package/src/drivers/signal-redis.test.ts +224 -0
  41. package/src/drivers/signal-redis.ts +835 -150
  42. package/src/elements/ai/approval.test.ts +15 -3
  43. package/src/elements/ai/approval.ts +15 -9
  44. package/src/elements/channel/runtime.ts +31 -4
  45. package/src/elements/channel/sql-ledger.test.ts +404 -0
  46. package/src/elements/channel/sql-ledger.ts +177 -33
  47. package/src/elements/clock/chaos-child.ts +12 -2
  48. package/src/elements/clock/durable.ts +11 -0
  49. package/src/elements/store/cache-bus.test.ts +121 -0
  50. package/src/elements/store/cache-bus.ts +278 -0
  51. package/src/elements/store/cache.test.ts +59 -0
  52. package/src/elements/store/cache.ts +157 -12
  53. package/src/elements/store/runtime.ts +44 -2
  54. package/src/elements/store/sql-errors.test.ts +131 -0
  55. package/src/elements/store/sql-errors.ts +39 -16
  56. package/src/elements/store/sql-nested-tx.test.ts +344 -0
  57. package/src/elements/store/sql-session.ts +183 -3
  58. package/src/kernel/app.ts +176 -28
  59. package/src/kernel/auto-registry.test.ts +5 -1
  60. package/src/kernel/boot-bind/signal.ts +13 -2
  61. package/src/kernel/boot-bind/store.ts +16 -1
  62. package/src/kernel/boot.test.ts +2 -2
  63. package/src/kernel/boot.ts +41 -11
  64. package/src/kernel/boundary-contract.ts +7 -0
  65. package/src/kernel/builtin-errors.ts +2 -0
  66. package/src/kernel/call.test.ts +1 -0
  67. package/src/kernel/fx-sql-handle.ts +7 -1
  68. package/src/kernel/horizontal-child.ts +54 -1
  69. package/src/kernel/horizontal.integration.test.ts +131 -7
  70. package/src/kernel/idempotency.test.ts +5 -6
  71. package/src/kernel/journal-boot.test.ts +83 -6
  72. package/src/kernel/journal.test.ts +44 -0
  73. package/src/kernel/journal.ts +171 -26
  74. package/src/mcp/authorization.ts +5 -11
  75. package/src/mcp/confirmation.ts +203 -38
  76. package/src/mcp/docs-server.ts +7 -2
  77. package/src/mcp/index.ts +4 -0
  78. package/src/mcp/mcp.test.ts +321 -33
  79. package/src/mcp/server.ts +8 -3
  80. package/src/mcp/session.ts +7 -0
  81. package/src/mcp/tools.ts +68 -19
  82. package/src/runtime/bun.ts +5 -0
  83. package/src/runtime/types.ts +5 -0
@@ -24,6 +24,7 @@ import { postgresDriver } from "../../drivers/postgres.ts";
24
24
  import { redisDriver } from "../../drivers/redis.ts";
25
25
  import { s3Driver } from "../../drivers/s3.ts";
26
26
  import type { FilesDriver, IndexDriver, KvDriver, SqlDriver } from "../../drivers/types.ts";
27
+ import { bunRedisCacheClient } from "../../elements/store/cache-bus.ts";
27
28
  import { createStoreRuntime, type StoreRuntime } from "../../elements/store.ts";
28
29
  import type { SqlRuntimeBinding } from "../../elements/store/runtime.ts";
29
30
  import type { StoreDecl } from "../../elements/store/declare.ts";
@@ -171,7 +172,14 @@ export function bindStore(
171
172
  const autoPush = resolveAutoPush(options);
172
173
  const domainDdl = resolveDomainDdlMode(env, autoPush);
173
174
 
174
- const store = createStoreRuntime({
175
+ let store!: StoreRuntime;
176
+ const cacheBus =
177
+ kvId === "redis" && kvUrl
178
+ ? { kind: "redis" as const, redis: bunRedisCacheClient(kvUrl) }
179
+ : sqlId === "postgres" && sqlUrl
180
+ ? { kind: "postgres" as const, sql: () => store.primarySql() }
181
+ : undefined;
182
+ store = createStoreRuntime({
175
183
  drivers: {
176
184
  sql: sqlDriverFor(sqlId),
177
185
  kv: kvDriverFor(kvId),
@@ -183,6 +191,13 @@ export function bindStore(
183
191
  files: filesBindings,
184
192
  index: indexBindings,
185
193
  now,
194
+ ...(options.cache?.maxEntries !== undefined
195
+ ? { cacheMaxEntries: options.cache.maxEntries }
196
+ : {}),
197
+ ...(options.cache?.defaultTtlMs !== undefined
198
+ ? { cacheDefaultTtlMs: options.cache.defaultTtlMs }
199
+ : {}),
200
+ ...(cacheBus ? { cacheBus } : {}),
186
201
  domainDdl,
187
202
  sqlUrl,
188
203
  });
@@ -285,8 +285,8 @@ describe("boot — cron autostart via real timer loop", () => {
285
285
  // Drive the wall-clock interval that boot installed — not clock.advance /
286
286
  // runNow / dispatchEvery (those prove the harness, not autostart).
287
287
  jest.advanceTimersByTime(1000);
288
- // Interval callback does `void clock.tick()`; drain the async store walk.
289
- for (let i = 0; i < 50; i++) await Promise.resolve();
288
+ // The tick is guarded and async. Wait until it settles, but not forever.
289
+ for (let i = 0; i < 50 && ran < 1; i++) await Promise.resolve();
290
290
 
291
291
  expect(ran).toBeGreaterThanOrEqual(1);
292
292
 
@@ -182,6 +182,13 @@ export interface BootOptions {
182
182
  readonly onDurableResume?: () => void | Promise<void>;
183
183
  /** Durable-run lease duration ms (default 30_000 — matches Signal claims). */
184
184
  readonly journalLeaseMs?: number;
185
+ /** Opt-in durable code version. Unset skips the stamp and the check. */
186
+ readonly codeVersion?: string;
187
+ /** Tier-1 cache bounds. Defaults: 10_000 entries, 60s TTL. */
188
+ readonly cache?: {
189
+ readonly maxEntries?: number;
190
+ readonly defaultTtlMs?: number;
191
+ };
185
192
  /** Injectable clock for test / frozen harnesses. */
186
193
  readonly now?: () => number;
187
194
  /**
@@ -539,16 +546,20 @@ export async function bootApplication(input: BootOptions = {}): Promise<BootResu
539
546
 
540
547
  // 4d. Scheduler — one timer drives clock ticks, durable-run resume, heartbeat.
541
548
  let schedulerTimer: ReturnType<typeof setInterval> | undefined;
549
+ const inflight = new Map<string, Promise<void>>();
550
+ let stopping = false;
542
551
  const startScheduler = options.startScheduler ?? env !== "test";
543
552
  const signalBus = signal?.bus ?? undefined;
544
553
  let refreshChannel: (() => Promise<void>) | undefined;
554
+ const pollCache = store?.cacheInvalidation === "postgres";
545
555
  if (
546
556
  startScheduler &&
547
557
  (clock !== undefined ||
548
558
  journal !== undefined ||
549
559
  instances !== undefined ||
550
560
  signalBus !== undefined ||
551
- needs.channel)
561
+ needs.channel ||
562
+ pollCache)
552
563
  ) {
553
564
  const period = options.schedulerIntervalMs ?? 1000;
554
565
  const clockRt = clock;
@@ -560,18 +571,30 @@ export async function bootApplication(input: BootOptions = {}): Promise<BootResu
560
571
  // Unexpected scheduler failures stay visible.
561
572
  console.error(err);
562
573
  };
574
+ const spawn = (name: string, task: () => Promise<unknown>): void => {
575
+ if (stopping || inflight.has(name)) return;
576
+ const run = Promise.resolve()
577
+ .then(task)
578
+ .then(() => undefined)
579
+ .catch(ignoreBenignSql)
580
+ .finally(() => {
581
+ inflight.delete(name);
582
+ });
583
+ inflight.set(name, run);
584
+ };
563
585
  schedulerTimer = setInterval(() => {
564
- if (signalBus) void signalBus.drain().catch(ignoreBenignSql);
565
- if (refreshChannel) void refreshChannel().catch(ignoreBenignSql);
566
- if (clockRt) void Promise.resolve(clockRt.tick()).catch(ignoreBenignSql);
567
- if (journal && durableResume) void Promise.resolve(durableResume()).catch(ignoreBenignSql);
568
- if (fleet) void fleet.maybeHeartbeat().catch(ignoreBenignSql);
586
+ if (signalBus) spawn("signal", () => signalBus.drain());
587
+ if (refreshChannel) spawn("channel", () => refreshChannel!());
588
+ if (clockRt) spawn("clock", () => Promise.resolve(clockRt.tick()));
589
+ if (journal && durableResume) spawn("durable", () => Promise.resolve(durableResume()));
590
+ if (fleet) spawn("fleet", () => fleet.maybeHeartbeat());
591
+ if (pollCache && store) spawn("cache", () => store.pollCacheInvalidations());
569
592
  const stem = ["run", "events"].join("-");
570
- void import(`../elements/ai/${stem}.ts`)
571
- .then((mod: typeof import("../elements/ai/run-events.ts")) =>
572
- mod.sweepInstalledAgentEvents(),
573
- )
574
- .catch(ignoreBenignSql);
593
+ spawn("agents", () =>
594
+ import(`../elements/ai/${stem}.ts`).then(
595
+ (mod: typeof import("../elements/ai/run-events.ts")) => mod.sweepInstalledAgentEvents(),
596
+ ),
597
+ );
575
598
  }, period);
576
599
  schedulerTimer.unref?.();
577
600
  }
@@ -653,7 +676,12 @@ export async function bootApplication(input: BootOptions = {}): Promise<BootResu
653
676
  instanceId,
654
677
  instances,
655
678
  capabilities,
679
+ /**
680
+ * Stop the scheduler interval and wake timers.
681
+ * Does not wait for a tick that has already started — {@link BootResult.close} does.
682
+ */
656
683
  stopScheduler() {
684
+ stopping = true;
657
685
  if (schedulerTimer !== undefined) {
658
686
  clearInterval(schedulerTimer);
659
687
  schedulerTimer = undefined;
@@ -661,11 +689,13 @@ export async function bootApplication(input: BootOptions = {}): Promise<BootResu
661
689
  clock?.stopWakes();
662
690
  },
663
691
  async close() {
692
+ stopping = true;
664
693
  if (schedulerTimer !== undefined) {
665
694
  clearInterval(schedulerTimer);
666
695
  schedulerTimer = undefined;
667
696
  }
668
697
  clock?.stopWakes();
698
+ await Promise.all([...inflight.values()]);
669
699
  await signal?.close();
670
700
  await vault?.close();
671
701
  await runs?.flush();
@@ -30,6 +30,13 @@ export interface BoundaryContract<
30
30
  * (`oke doctor --diff` / CI gate).
31
31
  */
32
32
  readonly breaking?: boolean;
33
+ /** Per-route body cap in bytes. Falls back to the app `maxRequestBodySize`. */
34
+ readonly maxBodySize?: number;
35
+ /**
36
+ * `required` (default) rejects a JSON-shaped body that is not
37
+ * `application/json` or `+json`. `any` keeps the previous parse-or-string behavior.
38
+ */
39
+ readonly jsonContentType?: "required" | "any";
33
40
  }
34
41
 
35
42
  /** Infer validated input from a {@link BoundaryContract} bag. */
@@ -15,6 +15,7 @@ export const BUILTIN_ERROR_STATUS = {
15
15
  Conflict: 409,
16
16
  ForeignKey: 409,
17
17
  UnsupportedMediaType: 415,
18
+ PayloadTooLarge: 413,
18
19
  RateLimited: 429,
19
20
  AuthRateLimited: 429,
20
21
  InvalidQuery: 400,
@@ -78,6 +79,7 @@ export type BuiltinErrorMap = {
78
79
  readonly Conflict: BuiltinErrorBag;
79
80
  readonly ForeignKey: BuiltinErrorBag;
80
81
  readonly UnsupportedMediaType: BuiltinErrorBag;
82
+ readonly PayloadTooLarge: BuiltinErrorBag;
81
83
  readonly RateLimited: { readonly retryAfterMs?: number };
82
84
  readonly AuthRateLimited: BuiltinErrorBag;
83
85
  readonly InvalidQuery: BuiltinErrorBag;
@@ -112,6 +112,7 @@ describe("fx.call — untriggered flows", () => {
112
112
  const a = await app.fetch(
113
113
  new Request("http://localhost/work", {
114
114
  method: "POST",
115
+ headers: { "content-type": "application/json" },
115
116
  body: JSON.stringify({ v: 10 }),
116
117
  }),
117
118
  );
@@ -17,7 +17,7 @@ import { schemaTableName, sqlTableRef } from "../manifest/sql-resource.ts";
17
17
  import type { CapabilityToken } from "./capability.ts";
18
18
  import { DryRunWriteIsolationError, isDryRun } from "./dry-run.ts";
19
19
  import type { EffectExternal } from "./effects.ts";
20
- import { runInSignalTransaction } from "./signal-tx.ts";
20
+ import { currentSignalTransaction, runInSignalTransaction } from "./signal-tx.ts";
21
21
 
22
22
  /** Capability-gated effect runner from {@link createFx}. */
23
23
  type Gated = <T>(
@@ -253,6 +253,12 @@ export function createGatedSqlHandle(options: GatedSqlHandleOptions): SqlStoreHa
253
253
  async transaction(fn) {
254
254
  refuseDryRunWrite();
255
255
  const h = await ensure();
256
+ // Already inside a signal transaction: do not begin or commit another
257
+ // one. sql-session opens a child staging scope and merges it; only the
258
+ // outermost commit publishes.
259
+ if (currentSignalTransaction()) {
260
+ return h.transaction(() => fn(api));
261
+ }
256
262
  const signalTx = options.beginSignal ? await options.beginSignal() : undefined;
257
263
  try {
258
264
  const result = await h.transaction(async () => {
@@ -20,6 +20,8 @@ import { dirname, join } from "node:path";
20
20
  import { createPostgresCronStore } from "../drivers/clock-postgres.ts";
21
21
  import { createPostgresJournalStore } from "../drivers/journal-postgres.ts";
22
22
  import { memorySignalDriver } from "../drivers/signal-memory.ts";
23
+ import { createBunSignalRedisClient, redisSignalDriver } from "../drivers/signal-redis.ts";
24
+ import { bunRedisCacheClient } from "../elements/store/cache-bus.ts";
23
25
  import { postgresDriver } from "../drivers/postgres.ts";
24
26
  import { redisDriver } from "../drivers/redis.ts";
25
27
  import { clock } from "../elements/clock/declare.ts";
@@ -99,14 +101,40 @@ clockRt.onCron(CRON, async () => {
99
101
  await Bun.write(path, prev + line);
100
102
  });
101
103
 
104
+ const redisSignals = process.env.OKE_HORIZONTAL_REDIS_SIGNALS === "1";
102
105
  const signalRt = createSignalRuntime({
103
- driver: memorySignalDriver,
106
+ driver: redisSignals ? redisSignalDriver : memorySignalDriver,
107
+ ...(redisSignals
108
+ ? { redis: createBunSignalRedisClient(redisUrl), compete: true, consumerId: instanceId }
109
+ : {}),
104
110
  durablePath: signalPath,
105
111
  leaseMs: Number.isFinite(leaseMs) ? leaseMs : 300,
106
112
  });
107
113
  const job = signal.once(SIG, { retries: 3, deadLetter: true, optional: true });
108
114
  signalRt.register(job);
115
+ const news = signal.broadcast("horizontal-news", { optional: true });
116
+ const feed = signal.live("horizontal-feed", { optional: true });
117
+ if (redisSignals) {
118
+ signalRt.register(news);
119
+ signalRt.register(feed);
120
+ }
109
121
  const bus = await signalRt.start();
122
+ if (redisSignals) {
123
+ await bus.subscribe("horizontal-news", `news-${instanceId}`, async (msg) => {
124
+ const line = `${JSON.stringify({ instanceId, kind: "broadcast", id: msg.id, at: Date.now() })}\n`;
125
+ const path = join(workDir, "broadcast.jsonl");
126
+ const prev = (await Bun.file(path).exists()) ? await Bun.file(path).text() : "";
127
+ await Bun.write(path, prev + line);
128
+ });
129
+ void (async () => {
130
+ for await (const event of bus.live("horizontal-feed")) {
131
+ const line = `${JSON.stringify({ instanceId, kind: "live", id: event.id, at: Date.now() })}\n`;
132
+ const path = join(workDir, "live.jsonl");
133
+ const prev = (await Bun.file(path).exists()) ? await Bun.file(path).text() : "";
134
+ await Bun.write(path, prev + line);
135
+ }
136
+ })();
137
+ }
110
138
  await bus.subscribe(SIG, `consumer-${instanceId}`, async (msg) => {
111
139
  const line = `${JSON.stringify({ instanceId, kind: "signal", id: msg.id, at: Date.now() })}\n`;
112
140
  const path = join(workDir, "signal.jsonl");
@@ -129,6 +157,7 @@ const storeRt = createStoreRuntime({
129
157
  drivers: { sql: postgresDriver },
130
158
  sql: { db: { name: "db", primary: { url: pgUrl } } },
131
159
  now: () => Date.now(),
160
+ cacheBus: { kind: "redis", redis: bunRedisCacheClient(redisUrl), origin: instanceId },
132
161
  });
133
162
  storeRt.register(db);
134
163
 
@@ -138,7 +167,16 @@ const ping = flow("horizontal.ping", {
138
167
  const rate = flow("horizontal.rate", {
139
168
  do: () => ({ ok: true as const, instanceId }),
140
169
  });
170
+ let cachedReads = 0;
171
+ const cached = flow("horizontal.cached", {
172
+ effects: { reads: ["sql:db"] },
173
+ do: () => {
174
+ cachedReads += 1;
175
+ return { reads: cachedReads, instanceId };
176
+ },
177
+ });
141
178
  const write = flow("horizontal.write", {
179
+ effects: { writes: ["sql:db"] },
142
180
  do: async (_input, fx) => {
143
181
  const id = okid();
144
182
  await fx
@@ -159,6 +197,18 @@ const emit = flow("horizontal.emit", {
159
197
  return { emitted: true as const };
160
198
  },
161
199
  });
200
+ const publishNews = flow("horizontal.news", {
201
+ do: async (_input, fx) => {
202
+ await fx.emit("horizontal-news", { from: instanceId, at: Date.now() });
203
+ return { emitted: true as const };
204
+ },
205
+ });
206
+ const publishFeed = flow("horizontal.feed", {
207
+ do: async (_input, fx) => {
208
+ await fx.emit("horizontal-feed", { from: instanceId, at: Date.now() });
209
+ return { emitted: true as const };
210
+ },
211
+ });
162
212
  const charge = flow("horizontal.charge", {
163
213
  durable: true,
164
214
  do: async (_input, fx) => {
@@ -193,8 +243,11 @@ const charge = flow("horizontal.charge", {
193
243
  const bindings: Binding[] = [
194
244
  { trigger: http.get("/ping").public(), flow: ping as AnyFlowDef },
195
245
  { trigger: http.get("/rate").gate(rateGate), flow: rate as AnyFlowDef },
246
+ { trigger: http.get("/cached").public(), flow: cached as AnyFlowDef },
196
247
  { trigger: http.post("/write").public(), flow: write as AnyFlowDef },
197
248
  { trigger: http.post("/emit").public(), flow: emit as AnyFlowDef },
249
+ { trigger: http.post("/news").public(), flow: publishNews as AnyFlowDef },
250
+ { trigger: http.post("/feed").public(), flow: publishFeed as AnyFlowDef },
198
251
  { trigger: http.post("/charge").public(), flow: charge as AnyFlowDef },
199
252
  ];
200
253
 
@@ -124,10 +124,26 @@ describe.skipIf(!LIVE)("horizontal — two OS processes, Postgres + Redis", () =
124
124
  const traffic = async () => {
125
125
  for (let i = 0; i < 8; i++) {
126
126
  await Promise.all([
127
- fetch(`${urlA}/write`, { method: "POST", body: "{}" }),
128
- fetch(`${urlB}/write`, { method: "POST", body: "{}" }),
129
- fetch(`${urlA}/emit`, { method: "POST", body: "{}" }),
130
- fetch(`${urlB}/emit`, { method: "POST", body: "{}" }),
127
+ fetch(`${urlA}/write`, {
128
+ method: "POST",
129
+ headers: { "content-type": "application/json" },
130
+ body: "{}",
131
+ }),
132
+ fetch(`${urlB}/write`, {
133
+ method: "POST",
134
+ headers: { "content-type": "application/json" },
135
+ body: "{}",
136
+ }),
137
+ fetch(`${urlA}/emit`, {
138
+ method: "POST",
139
+ headers: { "content-type": "application/json" },
140
+ body: "{}",
141
+ }),
142
+ fetch(`${urlB}/emit`, {
143
+ method: "POST",
144
+ headers: { "content-type": "application/json" },
145
+ body: "{}",
146
+ }),
131
147
  fetch(`${urlA}/ping`),
132
148
  fetch(`${urlB}/ping`),
133
149
  ]);
@@ -150,7 +166,11 @@ describe.skipIf(!LIVE)("horizontal — two OS processes, Postgres + Redis", () =
150
166
  expect(limited).toBeGreaterThanOrEqual(1);
151
167
 
152
168
  // (2) Start durable on A; hang mid-step.
153
- void fetch(`${urlA}/charge`, { method: "POST", body: "{}" });
169
+ void fetch(`${urlA}/charge`, {
170
+ method: "POST",
171
+ headers: { "content-type": "application/json" },
172
+ body: "{}",
173
+ });
154
174
  expect(await waitForFile(join(dir, "hang-inst-a.json"), 15_000)).toBe(true);
155
175
 
156
176
  // (5) Kill A mid combined scenario — B keeps serving.
@@ -177,8 +197,24 @@ describe.skipIf(!LIVE)("horizontal — two OS processes, Postgres + Redis", () =
177
197
  // Keep traffic going on survivor (no deadlock).
178
198
  await trafficPromise.catch(() => {});
179
199
  for (let i = 0; i < 4; i++) {
180
- expect((await fetch(`${urlB}/write`, { method: "POST", body: "{}" })).status).toBe(200);
181
- expect((await fetch(`${urlB}/emit`, { method: "POST", body: "{}" })).status).toBe(200);
200
+ expect(
201
+ (
202
+ await fetch(`${urlB}/write`, {
203
+ method: "POST",
204
+ headers: { "content-type": "application/json" },
205
+ body: "{}",
206
+ })
207
+ ).status,
208
+ ).toBe(200);
209
+ expect(
210
+ (
211
+ await fetch(`${urlB}/emit`, {
212
+ method: "POST",
213
+ headers: { "content-type": "application/json" },
214
+ body: "{}",
215
+ })
216
+ ).status,
217
+ ).toBe(200);
182
218
  }
183
219
 
184
220
  // (1) Cron fired exactly once across both instances.
@@ -220,6 +256,94 @@ describe.skipIf(!LIVE)("horizontal — two OS processes, Postgres + Redis", () =
220
256
  await cleanupC.close();
221
257
  }
222
258
  }, 90_000);
259
+
260
+ test("broadcast, live, and cache invalidation reach the other instance", async () => {
261
+ const pg = LIVE!.pg;
262
+ const redis = LIVE!.redis;
263
+ const dir = await mkdtemp(join(tmpdir(), "oke-horizontal-bus-"));
264
+ const signalPath = join(dir, "signal.json");
265
+ const post = (url: string) =>
266
+ fetch(url, {
267
+ method: "POST",
268
+ headers: { "content-type": "application/json" },
269
+ body: "{}",
270
+ });
271
+ const spawn = (instanceId: string, port: number) =>
272
+ Bun.spawn({
273
+ cmd: [
274
+ "bun",
275
+ childPath,
276
+ "serve",
277
+ instanceId,
278
+ String(port),
279
+ pg,
280
+ redis,
281
+ signalPath,
282
+ dir,
283
+ "5000",
284
+ ],
285
+ stdout: "pipe",
286
+ stderr: "pipe",
287
+ env: {
288
+ ...process.env,
289
+ DATABASE_URL: pg,
290
+ REDIS_URL: redis,
291
+ OKE_HORIZONTAL_REDIS_SIGNALS: "1",
292
+ },
293
+ });
294
+ const a = spawn("bus-a", 0);
295
+ const b = spawn("bus-b", 0);
296
+ try {
297
+ expect(await waitForFile(join(dir, "ready-bus-a.json"))).toBe(true);
298
+ expect(await waitForFile(join(dir, "ready-bus-b.json"))).toBe(true);
299
+ const readyA = (await Bun.file(join(dir, "ready-bus-a.json")).json()) as { port: number };
300
+ const readyB = (await Bun.file(join(dir, "ready-bus-b.json")).json()) as { port: number };
301
+ const urlA = `http://127.0.0.1:${readyA.port}`;
302
+ const urlB = `http://127.0.0.1:${readyB.port}`;
303
+ const first = (await (await fetch(`${urlA}/cached`)).json()) as { data: { reads: number } };
304
+ const second = (await (await fetch(`${urlA}/cached`)).json()) as { data: { reads: number } };
305
+ expect(first.data.reads).toBe(1);
306
+ expect(second.data.reads).toBe(1);
307
+ expect((await post(`${urlB}/write`)).status).toBe(200);
308
+ expect(
309
+ await waitFor(async () => {
310
+ const body = (await (await fetch(`${urlA}/cached`)).json()) as {
311
+ data: { reads: number };
312
+ };
313
+ return body.data.reads === 2;
314
+ }),
315
+ ).toBe(true);
316
+ expect((await post(`${urlA}/news`)).status).toBe(200);
317
+ expect((await post(`${urlA}/feed`)).status).toBe(200);
318
+ expect(
319
+ await waitFor(async () => {
320
+ const broadcast = await loadJsonl(join(dir, "broadcast.jsonl"));
321
+ const live = await loadJsonl(join(dir, "live.jsonl"));
322
+ const ids = (rows: Array<Record<string, unknown>>) =>
323
+ new Set(rows.map((row) => row.instanceId));
324
+ return (
325
+ ids(broadcast).has("bus-a") &&
326
+ ids(broadcast).has("bus-b") &&
327
+ ids(live).has("bus-a") &&
328
+ ids(live).has("bus-b")
329
+ );
330
+ }),
331
+ ).toBe(true);
332
+ } finally {
333
+ try {
334
+ a.kill(9);
335
+ } catch {
336
+ /* already dead */
337
+ }
338
+ try {
339
+ b.kill(9);
340
+ } catch {
341
+ /* ignore */
342
+ }
343
+ await Promise.allSettled([a.exited, b.exited]);
344
+ await rm(dir, { recursive: true, force: true });
345
+ }
346
+ }, 60_000);
223
347
  });
224
348
 
225
349
  if (!LIVE) {
@@ -349,13 +349,12 @@ describe("HTTP idempotency", () => {
349
349
  expect(await durableStore.idempotency!.forfeit(held!, held!.claimToken)).toBe(true);
350
350
  await Bun.sleep(120);
351
351
  const resumed = durable.fetch(post({ n: 1 }));
352
- expect(await waitFor(() => blockers.length === 2)).toBe(true);
353
- expect(step1).toEqual(["step1"]);
352
+ // The holder's heartbeat kept the lease, so the same process does not
353
+ // start a second session for the run still inside mid-flight.
354
+ expect((await resumed).status).toBe(409);
355
+ expect(blockers).toHaveLength(1);
354
356
  blockers[0]!();
355
- blockers[1]!();
356
- // The forfeited holder kept the old lease token. Its later write is rejected.
357
- await expect(hung).rejects.toThrow(/OKE1074/);
358
- expect((await resumed).status).toBe(200);
357
+ expect((await hung).status).toBe(200);
359
358
  expect(step1).toEqual(["step1"]);
360
359
  }, 30_000);
361
360
 
@@ -22,6 +22,7 @@ import {
22
22
  type PostgresJournalStore,
23
23
  } from "../drivers/journal-postgres.ts";
24
24
  import { oke, type OkeApp } from "./app.ts";
25
+ import { clearJournalHeartbeats } from "./journal.ts";
25
26
  import type { JournalRuntime } from "./boot-bind/journal.ts";
26
27
  import { flow, resetFlowSeq, type AnyFlowDef } from "./flow.ts";
27
28
  import type { Binding } from "./on.ts";
@@ -47,6 +48,7 @@ async function bootDurableApp(options: {
47
48
  readonly name: string;
48
49
  readonly journal: JournalRuntime;
49
50
  readonly bindings: readonly Binding[];
51
+ readonly now?: () => number;
50
52
  }): Promise<OkeApp> {
51
53
  const app = oke({
52
54
  name: options.name,
@@ -57,6 +59,7 @@ async function bootDurableApp(options: {
57
59
  // oke() in the process — sibling apps need their own copies.
58
60
  bindings: [...options.bindings],
59
61
  elements: { journal: options.journal },
62
+ ...(options.now ? { fx: { now: options.now } } : {}),
60
63
  });
61
64
  await app.boot();
62
65
  return app;
@@ -127,11 +130,18 @@ describe("journal boot — shared store, real boot paths (fake SQL)", () => {
127
130
  bindings,
128
131
  });
129
132
  // Start the run through the real HTTP path; it hangs inside step 2.
130
- void appA.fetch(new Request("http://localhost/charge", { method: "POST", body: "{}" }));
133
+ void appA.fetch(
134
+ new Request("http://localhost/charge", {
135
+ method: "POST",
136
+ headers: { "content-type": "application/json" },
137
+ body: "{}",
138
+ }),
139
+ );
131
140
  expect(await waitFor(async () => blockers.length === 1)).toBe(true);
132
141
  const runId = (await store.list())[0]!.id;
133
- // "Crash": no commit, no release — the lease (120ms) is left to expire.
134
- // appA is deliberately never stopped; its promise stays dangling.
142
+ // "Crash": the process dies, so its heartbeat timers die with it.
143
+ // The in-flight promise stays dangling and the lease (120ms) expires.
144
+ clearJournalHeartbeats();
135
145
 
136
146
  await Bun.sleep(300);
137
147
  const appB = await bootDurableApp({
@@ -165,7 +175,11 @@ describe("journal boot — shared store, real boot paths (fake SQL)", () => {
165
175
  bindings,
166
176
  });
167
177
  const parked = await appA.fetch(
168
- new Request("http://localhost/sleep", { method: "POST", body: "{}" }),
178
+ new Request("http://localhost/sleep", {
179
+ method: "POST",
180
+ headers: { "content-type": "application/json" },
181
+ body: "{}",
182
+ }),
169
183
  );
170
184
  expect(parked.status).toBe(204); // durable park — lease released, row sleeping
171
185
  const runId = (await store.list())[0]!.id;
@@ -198,7 +212,13 @@ describe("journal boot — shared store, real boot paths (fake SQL)", () => {
198
212
  journal: journalRuntime(store, "A", 5_000),
199
213
  bindings,
200
214
  });
201
- await appA.fetch(new Request("http://localhost/sleep", { method: "POST", body: "{}" }));
215
+ await appA.fetch(
216
+ new Request("http://localhost/sleep", {
217
+ method: "POST",
218
+ headers: { "content-type": "application/json" },
219
+ body: "{}",
220
+ }),
221
+ );
202
222
  await appA.stop();
203
223
 
204
224
  // Both survivors boot while the sleep is future (orphan scan skips it).
@@ -217,10 +237,61 @@ describe("journal boot — shared store, real boot paths (fake SQL)", () => {
217
237
  await Promise.all([appB.resumeDurable(), appC.resumeDurable()]);
218
238
 
219
239
  expect(confirms).toHaveLength(1);
240
+
220
241
  await appB.stop();
221
242
  await appC.stop();
222
243
  });
223
244
 
245
+ test("resume uses a live clock: a lease written after the tick sample is not already expired", async () => {
246
+ let clock = 10_000_000;
247
+ const store = await createPostgresJournalStore({ sql: createPostgresJournalFake() });
248
+ const confirms: string[] = [];
249
+ const bindings = [sleeperBinding("120ms", confirms)];
250
+ const acquiredAt: number[] = [];
251
+ const acquire = store.acquireLease.bind(store);
252
+ store.acquireLease = async (runId, instanceId, at, leaseMs) => {
253
+ acquiredAt.push(at);
254
+ return acquire(runId, instanceId, at, leaseMs);
255
+ };
256
+ const claim = store.claimDueSleep.bind(store);
257
+ store.claimDueSleep = async (instanceId, at, leaseMs) => {
258
+ const due = await claim(instanceId, at, leaseMs);
259
+ // The tick already sampled `at`. A live run clock must see this jump.
260
+ if (due) clock += 2_700;
261
+ return due;
262
+ };
263
+
264
+ const appA = await bootDurableApp({
265
+ name: "clock-a",
266
+ journal: journalRuntime(store, "A", 5_000),
267
+ bindings,
268
+ now: () => clock,
269
+ });
270
+ await appA.fetch(
271
+ new Request("http://localhost/sleep", {
272
+ method: "POST",
273
+ headers: { "content-type": "application/json" },
274
+ body: "{}",
275
+ }),
276
+ );
277
+ await appA.stop();
278
+
279
+ const appB = await bootDurableApp({
280
+ name: "clock-b",
281
+ journal: journalRuntime(store, "B", 5_000),
282
+ bindings,
283
+ now: () => clock,
284
+ });
285
+ const before = clock;
286
+ clock = before + 500;
287
+ acquiredAt.length = 0;
288
+ await appB.resumeDurable();
289
+
290
+ const stamped = acquiredAt.find((at) => at >= before + 2_700);
291
+ expect(stamped).toBeDefined();
292
+ await appB.stop();
293
+ });
294
+
224
295
  test("boot orphan scan skips runs held under a live lease", async () => {
225
296
  const store = await createPostgresJournalStore({ sql: createPostgresJournalFake() });
226
297
  const step1Calls: string[] = [];
@@ -233,7 +304,13 @@ describe("journal boot — shared store, real boot paths (fake SQL)", () => {
233
304
  journal: journalRuntime(store, "A", 30_000),
234
305
  bindings,
235
306
  });
236
- void appA.fetch(new Request("http://localhost/charge", { method: "POST", body: "{}" }));
307
+ void appA.fetch(
308
+ new Request("http://localhost/charge", {
309
+ method: "POST",
310
+ headers: { "content-type": "application/json" },
311
+ body: "{}",
312
+ }),
313
+ );
237
314
  expect(await waitFor(async () => blockers.length === 1)).toBe(true);
238
315
 
239
316
  // B boots while A's lease is live — must not steal the run.