okengine 0.23.1 → 0.23.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/site/content/docs/ai/mcp.mdx +3 -3
- package/site/content/docs/elements/signal/broadcast.mdx +4 -2
- package/site/content/docs/reference/configuration.mdx +16 -0
- package/site/content/docs/reference/errors.mdx +31 -19
- package/src/compiler/aot.test.ts +3 -2
- package/src/compiler/aot.ts +45 -1
- package/src/compiler/dynamic.ts +1 -1
- package/src/compiler/http-parse.test.ts +101 -0
- package/src/compiler/http-parse.ts +188 -9
- package/src/compiler/interpret.ts +25 -2
- package/src/console/ui-next/dist/assets/{access-page-C_N4qvai.js → access-page-zBMhZnFm.js} +1 -1
- package/src/console/ui-next/dist/assets/{agent-disclosure-Cq1TxniW.js → agent-disclosure-CBj9f7Ju.js} +1 -1
- package/src/console/ui-next/dist/assets/{cache-glyph-BaI4uxZz.js → cache-glyph-D3MY49h3.js} +1 -1
- package/src/console/ui-next/dist/assets/{call-pii-button-DNyK7IIY.js → call-pii-button-L9JxdBhp.js} +1 -1
- package/src/console/ui-next/dist/assets/{collapsible-CRhBJMZR.js → collapsible-BvFaX-Zt.js} +1 -1
- package/src/console/ui-next/dist/assets/{decisions-page-D9twj8Xy.js → decisions-page-DgB76m-X.js} +1 -1
- package/src/console/ui-next/dist/assets/{duration-tone-CsHWlVPU.js → duration-tone-l1DSJ2Vz.js} +1 -1
- package/src/console/ui-next/dist/assets/{flows-page-ZaTJAhR9.js → flows-page-Bb9t1lu6.js} +1 -1
- package/src/console/ui-next/dist/assets/{highlighted-json-DQ-EMAfz.js → highlighted-json-DMvsi3Ti.js} +1 -1
- package/src/console/ui-next/dist/assets/{http-method-Cn_Fl79s.js → http-method-BwhJBTcQ.js} +1 -1
- package/src/console/ui-next/dist/assets/{index-CmRFeWdT.js → index-RhV2jT_7.js} +3 -3
- package/src/console/ui-next/dist/assets/{observability-page-Cn6R_dXz.js → observability-page-BRfUILBq.js} +1 -1
- package/src/console/ui-next/dist/assets/{replica-lag-Bx6EZhSe.js → replica-lag-BsceAzIG.js} +1 -1
- package/src/console/ui-next/dist/assets/{request-meta-DmdiUrfh.js → request-meta-C6lTyCXp.js} +1 -1
- package/src/console/ui-next/dist/assets/{store-page-8P6vGXnn.js → store-page-YmE806bJ.js} +1 -1
- package/src/console/ui-next/dist/assets/{trace-detail-sheet-BfGvpFSv.js → trace-detail-sheet-Di4irS49.js} +1 -1
- package/src/console/ui-next/dist/assets/{tree-expand-toggle-CWdUHbc0.js → tree-expand-toggle-DYeL6Rye.js} +1 -1
- package/src/console/ui-next/dist/assets/{units-page-CZOFZzay.js → units-page-07agasRA.js} +1 -1
- package/src/console/ui-next/dist/assets/{vault-page-ki_ishmX.js → vault-page-CHBck4n_.js} +1 -1
- package/src/console/ui-next/dist/index.html +1 -1
- package/src/console/ui-next/seed-parked-approval.ts +5 -1
- package/src/drivers/journal-postgres.test.ts +84 -1
- package/src/drivers/journal-postgres.ts +282 -32
- package/src/drivers/postgres.test.ts +60 -0
- package/src/drivers/postgres.ts +79 -5
- package/src/drivers/signal-compete.test.ts +6 -2
- package/src/drivers/signal-postgres.test.ts +170 -0
- package/src/drivers/signal-postgres.ts +449 -196
- package/src/drivers/signal-redis.test.ts +224 -0
- package/src/drivers/signal-redis.ts +835 -150
- package/src/elements/ai/approval.test.ts +15 -3
- package/src/elements/ai/approval.ts +15 -9
- package/src/elements/channel/runtime.ts +31 -4
- package/src/elements/channel/sql-ledger.test.ts +404 -0
- package/src/elements/channel/sql-ledger.ts +177 -33
- package/src/elements/clock/chaos-child.ts +12 -2
- package/src/elements/clock/durable.ts +11 -0
- package/src/elements/store/cache-bus.test.ts +121 -0
- package/src/elements/store/cache-bus.ts +278 -0
- package/src/elements/store/cache.test.ts +59 -0
- package/src/elements/store/cache.ts +157 -12
- package/src/elements/store/runtime.ts +44 -2
- package/src/elements/store/sql-errors.test.ts +131 -0
- package/src/elements/store/sql-errors.ts +39 -16
- package/src/elements/store/sql-nested-tx.test.ts +344 -0
- package/src/elements/store/sql-session.ts +183 -3
- package/src/kernel/app.ts +176 -28
- package/src/kernel/auto-registry.test.ts +5 -1
- package/src/kernel/boot-bind/signal.ts +13 -2
- package/src/kernel/boot-bind/store.ts +16 -1
- package/src/kernel/boot.test.ts +2 -2
- package/src/kernel/boot.ts +41 -11
- package/src/kernel/boundary-contract.ts +7 -0
- package/src/kernel/builtin-errors.ts +2 -0
- package/src/kernel/call.test.ts +1 -0
- package/src/kernel/fx-sql-handle.ts +7 -1
- package/src/kernel/horizontal-child.ts +54 -1
- package/src/kernel/horizontal.integration.test.ts +131 -7
- package/src/kernel/idempotency.test.ts +5 -6
- package/src/kernel/journal-boot.test.ts +83 -6
- package/src/kernel/journal.test.ts +44 -0
- package/src/kernel/journal.ts +171 -26
- package/src/mcp/authorization.ts +5 -11
- package/src/mcp/confirmation.ts +203 -38
- package/src/mcp/docs-server.ts +7 -2
- package/src/mcp/index.ts +4 -0
- package/src/mcp/mcp.test.ts +321 -33
- package/src/mcp/server.ts +8 -3
- package/src/mcp/session.ts +7 -0
- package/src/mcp/tools.ts +68 -19
- package/src/runtime/bun.ts +5 -0
- package/src/runtime/types.ts +5 -0
|
@@ -24,6 +24,7 @@ import { postgresDriver } from "../../drivers/postgres.ts";
|
|
|
24
24
|
import { redisDriver } from "../../drivers/redis.ts";
|
|
25
25
|
import { s3Driver } from "../../drivers/s3.ts";
|
|
26
26
|
import type { FilesDriver, IndexDriver, KvDriver, SqlDriver } from "../../drivers/types.ts";
|
|
27
|
+
import { bunRedisCacheClient } from "../../elements/store/cache-bus.ts";
|
|
27
28
|
import { createStoreRuntime, type StoreRuntime } from "../../elements/store.ts";
|
|
28
29
|
import type { SqlRuntimeBinding } from "../../elements/store/runtime.ts";
|
|
29
30
|
import type { StoreDecl } from "../../elements/store/declare.ts";
|
|
@@ -171,7 +172,14 @@ export function bindStore(
|
|
|
171
172
|
const autoPush = resolveAutoPush(options);
|
|
172
173
|
const domainDdl = resolveDomainDdlMode(env, autoPush);
|
|
173
174
|
|
|
174
|
-
|
|
175
|
+
let store!: StoreRuntime;
|
|
176
|
+
const cacheBus =
|
|
177
|
+
kvId === "redis" && kvUrl
|
|
178
|
+
? { kind: "redis" as const, redis: bunRedisCacheClient(kvUrl) }
|
|
179
|
+
: sqlId === "postgres" && sqlUrl
|
|
180
|
+
? { kind: "postgres" as const, sql: () => store.primarySql() }
|
|
181
|
+
: undefined;
|
|
182
|
+
store = createStoreRuntime({
|
|
175
183
|
drivers: {
|
|
176
184
|
sql: sqlDriverFor(sqlId),
|
|
177
185
|
kv: kvDriverFor(kvId),
|
|
@@ -183,6 +191,13 @@ export function bindStore(
|
|
|
183
191
|
files: filesBindings,
|
|
184
192
|
index: indexBindings,
|
|
185
193
|
now,
|
|
194
|
+
...(options.cache?.maxEntries !== undefined
|
|
195
|
+
? { cacheMaxEntries: options.cache.maxEntries }
|
|
196
|
+
: {}),
|
|
197
|
+
...(options.cache?.defaultTtlMs !== undefined
|
|
198
|
+
? { cacheDefaultTtlMs: options.cache.defaultTtlMs }
|
|
199
|
+
: {}),
|
|
200
|
+
...(cacheBus ? { cacheBus } : {}),
|
|
186
201
|
domainDdl,
|
|
187
202
|
sqlUrl,
|
|
188
203
|
});
|
package/src/kernel/boot.test.ts
CHANGED
|
@@ -285,8 +285,8 @@ describe("boot — cron autostart via real timer loop", () => {
|
|
|
285
285
|
// Drive the wall-clock interval that boot installed — not clock.advance /
|
|
286
286
|
// runNow / dispatchEvery (those prove the harness, not autostart).
|
|
287
287
|
jest.advanceTimersByTime(1000);
|
|
288
|
-
//
|
|
289
|
-
for (let i = 0; i < 50; i++) await Promise.resolve();
|
|
288
|
+
// The tick is guarded and async. Wait until it settles, but not forever.
|
|
289
|
+
for (let i = 0; i < 50 && ran < 1; i++) await Promise.resolve();
|
|
290
290
|
|
|
291
291
|
expect(ran).toBeGreaterThanOrEqual(1);
|
|
292
292
|
|
package/src/kernel/boot.ts
CHANGED
|
@@ -182,6 +182,13 @@ export interface BootOptions {
|
|
|
182
182
|
readonly onDurableResume?: () => void | Promise<void>;
|
|
183
183
|
/** Durable-run lease duration ms (default 30_000 — matches Signal claims). */
|
|
184
184
|
readonly journalLeaseMs?: number;
|
|
185
|
+
/** Opt-in durable code version. Unset skips the stamp and the check. */
|
|
186
|
+
readonly codeVersion?: string;
|
|
187
|
+
/** Tier-1 cache bounds. Defaults: 10_000 entries, 60s TTL. */
|
|
188
|
+
readonly cache?: {
|
|
189
|
+
readonly maxEntries?: number;
|
|
190
|
+
readonly defaultTtlMs?: number;
|
|
191
|
+
};
|
|
185
192
|
/** Injectable clock for test / frozen harnesses. */
|
|
186
193
|
readonly now?: () => number;
|
|
187
194
|
/**
|
|
@@ -539,16 +546,20 @@ export async function bootApplication(input: BootOptions = {}): Promise<BootResu
|
|
|
539
546
|
|
|
540
547
|
// 4d. Scheduler — one timer drives clock ticks, durable-run resume, heartbeat.
|
|
541
548
|
let schedulerTimer: ReturnType<typeof setInterval> | undefined;
|
|
549
|
+
const inflight = new Map<string, Promise<void>>();
|
|
550
|
+
let stopping = false;
|
|
542
551
|
const startScheduler = options.startScheduler ?? env !== "test";
|
|
543
552
|
const signalBus = signal?.bus ?? undefined;
|
|
544
553
|
let refreshChannel: (() => Promise<void>) | undefined;
|
|
554
|
+
const pollCache = store?.cacheInvalidation === "postgres";
|
|
545
555
|
if (
|
|
546
556
|
startScheduler &&
|
|
547
557
|
(clock !== undefined ||
|
|
548
558
|
journal !== undefined ||
|
|
549
559
|
instances !== undefined ||
|
|
550
560
|
signalBus !== undefined ||
|
|
551
|
-
needs.channel
|
|
561
|
+
needs.channel ||
|
|
562
|
+
pollCache)
|
|
552
563
|
) {
|
|
553
564
|
const period = options.schedulerIntervalMs ?? 1000;
|
|
554
565
|
const clockRt = clock;
|
|
@@ -560,18 +571,30 @@ export async function bootApplication(input: BootOptions = {}): Promise<BootResu
|
|
|
560
571
|
// Unexpected scheduler failures stay visible.
|
|
561
572
|
console.error(err);
|
|
562
573
|
};
|
|
574
|
+
const spawn = (name: string, task: () => Promise<unknown>): void => {
|
|
575
|
+
if (stopping || inflight.has(name)) return;
|
|
576
|
+
const run = Promise.resolve()
|
|
577
|
+
.then(task)
|
|
578
|
+
.then(() => undefined)
|
|
579
|
+
.catch(ignoreBenignSql)
|
|
580
|
+
.finally(() => {
|
|
581
|
+
inflight.delete(name);
|
|
582
|
+
});
|
|
583
|
+
inflight.set(name, run);
|
|
584
|
+
};
|
|
563
585
|
schedulerTimer = setInterval(() => {
|
|
564
|
-
if (signalBus)
|
|
565
|
-
if (refreshChannel)
|
|
566
|
-
if (clockRt)
|
|
567
|
-
if (journal && durableResume)
|
|
568
|
-
if (fleet)
|
|
586
|
+
if (signalBus) spawn("signal", () => signalBus.drain());
|
|
587
|
+
if (refreshChannel) spawn("channel", () => refreshChannel!());
|
|
588
|
+
if (clockRt) spawn("clock", () => Promise.resolve(clockRt.tick()));
|
|
589
|
+
if (journal && durableResume) spawn("durable", () => Promise.resolve(durableResume()));
|
|
590
|
+
if (fleet) spawn("fleet", () => fleet.maybeHeartbeat());
|
|
591
|
+
if (pollCache && store) spawn("cache", () => store.pollCacheInvalidations());
|
|
569
592
|
const stem = ["run", "events"].join("-");
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
mod.sweepInstalledAgentEvents(),
|
|
573
|
-
)
|
|
574
|
-
|
|
593
|
+
spawn("agents", () =>
|
|
594
|
+
import(`../elements/ai/${stem}.ts`).then(
|
|
595
|
+
(mod: typeof import("../elements/ai/run-events.ts")) => mod.sweepInstalledAgentEvents(),
|
|
596
|
+
),
|
|
597
|
+
);
|
|
575
598
|
}, period);
|
|
576
599
|
schedulerTimer.unref?.();
|
|
577
600
|
}
|
|
@@ -653,7 +676,12 @@ export async function bootApplication(input: BootOptions = {}): Promise<BootResu
|
|
|
653
676
|
instanceId,
|
|
654
677
|
instances,
|
|
655
678
|
capabilities,
|
|
679
|
+
/**
|
|
680
|
+
* Stop the scheduler interval and wake timers.
|
|
681
|
+
* Does not wait for a tick that has already started — {@link BootResult.close} does.
|
|
682
|
+
*/
|
|
656
683
|
stopScheduler() {
|
|
684
|
+
stopping = true;
|
|
657
685
|
if (schedulerTimer !== undefined) {
|
|
658
686
|
clearInterval(schedulerTimer);
|
|
659
687
|
schedulerTimer = undefined;
|
|
@@ -661,11 +689,13 @@ export async function bootApplication(input: BootOptions = {}): Promise<BootResu
|
|
|
661
689
|
clock?.stopWakes();
|
|
662
690
|
},
|
|
663
691
|
async close() {
|
|
692
|
+
stopping = true;
|
|
664
693
|
if (schedulerTimer !== undefined) {
|
|
665
694
|
clearInterval(schedulerTimer);
|
|
666
695
|
schedulerTimer = undefined;
|
|
667
696
|
}
|
|
668
697
|
clock?.stopWakes();
|
|
698
|
+
await Promise.all([...inflight.values()]);
|
|
669
699
|
await signal?.close();
|
|
670
700
|
await vault?.close();
|
|
671
701
|
await runs?.flush();
|
|
@@ -30,6 +30,13 @@ export interface BoundaryContract<
|
|
|
30
30
|
* (`oke doctor --diff` / CI gate).
|
|
31
31
|
*/
|
|
32
32
|
readonly breaking?: boolean;
|
|
33
|
+
/** Per-route body cap in bytes. Falls back to the app `maxRequestBodySize`. */
|
|
34
|
+
readonly maxBodySize?: number;
|
|
35
|
+
/**
|
|
36
|
+
* `required` (default) rejects a JSON-shaped body that is not
|
|
37
|
+
* `application/json` or `+json`. `any` keeps the previous parse-or-string behavior.
|
|
38
|
+
*/
|
|
39
|
+
readonly jsonContentType?: "required" | "any";
|
|
33
40
|
}
|
|
34
41
|
|
|
35
42
|
/** Infer validated input from a {@link BoundaryContract} bag. */
|
|
@@ -15,6 +15,7 @@ export const BUILTIN_ERROR_STATUS = {
|
|
|
15
15
|
Conflict: 409,
|
|
16
16
|
ForeignKey: 409,
|
|
17
17
|
UnsupportedMediaType: 415,
|
|
18
|
+
PayloadTooLarge: 413,
|
|
18
19
|
RateLimited: 429,
|
|
19
20
|
AuthRateLimited: 429,
|
|
20
21
|
InvalidQuery: 400,
|
|
@@ -78,6 +79,7 @@ export type BuiltinErrorMap = {
|
|
|
78
79
|
readonly Conflict: BuiltinErrorBag;
|
|
79
80
|
readonly ForeignKey: BuiltinErrorBag;
|
|
80
81
|
readonly UnsupportedMediaType: BuiltinErrorBag;
|
|
82
|
+
readonly PayloadTooLarge: BuiltinErrorBag;
|
|
81
83
|
readonly RateLimited: { readonly retryAfterMs?: number };
|
|
82
84
|
readonly AuthRateLimited: BuiltinErrorBag;
|
|
83
85
|
readonly InvalidQuery: BuiltinErrorBag;
|
package/src/kernel/call.test.ts
CHANGED
|
@@ -17,7 +17,7 @@ import { schemaTableName, sqlTableRef } from "../manifest/sql-resource.ts";
|
|
|
17
17
|
import type { CapabilityToken } from "./capability.ts";
|
|
18
18
|
import { DryRunWriteIsolationError, isDryRun } from "./dry-run.ts";
|
|
19
19
|
import type { EffectExternal } from "./effects.ts";
|
|
20
|
-
import { runInSignalTransaction } from "./signal-tx.ts";
|
|
20
|
+
import { currentSignalTransaction, runInSignalTransaction } from "./signal-tx.ts";
|
|
21
21
|
|
|
22
22
|
/** Capability-gated effect runner from {@link createFx}. */
|
|
23
23
|
type Gated = <T>(
|
|
@@ -253,6 +253,12 @@ export function createGatedSqlHandle(options: GatedSqlHandleOptions): SqlStoreHa
|
|
|
253
253
|
async transaction(fn) {
|
|
254
254
|
refuseDryRunWrite();
|
|
255
255
|
const h = await ensure();
|
|
256
|
+
// Already inside a signal transaction: do not begin or commit another
|
|
257
|
+
// one. sql-session opens a child staging scope and merges it; only the
|
|
258
|
+
// outermost commit publishes.
|
|
259
|
+
if (currentSignalTransaction()) {
|
|
260
|
+
return h.transaction(() => fn(api));
|
|
261
|
+
}
|
|
256
262
|
const signalTx = options.beginSignal ? await options.beginSignal() : undefined;
|
|
257
263
|
try {
|
|
258
264
|
const result = await h.transaction(async () => {
|
|
@@ -20,6 +20,8 @@ import { dirname, join } from "node:path";
|
|
|
20
20
|
import { createPostgresCronStore } from "../drivers/clock-postgres.ts";
|
|
21
21
|
import { createPostgresJournalStore } from "../drivers/journal-postgres.ts";
|
|
22
22
|
import { memorySignalDriver } from "../drivers/signal-memory.ts";
|
|
23
|
+
import { createBunSignalRedisClient, redisSignalDriver } from "../drivers/signal-redis.ts";
|
|
24
|
+
import { bunRedisCacheClient } from "../elements/store/cache-bus.ts";
|
|
23
25
|
import { postgresDriver } from "../drivers/postgres.ts";
|
|
24
26
|
import { redisDriver } from "../drivers/redis.ts";
|
|
25
27
|
import { clock } from "../elements/clock/declare.ts";
|
|
@@ -99,14 +101,40 @@ clockRt.onCron(CRON, async () => {
|
|
|
99
101
|
await Bun.write(path, prev + line);
|
|
100
102
|
});
|
|
101
103
|
|
|
104
|
+
const redisSignals = process.env.OKE_HORIZONTAL_REDIS_SIGNALS === "1";
|
|
102
105
|
const signalRt = createSignalRuntime({
|
|
103
|
-
driver: memorySignalDriver,
|
|
106
|
+
driver: redisSignals ? redisSignalDriver : memorySignalDriver,
|
|
107
|
+
...(redisSignals
|
|
108
|
+
? { redis: createBunSignalRedisClient(redisUrl), compete: true, consumerId: instanceId }
|
|
109
|
+
: {}),
|
|
104
110
|
durablePath: signalPath,
|
|
105
111
|
leaseMs: Number.isFinite(leaseMs) ? leaseMs : 300,
|
|
106
112
|
});
|
|
107
113
|
const job = signal.once(SIG, { retries: 3, deadLetter: true, optional: true });
|
|
108
114
|
signalRt.register(job);
|
|
115
|
+
const news = signal.broadcast("horizontal-news", { optional: true });
|
|
116
|
+
const feed = signal.live("horizontal-feed", { optional: true });
|
|
117
|
+
if (redisSignals) {
|
|
118
|
+
signalRt.register(news);
|
|
119
|
+
signalRt.register(feed);
|
|
120
|
+
}
|
|
109
121
|
const bus = await signalRt.start();
|
|
122
|
+
if (redisSignals) {
|
|
123
|
+
await bus.subscribe("horizontal-news", `news-${instanceId}`, async (msg) => {
|
|
124
|
+
const line = `${JSON.stringify({ instanceId, kind: "broadcast", id: msg.id, at: Date.now() })}\n`;
|
|
125
|
+
const path = join(workDir, "broadcast.jsonl");
|
|
126
|
+
const prev = (await Bun.file(path).exists()) ? await Bun.file(path).text() : "";
|
|
127
|
+
await Bun.write(path, prev + line);
|
|
128
|
+
});
|
|
129
|
+
void (async () => {
|
|
130
|
+
for await (const event of bus.live("horizontal-feed")) {
|
|
131
|
+
const line = `${JSON.stringify({ instanceId, kind: "live", id: event.id, at: Date.now() })}\n`;
|
|
132
|
+
const path = join(workDir, "live.jsonl");
|
|
133
|
+
const prev = (await Bun.file(path).exists()) ? await Bun.file(path).text() : "";
|
|
134
|
+
await Bun.write(path, prev + line);
|
|
135
|
+
}
|
|
136
|
+
})();
|
|
137
|
+
}
|
|
110
138
|
await bus.subscribe(SIG, `consumer-${instanceId}`, async (msg) => {
|
|
111
139
|
const line = `${JSON.stringify({ instanceId, kind: "signal", id: msg.id, at: Date.now() })}\n`;
|
|
112
140
|
const path = join(workDir, "signal.jsonl");
|
|
@@ -129,6 +157,7 @@ const storeRt = createStoreRuntime({
|
|
|
129
157
|
drivers: { sql: postgresDriver },
|
|
130
158
|
sql: { db: { name: "db", primary: { url: pgUrl } } },
|
|
131
159
|
now: () => Date.now(),
|
|
160
|
+
cacheBus: { kind: "redis", redis: bunRedisCacheClient(redisUrl), origin: instanceId },
|
|
132
161
|
});
|
|
133
162
|
storeRt.register(db);
|
|
134
163
|
|
|
@@ -138,7 +167,16 @@ const ping = flow("horizontal.ping", {
|
|
|
138
167
|
const rate = flow("horizontal.rate", {
|
|
139
168
|
do: () => ({ ok: true as const, instanceId }),
|
|
140
169
|
});
|
|
170
|
+
let cachedReads = 0;
|
|
171
|
+
const cached = flow("horizontal.cached", {
|
|
172
|
+
effects: { reads: ["sql:db"] },
|
|
173
|
+
do: () => {
|
|
174
|
+
cachedReads += 1;
|
|
175
|
+
return { reads: cachedReads, instanceId };
|
|
176
|
+
},
|
|
177
|
+
});
|
|
141
178
|
const write = flow("horizontal.write", {
|
|
179
|
+
effects: { writes: ["sql:db"] },
|
|
142
180
|
do: async (_input, fx) => {
|
|
143
181
|
const id = okid();
|
|
144
182
|
await fx
|
|
@@ -159,6 +197,18 @@ const emit = flow("horizontal.emit", {
|
|
|
159
197
|
return { emitted: true as const };
|
|
160
198
|
},
|
|
161
199
|
});
|
|
200
|
+
const publishNews = flow("horizontal.news", {
|
|
201
|
+
do: async (_input, fx) => {
|
|
202
|
+
await fx.emit("horizontal-news", { from: instanceId, at: Date.now() });
|
|
203
|
+
return { emitted: true as const };
|
|
204
|
+
},
|
|
205
|
+
});
|
|
206
|
+
const publishFeed = flow("horizontal.feed", {
|
|
207
|
+
do: async (_input, fx) => {
|
|
208
|
+
await fx.emit("horizontal-feed", { from: instanceId, at: Date.now() });
|
|
209
|
+
return { emitted: true as const };
|
|
210
|
+
},
|
|
211
|
+
});
|
|
162
212
|
const charge = flow("horizontal.charge", {
|
|
163
213
|
durable: true,
|
|
164
214
|
do: async (_input, fx) => {
|
|
@@ -193,8 +243,11 @@ const charge = flow("horizontal.charge", {
|
|
|
193
243
|
const bindings: Binding[] = [
|
|
194
244
|
{ trigger: http.get("/ping").public(), flow: ping as AnyFlowDef },
|
|
195
245
|
{ trigger: http.get("/rate").gate(rateGate), flow: rate as AnyFlowDef },
|
|
246
|
+
{ trigger: http.get("/cached").public(), flow: cached as AnyFlowDef },
|
|
196
247
|
{ trigger: http.post("/write").public(), flow: write as AnyFlowDef },
|
|
197
248
|
{ trigger: http.post("/emit").public(), flow: emit as AnyFlowDef },
|
|
249
|
+
{ trigger: http.post("/news").public(), flow: publishNews as AnyFlowDef },
|
|
250
|
+
{ trigger: http.post("/feed").public(), flow: publishFeed as AnyFlowDef },
|
|
198
251
|
{ trigger: http.post("/charge").public(), flow: charge as AnyFlowDef },
|
|
199
252
|
];
|
|
200
253
|
|
|
@@ -124,10 +124,26 @@ describe.skipIf(!LIVE)("horizontal — two OS processes, Postgres + Redis", () =
|
|
|
124
124
|
const traffic = async () => {
|
|
125
125
|
for (let i = 0; i < 8; i++) {
|
|
126
126
|
await Promise.all([
|
|
127
|
-
fetch(`${urlA}/write`, {
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
127
|
+
fetch(`${urlA}/write`, {
|
|
128
|
+
method: "POST",
|
|
129
|
+
headers: { "content-type": "application/json" },
|
|
130
|
+
body: "{}",
|
|
131
|
+
}),
|
|
132
|
+
fetch(`${urlB}/write`, {
|
|
133
|
+
method: "POST",
|
|
134
|
+
headers: { "content-type": "application/json" },
|
|
135
|
+
body: "{}",
|
|
136
|
+
}),
|
|
137
|
+
fetch(`${urlA}/emit`, {
|
|
138
|
+
method: "POST",
|
|
139
|
+
headers: { "content-type": "application/json" },
|
|
140
|
+
body: "{}",
|
|
141
|
+
}),
|
|
142
|
+
fetch(`${urlB}/emit`, {
|
|
143
|
+
method: "POST",
|
|
144
|
+
headers: { "content-type": "application/json" },
|
|
145
|
+
body: "{}",
|
|
146
|
+
}),
|
|
131
147
|
fetch(`${urlA}/ping`),
|
|
132
148
|
fetch(`${urlB}/ping`),
|
|
133
149
|
]);
|
|
@@ -150,7 +166,11 @@ describe.skipIf(!LIVE)("horizontal — two OS processes, Postgres + Redis", () =
|
|
|
150
166
|
expect(limited).toBeGreaterThanOrEqual(1);
|
|
151
167
|
|
|
152
168
|
// (2) Start durable on A; hang mid-step.
|
|
153
|
-
void fetch(`${urlA}/charge`, {
|
|
169
|
+
void fetch(`${urlA}/charge`, {
|
|
170
|
+
method: "POST",
|
|
171
|
+
headers: { "content-type": "application/json" },
|
|
172
|
+
body: "{}",
|
|
173
|
+
});
|
|
154
174
|
expect(await waitForFile(join(dir, "hang-inst-a.json"), 15_000)).toBe(true);
|
|
155
175
|
|
|
156
176
|
// (5) Kill A mid combined scenario — B keeps serving.
|
|
@@ -177,8 +197,24 @@ describe.skipIf(!LIVE)("horizontal — two OS processes, Postgres + Redis", () =
|
|
|
177
197
|
// Keep traffic going on survivor (no deadlock).
|
|
178
198
|
await trafficPromise.catch(() => {});
|
|
179
199
|
for (let i = 0; i < 4; i++) {
|
|
180
|
-
expect(
|
|
181
|
-
|
|
200
|
+
expect(
|
|
201
|
+
(
|
|
202
|
+
await fetch(`${urlB}/write`, {
|
|
203
|
+
method: "POST",
|
|
204
|
+
headers: { "content-type": "application/json" },
|
|
205
|
+
body: "{}",
|
|
206
|
+
})
|
|
207
|
+
).status,
|
|
208
|
+
).toBe(200);
|
|
209
|
+
expect(
|
|
210
|
+
(
|
|
211
|
+
await fetch(`${urlB}/emit`, {
|
|
212
|
+
method: "POST",
|
|
213
|
+
headers: { "content-type": "application/json" },
|
|
214
|
+
body: "{}",
|
|
215
|
+
})
|
|
216
|
+
).status,
|
|
217
|
+
).toBe(200);
|
|
182
218
|
}
|
|
183
219
|
|
|
184
220
|
// (1) Cron fired exactly once across both instances.
|
|
@@ -220,6 +256,94 @@ describe.skipIf(!LIVE)("horizontal — two OS processes, Postgres + Redis", () =
|
|
|
220
256
|
await cleanupC.close();
|
|
221
257
|
}
|
|
222
258
|
}, 90_000);
|
|
259
|
+
|
|
260
|
+
test("broadcast, live, and cache invalidation reach the other instance", async () => {
|
|
261
|
+
const pg = LIVE!.pg;
|
|
262
|
+
const redis = LIVE!.redis;
|
|
263
|
+
const dir = await mkdtemp(join(tmpdir(), "oke-horizontal-bus-"));
|
|
264
|
+
const signalPath = join(dir, "signal.json");
|
|
265
|
+
const post = (url: string) =>
|
|
266
|
+
fetch(url, {
|
|
267
|
+
method: "POST",
|
|
268
|
+
headers: { "content-type": "application/json" },
|
|
269
|
+
body: "{}",
|
|
270
|
+
});
|
|
271
|
+
const spawn = (instanceId: string, port: number) =>
|
|
272
|
+
Bun.spawn({
|
|
273
|
+
cmd: [
|
|
274
|
+
"bun",
|
|
275
|
+
childPath,
|
|
276
|
+
"serve",
|
|
277
|
+
instanceId,
|
|
278
|
+
String(port),
|
|
279
|
+
pg,
|
|
280
|
+
redis,
|
|
281
|
+
signalPath,
|
|
282
|
+
dir,
|
|
283
|
+
"5000",
|
|
284
|
+
],
|
|
285
|
+
stdout: "pipe",
|
|
286
|
+
stderr: "pipe",
|
|
287
|
+
env: {
|
|
288
|
+
...process.env,
|
|
289
|
+
DATABASE_URL: pg,
|
|
290
|
+
REDIS_URL: redis,
|
|
291
|
+
OKE_HORIZONTAL_REDIS_SIGNALS: "1",
|
|
292
|
+
},
|
|
293
|
+
});
|
|
294
|
+
const a = spawn("bus-a", 0);
|
|
295
|
+
const b = spawn("bus-b", 0);
|
|
296
|
+
try {
|
|
297
|
+
expect(await waitForFile(join(dir, "ready-bus-a.json"))).toBe(true);
|
|
298
|
+
expect(await waitForFile(join(dir, "ready-bus-b.json"))).toBe(true);
|
|
299
|
+
const readyA = (await Bun.file(join(dir, "ready-bus-a.json")).json()) as { port: number };
|
|
300
|
+
const readyB = (await Bun.file(join(dir, "ready-bus-b.json")).json()) as { port: number };
|
|
301
|
+
const urlA = `http://127.0.0.1:${readyA.port}`;
|
|
302
|
+
const urlB = `http://127.0.0.1:${readyB.port}`;
|
|
303
|
+
const first = (await (await fetch(`${urlA}/cached`)).json()) as { data: { reads: number } };
|
|
304
|
+
const second = (await (await fetch(`${urlA}/cached`)).json()) as { data: { reads: number } };
|
|
305
|
+
expect(first.data.reads).toBe(1);
|
|
306
|
+
expect(second.data.reads).toBe(1);
|
|
307
|
+
expect((await post(`${urlB}/write`)).status).toBe(200);
|
|
308
|
+
expect(
|
|
309
|
+
await waitFor(async () => {
|
|
310
|
+
const body = (await (await fetch(`${urlA}/cached`)).json()) as {
|
|
311
|
+
data: { reads: number };
|
|
312
|
+
};
|
|
313
|
+
return body.data.reads === 2;
|
|
314
|
+
}),
|
|
315
|
+
).toBe(true);
|
|
316
|
+
expect((await post(`${urlA}/news`)).status).toBe(200);
|
|
317
|
+
expect((await post(`${urlA}/feed`)).status).toBe(200);
|
|
318
|
+
expect(
|
|
319
|
+
await waitFor(async () => {
|
|
320
|
+
const broadcast = await loadJsonl(join(dir, "broadcast.jsonl"));
|
|
321
|
+
const live = await loadJsonl(join(dir, "live.jsonl"));
|
|
322
|
+
const ids = (rows: Array<Record<string, unknown>>) =>
|
|
323
|
+
new Set(rows.map((row) => row.instanceId));
|
|
324
|
+
return (
|
|
325
|
+
ids(broadcast).has("bus-a") &&
|
|
326
|
+
ids(broadcast).has("bus-b") &&
|
|
327
|
+
ids(live).has("bus-a") &&
|
|
328
|
+
ids(live).has("bus-b")
|
|
329
|
+
);
|
|
330
|
+
}),
|
|
331
|
+
).toBe(true);
|
|
332
|
+
} finally {
|
|
333
|
+
try {
|
|
334
|
+
a.kill(9);
|
|
335
|
+
} catch {
|
|
336
|
+
/* already dead */
|
|
337
|
+
}
|
|
338
|
+
try {
|
|
339
|
+
b.kill(9);
|
|
340
|
+
} catch {
|
|
341
|
+
/* ignore */
|
|
342
|
+
}
|
|
343
|
+
await Promise.allSettled([a.exited, b.exited]);
|
|
344
|
+
await rm(dir, { recursive: true, force: true });
|
|
345
|
+
}
|
|
346
|
+
}, 60_000);
|
|
223
347
|
});
|
|
224
348
|
|
|
225
349
|
if (!LIVE) {
|
|
@@ -349,13 +349,12 @@ describe("HTTP idempotency", () => {
|
|
|
349
349
|
expect(await durableStore.idempotency!.forfeit(held!, held!.claimToken)).toBe(true);
|
|
350
350
|
await Bun.sleep(120);
|
|
351
351
|
const resumed = durable.fetch(post({ n: 1 }));
|
|
352
|
-
|
|
353
|
-
|
|
352
|
+
// The holder's heartbeat kept the lease, so the same process does not
|
|
353
|
+
// start a second session for the run still inside mid-flight.
|
|
354
|
+
expect((await resumed).status).toBe(409);
|
|
355
|
+
expect(blockers).toHaveLength(1);
|
|
354
356
|
blockers[0]!();
|
|
355
|
-
|
|
356
|
-
// The forfeited holder kept the old lease token. Its later write is rejected.
|
|
357
|
-
await expect(hung).rejects.toThrow(/OKE1074/);
|
|
358
|
-
expect((await resumed).status).toBe(200);
|
|
357
|
+
expect((await hung).status).toBe(200);
|
|
359
358
|
expect(step1).toEqual(["step1"]);
|
|
360
359
|
}, 30_000);
|
|
361
360
|
|
|
@@ -22,6 +22,7 @@ import {
|
|
|
22
22
|
type PostgresJournalStore,
|
|
23
23
|
} from "../drivers/journal-postgres.ts";
|
|
24
24
|
import { oke, type OkeApp } from "./app.ts";
|
|
25
|
+
import { clearJournalHeartbeats } from "./journal.ts";
|
|
25
26
|
import type { JournalRuntime } from "./boot-bind/journal.ts";
|
|
26
27
|
import { flow, resetFlowSeq, type AnyFlowDef } from "./flow.ts";
|
|
27
28
|
import type { Binding } from "./on.ts";
|
|
@@ -47,6 +48,7 @@ async function bootDurableApp(options: {
|
|
|
47
48
|
readonly name: string;
|
|
48
49
|
readonly journal: JournalRuntime;
|
|
49
50
|
readonly bindings: readonly Binding[];
|
|
51
|
+
readonly now?: () => number;
|
|
50
52
|
}): Promise<OkeApp> {
|
|
51
53
|
const app = oke({
|
|
52
54
|
name: options.name,
|
|
@@ -57,6 +59,7 @@ async function bootDurableApp(options: {
|
|
|
57
59
|
// oke() in the process — sibling apps need their own copies.
|
|
58
60
|
bindings: [...options.bindings],
|
|
59
61
|
elements: { journal: options.journal },
|
|
62
|
+
...(options.now ? { fx: { now: options.now } } : {}),
|
|
60
63
|
});
|
|
61
64
|
await app.boot();
|
|
62
65
|
return app;
|
|
@@ -127,11 +130,18 @@ describe("journal boot — shared store, real boot paths (fake SQL)", () => {
|
|
|
127
130
|
bindings,
|
|
128
131
|
});
|
|
129
132
|
// Start the run through the real HTTP path; it hangs inside step 2.
|
|
130
|
-
void appA.fetch(
|
|
133
|
+
void appA.fetch(
|
|
134
|
+
new Request("http://localhost/charge", {
|
|
135
|
+
method: "POST",
|
|
136
|
+
headers: { "content-type": "application/json" },
|
|
137
|
+
body: "{}",
|
|
138
|
+
}),
|
|
139
|
+
);
|
|
131
140
|
expect(await waitFor(async () => blockers.length === 1)).toBe(true);
|
|
132
141
|
const runId = (await store.list())[0]!.id;
|
|
133
|
-
// "Crash":
|
|
134
|
-
//
|
|
142
|
+
// "Crash": the process dies, so its heartbeat timers die with it.
|
|
143
|
+
// The in-flight promise stays dangling and the lease (120ms) expires.
|
|
144
|
+
clearJournalHeartbeats();
|
|
135
145
|
|
|
136
146
|
await Bun.sleep(300);
|
|
137
147
|
const appB = await bootDurableApp({
|
|
@@ -165,7 +175,11 @@ describe("journal boot — shared store, real boot paths (fake SQL)", () => {
|
|
|
165
175
|
bindings,
|
|
166
176
|
});
|
|
167
177
|
const parked = await appA.fetch(
|
|
168
|
-
new Request("http://localhost/sleep", {
|
|
178
|
+
new Request("http://localhost/sleep", {
|
|
179
|
+
method: "POST",
|
|
180
|
+
headers: { "content-type": "application/json" },
|
|
181
|
+
body: "{}",
|
|
182
|
+
}),
|
|
169
183
|
);
|
|
170
184
|
expect(parked.status).toBe(204); // durable park — lease released, row sleeping
|
|
171
185
|
const runId = (await store.list())[0]!.id;
|
|
@@ -198,7 +212,13 @@ describe("journal boot — shared store, real boot paths (fake SQL)", () => {
|
|
|
198
212
|
journal: journalRuntime(store, "A", 5_000),
|
|
199
213
|
bindings,
|
|
200
214
|
});
|
|
201
|
-
await appA.fetch(
|
|
215
|
+
await appA.fetch(
|
|
216
|
+
new Request("http://localhost/sleep", {
|
|
217
|
+
method: "POST",
|
|
218
|
+
headers: { "content-type": "application/json" },
|
|
219
|
+
body: "{}",
|
|
220
|
+
}),
|
|
221
|
+
);
|
|
202
222
|
await appA.stop();
|
|
203
223
|
|
|
204
224
|
// Both survivors boot while the sleep is future (orphan scan skips it).
|
|
@@ -217,10 +237,61 @@ describe("journal boot — shared store, real boot paths (fake SQL)", () => {
|
|
|
217
237
|
await Promise.all([appB.resumeDurable(), appC.resumeDurable()]);
|
|
218
238
|
|
|
219
239
|
expect(confirms).toHaveLength(1);
|
|
240
|
+
|
|
220
241
|
await appB.stop();
|
|
221
242
|
await appC.stop();
|
|
222
243
|
});
|
|
223
244
|
|
|
245
|
+
test("resume uses a live clock: a lease written after the tick sample is not already expired", async () => {
|
|
246
|
+
let clock = 10_000_000;
|
|
247
|
+
const store = await createPostgresJournalStore({ sql: createPostgresJournalFake() });
|
|
248
|
+
const confirms: string[] = [];
|
|
249
|
+
const bindings = [sleeperBinding("120ms", confirms)];
|
|
250
|
+
const acquiredAt: number[] = [];
|
|
251
|
+
const acquire = store.acquireLease.bind(store);
|
|
252
|
+
store.acquireLease = async (runId, instanceId, at, leaseMs) => {
|
|
253
|
+
acquiredAt.push(at);
|
|
254
|
+
return acquire(runId, instanceId, at, leaseMs);
|
|
255
|
+
};
|
|
256
|
+
const claim = store.claimDueSleep.bind(store);
|
|
257
|
+
store.claimDueSleep = async (instanceId, at, leaseMs) => {
|
|
258
|
+
const due = await claim(instanceId, at, leaseMs);
|
|
259
|
+
// The tick already sampled `at`. A live run clock must see this jump.
|
|
260
|
+
if (due) clock += 2_700;
|
|
261
|
+
return due;
|
|
262
|
+
};
|
|
263
|
+
|
|
264
|
+
const appA = await bootDurableApp({
|
|
265
|
+
name: "clock-a",
|
|
266
|
+
journal: journalRuntime(store, "A", 5_000),
|
|
267
|
+
bindings,
|
|
268
|
+
now: () => clock,
|
|
269
|
+
});
|
|
270
|
+
await appA.fetch(
|
|
271
|
+
new Request("http://localhost/sleep", {
|
|
272
|
+
method: "POST",
|
|
273
|
+
headers: { "content-type": "application/json" },
|
|
274
|
+
body: "{}",
|
|
275
|
+
}),
|
|
276
|
+
);
|
|
277
|
+
await appA.stop();
|
|
278
|
+
|
|
279
|
+
const appB = await bootDurableApp({
|
|
280
|
+
name: "clock-b",
|
|
281
|
+
journal: journalRuntime(store, "B", 5_000),
|
|
282
|
+
bindings,
|
|
283
|
+
now: () => clock,
|
|
284
|
+
});
|
|
285
|
+
const before = clock;
|
|
286
|
+
clock = before + 500;
|
|
287
|
+
acquiredAt.length = 0;
|
|
288
|
+
await appB.resumeDurable();
|
|
289
|
+
|
|
290
|
+
const stamped = acquiredAt.find((at) => at >= before + 2_700);
|
|
291
|
+
expect(stamped).toBeDefined();
|
|
292
|
+
await appB.stop();
|
|
293
|
+
});
|
|
294
|
+
|
|
224
295
|
test("boot orphan scan skips runs held under a live lease", async () => {
|
|
225
296
|
const store = await createPostgresJournalStore({ sql: createPostgresJournalFake() });
|
|
226
297
|
const step1Calls: string[] = [];
|
|
@@ -233,7 +304,13 @@ describe("journal boot — shared store, real boot paths (fake SQL)", () => {
|
|
|
233
304
|
journal: journalRuntime(store, "A", 30_000),
|
|
234
305
|
bindings,
|
|
235
306
|
});
|
|
236
|
-
void appA.fetch(
|
|
307
|
+
void appA.fetch(
|
|
308
|
+
new Request("http://localhost/charge", {
|
|
309
|
+
method: "POST",
|
|
310
|
+
headers: { "content-type": "application/json" },
|
|
311
|
+
body: "{}",
|
|
312
|
+
}),
|
|
313
|
+
);
|
|
237
314
|
expect(await waitFor(async () => blockers.length === 1)).toBe(true);
|
|
238
315
|
|
|
239
316
|
// B boots while A's lease is live — must not steal the run.
|