okengine 0.17.2 → 0.18.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -5
- package/manifest.v1.schema.json +61 -2
- package/package.json +18 -4
- package/site/content/docs/elements/clock.mdx +1 -1
- package/site/content/docs/elements/flow.mdx +25 -1
- package/site/content/docs/elements/store.mdx +287 -341
- package/site/content/docs/elements/vault.mdx +5 -5
- package/site/content/docs/get-started/installation.mdx +1 -2
- package/site/content/docs/get-started/introduction.mdx +54 -110
- package/site/content/docs/get-started/meta.json +9 -1
- package/site/content/docs/get-started/testing.mdx +328 -0
- package/site/content/docs/get-started/why.mdx +94 -70
- package/site/content/docs/index.mdx +1 -1
- package/site/content/docs/plugins/apple.mdx +151 -0
- package/site/content/docs/plugins/discord.mdx +139 -0
- package/site/content/docs/plugins/facebook.mdx +134 -0
- package/site/content/docs/plugins/figma.mdx +138 -0
- package/site/content/docs/plugins/github.mdx +138 -0
- package/site/content/docs/plugins/google.mdx +153 -0
- package/site/content/docs/plugins/index.mdx +47 -1
- package/site/content/docs/plugins/meta.json +10 -0
- package/site/content/docs/plugins/microsoft.mdx +151 -0
- package/site/content/docs/plugins/oauth.mdx +188 -0
- package/site/content/docs/plugins/x.mdx +125 -0
- package/site/content/docs/reference/cli.md +3 -2
- package/site/content/docs/reference/client.mdx +58 -1
- package/site/content/docs/reference/configuration.mdx +2 -4
- package/site/content/docs/reference/fx.mdx +3 -1
- package/site/content/docs/reference/index.mdx +0 -5
- package/site/content/docs/reference/meta.json +2 -2
- package/site/content/docs/reference/okid.mdx +137 -0
- package/src/auth/bindings.ts +1 -1
- package/src/auth/config.ts +9 -0
- package/src/auth/identity-sql.ts +314 -0
- package/src/auth/identity.ts +140 -2
- package/src/auth/index.ts +17 -1
- package/src/auth/method-context.ts +3 -0
- package/src/auth/oauth-as/cimd.ts +132 -0
- package/src/auth/oauth-as/crypto.test.ts +101 -0
- package/src/auth/oauth-as/crypto.ts +393 -0
- package/src/auth/oauth-as/errors.ts +68 -0
- package/src/auth/oauth-as/http.test.ts +419 -0
- package/src/auth/oauth-as/http.ts +842 -0
- package/src/auth/oauth-as/stores.ts +61 -0
- package/src/auth/oauth-as/tables.ts +142 -0
- package/src/auth/tables.ts +0 -11
- package/src/bench/README.md +83 -0
- package/src/bench/REPORT.md +176 -0
- package/src/bench/g01-rls-stamp.bench.ts +194 -0
- package/src/bench/g02-clock-per-tenant.bench.ts +158 -0
- package/src/bench/g03-signal-once.bench.ts +157 -0
- package/src/bench/g03-signal-reconnect.bench.ts +254 -0
- package/src/bench/g03-signal-sse-memory.bench.ts +191 -0
- package/src/bench/g04-auth-vault-hotpath.bench.ts +170 -0
- package/src/bench/g05-sustained-full.bench.ts +265 -0
- package/src/bench/g06-mixed-load.bench.ts +260 -0
- package/src/bench/g07-vault-crypto.bench.ts +100 -0
- package/src/bench/g07-vault-rotate-under-read.bench.ts +285 -0
- package/src/bench/g08-conn-oversubscribe.bench.ts +194 -0
- package/src/bench/g08-store-kv-durable.bench.ts +133 -0
- package/src/bench/g08-store-sql.bench.ts +178 -0
- package/src/bench/g09-journal-sustained.bench.ts +203 -0
- package/src/bench/g10-observability-contention.bench.ts +246 -0
- package/src/bench/g11-cold-start-cycle.bench.ts +164 -0
- package/src/bench/g13-elements.bench.ts +427 -0
- package/src/bench/g14-graceful-shutdown.bench.ts +244 -0
- package/src/bench/g15-postgres-degradation.bench.ts +264 -0
- package/src/bench/g16-live-query-fanout.bench.ts +206 -0
- package/src/bench/lib/event-loop-lag.ts +26 -0
- package/src/bench/lib/infra.ts +60 -0
- package/src/bench/lib/report.ts +52 -0
- package/src/bench/lib/rss-sampler.ts +61 -0
- package/src/bench/lib/signal-pg.ts +88 -0
- package/src/bench/load-app.ts +337 -0
- package/src/bench/load-child.ts +108 -0
- package/src/bench/smoke.bench.ts +43 -0
- package/src/cli/competitor-mention-removal.test.ts +28 -0
- package/src/cli/doctor-fd.ts +117 -0
- package/src/cli/doctor.test.ts +192 -0
- package/src/cli/doctor.ts +129 -1
- package/src/client/create.ts +95 -1
- package/src/client/index.ts +9 -2
- package/src/client/transport.ts +11 -4
- package/src/client/use-live-query.ts +154 -0
- package/src/client-react/index.ts +15 -1
- package/src/client-react/live-resource.ts +246 -0
- package/src/client-react/use-live-query.test.ts +475 -0
- package/src/client-react/use-live-query.ts +530 -0
- package/src/compiler/extract.test.ts +518 -0
- package/src/compiler/extract.ts +386 -19
- package/src/console/server/invoke-user-flow.ts +2 -1
- package/src/console/ui-next/dist/assets/{access-page-DnWbnGzq.js → access-page-DY4N6nnk.js} +1 -1
- package/src/console/ui-next/dist/assets/{flows-page-BiZ4-6yQ.js → flows-page-CsPDMrVM.js} +1 -1
- package/src/console/ui-next/dist/assets/{index-C8NRK2R-.js → index-CcTDXHuz.js} +3 -3
- package/src/console/ui-next/dist/assets/{observability-page-CrB6vd1T.js → observability-page-CKR595wP.js} +1 -1
- package/src/console/ui-next/dist/assets/{store-page-CS5-aETQ.js → store-page-02xOiqIK.js} +3 -3
- package/src/console/ui-next/dist/assets/{units-page-CjtdlW8l.js → units-page-CpPFFKyE.js} +1 -1
- package/src/console/ui-next/dist/assets/{vault-page-C6Xxm9SA.js → vault-page-BsMf-9_W.js} +1 -1
- package/src/console/ui-next/dist/index.html +1 -1
- package/src/console/ui-next/src/features/store/lib/fields-from-table.ts +36 -2
- package/src/drivers/cdc-outbox.ts +389 -0
- package/src/drivers/memory.ts +20 -0
- package/src/drivers/oauth-apple.ts +156 -0
- package/src/drivers/oauth-discord.ts +79 -0
- package/src/drivers/oauth-facebook.ts +80 -0
- package/src/drivers/oauth-figma.ts +116 -0
- package/src/drivers/oauth-github.ts +92 -0
- package/src/drivers/oauth-google.ts +142 -0
- package/src/drivers/oauth-microsoft.ts +174 -0
- package/src/drivers/oauth-oidc.ts +293 -0
- package/src/drivers/oauth-shared.ts +326 -0
- package/src/drivers/oauth-types.ts +159 -0
- package/src/drivers/oauth-x.ts +77 -0
- package/src/drivers/oauth2-common.ts +95 -0
- package/src/drivers/oauth2-token.ts +61 -0
- package/src/drivers/pg-rls-row-passes.ts +251 -0
- package/src/drivers/pg-rls.ts +2 -0
- package/src/drivers/postgres.ts +45 -2
- package/src/drivers/signal-postgres.ts +2 -1
- package/src/elements/channel/runtime.ts +29 -2
- package/src/elements/channel.test.ts +52 -0
- package/src/elements/gate/boot.ts +29 -2
- package/src/elements/store/emit-drizzle.ts +147 -14
- package/src/elements/store/field-ddl.test.ts +118 -0
- package/src/elements/store/field-types.test.ts +455 -0
- package/src/elements/store/list-query.golden.json +777 -0
- package/src/elements/store/list-query.parity.test.ts +396 -0
- package/src/elements/store/list-query.ts +792 -0
- package/src/elements/store/live-default.test.ts +136 -0
- package/src/elements/store/live-http.test.ts +160 -0
- package/src/elements/store/live-isolation.test.ts +291 -0
- package/src/elements/store/live-query-runtime.test.ts +323 -0
- package/src/elements/store/live-query-runtime.ts +403 -0
- package/src/elements/store/live-query-server.test.ts +377 -0
- package/src/elements/store/live-query-server.ts +102 -0
- package/src/elements/store/live-query.ts +97 -0
- package/src/elements/store/resource.ts +189 -680
- package/src/elements/store/rls-row-passes-policies.parity.test.ts +665 -0
- package/src/elements/store/schema-decl.ts +539 -41
- package/src/elements/store/sql-rls-stamp.test.ts +27 -0
- package/src/elements/store/sql-session.ts +297 -35
- package/src/elements/store/table.ts +102 -21
- package/src/elements/store.test.ts +3 -1
- package/src/elements/store.ts +12 -1
- package/src/elements/vault/chaos-child.ts +74 -1
- package/src/elements/vault/chaos.test.ts +4 -2
- package/src/elements/vault/storage.ts +4 -2
- package/src/index.ts +4 -1
- package/src/kernel/app-auth.ts +1 -0
- package/src/kernel/app.ts +116 -2
- package/src/kernel/auth-sharing.test.ts +196 -0
- package/src/kernel/boot.test.ts +3 -3
- package/src/kernel/errors.ts +8 -0
- package/src/kernel/fx.test.ts +1 -0
- package/src/kernel/fx.ts +14 -2
- package/src/kernel/horizontal-child.ts +2 -1
- package/src/kernel/http-resource.ts +33 -7
- package/src/kernel/identity-host-persist.test.ts +119 -0
- package/src/kernel/instance-id.ts +4 -2
- package/src/kernel/journal.ts +2 -1
- package/src/kernel/mcp-tool.test.ts +95 -0
- package/src/kernel/on.ts +9 -0
- package/src/kernel/realtime-bind.ts +326 -0
- package/src/kernel/resource-live.ts +117 -0
- package/src/kernel/triggers.ts +86 -4
- package/src/manifest/diff.ts +37 -0
- package/src/manifest/types.ts +64 -2
- package/src/okid.bench.test.ts +64 -0
- package/src/okid.test.ts +338 -0
- package/src/okid.ts +245 -0
- package/src/plugins/anonymous.ts +19 -1
- package/src/plugins/auth/shared.ts +15 -0
- package/src/plugins/index.ts +2 -0
- package/src/plugins/magic-link.ts +10 -8
- package/src/plugins/mcp-oauth.ts +208 -0
- package/src/plugins/oauth/flow-store.ts +117 -0
- package/src/plugins/oauth/link.ts +69 -0
- package/src/plugins/oauth/shared.ts +108 -0
- package/src/plugins/oauth/token-vault.ts +100 -0
- package/src/plugins/oauth.security.test.ts +535 -0
- package/src/plugins/oauth.ts +532 -0
- package/src/plugins/otp.ts +48 -6
- package/src/plugins/passkey.ts +20 -1
- package/src/plugins/two-factor.ts +11 -0
- package/src/plugins/username.ts +40 -7
- package/src/release/build-lib.ts +7 -1
- package/src/release/measure.ts +1 -0
- package/src/release/official-plugins.ts +4 -1
- package/src/runs/collect.ts +2 -1
- package/src/runs/drivers/files.ts +2 -1
- package/src/test/create-test-app.ts +114 -5
- package/src/test/export-bundle.test.ts +33 -0
- package/src/test/live-signals.test.ts +83 -0
- package/src/test/tenant-isolation.test.ts +175 -0
- package/src/testing.ts +26 -0
- package/src/upgrade/codemods.ts +1 -1
- package/site/content/docs/reference/migrating-environments.mdx +0 -158
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* G3c — mass reconnect storm + `nextResubscribeDelay` backoff verification
|
|
3
|
+
* (`src/client/live.ts`: 500 ms floor → 30 s ceiling).
|
|
4
|
+
*
|
|
5
|
+
* Part 1 (pure): the backoff sequence clamps to ≥500 ms, doubles, and caps at
|
|
6
|
+
* exactly 30 000 ms.
|
|
7
|
+
*
|
|
8
|
+
* Part 2 (live): boot `load-child serve`, open N SSE subscriber loops with
|
|
9
|
+
* auto-resubscribe driven by `nextResubscribeDelay`, SIGKILL the server mid-
|
|
10
|
+
* stream, restart it, and observe the reconnect storm: every client must come
|
|
11
|
+
* back and every observed backoff must stay within [500, 30 000] ms while
|
|
12
|
+
* growing monotonically per client until success.
|
|
13
|
+
*
|
|
14
|
+
* Run: OKE_BENCH=1 OKE_TEST_POSTGRES=1 DATABASE_URL=… OKE_TEST_REDIS_URL=… bun test ./src/bench/g03-signal-reconnect.bench.ts --timeout 180000
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { describe, expect, test } from "bun:test";
|
|
18
|
+
import { LIVE_PG } from "./lib/infra.ts";
|
|
19
|
+
import { DISCLAIMER, HARDWARE, percentile, writeArtifact } from "./lib/report.ts";
|
|
20
|
+
import {
|
|
21
|
+
LIVE_RESUBSCRIBE_INITIAL_MS,
|
|
22
|
+
LIVE_RESUBSCRIBE_MAX_MS,
|
|
23
|
+
nextResubscribeDelay,
|
|
24
|
+
} from "../client/live.ts";
|
|
25
|
+
|
|
26
|
+
const CAL = process.env.OKE_BENCH_CAL === "1";
|
|
27
|
+
const PORT = 6672;
|
|
28
|
+
const BASE = `http://127.0.0.1:${PORT}`;
|
|
29
|
+
const SIGNAL = "bench-live";
|
|
30
|
+
const CLIENTS = CAL ? 50 : 200;
|
|
31
|
+
|
|
32
|
+
type ServeProc = Bun.Subprocess<"ignore", "pipe", "ignore">;
|
|
33
|
+
|
|
34
|
+
function spawnServe(): ServeProc {
|
|
35
|
+
return Bun.spawn(["bun", "run", "src/bench/load-child.ts", "serve", String(PORT)], {
|
|
36
|
+
stdout: "pipe",
|
|
37
|
+
stderr: "ignore",
|
|
38
|
+
env: process.env,
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** One auto-resubscribing SSE client; records every backoff it sleeps. */
|
|
43
|
+
async function reconnectClient(
|
|
44
|
+
target: string,
|
|
45
|
+
stop: Promise<void>,
|
|
46
|
+
delays: number[],
|
|
47
|
+
): Promise<void> {
|
|
48
|
+
let delay = LIVE_RESUBSCRIBE_INITIAL_MS;
|
|
49
|
+
let stopped = false;
|
|
50
|
+
void stop.then(() => {
|
|
51
|
+
stopped = true;
|
|
52
|
+
});
|
|
53
|
+
for (;;) {
|
|
54
|
+
if (stopped) return;
|
|
55
|
+
const ac = new AbortController();
|
|
56
|
+
stop.then(
|
|
57
|
+
() => ac.abort(),
|
|
58
|
+
() => ac.abort(),
|
|
59
|
+
);
|
|
60
|
+
try {
|
|
61
|
+
const res = await fetch(target, {
|
|
62
|
+
signal: ac.signal,
|
|
63
|
+
headers: { accept: "text/event-stream" },
|
|
64
|
+
});
|
|
65
|
+
if (res.body) {
|
|
66
|
+
const reader = res.body.getReader();
|
|
67
|
+
for (;;) {
|
|
68
|
+
const { done } = await reader.read();
|
|
69
|
+
if (done) break;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
} catch {
|
|
73
|
+
/* drop — reconnect below */
|
|
74
|
+
}
|
|
75
|
+
if (stopped || ac.signal.aborted) return;
|
|
76
|
+
// Server is down or dropped us: back off, then retry.
|
|
77
|
+
delays.push(delay);
|
|
78
|
+
await Bun.sleep(delay);
|
|
79
|
+
delay = nextResubscribeDelay(delay);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Resolve the ready line's pid from a serve child, then keep draining stdout. */
|
|
84
|
+
async function readReadyPid(proc: ServeProc): Promise<number> {
|
|
85
|
+
const reader = proc.stdout.getReader();
|
|
86
|
+
const dec = new TextDecoder();
|
|
87
|
+
let buf = "";
|
|
88
|
+
for (;;) {
|
|
89
|
+
const { done, value } = await reader.read();
|
|
90
|
+
if (done) throw new Error("server died before ready");
|
|
91
|
+
buf += dec.decode(value, { stream: true });
|
|
92
|
+
const nl = buf.indexOf("\n");
|
|
93
|
+
if (nl >= 0) {
|
|
94
|
+
const line = buf.slice(0, nl).trim();
|
|
95
|
+
if (line.startsWith("{")) {
|
|
96
|
+
void (async () => {
|
|
97
|
+
try {
|
|
98
|
+
for (;;) {
|
|
99
|
+
const r = await reader.read();
|
|
100
|
+
if (r.done) break;
|
|
101
|
+
}
|
|
102
|
+
} catch {
|
|
103
|
+
/* closed */
|
|
104
|
+
}
|
|
105
|
+
})();
|
|
106
|
+
const parsed = JSON.parse(line) as { pid: number };
|
|
107
|
+
return parsed.pid;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function promiseOf<T>(): { promise: Promise<T>; resolve: (v: T) => void } {
|
|
114
|
+
let resolve!: (v: T) => void;
|
|
115
|
+
const promise = new Promise<T>((r) => {
|
|
116
|
+
resolve = r;
|
|
117
|
+
});
|
|
118
|
+
return { promise, resolve };
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** Established server-side sockets on the bench port for one pid. */
|
|
122
|
+
function countConns(pid: number): number {
|
|
123
|
+
try {
|
|
124
|
+
const out = Bun.spawnSync([
|
|
125
|
+
"sh",
|
|
126
|
+
"-c",
|
|
127
|
+
`lsof -a -p ${pid} -iTCP:${PORT} -sTCP:ESTABLISHED 2>/dev/null | tail -n +2 | wc -l`,
|
|
128
|
+
]);
|
|
129
|
+
return Number(out.stdout.toString().trim());
|
|
130
|
+
} catch {
|
|
131
|
+
return 0;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
describe.skipIf(!process.env.OKE_BENCH || !LIVE_PG)("G3c — reconnect storm + backoff cap", () => {
|
|
136
|
+
test("nextResubscribeDelay floor/doubling/ceiling (pure)", () => {
|
|
137
|
+
expect(LIVE_RESUBSCRIBE_INITIAL_MS).toBe(500);
|
|
138
|
+
expect(LIVE_RESUBSCRIBE_MAX_MS).toBe(30_000);
|
|
139
|
+
// Floor: any input clamps up to the 500 ms initial before doubling.
|
|
140
|
+
expect(nextResubscribeDelay(0)).toBe(1000); // max(0, 500) * 2
|
|
141
|
+
expect(nextResubscribeDelay(250)).toBe(1000); // clamped to floor first
|
|
142
|
+
expect(nextResubscribeDelay(500)).toBe(1000);
|
|
143
|
+
// Ceiling: never exceeds 30 s even from large inputs.
|
|
144
|
+
expect(nextResubscribeDelay(20_000)).toBe(30_000);
|
|
145
|
+
expect(nextResubscribeDelay(30_000)).toBe(30_000);
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
test("backoff sequence stays within [500ms floor, 30s ceiling]", () => {
|
|
149
|
+
let d = LIVE_RESUBSCRIBE_INITIAL_MS;
|
|
150
|
+
const seq: number[] = [d];
|
|
151
|
+
for (let i = 0; i < 12; i++) {
|
|
152
|
+
d = nextResubscribeDelay(d);
|
|
153
|
+
seq.push(d);
|
|
154
|
+
expect(d).toBeGreaterThanOrEqual(LIVE_RESUBSCRIBE_INITIAL_MS);
|
|
155
|
+
expect(d).toBeLessThanOrEqual(LIVE_RESUBSCRIBE_MAX_MS);
|
|
156
|
+
}
|
|
157
|
+
expect(seq[1]).toBe(1000);
|
|
158
|
+
expect(seq.at(-1)).toBe(30_000);
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
test(
|
|
162
|
+
`${CLIENTS} clients survive SIGKILL + restart via capped backoff`,
|
|
163
|
+
async () => {
|
|
164
|
+
const servers: ServeProc[] = [];
|
|
165
|
+
try {
|
|
166
|
+
let server = spawnServe();
|
|
167
|
+
servers.push(server);
|
|
168
|
+
let serverPid = await readReadyPid(server);
|
|
169
|
+
|
|
170
|
+
const target = `${BASE}/_oke/live/${SIGNAL}`;
|
|
171
|
+
const allDelays: number[][] = Array.from({ length: CLIENTS }, () => []);
|
|
172
|
+
const stop = promiseOf<void>();
|
|
173
|
+
const clients = Array.from({ length: CLIENTS }, (_, i) =>
|
|
174
|
+
reconnectClient(target, stop.promise, allDelays[i]!),
|
|
175
|
+
);
|
|
176
|
+
|
|
177
|
+
// Give everyone a live connection first.
|
|
178
|
+
await Bun.sleep(CAL ? 2_000 : 4_000);
|
|
179
|
+
expect(countConns(serverPid)).toBeGreaterThan(CLIENTS * 0.5);
|
|
180
|
+
|
|
181
|
+
// Abrupt kill — no graceful close, every socket drops at once.
|
|
182
|
+
server.kill();
|
|
183
|
+
await server.exited.catch(() => {});
|
|
184
|
+
|
|
185
|
+
await Bun.sleep(CAL ? 1_000 : 2_000); // clients enter backoff
|
|
186
|
+
|
|
187
|
+
// Restart; the storm should land within ≤ a few backoff rounds.
|
|
188
|
+
server = spawnServe();
|
|
189
|
+
servers.push(server);
|
|
190
|
+
serverPid = await readReadyPid(server);
|
|
191
|
+
|
|
192
|
+
const settleMs = CAL ? 8_000 : 35_000;
|
|
193
|
+
await Bun.sleep(settleMs);
|
|
194
|
+
|
|
195
|
+
// Count re-established sockets while clients are still connected.
|
|
196
|
+
const reconnected = countConns(serverPid);
|
|
197
|
+
|
|
198
|
+
stop.resolve();
|
|
199
|
+
await Promise.race([Promise.all(clients), Bun.sleep(10_000)]);
|
|
200
|
+
|
|
201
|
+
const flat = allDelays.flat();
|
|
202
|
+
const minD = flat.length ? Math.min(...flat) : -1;
|
|
203
|
+
const maxD = flat.length ? Math.max(...flat) : -1;
|
|
204
|
+
|
|
205
|
+
const latencies = allDelays.map((d, i) => ({
|
|
206
|
+
client: i,
|
|
207
|
+
attempts: d.length,
|
|
208
|
+
}));
|
|
209
|
+
const activeClients = latencies.filter((l) => l.attempts > 0).length;
|
|
210
|
+
|
|
211
|
+
const issues: string[] = [];
|
|
212
|
+
if (minD < LIVE_RESUBSCRIBE_INITIAL_MS) issues.push(`backoff below floor: ${minD}ms`);
|
|
213
|
+
if (maxD > LIVE_RESUBSCRIBE_MAX_MS) issues.push(`backoff above ceiling: ${maxD}ms`);
|
|
214
|
+
if (reconnected < CLIENTS * 0.9) {
|
|
215
|
+
issues.push(`only ${reconnected}/${CLIENTS} sockets re-established after restart`);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
const metrics: Record<string, number> = {
|
|
219
|
+
clients: CLIENTS,
|
|
220
|
+
backoffSamples: flat.length,
|
|
221
|
+
backoffMinMs: minD,
|
|
222
|
+
backoffMaxMs: maxD,
|
|
223
|
+
backoffP99Ms: percentile(flat, 99),
|
|
224
|
+
clientsThatRetried: activeClients,
|
|
225
|
+
socketsReconnected: reconnected,
|
|
226
|
+
};
|
|
227
|
+
console.log("[G3c] metrics:", JSON.stringify(metrics));
|
|
228
|
+
|
|
229
|
+
const path = await writeArtifact({
|
|
230
|
+
group: "G3c-signal-reconnect",
|
|
231
|
+
hardware: HARDWARE,
|
|
232
|
+
disclaimer: DISCLAIMER,
|
|
233
|
+
command:
|
|
234
|
+
"OKE_BENCH=1 OKE_TEST_POSTGRES=1 DATABASE_URL=$DATABASE_URL OKE_TEST_REDIS_URL=$OKE_TEST_REDIS_URL bun test ./src/bench/g03-signal-reconnect.bench.ts --timeout 180000",
|
|
235
|
+
metrics,
|
|
236
|
+
issues,
|
|
237
|
+
fixes: [],
|
|
238
|
+
remeasured: null,
|
|
239
|
+
});
|
|
240
|
+
console.log(`[G3c] artifact: ${path}`);
|
|
241
|
+
|
|
242
|
+
expect(minD).toBeGreaterThanOrEqual(LIVE_RESUBSCRIBE_INITIAL_MS);
|
|
243
|
+
expect(maxD).toBeLessThanOrEqual(LIVE_RESUBSCRIBE_MAX_MS);
|
|
244
|
+
expect(reconnected).toBeGreaterThanOrEqual(CLIENTS * 0.5);
|
|
245
|
+
} finally {
|
|
246
|
+
for (const s of servers) {
|
|
247
|
+
s.kill();
|
|
248
|
+
await s.exited.catch(() => {});
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
},
|
|
252
|
+
CAL ? 40_000 : 120_000,
|
|
253
|
+
);
|
|
254
|
+
});
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* G3b — SSE subscriber memory + fd growth, with the G12 doctor-fd re-verify.
|
|
3
|
+
*
|
|
4
|
+
* Boots `load-child serve`, attaches 100 SSE subscribers via
|
|
5
|
+
* `load-child flood-sse`, holds, scales to 500 total, and samples RSS + open
|
|
6
|
+
* fds every 5 s for ≥60 s (`lib/rss-sampler.ts`). Cross-checks G12's
|
|
7
|
+
* `estimatePeakFds` per-subscriber constant against observed server-side fd
|
|
8
|
+
* growth; a >20% delta is recorded in `issues` (and fixed in
|
|
9
|
+
* `src/cli/doctor-fd.ts`).
|
|
10
|
+
*
|
|
11
|
+
* Run: OKE_BENCH=1 OKE_TEST_POSTGRES=1 DATABASE_URL=… OKE_TEST_REDIS_URL=… bun test ./src/bench/g03-signal-sse-memory.bench.ts --timeout 240000
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { describe, expect, test } from "bun:test";
|
|
15
|
+
import { estimatePeakFds, FD_COST_PER_SUBSCRIBER } from "../cli/doctor-fd.ts";
|
|
16
|
+
import { LIVE_PG, waitFor } from "./lib/infra.ts";
|
|
17
|
+
import { rssSlopeMbPerMin, startSampler } from "./lib/rss-sampler.ts";
|
|
18
|
+
import { DISCLAIMER, HARDWARE, writeArtifact } from "./lib/report.ts";
|
|
19
|
+
|
|
20
|
+
const CAL = process.env.OKE_BENCH_CAL === "1";
|
|
21
|
+
const PORT = 6671;
|
|
22
|
+
const BASE = `http://127.0.0.1:${PORT}`;
|
|
23
|
+
const SIGNAL = "bench-live";
|
|
24
|
+
const FIRST_WAVE = 100;
|
|
25
|
+
const SECOND_WAVE = 400;
|
|
26
|
+
const HOLD_MS = CAL ? 8_000 : 30_000;
|
|
27
|
+
|
|
28
|
+
interface Ready {
|
|
29
|
+
ready: boolean;
|
|
30
|
+
pid: number;
|
|
31
|
+
port: number;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
type Server = Bun.Subprocess<"ignore", "pipe", "pipe">;
|
|
35
|
+
|
|
36
|
+
function spawnChild(args: string[]): Server {
|
|
37
|
+
return Bun.spawn(["bun", "run", "src/bench/load-child.ts", ...args], {
|
|
38
|
+
stdout: "pipe",
|
|
39
|
+
stderr: "pipe",
|
|
40
|
+
env: process.env,
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Read the first JSON stdout line (the ready line), then keep draining. */
|
|
45
|
+
async function readReady(proc: Server): Promise<Ready> {
|
|
46
|
+
const reader = proc.stdout.getReader();
|
|
47
|
+
const dec = new TextDecoder();
|
|
48
|
+
let buf = "";
|
|
49
|
+
for (;;) {
|
|
50
|
+
const { done, value } = await reader.read();
|
|
51
|
+
if (done) throw new Error("child exited before ready line");
|
|
52
|
+
buf += dec.decode(value, { stream: true });
|
|
53
|
+
const nl = buf.indexOf("\n");
|
|
54
|
+
if (nl >= 0) {
|
|
55
|
+
const line = buf.slice(0, nl).trim();
|
|
56
|
+
if (line.startsWith("{")) {
|
|
57
|
+
void (async () => {
|
|
58
|
+
try {
|
|
59
|
+
for (;;) {
|
|
60
|
+
const r = await reader.read();
|
|
61
|
+
if (r.done) break;
|
|
62
|
+
}
|
|
63
|
+
} catch {
|
|
64
|
+
/* closed */
|
|
65
|
+
}
|
|
66
|
+
})();
|
|
67
|
+
return JSON.parse(line) as Ready;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
describe.skipIf(!process.env.OKE_BENCH || !LIVE_PG)("G3b — SSE memory + fd growth", () => {
|
|
74
|
+
test(
|
|
75
|
+
`${FIRST_WAVE}→${FIRST_WAVE + SECOND_WAVE} SSE subs — RSS/fd sampling + G12 fd cross-check`,
|
|
76
|
+
async () => {
|
|
77
|
+
const server = spawnChild(["serve", String(PORT)]);
|
|
78
|
+
let serverPid = -1;
|
|
79
|
+
let flood1: Server | undefined;
|
|
80
|
+
let flood2: Server | undefined;
|
|
81
|
+
try {
|
|
82
|
+
const ready = await readReady(server);
|
|
83
|
+
serverPid = ready.pid;
|
|
84
|
+
expect(ready.ready).toBe(true);
|
|
85
|
+
|
|
86
|
+
const sampler = startSampler(serverPid, 5_000);
|
|
87
|
+
await Bun.sleep(3_000); // settle baseline
|
|
88
|
+
|
|
89
|
+
// Wave 1: 100 subscribers.
|
|
90
|
+
flood1 = spawnChild(["flood-sse", BASE, SIGNAL, String(FIRST_WAVE)]);
|
|
91
|
+
const ok100 = await waitFor(async () => countSse(serverPid) >= FIRST_WAVE, 30_000);
|
|
92
|
+
await Bun.sleep(HOLD_MS / 2);
|
|
93
|
+
const fdsAt100 = avgFds(sampler.samples());
|
|
94
|
+
|
|
95
|
+
// Wave 2: scale to 500 total.
|
|
96
|
+
flood2 = spawnChild(["flood-sse", BASE, SIGNAL, String(SECOND_WAVE)]);
|
|
97
|
+
const ok500 = await waitFor(
|
|
98
|
+
async () => countSse(serverPid) >= FIRST_WAVE + SECOND_WAVE - 5,
|
|
99
|
+
60_000,
|
|
100
|
+
);
|
|
101
|
+
await Bun.sleep(HOLD_MS);
|
|
102
|
+
|
|
103
|
+
const all = [...sampler.samples()];
|
|
104
|
+
const rssSlope = rssSlopeMbPerMin(all);
|
|
105
|
+
const fdsAt500 = avgFds(all, 4);
|
|
106
|
+
const addedSubs = SECOND_WAVE;
|
|
107
|
+
const observed = Math.max(0, (fdsAt500 - fdsAt100) / addedSubs);
|
|
108
|
+
|
|
109
|
+
await sampler.stop();
|
|
110
|
+
|
|
111
|
+
// G12 cross-check: compare the formula's per-subscriber fd constant
|
|
112
|
+
// against observed server-side fd growth per added subscriber.
|
|
113
|
+
const baseNoManifest = estimatePeakFds(null);
|
|
114
|
+
const g12PerSub = FD_COST_PER_SUBSCRIBER;
|
|
115
|
+
|
|
116
|
+
const denom = observed < 0.05 ? 0.05 : observed;
|
|
117
|
+
const deltaPct = Number((((g12PerSub - denom) / denom) * 100).toFixed(1));
|
|
118
|
+
|
|
119
|
+
const issues: string[] = [];
|
|
120
|
+
if (!ok100) issues.push(`wave-1 did not reach ${FIRST_WAVE} concurrent SSE (server side)`);
|
|
121
|
+
if (!ok500) {
|
|
122
|
+
issues.push(`wave-2 did not reach ~${FIRST_WAVE + SECOND_WAVE} concurrent SSE`);
|
|
123
|
+
}
|
|
124
|
+
if (Math.abs(deltaPct) > 20) {
|
|
125
|
+
issues.push(
|
|
126
|
+
`G12 fd cross-check OFF: estimatePeakFds budgets ${g12PerSub} fds/subscriber, observed ${observed.toFixed(2)} (delta ${deltaPct}%) — constants need reconciliation`,
|
|
127
|
+
);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
const metrics: Record<string, number> = {
|
|
131
|
+
targetSubscribers: FIRST_WAVE + SECOND_WAVE,
|
|
132
|
+
fdsAt100Avg: Number(fdsAt100.toFixed(1)),
|
|
133
|
+
fdsAt500Avg: Number(fdsAt500.toFixed(1)),
|
|
134
|
+
fdsPerSubscriberObserved: Number(observed.toFixed(2)),
|
|
135
|
+
g12EstimateFdsPerSubscriber: g12PerSub,
|
|
136
|
+
g12DeltaPct: deltaPct,
|
|
137
|
+
g12EstimatedNeedNoManifest: baseNoManifest,
|
|
138
|
+
rssSlopeMbPerMinFullRun: rssSlope,
|
|
139
|
+
rssMaxMb: Number(Math.max(...all.map((s) => s.rssMb)).toFixed(1)),
|
|
140
|
+
samples: all.length,
|
|
141
|
+
};
|
|
142
|
+
console.log("[G3b] metrics:", JSON.stringify(metrics));
|
|
143
|
+
|
|
144
|
+
const path = await writeArtifact({
|
|
145
|
+
group: "G3b-signal-sse-memory",
|
|
146
|
+
hardware: HARDWARE,
|
|
147
|
+
disclaimer: DISCLAIMER,
|
|
148
|
+
command:
|
|
149
|
+
"OKE_BENCH=1 OKE_TEST_POSTGRES=1 DATABASE_URL=$DATABASE_URL OKE_TEST_REDIS_URL=$OKE_TEST_REDIS_URL bun test ./src/bench/g03-signal-sse-memory.bench.ts --timeout 240000",
|
|
150
|
+
metrics,
|
|
151
|
+
issues,
|
|
152
|
+
fixes: [],
|
|
153
|
+
remeasured: null,
|
|
154
|
+
});
|
|
155
|
+
console.log(`[G3b] artifact: ${path}`);
|
|
156
|
+
|
|
157
|
+
expect(all.length).toBeGreaterThanOrEqual(CAL ? 3 : 10);
|
|
158
|
+
expect(fdsAt500).toBeGreaterThan(fdsAt100);
|
|
159
|
+
} finally {
|
|
160
|
+
flood1?.kill();
|
|
161
|
+
flood2?.kill();
|
|
162
|
+
server.kill();
|
|
163
|
+
await server.exited.catch(() => {});
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
expect(serverPid).toBeGreaterThan(0);
|
|
167
|
+
},
|
|
168
|
+
CAL ? 90_000 : 200_000,
|
|
169
|
+
);
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
/** Established TCP connections to the bench port held by the server pid. */
|
|
173
|
+
function countSse(pid: number): number {
|
|
174
|
+
try {
|
|
175
|
+
const out = Bun.spawnSync([
|
|
176
|
+
"sh",
|
|
177
|
+
"-c",
|
|
178
|
+
`lsof -a -p ${pid} -iTCP:${PORT} -sTCP:ESTABLISHED 2>/dev/null | tail -n +2 | wc -l`,
|
|
179
|
+
]);
|
|
180
|
+
return Number(out.stdout.toString().trim());
|
|
181
|
+
} catch {
|
|
182
|
+
return 0;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** Average open-fd count over the most recent `n` non-zero samples. */
|
|
187
|
+
function avgFds(samples: readonly { fds: number }[], n = 4): number {
|
|
188
|
+
const win = samples.slice(-n).filter((s) => s.fds > 0);
|
|
189
|
+
if (win.length === 0) return 0;
|
|
190
|
+
return win.reduce((a, s) => a + s.fds, 0) / win.length;
|
|
191
|
+
}
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* G4 — auth / vault hot path (pure CPU, no infra).
|
|
3
|
+
*
|
|
4
|
+
* • `authenticateApiKey` at 1k → 10k → 50k iterations (in-memory store)
|
|
5
|
+
* • raw HMAC-SHA-256 verify (`hashApiKeySecret`) isolated
|
|
6
|
+
* • A/B: boot-bag `vaultRuntime.read` vs request-time `fx.vault.get`
|
|
7
|
+
* over ≥10k reads each → added p99 of the fx wrapper
|
|
8
|
+
*
|
|
9
|
+
* Run: bun test ./src/bench/g04-auth-vault-hotpath.bench.ts
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { describe, expect, test } from "bun:test";
|
|
13
|
+
import {
|
|
14
|
+
authenticateApiKey,
|
|
15
|
+
createApiKey,
|
|
16
|
+
createApiKeyStore,
|
|
17
|
+
hashApiKeySecret,
|
|
18
|
+
} from "../auth/api-keys.ts";
|
|
19
|
+
import { createVaultRuntime } from "../elements/vault/runtime.ts";
|
|
20
|
+
import { vault } from "../elements/vault/declare.ts";
|
|
21
|
+
import { createFx } from "../kernel/fx.ts";
|
|
22
|
+
import { DISCLAIMER, HARDWARE, summarize, writeArtifact } from "./lib/report.ts";
|
|
23
|
+
|
|
24
|
+
const CAL = process.env.OKE_BENCH_CAL === "1";
|
|
25
|
+
const AUTH_ITERS: readonly number[] = CAL ? [1_000] : [1_000, 10_000, 50_000];
|
|
26
|
+
const VAULT_READS = CAL ? 2_000 : 20_000;
|
|
27
|
+
const SECRET_NAME = "OKE_BENCH_G4_SECRET";
|
|
28
|
+
|
|
29
|
+
/** Time an async op `iters` times; returns per-op latency samples in ms. */
|
|
30
|
+
async function timeLoop(iters: number, op: () => Promise<unknown>): Promise<number[]> {
|
|
31
|
+
const samples: number[] = new Array(iters);
|
|
32
|
+
for (let i = 0; i < iters; i += 1) {
|
|
33
|
+
const s = performance.now();
|
|
34
|
+
await op();
|
|
35
|
+
samples[i] = performance.now() - s;
|
|
36
|
+
}
|
|
37
|
+
return samples;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
describe.skipIf(!process.env.OKE_BENCH)("G4 — auth/vault hot path", () => {
|
|
41
|
+
test(
|
|
42
|
+
"authenticateApiKey + HMAC sweep",
|
|
43
|
+
async () => {
|
|
44
|
+
const store = createApiKeyStore();
|
|
45
|
+
const creatorScopes = new Set(["read", "write"]);
|
|
46
|
+
const created = await createApiKey(store, {
|
|
47
|
+
plane: "user" as never,
|
|
48
|
+
name: "bench-key",
|
|
49
|
+
scopes: ["read"],
|
|
50
|
+
creatorId: "bench",
|
|
51
|
+
creatorScopes,
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
const metrics: Record<string, number> = {};
|
|
55
|
+
const issues: string[] = [];
|
|
56
|
+
|
|
57
|
+
// Raw HMAC verify (the inner hash of every auth attempt).
|
|
58
|
+
const hmac = await timeLoop(AUTH_ITERS[0]!, () => hashApiKeySecret(created.secret));
|
|
59
|
+
const hmacS = summarize(hmac);
|
|
60
|
+
metrics["hmac.p50Ms"] = hmacS.p50Ms;
|
|
61
|
+
metrics["hmac.p99Ms"] = hmacS.p99Ms;
|
|
62
|
+
metrics["hmac.opsPerSec"] = hmacS.opsPerSec;
|
|
63
|
+
|
|
64
|
+
// Full authenticateApiKey (hash + scan + persist no-op).
|
|
65
|
+
for (const iters of AUTH_ITERS) {
|
|
66
|
+
const samples = await timeLoop(iters, async () => {
|
|
67
|
+
const row = await authenticateApiKey(store, created.secret);
|
|
68
|
+
if (!row) throw new Error("authenticateApiKey returned null");
|
|
69
|
+
});
|
|
70
|
+
const s = summarize(samples);
|
|
71
|
+
metrics[`auth.n${iters}.p50Ms`] = s.p50Ms;
|
|
72
|
+
metrics[`auth.n${iters}.p99Ms`] = s.p99Ms;
|
|
73
|
+
metrics[`auth.n${iters}.opsPerSec`] = s.opsPerSec;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
console.log("[G4] auth metrics:", JSON.stringify(metrics));
|
|
77
|
+
|
|
78
|
+
// Sanity: wrong secret must not match.
|
|
79
|
+
expect(await authenticateApiKey(store, "oke_bogus")).toBeNull();
|
|
80
|
+
|
|
81
|
+
const path = await writeArtifact({
|
|
82
|
+
group: "G4-auth",
|
|
83
|
+
hardware: HARDWARE,
|
|
84
|
+
disclaimer: DISCLAIMER,
|
|
85
|
+
command: "OKE_BENCH=1 bun test ./src/bench/g04-auth-vault-hotpath.bench.ts",
|
|
86
|
+
metrics,
|
|
87
|
+
issues,
|
|
88
|
+
fixes: [],
|
|
89
|
+
remeasured: null,
|
|
90
|
+
});
|
|
91
|
+
console.log(`[G4] auth artifact: ${path}`);
|
|
92
|
+
},
|
|
93
|
+
CAL ? 60_000 : 180_000,
|
|
94
|
+
);
|
|
95
|
+
|
|
96
|
+
test(
|
|
97
|
+
"vault boot-bag read vs request-time fx.vault.get",
|
|
98
|
+
async () => {
|
|
99
|
+
const vaultRt = createVaultRuntime({
|
|
100
|
+
secrets: [vault.config(SECRET_NAME, { dev: "bench-value-0123456789" })],
|
|
101
|
+
allowDevFallbacks: true,
|
|
102
|
+
});
|
|
103
|
+
await vaultRt.boot();
|
|
104
|
+
|
|
105
|
+
// A: direct boot-bag read (sync).
|
|
106
|
+
const bootSamples: number[] = [];
|
|
107
|
+
for (let i = 0; i < VAULT_READS; i += 1) {
|
|
108
|
+
const s = performance.now();
|
|
109
|
+
const v = vaultRt.read(SECRET_NAME);
|
|
110
|
+
if (v !== "bench-value-0123456789") throw new Error("boot-bag read mismatch");
|
|
111
|
+
bootSamples.push(performance.now() - s);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// B: per-request fx.vault.get (capability + effect ledger + redaction hook).
|
|
115
|
+
const fx = createFx({ flow: "bench.g4", vaultRuntime: vaultRt });
|
|
116
|
+
const fxBaseline: number[] = [];
|
|
117
|
+
for (let i = 0; i < VAULT_READS; i += 1) {
|
|
118
|
+
const s = performance.now();
|
|
119
|
+
const v = await fx.vault.get(SECRET_NAME);
|
|
120
|
+
if (v.reveal() !== "bench-value-0123456789") throw new Error("fx read mismatch");
|
|
121
|
+
fxBaseline.push(performance.now() - s);
|
|
122
|
+
}
|
|
123
|
+
// Second fx pass to warm JSC before trusting percentiles.
|
|
124
|
+
const fxSamples: number[] = [];
|
|
125
|
+
for (let i = 0; i < VAULT_READS; i += 1) {
|
|
126
|
+
const s = performance.now();
|
|
127
|
+
await fx.vault.get(SECRET_NAME);
|
|
128
|
+
fxSamples.push(performance.now() - s);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const a = summarize(bootSamples);
|
|
132
|
+
const b = summarize(fxSamples);
|
|
133
|
+
const addedP99Ms = Number((b.p99Ms - a.p99Ms).toFixed(3));
|
|
134
|
+
const metrics: Record<string, number> = {
|
|
135
|
+
"bootBag.reads": VAULT_READS,
|
|
136
|
+
"bootBag.p50Ms": a.p50Ms,
|
|
137
|
+
"bootBag.p99Ms": a.p99Ms,
|
|
138
|
+
"bootBag.opsPerSec": a.opsPerSec,
|
|
139
|
+
"fx.get.reads": VAULT_READS,
|
|
140
|
+
"fx.get.p50Ms": b.p50Ms,
|
|
141
|
+
"fx.get.p99Ms": b.p99Ms,
|
|
142
|
+
"fx.get.opsPerSec": b.opsPerSec,
|
|
143
|
+
addedP99Ms: addedP99Ms,
|
|
144
|
+
addedP50Ms: Number((b.p50Ms - a.p50Ms).toFixed(3)),
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
console.log("[G4] vault metrics:", JSON.stringify(metrics));
|
|
148
|
+
void fxBaseline;
|
|
149
|
+
|
|
150
|
+
const issues: string[] = [];
|
|
151
|
+
if (addedP99Ms > 1) {
|
|
152
|
+
issues.push(`fx.vault.get adds ${addedP99Ms}ms p99 over boot-bag read (>1ms budget watch)`);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
const path = await writeArtifact({
|
|
156
|
+
group: "G4-vault",
|
|
157
|
+
hardware: HARDWARE,
|
|
158
|
+
disclaimer: DISCLAIMER,
|
|
159
|
+
command: "OKE_BENCH=1 bun test ./src/bench/g04-auth-vault-hotpath.bench.ts",
|
|
160
|
+
metrics,
|
|
161
|
+
issues,
|
|
162
|
+
fixes: [],
|
|
163
|
+
remeasured: null,
|
|
164
|
+
});
|
|
165
|
+
console.log(`[G4] vault artifact: ${path}`);
|
|
166
|
+
expect(b.opsPerSec).toBeGreaterThan(0);
|
|
167
|
+
},
|
|
168
|
+
CAL ? 30_000 : 120_000,
|
|
169
|
+
);
|
|
170
|
+
});
|