@steve31415/baselib 3.45.0 → 3.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/hub/index.d.ts +3 -0
- package/dist/hub/index.js +3 -0
- package/dist/hub/notify.d.ts +52 -0
- package/dist/hub/notify.js +112 -0
- package/dist/hub/server.d.ts +66 -0
- package/dist/hub/server.js +490 -0
- package/dist/hub/types.d.ts +25 -0
- package/dist/hub/types.js +47 -0
- package/dist/s2s.d.ts +12 -1
- package/dist/s2s.js +86 -12
- package/dist/sync/broadcaster.d.ts +4 -0
- package/dist/sync/broadcaster.js +6 -0
- package/dist/sync/db-bridge.d.ts +11 -5
- package/dist/sync/db-bridge.js +35 -11
- package/package.json +6 -1
package/README.md
CHANGED
|
@@ -6,7 +6,7 @@ What it provides and why: `docs/SPEC.md`. How it's put together:
|
|
|
6
6
|
`~/migration/research/base-services-design.md`.
|
|
7
7
|
|
|
8
8
|
Server subpath exports: `config`, `log`, `auth`, `oauth-resource`, `connector`, `s2s`, `db`, `http`, `sync`,
|
|
9
|
-
`app-update`, `llm`, `email`, and `
|
|
9
|
+
`app-update`, `llm`, `email`, `twitter`, and `hub` (the connection hub's server core and app notifier). Browser exports: `log-browser`, `rum`,
|
|
10
10
|
`slow-fetch`, `sync-browser`, and `app-update-browser`.
|
|
11
11
|
|
|
12
12
|
`sync` provides the Postgres event-log and server protocol primitives;
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type { Logger } from '../log.js';
|
|
2
|
+
import { serviceFetch } from '../s2s.js';
|
|
3
|
+
import type { Hint } from './types.js';
|
|
4
|
+
export declare const NOTIFY_HUB_TIMEOUT_MS = 1000;
|
|
5
|
+
/** Consecutive failures that open the circuit. */
|
|
6
|
+
export declare const HUB_BREAKER_FAILURES = 3;
|
|
7
|
+
/** How long an open circuit skips the hub before trying again. */
|
|
8
|
+
export declare const HUB_BREAKER_OPEN_MS = 30000;
|
|
9
|
+
export type AppHint = Omit<Hint, 'app'>;
|
|
10
|
+
export interface HubNotifierOptions {
|
|
11
|
+
/** The hub's canonical URL (also the OIDC audience). */
|
|
12
|
+
hubUrl: string;
|
|
13
|
+
/** This app's name; the hub checks it against the caller's service account. */
|
|
14
|
+
app: string;
|
|
15
|
+
logger: Logger;
|
|
16
|
+
timeoutMs?: number;
|
|
17
|
+
breakerFailures?: number;
|
|
18
|
+
breakerOpenMs?: number;
|
|
19
|
+
/** Injectable for tests (default baselib serviceFetch). */
|
|
20
|
+
fetch?: typeof serviceFetch;
|
|
21
|
+
now?: () => number;
|
|
22
|
+
}
|
|
23
|
+
export declare class HubNotifier {
|
|
24
|
+
private readonly url;
|
|
25
|
+
private readonly audience;
|
|
26
|
+
private readonly app;
|
|
27
|
+
private readonly logger;
|
|
28
|
+
private readonly timeoutMs;
|
|
29
|
+
private readonly breakerFailures;
|
|
30
|
+
private readonly breakerOpenMs;
|
|
31
|
+
private readonly fetch;
|
|
32
|
+
private readonly now;
|
|
33
|
+
private consecutiveFailures;
|
|
34
|
+
private openUntil;
|
|
35
|
+
private skipped;
|
|
36
|
+
constructor(opts: HubNotifierOptions);
|
|
37
|
+
/** Send one hint. Never throws; resolves within the bound. */
|
|
38
|
+
notify(hint: AppHint): Promise<void>;
|
|
39
|
+
/** Hints for one request, coalesced: add() as events commit, then await
|
|
40
|
+
* flush() once, after the commit, before answering. */
|
|
41
|
+
batch(): HubNotifyBatch;
|
|
42
|
+
private send;
|
|
43
|
+
}
|
|
44
|
+
export declare class HubNotifyBatch {
|
|
45
|
+
private readonly notifier;
|
|
46
|
+
private readonly hints;
|
|
47
|
+
constructor(notifier: HubNotifier);
|
|
48
|
+
add(hint: AppHint): void;
|
|
49
|
+
/** Send every distinct hint (concurrently, each within the notifier's
|
|
50
|
+
* bound) and empty the batch. Never throws. */
|
|
51
|
+
flush(): Promise<void>;
|
|
52
|
+
}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
// The app side of the connection hub (design §5.3, review §14.3 S1/S7):
|
|
2
|
+
// after every commit, whoever made it, the app tells the hub which user's
|
|
3
|
+
// clients should go and ask. Rules: called after commit and awaited inside
|
|
4
|
+
// the request (a throttled instance still has CPU then); one attempt bounded
|
|
5
|
+
// at 1 s including the ID-token mint, no inline retry; a failure is a WARN and
|
|
6
|
+
// never fails the write; after repeated failures the notifier skips the hub
|
|
7
|
+
// for a while (a hub outage must not add 1 s to every write). Fire it on an
|
|
8
|
+
// apply outcome of `duplicate` as well as `applied`: a retried POST whose first
|
|
9
|
+
// attempt secretly succeeded must still hint. Lost hints are tolerated: every
|
|
10
|
+
// client re-checks on its own timer.
|
|
11
|
+
import { serializeError } from '../log.js';
|
|
12
|
+
import { serviceFetch } from '../s2s.js';
|
|
13
|
+
export const NOTIFY_HUB_TIMEOUT_MS = 1_000;
|
|
14
|
+
/** Consecutive failures that open the circuit. */
|
|
15
|
+
export const HUB_BREAKER_FAILURES = 3;
|
|
16
|
+
/** How long an open circuit skips the hub before trying again. */
|
|
17
|
+
export const HUB_BREAKER_OPEN_MS = 30_000;
|
|
18
|
+
export class HubNotifier {
|
|
19
|
+
url;
|
|
20
|
+
audience;
|
|
21
|
+
app;
|
|
22
|
+
logger;
|
|
23
|
+
timeoutMs;
|
|
24
|
+
breakerFailures;
|
|
25
|
+
breakerOpenMs;
|
|
26
|
+
fetch;
|
|
27
|
+
now;
|
|
28
|
+
consecutiveFailures = 0;
|
|
29
|
+
openUntil = -Infinity;
|
|
30
|
+
skipped = 0;
|
|
31
|
+
constructor(opts) {
|
|
32
|
+
this.audience = opts.hubUrl.replace(/\/$/, '');
|
|
33
|
+
this.url = `${this.audience}/api/notify`;
|
|
34
|
+
this.app = opts.app;
|
|
35
|
+
this.logger = opts.logger;
|
|
36
|
+
this.timeoutMs = opts.timeoutMs ?? NOTIFY_HUB_TIMEOUT_MS;
|
|
37
|
+
this.breakerFailures = opts.breakerFailures ?? HUB_BREAKER_FAILURES;
|
|
38
|
+
this.breakerOpenMs = opts.breakerOpenMs ?? HUB_BREAKER_OPEN_MS;
|
|
39
|
+
this.fetch = opts.fetch ?? serviceFetch;
|
|
40
|
+
this.now = opts.now ?? (() => performance.now());
|
|
41
|
+
}
|
|
42
|
+
/** Send one hint. Never throws; resolves within the bound. */
|
|
43
|
+
async notify(hint) {
|
|
44
|
+
if (this.now() < this.openUntil) {
|
|
45
|
+
this.skipped++;
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
const failure = await this.send(hint);
|
|
49
|
+
if (failure === null) {
|
|
50
|
+
if (this.consecutiveFailures >= this.breakerFailures) {
|
|
51
|
+
this.logger.info('hub notify recovered', { skipped: this.skipped });
|
|
52
|
+
}
|
|
53
|
+
this.consecutiveFailures = 0;
|
|
54
|
+
this.skipped = 0;
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
this.consecutiveFailures++;
|
|
58
|
+
const fields = { kind: hint.kind, consecutiveFailures: this.consecutiveFailures, ...failure.fields };
|
|
59
|
+
if (failure.bug)
|
|
60
|
+
this.logger.error('hub notify refused', fields);
|
|
61
|
+
else
|
|
62
|
+
this.logger.warn('hub notify failed', fields);
|
|
63
|
+
if (this.consecutiveFailures >= this.breakerFailures) {
|
|
64
|
+
this.openUntil = this.now() + this.breakerOpenMs;
|
|
65
|
+
if (this.consecutiveFailures === this.breakerFailures) {
|
|
66
|
+
this.logger.warn('hub notify circuit open', { openMs: this.breakerOpenMs });
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
/** Hints for one request, coalesced: add() as events commit, then await
|
|
71
|
+
* flush() once, after the commit, before answering. */
|
|
72
|
+
batch() {
|
|
73
|
+
return new HubNotifyBatch(this);
|
|
74
|
+
}
|
|
75
|
+
async send(hint) {
|
|
76
|
+
try {
|
|
77
|
+
const res = await this.fetch(this.url, {
|
|
78
|
+
audience: this.audience,
|
|
79
|
+
method: 'POST',
|
|
80
|
+
headers: { 'content-type': 'application/json' },
|
|
81
|
+
body: JSON.stringify({ ...hint, app: this.app }),
|
|
82
|
+
signal: AbortSignal.timeout(this.timeoutMs),
|
|
83
|
+
});
|
|
84
|
+
if (res.status === 202)
|
|
85
|
+
return null;
|
|
86
|
+
// 400/401/403/413: the hint or the grant is wrong, not the network.
|
|
87
|
+
return { bug: [400, 401, 403, 413].includes(res.status), fields: { status: res.status } };
|
|
88
|
+
}
|
|
89
|
+
catch (error) {
|
|
90
|
+
return { bug: false, fields: { error: serializeError(error) } };
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
export class HubNotifyBatch {
|
|
95
|
+
notifier;
|
|
96
|
+
hints = new Map();
|
|
97
|
+
constructor(notifier) {
|
|
98
|
+
this.notifier = notifier;
|
|
99
|
+
}
|
|
100
|
+
add(hint) {
|
|
101
|
+
const key = JSON.stringify([hint.user, hint.kind, hint.ref ?? null, hint.payload ?? null]);
|
|
102
|
+
if (!this.hints.has(key))
|
|
103
|
+
this.hints.set(key, hint);
|
|
104
|
+
}
|
|
105
|
+
/** Send every distinct hint (concurrently, each within the notifier's
|
|
106
|
+
* bound) and empty the batch. Never throws. */
|
|
107
|
+
async flush() {
|
|
108
|
+
const hints = [...this.hints.values()];
|
|
109
|
+
this.hints.clear();
|
|
110
|
+
await Promise.all(hints.map((hint) => this.notifier.notify(hint)));
|
|
111
|
+
}
|
|
112
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import type { Hono } from 'hono';
|
|
2
|
+
import type { UpgradeWebSocket } from 'hono/ws';
|
|
3
|
+
import type pg from 'pg';
|
|
4
|
+
import { type RequireIdentityOptions } from '../auth.js';
|
|
5
|
+
import { type Logger } from '../log.js';
|
|
6
|
+
import type { Grants } from '../s2s.js';
|
|
7
|
+
import { type DbBroadcastBridgeOptions } from '../sync/db-bridge.js';
|
|
8
|
+
/** The hub database's one table (the hub repo's first migration carries the
|
|
9
|
+
* same statements). */
|
|
10
|
+
export declare const HUB_SCHEMA_SQL = "\ncreate table if not exists daemon_tokens (\n app text not null,\n daemon_id text not null,\n token_hash text not null unique,\n expires_at timestamptz not null,\n registered_at timestamptz not null default now(),\n primary key (app, daemon_id)\n);\ncreate index if not exists daemon_tokens_expires_at on daemon_tokens (expires_at);\n";
|
|
11
|
+
export declare const HUB_CHANNEL = "hub_hints";
|
|
12
|
+
/** A registered token's lifetime, on the database clock. */
|
|
13
|
+
export declare const DAEMON_TOKEN_TTL_SECONDS = 3600;
|
|
14
|
+
/** Distinct daemon ids per app with live tokens: a backstop against a daemon
|
|
15
|
+
* minting in a loop (each process upserts one row by its daemon id). */
|
|
16
|
+
export declare const DAEMON_IDS_PER_APP_MAX = 20;
|
|
17
|
+
/** The cross-copy publish's own bound; it runs after the 202. */
|
|
18
|
+
export declare const HUB_PUBLISH_TIMEOUT_MS = 2000;
|
|
19
|
+
export declare const DAEMON_WAIT_MAX_SECONDS = 55;
|
|
20
|
+
export declare const DAEMON_LOOKBACK_MAX_MS = 60000;
|
|
21
|
+
/** 401s per client address per minute on the token path before further
|
|
22
|
+
* tokens from it are refused without a database lookup. */
|
|
23
|
+
export declare const TOKEN_FAILURES_PER_MIN = 20;
|
|
24
|
+
export interface HubOptions {
|
|
25
|
+
pool: pg.Pool;
|
|
26
|
+
logger: Logger;
|
|
27
|
+
/** The hub's canonical URL, e.g. https://hub.apps.snewman.net: the OIDC
|
|
28
|
+
* audience, and its host is the load-balancer host for client addresses. */
|
|
29
|
+
audience: string;
|
|
30
|
+
/** The hub's grants: every app's service account on POST /api/notify and
|
|
31
|
+
* POST /api/register-daemon; the coding agent is always admitted. */
|
|
32
|
+
grants: Grants;
|
|
33
|
+
/** Service accounts are `<app>-svc@<serviceAccountDomain>`; the part
|
|
34
|
+
* before `-svc@` names the caller's app (test mode: e.g. 'test'). */
|
|
35
|
+
serviceAccountDomain: string;
|
|
36
|
+
/** Pages are `https://<app>.<appDomain>`; a socket for app A is admitted
|
|
37
|
+
* only from that origin. */
|
|
38
|
+
appDomain: string;
|
|
39
|
+
/** Also admit `http://localhost:<port>` and `http://127.0.0.1:<port>`
|
|
40
|
+
* origins (local development and Playwright). */
|
|
41
|
+
allowLocalOrigins?: boolean;
|
|
42
|
+
upgradeWebSocket: UpgradeWebSocket;
|
|
43
|
+
/** Passed through to requireIdentity (authUrl, whoamiFetch, verifyClient,
|
|
44
|
+
* rateLimitPerMin…). The hub admits invited users for sockets. */
|
|
45
|
+
identity?: Partial<Omit<RequireIdentityOptions, 'audience' | 'grants' | 'logger' | 'userAccess'>>;
|
|
46
|
+
/** Injectable for tests. */
|
|
47
|
+
bridge?: Partial<Pick<DbBroadcastBridgeOptions, 'heartbeatMs' | 'heartbeatTimeoutMs' | 'reconnectDelaysMs'>>;
|
|
48
|
+
/** Monotonic milliseconds (tests). */
|
|
49
|
+
now?: () => number;
|
|
50
|
+
}
|
|
51
|
+
export interface Hub {
|
|
52
|
+
/** Start the cross-copy bridge. Call before the server accepts
|
|
53
|
+
* connections; never rejects (a failed first connect enters the bridge's
|
|
54
|
+
* reconnect ladder). */
|
|
55
|
+
start(): Promise<void>;
|
|
56
|
+
/** SIGTERM: stop admitting, close browser sockets with 1012, answer every
|
|
57
|
+
* parked wait 204, wait (bounded) for in-flight publishes, stop the
|
|
58
|
+
* bridge. The caller then ends the pool and flushes the logger. */
|
|
59
|
+
stop(): Promise<void>;
|
|
60
|
+
/** Local counts, for logs and tests. */
|
|
61
|
+
stats(): {
|
|
62
|
+
sockets: number;
|
|
63
|
+
waits: number;
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
export declare function createHub(app: Hono, opts: HubOptions): Hub;
|
|
@@ -0,0 +1,490 @@
|
|
|
1
|
+
// The connection hub's server core (design: ~/migration/research/
|
|
2
|
+
// connection-hub-design-2026-10-30.md §5, as refined by §14 and its review
|
|
3
|
+
// dispositions §14.3, which bind). The `hub` repo is a thin wrapper: it builds
|
|
4
|
+
// the pool, logger and WebSocket upgrader, mounts this, and starts the server.
|
|
5
|
+
//
|
|
6
|
+
// One always-on service holds the fleet's long-lived connections so the apps
|
|
7
|
+
// need not: browser pages park a socket here and daemons park their idle
|
|
8
|
+
// waits here; apps POST a hint after every commit and the hub fans it out.
|
|
9
|
+
// Hints are information only ("go ask the app"): the hub stores none, a hint
|
|
10
|
+
// with no listener is gone, and every client re-checks on its own timer.
|
|
11
|
+
//
|
|
12
|
+
// Copies. Any number of hub copies may run (two is normal). Each delivers a
|
|
13
|
+
// hint to its own sockets and waits, then publishes it on the hub database's
|
|
14
|
+
// `hub_hints` channel tagged with its bridge's origin; the other copies relay
|
|
15
|
+
// it to theirs and skip their own. State per copy is memory only and
|
|
16
|
+
// rebuildable from nothing, apart from the daemon-token table (§14.1), which
|
|
17
|
+
// both copies read.
|
|
18
|
+
//
|
|
19
|
+
// Routes:
|
|
20
|
+
// POST /api/notify app → hub, OIDC; the hint's app must be the
|
|
21
|
+
// caller's own (from its service account)
|
|
22
|
+
// POST /api/register-daemon app → hub, OIDC; registers a daemon hub token
|
|
23
|
+
// GET /api/daemon/wait daemon, X-Hub-Token (or a granted service
|
|
24
|
+
// caller, for the dark-run curl); answers on a
|
|
25
|
+
// `work` hint, else 204 after `wait` seconds
|
|
26
|
+
// GET /api/ws?app=<app> browser WebSocket, fleet cookie, Origin
|
|
27
|
+
// must be that app's page
|
|
28
|
+
import { createHash } from 'node:crypto';
|
|
29
|
+
import { createMiddleware } from 'hono/factory';
|
|
30
|
+
import { requireIdentity } from '../auth.js';
|
|
31
|
+
import { forwardedClientIp } from '../http.js';
|
|
32
|
+
import { serializeError } from '../log.js';
|
|
33
|
+
import { DbBroadcastBridge } from '../sync/db-bridge.js';
|
|
34
|
+
import { checkHint, isAppName } from './types.js';
|
|
35
|
+
/** The hub database's one table (the hub repo's first migration carries the
|
|
36
|
+
* same statements). */
|
|
37
|
+
export const HUB_SCHEMA_SQL = `
|
|
38
|
+
create table if not exists daemon_tokens (
|
|
39
|
+
app text not null,
|
|
40
|
+
daemon_id text not null,
|
|
41
|
+
token_hash text not null unique,
|
|
42
|
+
expires_at timestamptz not null,
|
|
43
|
+
registered_at timestamptz not null default now(),
|
|
44
|
+
primary key (app, daemon_id)
|
|
45
|
+
);
|
|
46
|
+
create index if not exists daemon_tokens_expires_at on daemon_tokens (expires_at);
|
|
47
|
+
`;
|
|
48
|
+
export const HUB_CHANNEL = 'hub_hints';
|
|
49
|
+
/** A registered token's lifetime, on the database clock. */
|
|
50
|
+
export const DAEMON_TOKEN_TTL_SECONDS = 3600;
|
|
51
|
+
/** Distinct daemon ids per app with live tokens: a backstop against a daemon
|
|
52
|
+
* minting in a loop (each process upserts one row by its daemon id). */
|
|
53
|
+
export const DAEMON_IDS_PER_APP_MAX = 20;
|
|
54
|
+
/** The cross-copy publish's own bound; it runs after the 202. */
|
|
55
|
+
export const HUB_PUBLISH_TIMEOUT_MS = 2_000;
|
|
56
|
+
export const DAEMON_WAIT_MAX_SECONDS = 55;
|
|
57
|
+
export const DAEMON_LOOKBACK_MAX_MS = 60_000;
|
|
58
|
+
/** 401s per client address per minute on the token path before further
|
|
59
|
+
* tokens from it are refused without a database lookup. */
|
|
60
|
+
export const TOKEN_FAILURES_PER_MIN = 20;
|
|
61
|
+
const TOKEN_FORMAT = /^hubt_[A-Za-z0-9_-]{43}$/;
|
|
62
|
+
const TOKEN_HASH_FORMAT = /^[0-9a-f]{64}$/;
|
|
63
|
+
const DAEMON_ID_FORMAT = /^[A-Za-z0-9_-]{8,64}$/;
|
|
64
|
+
export function createHub(app, opts) {
|
|
65
|
+
const { pool, logger } = opts;
|
|
66
|
+
const now = opts.now ?? (() => performance.now());
|
|
67
|
+
const canonicalHost = new URL(opts.audience).host;
|
|
68
|
+
const identity = requireIdentity({
|
|
69
|
+
rateLimitPerMin: 6_000,
|
|
70
|
+
...opts.identity,
|
|
71
|
+
audience: opts.audience,
|
|
72
|
+
grants: opts.grants,
|
|
73
|
+
logger,
|
|
74
|
+
userAccess: 'invited',
|
|
75
|
+
});
|
|
76
|
+
const serviceAccount = new RegExp(`^([a-z][a-z0-9-]*)-svc@${escapeRegExp(opts.serviceAccountDomain)}$`);
|
|
77
|
+
// app → user → sockets
|
|
78
|
+
const sockets = new Map();
|
|
79
|
+
const waits = new Map();
|
|
80
|
+
// app → when this copy last saw a `work` hint for it (local or relayed)
|
|
81
|
+
const lastWork = new Map();
|
|
82
|
+
// when the bridge last recovered: hints sent while it was deaf are unknowable
|
|
83
|
+
let lastBlanketAt = -Infinity;
|
|
84
|
+
const publishes = new Set();
|
|
85
|
+
const tokenFailures = new FailureCounter(TOKEN_FAILURES_PER_MIN);
|
|
86
|
+
let stopping = false;
|
|
87
|
+
const bridge = new DbBroadcastBridge({
|
|
88
|
+
pool,
|
|
89
|
+
channel: HUB_CHANNEL,
|
|
90
|
+
logger,
|
|
91
|
+
ignoreOwn: true,
|
|
92
|
+
...opts.bridge,
|
|
93
|
+
onMessage: (message) => {
|
|
94
|
+
const check = checkHint(message?.hint);
|
|
95
|
+
if (!check.ok) {
|
|
96
|
+
logger.warn('hub relayed hint invalid', { error: check.error });
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
deliverLocal(check.hint);
|
|
100
|
+
},
|
|
101
|
+
onRecovered: () => blanket(),
|
|
102
|
+
});
|
|
103
|
+
function deliverLocal(hint) {
|
|
104
|
+
if (hint.kind === 'work') {
|
|
105
|
+
// Work hints go to daemon waits only, never to a browser socket
|
|
106
|
+
// whatever its app (review B1).
|
|
107
|
+
lastWork.set(hint.app, { at: now(), ...(hint.ref !== undefined ? { ref: hint.ref } : {}) });
|
|
108
|
+
for (const waiter of [...(waits.get(hint.app) ?? [])])
|
|
109
|
+
waiter.finish(workResponse(hint.ref));
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
const users = sockets.get(hint.app);
|
|
113
|
+
if (!users)
|
|
114
|
+
return;
|
|
115
|
+
const targets = hint.user === '*' ? [...users.values()].flatMap((set) => [...set]) : [...(users.get(hint.user) ?? [])];
|
|
116
|
+
if (targets.length === 0)
|
|
117
|
+
return;
|
|
118
|
+
const frame = JSON.stringify({ type: 'hint', ...hint });
|
|
119
|
+
for (const socket of targets)
|
|
120
|
+
send(socket, frame);
|
|
121
|
+
}
|
|
122
|
+
function send(socket, frame) {
|
|
123
|
+
try {
|
|
124
|
+
socket.ws.send(frame);
|
|
125
|
+
}
|
|
126
|
+
catch (error) {
|
|
127
|
+
logger.warn('hub socket send failed', { app: socket.app, error: serializeError(error) });
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
function blanket() {
|
|
131
|
+
lastBlanketAt = now();
|
|
132
|
+
for (const [appName, users] of sockets) {
|
|
133
|
+
for (const [user, set] of users) {
|
|
134
|
+
const frame = JSON.stringify({ type: 'hint', app: appName, user, kind: 'sync_available' });
|
|
135
|
+
for (const socket of set)
|
|
136
|
+
send(socket, frame);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
for (const set of waits.values())
|
|
140
|
+
for (const waiter of [...set])
|
|
141
|
+
waiter.finish(workResponse());
|
|
142
|
+
}
|
|
143
|
+
function publish(hint) {
|
|
144
|
+
const p = withinMs(bridge.notify({ hint }), HUB_PUBLISH_TIMEOUT_MS, 'hub publish')
|
|
145
|
+
.catch((error) => {
|
|
146
|
+
logger.warn('hub publish failed', { app: hint.app, kind: hint.kind, error: serializeError(error) });
|
|
147
|
+
})
|
|
148
|
+
.finally(() => publishes.delete(p));
|
|
149
|
+
publishes.add(p);
|
|
150
|
+
}
|
|
151
|
+
/** requireIdentity without letting it end the chain: whether it admitted
|
|
152
|
+
* the request (identity set), else the response it chose. */
|
|
153
|
+
async function runIdentity(c) {
|
|
154
|
+
let admitted = false;
|
|
155
|
+
const response = await identity(c, async () => {
|
|
156
|
+
admitted = true;
|
|
157
|
+
});
|
|
158
|
+
return { admitted, response: response ?? undefined };
|
|
159
|
+
}
|
|
160
|
+
/** The caller's app, from its service account; null for any other caller
|
|
161
|
+
* (the coding agent, a browser). */
|
|
162
|
+
function callerApp(c) {
|
|
163
|
+
const id = c.var.identity;
|
|
164
|
+
if (id?.type !== 's2s')
|
|
165
|
+
return null;
|
|
166
|
+
return serviceAccount.exec(id.email)?.[1] ?? null;
|
|
167
|
+
}
|
|
168
|
+
// ---- app → hub ----
|
|
169
|
+
app.post('/api/notify', identity, async (c) => {
|
|
170
|
+
const caller = callerApp(c);
|
|
171
|
+
if (!caller)
|
|
172
|
+
return c.json({ error: 'app service accounts only' }, 403);
|
|
173
|
+
const check = checkHint(await c.req.json().catch(() => null));
|
|
174
|
+
if (!check.ok)
|
|
175
|
+
return c.json({ error: check.error }, check.status);
|
|
176
|
+
if (check.hint.app !== caller) {
|
|
177
|
+
logger.warn('hub notify for another app refused', { caller, app: check.hint.app });
|
|
178
|
+
return c.json({ error: 'an app may hint only its own clients' }, 403);
|
|
179
|
+
}
|
|
180
|
+
deliverLocal(check.hint);
|
|
181
|
+
// After local delivery and off the response path, with its own bound, so
|
|
182
|
+
// a slow hub pool never trips the apps' 1 s notify bound (review S1).
|
|
183
|
+
publish(check.hint);
|
|
184
|
+
return c.body(null, 202);
|
|
185
|
+
});
|
|
186
|
+
app.post('/api/register-daemon', identity, async (c) => {
|
|
187
|
+
const caller = callerApp(c);
|
|
188
|
+
if (!caller)
|
|
189
|
+
return c.json({ error: 'app service accounts only' }, 403);
|
|
190
|
+
const body = (await c.req.json().catch(() => null));
|
|
191
|
+
const tokenHash = body?.tokenHash;
|
|
192
|
+
const daemonId = body?.daemonId;
|
|
193
|
+
if (typeof tokenHash !== 'string' || !TOKEN_HASH_FORMAT.test(tokenHash))
|
|
194
|
+
return c.json({ error: 'invalid tokenHash' }, 400);
|
|
195
|
+
if (typeof daemonId !== 'string' || !DAEMON_ID_FORMAT.test(daemonId))
|
|
196
|
+
return c.json({ error: 'invalid daemonId' }, 400);
|
|
197
|
+
const client = await pool.connect();
|
|
198
|
+
let ok = false;
|
|
199
|
+
try {
|
|
200
|
+
await client.query('begin');
|
|
201
|
+
// Serializes this app's registrations, so the cap is exact.
|
|
202
|
+
await client.query(`select pg_advisory_xact_lock(hashtext('hub_register:' || $1))`, [caller]);
|
|
203
|
+
const { rows } = await client.query(`select count(*)::int as n from daemon_tokens where app = $1 and daemon_id <> $2 and expires_at > now()`, [caller, daemonId]);
|
|
204
|
+
if (rows[0].n >= DAEMON_IDS_PER_APP_MAX) {
|
|
205
|
+
await client.query('rollback');
|
|
206
|
+
ok = true;
|
|
207
|
+
logger.error('hub daemon registration cap reached', { app: caller, daemons: rows[0].n });
|
|
208
|
+
return c.json({ error: 'too many daemons registered for this app' }, 429);
|
|
209
|
+
}
|
|
210
|
+
// Upsert by daemon id: a retry or an orphaned register overwrites the
|
|
211
|
+
// process's one row, and its previous token dies at once (review S3).
|
|
212
|
+
await client.query(`insert into daemon_tokens (app, daemon_id, token_hash, expires_at)
|
|
213
|
+
values ($1, $2, $3, now() + make_interval(secs => $4))
|
|
214
|
+
on conflict (app, daemon_id) do update
|
|
215
|
+
set token_hash = excluded.token_hash, expires_at = excluded.expires_at, registered_at = now()`, [caller, daemonId, tokenHash, DAEMON_TOKEN_TTL_SECONDS]);
|
|
216
|
+
await client.query(`delete from daemon_tokens where expires_at < now() - interval '1 day'`);
|
|
217
|
+
await client.query('commit');
|
|
218
|
+
ok = true;
|
|
219
|
+
}
|
|
220
|
+
finally {
|
|
221
|
+
if (!ok)
|
|
222
|
+
await client.query('rollback').catch(() => { });
|
|
223
|
+
client.release(ok ? undefined : true);
|
|
224
|
+
}
|
|
225
|
+
return c.json({ ttlSeconds: DAEMON_TOKEN_TTL_SECONDS });
|
|
226
|
+
});
|
|
227
|
+
// ---- daemon waits ----
|
|
228
|
+
/** Resolves the waiting app: from a daemon hub token, or (no token) from
|
|
229
|
+
* `?app=` for a granted service caller. Cookies are refused (review B1). */
|
|
230
|
+
const daemonAuth = createMiddleware(async (c, next) => {
|
|
231
|
+
const token = c.req.header('x-hub-token');
|
|
232
|
+
if (token === undefined) {
|
|
233
|
+
const { admitted, response } = await runIdentity(c);
|
|
234
|
+
if (!admitted)
|
|
235
|
+
return response;
|
|
236
|
+
if (c.var.identity?.type !== 's2s')
|
|
237
|
+
return c.json({ error: 'service callers only' }, 403);
|
|
238
|
+
const named = c.req.query('app');
|
|
239
|
+
if (!isAppName(named))
|
|
240
|
+
return c.json({ error: 'app required' }, 400);
|
|
241
|
+
c.set('hubApp', named);
|
|
242
|
+
await next();
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
if (c.req.header('authorization') !== undefined) {
|
|
246
|
+
// A daemon's hub client never sends Authorization; both together means
|
|
247
|
+
// an app credential is travelling to the hub (review S4).
|
|
248
|
+
logger.error('hub daemon wait with both X-Hub-Token and Authorization', {});
|
|
249
|
+
return c.json({ error: 'send X-Hub-Token or Authorization, not both' }, 400);
|
|
250
|
+
}
|
|
251
|
+
const ip = forwardedClientIp(c, canonicalHost);
|
|
252
|
+
if (tokenFailures.blocked(ip, now()))
|
|
253
|
+
return c.json({ error: 'too many invalid tokens' }, 429);
|
|
254
|
+
if (!TOKEN_FORMAT.test(token)) {
|
|
255
|
+
tokenFailures.fail(ip, now());
|
|
256
|
+
return c.json({ error: 'invalid token' }, 401);
|
|
257
|
+
}
|
|
258
|
+
let found;
|
|
259
|
+
try {
|
|
260
|
+
const { rows } = await pool.query(`select app from daemon_tokens where token_hash = $1 and expires_at > now()`, [createHash('sha256').update(token).digest('hex')]);
|
|
261
|
+
found = rows[0]?.app;
|
|
262
|
+
}
|
|
263
|
+
catch (error) {
|
|
264
|
+
// 503, not 401: a hub database outage must not make every daemon
|
|
265
|
+
// re-mint; the daemon treats 503 as a hub failure.
|
|
266
|
+
logger.warn('hub token lookup failed', { error: serializeError(error) });
|
|
267
|
+
return c.json({ error: 'token lookup unavailable' }, 503);
|
|
268
|
+
}
|
|
269
|
+
if (!found) {
|
|
270
|
+
tokenFailures.fail(ip, now());
|
|
271
|
+
return c.json({ error: 'invalid or expired token' }, 401);
|
|
272
|
+
}
|
|
273
|
+
c.set('hubApp', found);
|
|
274
|
+
await next();
|
|
275
|
+
});
|
|
276
|
+
app.get('/api/daemon/wait', daemonAuth, (c) => {
|
|
277
|
+
const appName = c.var.hubApp;
|
|
278
|
+
const waitSeconds = intParam(c.req.query('wait'), 0, DAEMON_WAIT_MAX_SECONDS, 50);
|
|
279
|
+
const lookbackMs = intParam(c.req.query('lookbackMs'), 0, DAEMON_LOOKBACK_MAX_MS, 0);
|
|
280
|
+
if (stopping)
|
|
281
|
+
return c.body(null, 204);
|
|
282
|
+
// A work hint between the daemon's direct request and this wait being
|
|
283
|
+
// parked would otherwise be lost to the 30 s timer (review S2).
|
|
284
|
+
const t = now();
|
|
285
|
+
const last = lastWork.get(appName);
|
|
286
|
+
if (lookbackMs > 0 && last && t - last.at <= lookbackMs)
|
|
287
|
+
return workResponse(last.ref);
|
|
288
|
+
if (lookbackMs > 0 && t - lastBlanketAt <= lookbackMs)
|
|
289
|
+
return workResponse();
|
|
290
|
+
return new Promise((resolve) => {
|
|
291
|
+
let set = waits.get(appName);
|
|
292
|
+
if (!set)
|
|
293
|
+
waits.set(appName, (set = new Set()));
|
|
294
|
+
const waiters = set;
|
|
295
|
+
const signal = c.req.raw.signal;
|
|
296
|
+
const waiter = {
|
|
297
|
+
app: appName,
|
|
298
|
+
finish: (response) => {
|
|
299
|
+
if (!waiters.delete(waiter))
|
|
300
|
+
return;
|
|
301
|
+
if (waiters.size === 0 && waits.get(appName) === waiters)
|
|
302
|
+
waits.delete(appName);
|
|
303
|
+
clearTimeout(timer);
|
|
304
|
+
signal.removeEventListener('abort', onAbort);
|
|
305
|
+
resolve(response);
|
|
306
|
+
},
|
|
307
|
+
};
|
|
308
|
+
const onAbort = () => waiter.finish(new Response(null, { status: 204 }));
|
|
309
|
+
const timer = setTimeout(() => waiter.finish(new Response(null, { status: 204 })), waitSeconds * 1000);
|
|
310
|
+
waiters.add(waiter);
|
|
311
|
+
if (signal.aborted)
|
|
312
|
+
onAbort();
|
|
313
|
+
else
|
|
314
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
315
|
+
});
|
|
316
|
+
});
|
|
317
|
+
// ---- browser sockets ----
|
|
318
|
+
// requireIdentity's refusal (401/403/503), kept so the socket can open and
|
|
319
|
+
// close with a code the client understands rather than fail its handshake.
|
|
320
|
+
const refusals = new WeakMap();
|
|
321
|
+
const originCheck = createMiddleware(async (c, next) => {
|
|
322
|
+
const appName = c.req.query('app');
|
|
323
|
+
if (!isAppName(appName))
|
|
324
|
+
return c.json({ error: 'app required' }, 400);
|
|
325
|
+
if (!originAllowed(c.req.header('origin'), appName, opts)) {
|
|
326
|
+
logger.warn('hub socket origin refused', { app: appName, origin: c.req.header('origin') ?? null });
|
|
327
|
+
return c.json({ error: 'origin not allowed' }, 403);
|
|
328
|
+
}
|
|
329
|
+
await next();
|
|
330
|
+
});
|
|
331
|
+
const socketIdentity = createMiddleware(async (c, next) => {
|
|
332
|
+
const { admitted, response } = await runIdentity(c);
|
|
333
|
+
if (!admitted)
|
|
334
|
+
refusals.set(c, response?.status ?? 401);
|
|
335
|
+
await next();
|
|
336
|
+
});
|
|
337
|
+
app.get('/api/ws', originCheck, socketIdentity, opts.upgradeWebSocket((c) => {
|
|
338
|
+
const appName = c.req.query('app');
|
|
339
|
+
const refusal = refusals.get(c);
|
|
340
|
+
const id = c.var.identity;
|
|
341
|
+
const user = refusal === undefined && id?.type === 'user' ? id.email : null;
|
|
342
|
+
let socket = null;
|
|
343
|
+
const remove = () => {
|
|
344
|
+
if (!socket)
|
|
345
|
+
return;
|
|
346
|
+
const users = sockets.get(socket.app);
|
|
347
|
+
const set = users?.get(socket.user);
|
|
348
|
+
set?.delete(socket);
|
|
349
|
+
if (set?.size === 0)
|
|
350
|
+
users.delete(socket.user);
|
|
351
|
+
if (users?.size === 0)
|
|
352
|
+
sockets.delete(socket.app);
|
|
353
|
+
socket = null;
|
|
354
|
+
};
|
|
355
|
+
return {
|
|
356
|
+
onOpen: (_evt, ws) => {
|
|
357
|
+
if (stopping) {
|
|
358
|
+
ws.close(1012, 'hub restarting');
|
|
359
|
+
return;
|
|
360
|
+
}
|
|
361
|
+
if (!user) {
|
|
362
|
+
// 4403: identity missing or expired (the client re-checks its
|
|
363
|
+
// session); 1013: the auth service is unavailable, try again.
|
|
364
|
+
if (refusal === 503)
|
|
365
|
+
ws.close(1013, 'auth unavailable');
|
|
366
|
+
else
|
|
367
|
+
ws.close(4403, 'user identity required');
|
|
368
|
+
return;
|
|
369
|
+
}
|
|
370
|
+
socket = { ws, app: appName, user };
|
|
371
|
+
let users = sockets.get(appName);
|
|
372
|
+
if (!users)
|
|
373
|
+
sockets.set(appName, (users = new Map()));
|
|
374
|
+
let set = users.get(user);
|
|
375
|
+
if (!set)
|
|
376
|
+
users.set(user, (set = new Set()));
|
|
377
|
+
set.add(socket);
|
|
378
|
+
},
|
|
379
|
+
onMessage: (evt, ws) => {
|
|
380
|
+
if (!socket)
|
|
381
|
+
return;
|
|
382
|
+
let message;
|
|
383
|
+
try {
|
|
384
|
+
message = JSON.parse(String(evt.data));
|
|
385
|
+
}
|
|
386
|
+
catch {
|
|
387
|
+
return;
|
|
388
|
+
}
|
|
389
|
+
// Unknown frames and fields are ignored on both sides: hub
|
|
390
|
+
// versions overlap during a hub deploy.
|
|
391
|
+
if (message?.type === 'ping')
|
|
392
|
+
ws.send(JSON.stringify({ type: 'pong' }));
|
|
393
|
+
else if (message?.type === 'connect')
|
|
394
|
+
ws.send(JSON.stringify({ type: 'ack' }));
|
|
395
|
+
},
|
|
396
|
+
onClose: remove,
|
|
397
|
+
onError: remove,
|
|
398
|
+
};
|
|
399
|
+
}));
|
|
400
|
+
return {
|
|
401
|
+
start: () => bridge.start(),
|
|
402
|
+
async stop() {
|
|
403
|
+
stopping = true;
|
|
404
|
+
for (const users of sockets.values()) {
|
|
405
|
+
for (const set of users.values()) {
|
|
406
|
+
for (const socket of set) {
|
|
407
|
+
try {
|
|
408
|
+
socket.ws.close(1012, 'hub restarting');
|
|
409
|
+
}
|
|
410
|
+
catch {
|
|
411
|
+
// already closing
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
sockets.clear();
|
|
417
|
+
for (const set of waits.values())
|
|
418
|
+
for (const waiter of [...set])
|
|
419
|
+
waiter.finish(new Response(null, { status: 204 }));
|
|
420
|
+
await Promise.allSettled([...publishes]);
|
|
421
|
+
await bridge.stop();
|
|
422
|
+
},
|
|
423
|
+
stats() {
|
|
424
|
+
let socketCount = 0;
|
|
425
|
+
for (const users of sockets.values())
|
|
426
|
+
for (const set of users.values())
|
|
427
|
+
socketCount += set.size;
|
|
428
|
+
let waitCount = 0;
|
|
429
|
+
for (const set of waits.values())
|
|
430
|
+
waitCount += set.size;
|
|
431
|
+
return { sockets: socketCount, waits: waitCount };
|
|
432
|
+
},
|
|
433
|
+
};
|
|
434
|
+
}
|
|
435
|
+
function workResponse(ref) {
|
|
436
|
+
return Response.json({ kind: 'work', ...(ref !== undefined ? { ref } : {}) });
|
|
437
|
+
}
|
|
438
|
+
function originAllowed(origin, appName, opts) {
|
|
439
|
+
if (!origin)
|
|
440
|
+
return false;
|
|
441
|
+
if (origin === `https://${appName}.${opts.appDomain}`)
|
|
442
|
+
return true;
|
|
443
|
+
return opts.allowLocalOrigins === true && /^http:\/\/(localhost|127\.0\.0\.1)(:\d+)?$/.test(origin);
|
|
444
|
+
}
|
|
445
|
+
function intParam(value, min, max, fallback) {
|
|
446
|
+
if (value === undefined || !/^\d+$/.test(value))
|
|
447
|
+
return fallback;
|
|
448
|
+
return Math.min(max, Math.max(min, Number(value)));
|
|
449
|
+
}
|
|
450
|
+
function escapeRegExp(s) {
|
|
451
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
452
|
+
}
|
|
453
|
+
/** `promise`, or a rejection once `ms` pass; the work itself continues. */
|
|
454
|
+
function withinMs(promise, ms, what) {
|
|
455
|
+
let timer;
|
|
456
|
+
const timeout = new Promise((_, reject) => {
|
|
457
|
+
timer = setTimeout(() => reject(new Error(`${what} timed out after ${ms} ms`)), ms);
|
|
458
|
+
});
|
|
459
|
+
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
|
|
460
|
+
}
|
|
461
|
+
/** Failed token attempts per client address in a fixed one-minute window.
|
|
462
|
+
* Keys are chosen by unauthenticated callers, so the map is bounded. */
|
|
463
|
+
class FailureCounter {
|
|
464
|
+
perMinute;
|
|
465
|
+
windows = new Map();
|
|
466
|
+
constructor(perMinute) {
|
|
467
|
+
this.perMinute = perMinute;
|
|
468
|
+
}
|
|
469
|
+
blocked(key, t) {
|
|
470
|
+
const w = this.windows.get(key);
|
|
471
|
+
return w !== undefined && t - w.start < 60_000 && w.count >= this.perMinute;
|
|
472
|
+
}
|
|
473
|
+
fail(key, t) {
|
|
474
|
+
const w = this.windows.get(key);
|
|
475
|
+
if (w && t - w.start < 60_000) {
|
|
476
|
+
w.count++;
|
|
477
|
+
return;
|
|
478
|
+
}
|
|
479
|
+
this.windows.delete(key);
|
|
480
|
+
if (this.windows.size >= 10_000) {
|
|
481
|
+
for (const [k, old] of this.windows) {
|
|
482
|
+
if (t - old.start >= 60_000 || this.windows.size >= 10_000)
|
|
483
|
+
this.windows.delete(k);
|
|
484
|
+
else
|
|
485
|
+
break;
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
this.windows.set(key, { start: t, count: 1 });
|
|
489
|
+
}
|
|
490
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
export type HintKind = 'sync_available' | 'build_changed' | 'notification' | 'work';
|
|
2
|
+
export interface Hint {
|
|
3
|
+
app: string;
|
|
4
|
+
/** A user's email, or '*' (every user of the app: build_changed, work). */
|
|
5
|
+
user: string;
|
|
6
|
+
kind: HintKind;
|
|
7
|
+
ref?: string;
|
|
8
|
+
payload?: Record<string, unknown>;
|
|
9
|
+
}
|
|
10
|
+
export declare const HINT_KINDS: readonly HintKind[];
|
|
11
|
+
/** The largest `payload`, serialized. NOTIFY allows 8000 bytes. */
|
|
12
|
+
export declare const HINT_PAYLOAD_MAX_BYTES = 1024;
|
|
13
|
+
export declare const HINT_REF_MAX_LENGTH = 200;
|
|
14
|
+
export declare function isAppName(value: unknown): value is string;
|
|
15
|
+
export type HintCheck = {
|
|
16
|
+
ok: true;
|
|
17
|
+
hint: Hint;
|
|
18
|
+
} | {
|
|
19
|
+
ok: false;
|
|
20
|
+
status: 400 | 413;
|
|
21
|
+
error: string;
|
|
22
|
+
};
|
|
23
|
+
/** Validate an untrusted hint. `build_changed` and `work` go to every user
|
|
24
|
+
* ('*'); `sync_available` and `notification` name one. */
|
|
25
|
+
export declare function checkHint(value: unknown): HintCheck;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
// The connection hub's one message shape (design: ~/migration/research/
|
|
2
|
+
// connection-hub-design-2026-10-30.md §5.2). A hint means only "go ask the
|
|
3
|
+
// app": it carries ids and kinds, never content, and duplicates, reordering
|
|
4
|
+
// and drops are all harmless because every client re-checks on a timer.
|
|
5
|
+
export const HINT_KINDS = ['sync_available', 'build_changed', 'notification', 'work'];
|
|
6
|
+
/** The largest `payload`, serialized. NOTIFY allows 8000 bytes. */
|
|
7
|
+
export const HINT_PAYLOAD_MAX_BYTES = 1024;
|
|
8
|
+
export const HINT_REF_MAX_LENGTH = 200;
|
|
9
|
+
const APP_NAME = /^[a-z][a-z0-9-]{0,39}$/;
|
|
10
|
+
export function isAppName(value) {
|
|
11
|
+
return typeof value === 'string' && APP_NAME.test(value);
|
|
12
|
+
}
|
|
13
|
+
/** Validate an untrusted hint. `build_changed` and `work` go to every user
|
|
14
|
+
* ('*'); `sync_available` and `notification` name one. */
|
|
15
|
+
export function checkHint(value) {
|
|
16
|
+
if (value === null || typeof value !== 'object' || Array.isArray(value))
|
|
17
|
+
return bad('hint must be an object');
|
|
18
|
+
const v = value;
|
|
19
|
+
if (!isAppName(v.app))
|
|
20
|
+
return bad('invalid app');
|
|
21
|
+
if (typeof v.kind !== 'string' || !HINT_KINDS.includes(v.kind))
|
|
22
|
+
return bad('invalid kind');
|
|
23
|
+
const kind = v.kind;
|
|
24
|
+
if (typeof v.user !== 'string' || v.user.length === 0 || v.user.length > 320)
|
|
25
|
+
return bad('invalid user');
|
|
26
|
+
const broadcast = kind === 'build_changed' || kind === 'work';
|
|
27
|
+
if (broadcast !== (v.user === '*'))
|
|
28
|
+
return bad(`${kind} takes ${broadcast ? "user '*'" : 'one user'}`);
|
|
29
|
+
if (v.ref !== undefined && (typeof v.ref !== 'string' || v.ref.length > HINT_REF_MAX_LENGTH))
|
|
30
|
+
return bad('invalid ref');
|
|
31
|
+
if (v.payload !== undefined) {
|
|
32
|
+
if (v.payload === null || typeof v.payload !== 'object' || Array.isArray(v.payload))
|
|
33
|
+
return bad('invalid payload');
|
|
34
|
+
if (Buffer.byteLength(JSON.stringify(v.payload)) > HINT_PAYLOAD_MAX_BYTES) {
|
|
35
|
+
return { ok: false, status: 413, error: `payload over ${HINT_PAYLOAD_MAX_BYTES} bytes` };
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
const hint = { app: v.app, user: v.user, kind };
|
|
39
|
+
if (v.ref !== undefined)
|
|
40
|
+
hint.ref = v.ref;
|
|
41
|
+
if (v.payload !== undefined)
|
|
42
|
+
hint.payload = v.payload;
|
|
43
|
+
return { ok: true, hint };
|
|
44
|
+
}
|
|
45
|
+
function bad(error) {
|
|
46
|
+
return { ok: false, status: 400, error };
|
|
47
|
+
}
|
package/dist/s2s.d.ts
CHANGED
|
@@ -67,9 +67,20 @@ export declare function resolveS2sCaller(opts: S2sOptions & {
|
|
|
67
67
|
export declare function s2sAuth(opts: S2sOptions): import("hono").MiddlewareHandler<any, string, {}, Response | (Response & import("hono").TypedResponse<{
|
|
68
68
|
error: string;
|
|
69
69
|
}, 429 | 401 | 403, "json">)>;
|
|
70
|
+
/** A cached ID token is replaced this long before its `exp`, as
|
|
71
|
+
* google-auth-library's own IdTokenClient does (its eager-refresh default).
|
|
72
|
+
* Tokens live an hour. */
|
|
73
|
+
export declare const ID_TOKEN_REFRESH_MARGIN_MS: number;
|
|
74
|
+
/** The longest one mint may take. On Cloud Run gcp-metadata sends the
|
|
75
|
+
* metadata-server request with no timeout at all; a healthy mint takes
|
|
76
|
+
* milliseconds. */
|
|
77
|
+
export declare const ID_TOKEN_MINT_TIMEOUT_MS = 5000;
|
|
70
78
|
/** fetch() with an OIDC identity token for the target's canonical audience.
|
|
71
79
|
* In test mode, sends the synthetic X-Test-S2S-Caller header instead
|
|
72
|
-
* (identity from TEST_S2S_IDENTITY, default 'test-service@test').
|
|
80
|
+
* (identity from TEST_S2S_IDENTITY, default 'test-service@test'). The token
|
|
81
|
+
* is cached per audience until ID_TOKEN_REFRESH_MARGIN_MS before it
|
|
82
|
+
* expires; concurrent calls share one mint, which is bounded by
|
|
83
|
+
* ID_TOKEN_MINT_TIMEOUT_MS, and `init.signal` also ends the wait for it. */
|
|
73
84
|
export declare function serviceFetch(url: string, opts: {
|
|
74
85
|
audience: string;
|
|
75
86
|
} & RequestInit): Promise<Response>;
|
package/dist/s2s.js
CHANGED
|
@@ -182,10 +182,26 @@ export function s2sAuth(opts) {
|
|
|
182
182
|
}
|
|
183
183
|
// ---------------------------------------------------------------------------
|
|
184
184
|
// Caller side
|
|
185
|
-
|
|
185
|
+
/** A cached ID token is replaced this long before its `exp`, as
|
|
186
|
+
* google-auth-library's own IdTokenClient does (its eager-refresh default).
|
|
187
|
+
* Tokens live an hour. */
|
|
188
|
+
export const ID_TOKEN_REFRESH_MARGIN_MS = 5 * 60_000;
|
|
189
|
+
/** The longest one mint may take. On Cloud Run gcp-metadata sends the
|
|
190
|
+
* metadata-server request with no timeout at all; a healthy mint takes
|
|
191
|
+
* milliseconds. */
|
|
192
|
+
export const ID_TOKEN_MINT_TIMEOUT_MS = 5_000;
|
|
193
|
+
// serviceFetch calls the provider directly, which bypasses IdTokenClient's
|
|
194
|
+
// token cache (it lives in getRequestMetadataAsync), so the cache, the
|
|
195
|
+
// single flight and the bound are kept here. Failures are never cached.
|
|
196
|
+
let idTokenProvider;
|
|
197
|
+
const idTokens = new Map();
|
|
198
|
+
const idTokenMints = new Map();
|
|
186
199
|
/** fetch() with an OIDC identity token for the target's canonical audience.
|
|
187
200
|
* In test mode, sends the synthetic X-Test-S2S-Caller header instead
|
|
188
|
-
* (identity from TEST_S2S_IDENTITY, default 'test-service@test').
|
|
201
|
+
* (identity from TEST_S2S_IDENTITY, default 'test-service@test'). The token
|
|
202
|
+
* is cached per audience until ID_TOKEN_REFRESH_MARGIN_MS before it
|
|
203
|
+
* expires; concurrent calls share one mint, which is bounded by
|
|
204
|
+
* ID_TOKEN_MINT_TIMEOUT_MS, and `init.signal` also ends the wait for it. */
|
|
189
205
|
export async function serviceFetch(url, opts) {
|
|
190
206
|
const { audience, ...init } = opts;
|
|
191
207
|
const headers = new Headers(init.headers);
|
|
@@ -193,17 +209,75 @@ export async function serviceFetch(url, opts) {
|
|
|
193
209
|
headers.set('x-test-s2s-caller', process.env.TEST_S2S_IDENTITY ?? 'test-service@test');
|
|
194
210
|
}
|
|
195
211
|
else {
|
|
196
|
-
|
|
197
|
-
if (!clientPromise) {
|
|
198
|
-
clientPromise = (async () => {
|
|
199
|
-
const { GoogleAuth } = await import('google-auth-library');
|
|
200
|
-
return new GoogleAuth().getIdTokenClient(audience);
|
|
201
|
-
})();
|
|
202
|
-
idTokenClients.set(audience, clientPromise);
|
|
203
|
-
}
|
|
204
|
-
const client = await clientPromise;
|
|
205
|
-
const token = await client.idTokenProvider.fetchIdToken(audience);
|
|
212
|
+
const token = await abortable(idTokenFor(audience), init.signal);
|
|
206
213
|
headers.set('authorization', `Bearer ${token}`);
|
|
207
214
|
}
|
|
208
215
|
return fetch(url, { ...init, headers });
|
|
209
216
|
}
|
|
217
|
+
function idTokenFor(audience) {
|
|
218
|
+
const cached = idTokens.get(audience);
|
|
219
|
+
if (cached && Date.now() < cached.refreshAt)
|
|
220
|
+
return Promise.resolve(cached.token);
|
|
221
|
+
let mint = idTokenMints.get(audience);
|
|
222
|
+
if (!mint) {
|
|
223
|
+
mint = withinMs(mintIdToken(audience), ID_TOKEN_MINT_TIMEOUT_MS, `ID token mint for ${audience}`)
|
|
224
|
+
.then((token) => {
|
|
225
|
+
const expiresAt = idTokenExpiry(token);
|
|
226
|
+
if (expiresAt !== undefined) {
|
|
227
|
+
idTokens.set(audience, { token, refreshAt: expiresAt - ID_TOKEN_REFRESH_MARGIN_MS });
|
|
228
|
+
}
|
|
229
|
+
return token;
|
|
230
|
+
})
|
|
231
|
+
.finally(() => idTokenMints.delete(audience));
|
|
232
|
+
idTokenMints.set(audience, mint);
|
|
233
|
+
}
|
|
234
|
+
return mint;
|
|
235
|
+
}
|
|
236
|
+
async function mintIdToken(audience) {
|
|
237
|
+
idTokenProvider ??= (async () => {
|
|
238
|
+
const { GoogleAuth } = await import('google-auth-library');
|
|
239
|
+
return (await new GoogleAuth().getIdTokenClient(audience)).idTokenProvider;
|
|
240
|
+
})();
|
|
241
|
+
const provider = idTokenProvider;
|
|
242
|
+
// A failed credential lookup is retried by the next call.
|
|
243
|
+
provider.catch(() => {
|
|
244
|
+
if (idTokenProvider === provider)
|
|
245
|
+
idTokenProvider = undefined;
|
|
246
|
+
});
|
|
247
|
+
return (await provider).fetchIdToken(audience);
|
|
248
|
+
}
|
|
249
|
+
/** Our own token's `exp` in ms, unverified; undefined if unreadable. */
|
|
250
|
+
function idTokenExpiry(token) {
|
|
251
|
+
try {
|
|
252
|
+
const exp = JSON.parse(Buffer.from(token.split('.')[1] ?? '', 'base64url').toString()).exp;
|
|
253
|
+
return typeof exp === 'number' ? exp * 1000 : undefined;
|
|
254
|
+
}
|
|
255
|
+
catch {
|
|
256
|
+
return undefined;
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
/** `promise`, or a rejection once `ms` pass. The work itself cannot be
|
|
260
|
+
* cancelled (fetchIdToken takes no signal); it is only no longer awaited. */
|
|
261
|
+
function withinMs(promise, ms, what) {
|
|
262
|
+
let timer;
|
|
263
|
+
const timeout = new Promise((_, reject) => {
|
|
264
|
+
timer = setTimeout(() => reject(new Error(`${what} timed out after ${ms} ms`)), ms);
|
|
265
|
+
});
|
|
266
|
+
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
|
|
267
|
+
}
|
|
268
|
+
/** `promise`, or a rejection with the signal's reason when it aborts (as
|
|
269
|
+
* fetch rejects), leaving the promise itself to its other waiters. Always
|
|
270
|
+
* subscribes to `promise`, so a mint no one waits for any more never
|
|
271
|
+
* rejects unhandled. */
|
|
272
|
+
function abortable(promise, signal) {
|
|
273
|
+
if (!signal)
|
|
274
|
+
return promise;
|
|
275
|
+
return new Promise((resolve, reject) => {
|
|
276
|
+
const onAbort = () => reject(signal.reason);
|
|
277
|
+
if (signal.aborted)
|
|
278
|
+
onAbort();
|
|
279
|
+
else
|
|
280
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
281
|
+
promise.then(resolve, reject).finally(() => signal.removeEventListener('abort', onAbort));
|
|
282
|
+
});
|
|
283
|
+
}
|
|
@@ -6,4 +6,8 @@ export declare class ScopeBroadcaster<T> {
|
|
|
6
6
|
* after the remaining listeners have received the message. */
|
|
7
7
|
publish(scopeKey: string, message: T, except?: (message: T) => void): void;
|
|
8
8
|
count(scopeKey: string): number;
|
|
9
|
+
/** The scopes with at least one listener: for an app that must
|
|
10
|
+
* re-announce to every local scope, e.g. after its database bridge may
|
|
11
|
+
* have missed notifications. */
|
|
12
|
+
scopes(): string[];
|
|
9
13
|
}
|
package/dist/sync/broadcaster.js
CHANGED
|
@@ -44,4 +44,10 @@ export class ScopeBroadcaster {
|
|
|
44
44
|
count(scopeKey) {
|
|
45
45
|
return this.listeners.get(scopeKey)?.size ?? 0;
|
|
46
46
|
}
|
|
47
|
+
/** The scopes with at least one listener: for an app that must
|
|
48
|
+
* re-announce to every local scope, e.g. after its database bridge may
|
|
49
|
+
* have missed notifications. */
|
|
50
|
+
scopes() {
|
|
51
|
+
return [...this.listeners.keys()];
|
|
52
|
+
}
|
|
47
53
|
}
|
package/dist/sync/db-bridge.d.ts
CHANGED
|
@@ -22,9 +22,11 @@ export interface DbBroadcastBridgeOptions {
|
|
|
22
22
|
/** Each notification's parsed JSON payload. An invalid payload is logged
|
|
23
23
|
* and skipped; a throw is logged and does not reach pg. */
|
|
24
24
|
onMessage: (message: unknown) => void;
|
|
25
|
-
/** Called
|
|
26
|
-
*
|
|
27
|
-
*
|
|
25
|
+
/** Called on every connect after start() has returned: a reconnect after
|
|
26
|
+
* a loss, or the first success after a failed first attempt (not on a
|
|
27
|
+
* first attempt that succeeds). Notifications sent while nothing was
|
|
28
|
+
* listening are lost and unknowable, so the app re-announces availability
|
|
29
|
+
* to every local scope. */
|
|
28
30
|
onRecovered: () => void;
|
|
29
31
|
/** Skip messages whose top-level `origin` is this bridge's `originId`,
|
|
30
32
|
* for an app that delivers its own writes locally and publishes them via
|
|
@@ -50,12 +52,16 @@ export declare class DbBroadcastBridge {
|
|
|
50
52
|
private listener;
|
|
51
53
|
private stopped;
|
|
52
54
|
private connecting;
|
|
53
|
-
|
|
55
|
+
/** Set once start() has returned. From then on the app may have sessions
|
|
56
|
+
* that a missed notification leaves stale, so every later connect is a
|
|
57
|
+
* recovery. */
|
|
58
|
+
private startReturned;
|
|
54
59
|
private reconnectAttempt;
|
|
55
60
|
private reconnectTimer;
|
|
56
61
|
private heartbeatTimer;
|
|
57
62
|
constructor(options: DbBroadcastBridgeOptions);
|
|
58
|
-
/**
|
|
63
|
+
/** Makes the first connect attempt; never rejects. A failure is logged
|
|
64
|
+
* and retried on the reconnect ladder (see the header). */
|
|
59
65
|
start(): Promise<void>;
|
|
60
66
|
/** Idempotent. Releases the listener client back to the pool; call it
|
|
61
67
|
* before the pool ends. Nothing is delivered after it resolves. */
|
package/dist/sync/db-bridge.js
CHANGED
|
@@ -4,6 +4,16 @@
|
|
|
4
4
|
// (another copy, the old revision during a deploy, a job) reaches every
|
|
5
5
|
// copy's sessions. Rules: ~/plasticine-way/docs/DATABASE.md "Cross-instance
|
|
6
6
|
// live delivery: LISTEN/NOTIFY". Moved here from todo's src/db-broadcast.ts.
|
|
7
|
+
//
|
|
8
|
+
// Lifecycle contract. start() makes the first connect attempt and resolves
|
|
9
|
+
// when it ends, whether or not it connected; it never rejects. A failed first
|
|
10
|
+
// attempt is logged as a loss (WARN 'database broadcast listener lost',
|
|
11
|
+
// `atStart: true`) and enters the same reconnect ladder as a later loss, so a
|
|
12
|
+
// process booting while the database is unreachable neither crash-loops on a
|
|
13
|
+
// top-level `await start()` nor stays deaf for life. Every connect after
|
|
14
|
+
// start() has returned (after a loss, or after a failed first attempt) logs
|
|
15
|
+
// INFO 'database broadcast listener recovered' and calls onRecovered: the app
|
|
16
|
+
// may have been serving sessions while nothing was listening.
|
|
7
17
|
import { randomUUID } from 'node:crypto';
|
|
8
18
|
/** How often the bridge proves its LISTEN connection alive with a `select
|
|
9
19
|
* 1`. Cloud SQL drops idle connections silently, the error surfaces only on
|
|
@@ -31,7 +41,10 @@ export class DbBroadcastBridge {
|
|
|
31
41
|
listener = null;
|
|
32
42
|
stopped = true;
|
|
33
43
|
connecting = false;
|
|
34
|
-
|
|
44
|
+
/** Set once start() has returned. From then on the app may have sessions
|
|
45
|
+
* that a missed notification leaves stale, so every later connect is a
|
|
46
|
+
* recovery. */
|
|
47
|
+
startReturned = false;
|
|
35
48
|
reconnectAttempt = 0;
|
|
36
49
|
reconnectTimer = null;
|
|
37
50
|
heartbeatTimer = null;
|
|
@@ -49,12 +62,28 @@ export class DbBroadcastBridge {
|
|
|
49
62
|
this.heartbeatTimeoutMs = options.heartbeatTimeoutMs ?? DB_BRIDGE_HEARTBEAT_TIMEOUT_MS;
|
|
50
63
|
this.reconnectDelaysMs = options.reconnectDelaysMs ?? DB_BRIDGE_RECONNECT_DELAYS_MS;
|
|
51
64
|
}
|
|
52
|
-
/**
|
|
65
|
+
/** Makes the first connect attempt; never rejects. A failure is logged
|
|
66
|
+
* and retried on the reconnect ladder (see the header). */
|
|
53
67
|
async start() {
|
|
54
68
|
if (!this.stopped)
|
|
55
69
|
return;
|
|
56
70
|
this.stopped = false;
|
|
57
|
-
|
|
71
|
+
try {
|
|
72
|
+
await this.connectListener();
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
if (!this.stopped) {
|
|
76
|
+
this.logger.warn('database broadcast listener lost', {
|
|
77
|
+
channel: this.channel,
|
|
78
|
+
error: String(error),
|
|
79
|
+
atStart: true,
|
|
80
|
+
});
|
|
81
|
+
this.scheduleReconnect();
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
finally {
|
|
85
|
+
this.startReturned = true;
|
|
86
|
+
}
|
|
58
87
|
}
|
|
59
88
|
/** Idempotent. Releases the listener client back to the pool; call it
|
|
60
89
|
* before the pool ends. Nothing is delivered after it resolves. */
|
|
@@ -86,7 +115,7 @@ export class DbBroadcastBridge {
|
|
|
86
115
|
const payload = this.ignoreOwn ? { ...message, origin: this.originId } : message;
|
|
87
116
|
await queryable.query('select pg_notify($1, $2)', [this.channel, JSON.stringify(payload)]);
|
|
88
117
|
}
|
|
89
|
-
async connectListener(
|
|
118
|
+
async connectListener() {
|
|
90
119
|
if (this.stopped || this.connecting || this.listener)
|
|
91
120
|
return;
|
|
92
121
|
this.connecting = true;
|
|
@@ -122,16 +151,11 @@ export class DbBroadcastBridge {
|
|
|
122
151
|
this.adjustPoolMin(1);
|
|
123
152
|
client.on('notification', (notification) => this.onNotification(notification.payload));
|
|
124
153
|
this.scheduleHeartbeat(client);
|
|
125
|
-
const recovered = this.connectedOnce;
|
|
126
|
-
this.connectedOnce = true;
|
|
127
154
|
this.reconnectAttempt = 0;
|
|
128
|
-
if (
|
|
155
|
+
if (this.startReturned) {
|
|
129
156
|
this.logger.info('database broadcast listener recovered', { channel: this.channel });
|
|
130
157
|
this.callApp('onRecovered', () => this.onRecovered());
|
|
131
158
|
}
|
|
132
|
-
else if (!initial) {
|
|
133
|
-
this.logger.info('database broadcast listener connected', { channel: this.channel });
|
|
134
|
-
}
|
|
135
159
|
}
|
|
136
160
|
finally {
|
|
137
161
|
this.connecting = false;
|
|
@@ -193,7 +217,7 @@ export class DbBroadcastBridge {
|
|
|
193
217
|
const delay = delays[Math.min(this.reconnectAttempt++, delays.length - 1)];
|
|
194
218
|
this.reconnectTimer = setTimeout(() => {
|
|
195
219
|
this.reconnectTimer = null;
|
|
196
|
-
void this.connectListener(
|
|
220
|
+
void this.connectListener().catch((error) => {
|
|
197
221
|
this.logger.warn('database broadcast reconnect failed', { channel: this.channel, error: String(error) });
|
|
198
222
|
this.scheduleReconnect();
|
|
199
223
|
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@steve31415/baselib",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.47.0",
|
|
4
4
|
"description": "Plasticine new-world shared platform library: logging, auth, service-to-service auth, db, HTTP, sync, app updates",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -65,6 +65,10 @@
|
|
|
65
65
|
"types": "./dist/sync-browser/index.d.ts",
|
|
66
66
|
"default": "./dist/sync-browser/index.js"
|
|
67
67
|
},
|
|
68
|
+
"./hub": {
|
|
69
|
+
"types": "./dist/hub/index.d.ts",
|
|
70
|
+
"default": "./dist/hub/index.js"
|
|
71
|
+
},
|
|
68
72
|
"./app-update": {
|
|
69
73
|
"types": "./dist/app-update/index.d.ts",
|
|
70
74
|
"default": "./dist/app-update/index.js"
|
|
@@ -116,6 +120,7 @@
|
|
|
116
120
|
"zod": "^4.6.5"
|
|
117
121
|
},
|
|
118
122
|
"devDependencies": {
|
|
123
|
+
"@hono/node-ws": "^1.3.1",
|
|
119
124
|
"@types/node": "^22.0.0",
|
|
120
125
|
"@types/pg": "^8.11.0",
|
|
121
126
|
"fake-indexeddb": "^6.2.5",
|