@volter/twin-world 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/known-external-services.json +1192 -0
- package/package.json +12 -5
- package/src/app-url.ts +175 -0
- package/src/attach.ts +113 -0
- package/src/browser-proxy-cli.ts +2 -1
- package/src/changeset.ts +416 -0
- package/src/cli.ts +589 -10
- package/src/covers.ts +724 -0
- package/src/fixture-env.ts +207 -0
- package/src/host-cli.ts +2 -1
- package/src/host-worker.ts +4 -3
- package/src/host.ts +26 -4
- package/src/index.ts +97 -1
- package/src/init.ts +1142 -0
- package/src/inject-map.ts +70 -0
- package/src/managed-infra-cli.ts +208 -0
- package/src/pack-facts.ts +136 -0
- package/src/pglite-backing.ts +125 -0
- package/src/pglite-host.mjs +147 -0
- package/src/prerequisites.ts +15 -58
- package/src/project-inspect.ts +683 -0
- package/src/redirect-proxy.ts +109 -85
- package/src/reflect.ts +443 -0
- package/src/resource-holder.ts +11 -0
- package/src/resources.ts +160 -0
- package/src/runtime-test-support.ts +208 -0
- package/src/runtime.ts +667 -106
- package/src/schema.ts +67 -0
- package/src/serve.ts +102 -0
- package/src/tail.ts +203 -0
package/src/reflect.ts
ADDED
|
@@ -0,0 +1,443 @@
|
|
|
1
|
+
// The reflect attachment (docs/ATTACH.md): attach an UNMODIFIED consumer —
|
|
2
|
+
// any language, any binary, no proxy env, no injector — by answering routed
|
|
3
|
+
// vendor hosts' DNS with the world's own TLS front. The consumer's ordinary
|
|
4
|
+
// `https://api.stripe.com` call roundtrips into the front, which terminates
|
|
5
|
+
// TLS with the session CA's per-host leaf (exactly the redirect-proxy's
|
|
6
|
+
// machinery) and forwards to the twin. Hosts that are not routed resolve real
|
|
7
|
+
// and never touch the world: passthrough is the default.
|
|
8
|
+
//
|
|
9
|
+
// Two small servers, both world-side ("the door belongs to the world"):
|
|
10
|
+
// • the FRONT — a TCP listener that peeks the TLS ClientHello SNI and pipes
|
|
11
|
+
// the raw bytes to a per-host leaf-cert https server (the same
|
|
12
|
+
// one-server-per-host trick redirect-proxy uses for CONNECT, reused here
|
|
13
|
+
// because Bun does not honor SNICallback).
|
|
14
|
+
// • the RESOLVER — a UDP DNS server ("the ear", pointed at by the attacher's
|
|
15
|
+
// environment, e.g. `docker run --dns`): A-answers routed hosts with the
|
|
16
|
+
// front's address (TTL 0 — a route flip lands on the next lookup),
|
|
17
|
+
// empty-answers AAAA for routed hosts (else real IPv6 wins), and forwards
|
|
18
|
+
// everything else to the real upstream untouched.
|
|
19
|
+
//
|
|
20
|
+
// ROUTES are attachment-scoped state (docs/ATTACH.md): one host per line in
|
|
21
|
+
// the instance's reflect-routes file, re-read live. The world stays ignorant
|
|
22
|
+
// of who is attached; routes only select interception. v1 routes are exact
|
|
23
|
+
// lowercase hostnames (wildcard families like *.atlassian.net: route the
|
|
24
|
+
// specific hosts the consumer uses).
|
|
25
|
+
//
|
|
26
|
+
// Per CLAUDE.md the runtime stays a lifecycle primitive: `reflect` runs in the
|
|
27
|
+
// foreground and supervises nothing; composition (containers, --dns wiring)
|
|
28
|
+
// lives in the cookbook recipe.
|
|
29
|
+
import dgram from 'node:dgram';
|
|
30
|
+
import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs';
|
|
31
|
+
import { connect as netConnect, createServer as createNetServer, type Server as NetServer, type Socket } from 'node:net';
|
|
32
|
+
import { join } from 'node:path';
|
|
33
|
+
import { activeVendorMap, ensureCa, leafCertFor, noTwinMessage, proxyTargetFor } from './redirect-proxy.ts';
|
|
34
|
+
import { instanceDir } from './runtime.ts';
|
|
35
|
+
|
|
36
|
+
// ---------------------------------------------------------------------------
|
|
37
|
+
// Routes — attachment-scoped, file-backed, re-read live by the resolver.
|
|
38
|
+
// ---------------------------------------------------------------------------
|
|
39
|
+
|
|
40
|
+
export function reflectRoutesPath(root: string, name: string): string {
|
|
41
|
+
return join(instanceDir(root, name), 'reflect-routes');
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function readReflectRoutes(path: string): Set<string> {
|
|
45
|
+
if (!existsSync(path)) return new Set();
|
|
46
|
+
return new Set(
|
|
47
|
+
readFileSync(path, 'utf8')
|
|
48
|
+
.split('\n')
|
|
49
|
+
.map((line) => line.trim().toLowerCase())
|
|
50
|
+
.filter((line) => line.length > 0 && !line.startsWith('#')),
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export function writeReflectRoutes(path: string, routes: Set<string>): void {
|
|
55
|
+
writeFileSync(path, `${[...routes].sort().join('\n')}\n`);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// ---------------------------------------------------------------------------
|
|
59
|
+
// SNI peek — minimal ClientHello parser (records buffered until complete).
|
|
60
|
+
// ---------------------------------------------------------------------------
|
|
61
|
+
|
|
62
|
+
/** Extract the SNI server name from a buffered TLS ClientHello, `undefined` if
|
|
63
|
+
* the buffer is not yet a complete handshake record or carries no SNI. */
|
|
64
|
+
export function parseSni(buffer: Buffer): string | undefined {
|
|
65
|
+
if (buffer.length < 5 || buffer[0] !== 0x16) return undefined; // not a TLS handshake record
|
|
66
|
+
const recordLength = buffer.readUInt16BE(3);
|
|
67
|
+
if (buffer.length < 5 + recordLength) return undefined; // incomplete — keep buffering
|
|
68
|
+
let offset = 5;
|
|
69
|
+
if (buffer[offset] !== 0x01) return undefined; // not a ClientHello
|
|
70
|
+
offset += 4; // handshake type + 24-bit length
|
|
71
|
+
offset += 2 + 32; // client version + random
|
|
72
|
+
const sessionIdLength = buffer[offset]!;
|
|
73
|
+
offset += 1 + sessionIdLength;
|
|
74
|
+
const cipherLength = buffer.readUInt16BE(offset);
|
|
75
|
+
offset += 2 + cipherLength;
|
|
76
|
+
const compressionLength = buffer[offset]!;
|
|
77
|
+
offset += 1 + compressionLength;
|
|
78
|
+
if (offset + 2 > buffer.length) return undefined;
|
|
79
|
+
const extensionsEnd = offset + 2 + buffer.readUInt16BE(offset);
|
|
80
|
+
offset += 2;
|
|
81
|
+
while (offset + 4 <= extensionsEnd && offset + 4 <= buffer.length) {
|
|
82
|
+
const extensionType = buffer.readUInt16BE(offset);
|
|
83
|
+
const extensionLength = buffer.readUInt16BE(offset + 2);
|
|
84
|
+
offset += 4;
|
|
85
|
+
if (extensionType === 0x0000) {
|
|
86
|
+
// server_name: list length (2), entry type (1, 0 = hostname), name length (2), name
|
|
87
|
+
const nameLength = buffer.readUInt16BE(offset + 3);
|
|
88
|
+
return buffer.subarray(offset + 5, offset + 5 + nameLength).toString('ascii').toLowerCase();
|
|
89
|
+
}
|
|
90
|
+
offset += extensionLength;
|
|
91
|
+
}
|
|
92
|
+
return undefined;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// ---------------------------------------------------------------------------
|
|
96
|
+
// The front.
|
|
97
|
+
// ---------------------------------------------------------------------------
|
|
98
|
+
|
|
99
|
+
export interface ReflectFrontOptions {
|
|
100
|
+
/** live env source — the world's twin map, exactly like the redirect proxy */
|
|
101
|
+
envLoader: () => Record<string, string | undefined>;
|
|
102
|
+
tlsDir: string;
|
|
103
|
+
host?: string;
|
|
104
|
+
port?: number;
|
|
105
|
+
/** additional SNI names to terminate (a served world's ADVERTISED hostname):
|
|
106
|
+
* the per-host handler already routes by Host header, so env-attached
|
|
107
|
+
* traffic arriving with SNI=<advertised> and Host=<vendor> lands in the
|
|
108
|
+
* right twin through the one door (docs/ATTACH.md, remote worlds). */
|
|
109
|
+
extraHosts?: string[];
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export interface ReflectFrontHandle {
|
|
113
|
+
host: string;
|
|
114
|
+
port: number;
|
|
115
|
+
caCertPath: string;
|
|
116
|
+
close(): Promise<void>;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export async function startReflectFront(options: ReflectFrontOptions): Promise<ReflectFrontHandle> {
|
|
120
|
+
const host = options.host ?? '0.0.0.0';
|
|
121
|
+
const ca = ensureCa(options.tlsDir);
|
|
122
|
+
const openSockets = new Set<Socket>();
|
|
123
|
+
const hostServers = new Map<string, ReturnType<typeof Bun.serve>>();
|
|
124
|
+
const hostPorts = new Map<string, number>();
|
|
125
|
+
|
|
126
|
+
// One TLS server per vendor host (the leaf cert names it), behind the SNI door. Bun's own server: it
|
|
127
|
+
// terminates TLS, forwards each request to the twin, and relays a websocket upgrade (a bot's gateway,
|
|
128
|
+
// a streaming API) to the twin's websocket — under Bun, node:http's 'upgrade' socket cannot write back
|
|
129
|
+
// to the client, so the relay is message-level, both ways.
|
|
130
|
+
const ensureHostServer = async (vendorHost: string): Promise<number> => {
|
|
131
|
+
const existing = hostPorts.get(vendorHost);
|
|
132
|
+
if (existing) return existing;
|
|
133
|
+
const leaf = leafCertFor(vendorHost, ca);
|
|
134
|
+
const resolveTwin = (req: Request): { origin: string; path: string } | { refuse: Response } => {
|
|
135
|
+
const url = new URL(req.url);
|
|
136
|
+
const hostHeader = (req.headers.get('host') ?? vendorHost).split(':')[0] ?? vendorHost;
|
|
137
|
+
const env = options.envLoader();
|
|
138
|
+
const prefixMatch = url.pathname.match(/^\/__vendor\/([a-z0-9-]+)(\/.*)?$/);
|
|
139
|
+
if (prefixMatch) {
|
|
140
|
+
const origin = activeVendorMap(env)[prefixMatch[1]!];
|
|
141
|
+
if (origin === undefined) return { refuse: new Response('no twin for vendor', { status: 502 }) };
|
|
142
|
+
return { origin, path: `${prefixMatch[2] ?? '/'}${url.search}` };
|
|
143
|
+
}
|
|
144
|
+
const twin = proxyTargetFor(hostHeader, env, url.pathname) ?? proxyTargetFor(vendorHost, env, url.pathname);
|
|
145
|
+
if (!twin) return { refuse: new Response(noTwinMessage(hostHeader, url.pathname), { status: 502, headers: { 'content-type': 'text/plain; charset=utf-8' } }) };
|
|
146
|
+
return { origin: twin.origin, path: `${url.pathname}${url.search}` };
|
|
147
|
+
};
|
|
148
|
+
type Relay = { upstreamUrl: string; upstream?: WebSocket; pending: Array<string | Uint8Array> };
|
|
149
|
+
const hostServer = Bun.serve<Relay>({
|
|
150
|
+
hostname: '127.0.0.1',
|
|
151
|
+
port: 0,
|
|
152
|
+
tls: { key: leaf.key, cert: leaf.cert },
|
|
153
|
+
idleTimeout: 255,
|
|
154
|
+
async fetch(req, server) {
|
|
155
|
+
const target = resolveTwin(req);
|
|
156
|
+
if ('refuse' in target) return target.refuse;
|
|
157
|
+
let origin: URL;
|
|
158
|
+
try { origin = new URL(target.origin); } catch { return new Response('bad twin origin', { status: 502 }); }
|
|
159
|
+
if (req.headers.get('upgrade')?.toLowerCase() === 'websocket') {
|
|
160
|
+
const upstreamUrl = `ws://${origin.hostname}:${origin.port || 80}${target.path}`;
|
|
161
|
+
return server.upgrade(req, { data: { upstreamUrl, pending: [] } }) ? undefined as unknown as Response : new Response('upgrade failed', { status: 400 });
|
|
162
|
+
}
|
|
163
|
+
const headers = new Headers(req.headers);
|
|
164
|
+
headers.delete('proxy-connection');
|
|
165
|
+
headers.delete('accept-encoding');
|
|
166
|
+
headers.delete('host');
|
|
167
|
+
try {
|
|
168
|
+
const upstream = await fetch(`http://${origin.hostname}:${origin.port || 80}${target.path}`, { method: req.method, headers, body: req.body, redirect: 'manual' });
|
|
169
|
+
const out = new Headers(upstream.headers);
|
|
170
|
+
out.delete('content-encoding');
|
|
171
|
+
out.delete('content-length');
|
|
172
|
+
return new Response(upstream.body, { status: upstream.status, headers: out });
|
|
173
|
+
} catch (error) {
|
|
174
|
+
return new Response(`twin unreachable: ${error instanceof Error ? error.message : String(error)}`, { status: 502 });
|
|
175
|
+
}
|
|
176
|
+
},
|
|
177
|
+
websocket: {
|
|
178
|
+
open(ws) {
|
|
179
|
+
const upstream = new WebSocket(ws.data.upstreamUrl);
|
|
180
|
+
ws.data.upstream = upstream;
|
|
181
|
+
upstream.onopen = () => { for (const m of ws.data.pending) upstream.send(m); ws.data.pending = []; };
|
|
182
|
+
upstream.onmessage = (event) => { ws.send(typeof event.data === 'string' ? event.data : new Uint8Array(event.data as ArrayBuffer)); };
|
|
183
|
+
upstream.onclose = (event) => { try { ws.close(event.code, event.reason); } catch { /* already closed */ } };
|
|
184
|
+
upstream.onerror = () => { try { ws.close(1011, 'twin websocket error'); } catch { /* already closed */ } };
|
|
185
|
+
},
|
|
186
|
+
message(ws, message) {
|
|
187
|
+
const data = typeof message === 'string' ? message : new Uint8Array(message);
|
|
188
|
+
const upstream = ws.data.upstream;
|
|
189
|
+
if (upstream && upstream.readyState === WebSocket.OPEN) upstream.send(data); else ws.data.pending.push(data);
|
|
190
|
+
},
|
|
191
|
+
close(ws, code, reason) { try { ws.data.upstream?.close(code, reason); } catch { /* already closed */ } },
|
|
192
|
+
},
|
|
193
|
+
});
|
|
194
|
+
const hostPort = Number(hostServer.port);
|
|
195
|
+
hostServers.set(vendorHost, hostServer);
|
|
196
|
+
hostPorts.set(vendorHost, hostPort);
|
|
197
|
+
return hostPort;
|
|
198
|
+
};
|
|
199
|
+
|
|
200
|
+
const server: NetServer = createNetServer((clientSocket: Socket) => {
|
|
201
|
+
openSockets.add(clientSocket);
|
|
202
|
+
clientSocket.on('close', () => openSockets.delete(clientSocket));
|
|
203
|
+
clientSocket.on('error', () => clientSocket.destroy());
|
|
204
|
+
let buffered = Buffer.alloc(0);
|
|
205
|
+
const onData = (chunk: Buffer) => {
|
|
206
|
+
buffered = Buffer.concat([buffered, chunk]);
|
|
207
|
+
const sni = parseSni(buffered);
|
|
208
|
+
if (sni === undefined) {
|
|
209
|
+
if (buffered.length > 65_536) clientSocket.destroy(); // not a ClientHello — give up
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
clientSocket.removeListener('data', onData);
|
|
213
|
+
const twin = proxyTargetFor(sni, options.envLoader());
|
|
214
|
+
if (!twin && !(options.extraHosts ?? []).includes(sni)) {
|
|
215
|
+
// only routed hosts (or the advertised name) are pointed here by the
|
|
216
|
+
// resolver/manifest; anything else is refused, never blind-forwarded
|
|
217
|
+
// (the front is a door to twins, not a general proxy)
|
|
218
|
+
clientSocket.destroy();
|
|
219
|
+
return;
|
|
220
|
+
}
|
|
221
|
+
ensureHostServer(sni)
|
|
222
|
+
.then((port) => {
|
|
223
|
+
const upstream: Socket = netConnect(port, '127.0.0.1');
|
|
224
|
+
openSockets.add(upstream);
|
|
225
|
+
upstream.on('close', () => openSockets.delete(upstream));
|
|
226
|
+
upstream.on('error', () => clientSocket.destroy());
|
|
227
|
+
clientSocket.on('error', () => upstream.destroy());
|
|
228
|
+
upstream.write(buffered);
|
|
229
|
+
upstream.pipe(clientSocket);
|
|
230
|
+
clientSocket.pipe(upstream);
|
|
231
|
+
})
|
|
232
|
+
.catch(() => clientSocket.destroy());
|
|
233
|
+
};
|
|
234
|
+
clientSocket.on('data', onData);
|
|
235
|
+
});
|
|
236
|
+
|
|
237
|
+
await new Promise<void>((resolveListen, reject) => {
|
|
238
|
+
server.once('error', reject);
|
|
239
|
+
server.listen(options.port ?? 0, host, () => resolveListen());
|
|
240
|
+
});
|
|
241
|
+
const address = server.address();
|
|
242
|
+
const port = typeof address === 'object' && address ? address.port : (options.port ?? 0);
|
|
243
|
+
|
|
244
|
+
return {
|
|
245
|
+
host,
|
|
246
|
+
port,
|
|
247
|
+
caCertPath: ca.caCert,
|
|
248
|
+
async close() {
|
|
249
|
+
for (const socket of openSockets) socket.destroy();
|
|
250
|
+
await new Promise<void>((resolveClose) => server.close(() => resolveClose()));
|
|
251
|
+
for (const hostServer of hostServers.values()) hostServer.stop(true);
|
|
252
|
+
},
|
|
253
|
+
};
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
// ---------------------------------------------------------------------------
|
|
257
|
+
// The resolver.
|
|
258
|
+
// ---------------------------------------------------------------------------
|
|
259
|
+
|
|
260
|
+
export interface ReflectResolverOptions {
|
|
261
|
+
/** live route set — re-read per query so a route flip lands immediately */
|
|
262
|
+
routesLoader: () => Set<string>;
|
|
263
|
+
/** the address routed hosts resolve to (the front's reachable IP) */
|
|
264
|
+
targetIp: string;
|
|
265
|
+
upstream?: string;
|
|
266
|
+
/** upstream port — default 53; tests point at a fake upstream on a high port */
|
|
267
|
+
upstreamPort?: number;
|
|
268
|
+
host?: string;
|
|
269
|
+
port?: number;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
export interface ReflectResolverHandle {
|
|
273
|
+
port: number;
|
|
274
|
+
close(): Promise<void>;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
function dnsQuestion(message: Buffer): { name: string; qtype: number; questionEnd: number } | undefined {
|
|
278
|
+
if (message.length < 17) return undefined;
|
|
279
|
+
const labels: string[] = [];
|
|
280
|
+
let offset = 12;
|
|
281
|
+
while (offset < message.length) {
|
|
282
|
+
const length = message[offset]!;
|
|
283
|
+
if (length === 0) {
|
|
284
|
+
offset += 1;
|
|
285
|
+
break;
|
|
286
|
+
}
|
|
287
|
+
labels.push(message.subarray(offset + 1, offset + 1 + length).toString('ascii'));
|
|
288
|
+
offset += 1 + length;
|
|
289
|
+
}
|
|
290
|
+
if (offset + 4 > message.length) return undefined;
|
|
291
|
+
return { name: labels.join('.').toLowerCase(), qtype: message.readUInt16BE(offset), questionEnd: offset + 4 };
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
function dnsHeader(query: Buffer, answerCount: number): Buffer {
|
|
295
|
+
const header = Buffer.alloc(12);
|
|
296
|
+
query.copy(header, 0, 0, 2); // ID
|
|
297
|
+
header.writeUInt16BE(0x8180, 2); // response, RD|RA, NOERROR
|
|
298
|
+
header.writeUInt16BE(1, 4); // QDCOUNT
|
|
299
|
+
header.writeUInt16BE(answerCount, 6); // ANCOUNT
|
|
300
|
+
return header;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
function dnsAnswerA(query: Buffer, questionEnd: number, ip: string): Buffer {
|
|
304
|
+
const answer = Buffer.alloc(16);
|
|
305
|
+
answer.writeUInt16BE(0xc00c, 0); // name: pointer to the question
|
|
306
|
+
answer.writeUInt16BE(1, 2); // TYPE A
|
|
307
|
+
answer.writeUInt16BE(1, 4); // CLASS IN
|
|
308
|
+
answer.writeUInt32BE(0, 6); // TTL 0 — activation flips on the next lookup
|
|
309
|
+
answer.writeUInt16BE(4, 10);
|
|
310
|
+
const parts = ip.split('.').map(Number);
|
|
311
|
+
for (let index = 0; index < 4; index++) answer[12 + index] = parts[index]!;
|
|
312
|
+
return Buffer.concat([dnsHeader(query, 1), query.subarray(12, questionEnd), answer]);
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
function dnsAnswerEmpty(query: Buffer, questionEnd: number): Buffer {
|
|
316
|
+
return Buffer.concat([dnsHeader(query, 0), query.subarray(12, questionEnd)]);
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
export async function startReflectResolver(options: ReflectResolverOptions): Promise<ReflectResolverHandle> {
|
|
320
|
+
const upstream = options.upstream ?? '8.8.8.8';
|
|
321
|
+
const server = dgram.createSocket('udp4');
|
|
322
|
+
server.on('message', (message, rinfo) => {
|
|
323
|
+
const question = dnsQuestion(message);
|
|
324
|
+
if (!question) return;
|
|
325
|
+
if (options.routesLoader().has(question.name)) {
|
|
326
|
+
const reply =
|
|
327
|
+
question.qtype === 1
|
|
328
|
+
? dnsAnswerA(message, question.questionEnd, options.targetIp)
|
|
329
|
+
: dnsAnswerEmpty(message, question.questionEnd); // AAAA etc.: empty, so the reflected A wins
|
|
330
|
+
server.send(reply, rinfo.port, rinfo.address);
|
|
331
|
+
return;
|
|
332
|
+
}
|
|
333
|
+
// passthrough: relay to the real upstream untouched
|
|
334
|
+
const forwarder = dgram.createSocket('udp4');
|
|
335
|
+
const timer = setTimeout(() => forwarder.close(), 3_000);
|
|
336
|
+
forwarder.on('message', (response) => {
|
|
337
|
+
clearTimeout(timer);
|
|
338
|
+
server.send(response, rinfo.port, rinfo.address);
|
|
339
|
+
forwarder.close();
|
|
340
|
+
});
|
|
341
|
+
forwarder.send(message, options.upstreamPort ?? 53, upstream);
|
|
342
|
+
});
|
|
343
|
+
await new Promise<void>((resolveListen, reject) => {
|
|
344
|
+
server.once('error', reject);
|
|
345
|
+
server.bind(options.port ?? 0, options.host ?? '0.0.0.0', () => resolveListen());
|
|
346
|
+
});
|
|
347
|
+
return {
|
|
348
|
+
port: server.address().port,
|
|
349
|
+
async close() {
|
|
350
|
+
await new Promise<void>((resolveClose) => server.close(() => resolveClose()));
|
|
351
|
+
},
|
|
352
|
+
};
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
// ---------------------------------------------------------------------------
|
|
356
|
+
// The reflect manifest — what a running `volter-world reflect` listens on, published for attachers
|
|
357
|
+
// (`attach --via reflect`). Written by the verb on start, removed on exit; absent = no front running.
|
|
358
|
+
// ---------------------------------------------------------------------------
|
|
359
|
+
export interface ReflectManifest {
|
|
360
|
+
/** the address routed hosts resolve to — where attachers reach the front (the verb's --target-ip) */
|
|
361
|
+
targetIp: string;
|
|
362
|
+
/** the address attachers reach the resolver at (--resolver-ip; default the target): on some Docker hosts
|
|
363
|
+
* the gateway address answers DNS itself (colima's VM does), so the resolver needs another host address */
|
|
364
|
+
resolverIp: string;
|
|
365
|
+
frontPort: number;
|
|
366
|
+
resolverPort: number;
|
|
367
|
+
caCertPath: string;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
export function reflectManifestPath(root: string, name: string): string {
|
|
371
|
+
return join(instanceDir(root, name), 'reflect.json');
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
export function writeReflectManifest(root: string, name: string, manifest: ReflectManifest): void {
|
|
375
|
+
writeFileSync(reflectManifestPath(root, name), `${JSON.stringify(manifest, null, 2)}\n`);
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
export function readReflectManifest(root: string, name: string): ReflectManifest | undefined {
|
|
379
|
+
const path = reflectManifestPath(root, name);
|
|
380
|
+
if (!existsSync(path)) return undefined;
|
|
381
|
+
const parsed = JSON.parse(readFileSync(path, 'utf8')) as Partial<ReflectManifest>;
|
|
382
|
+
if (typeof parsed.targetIp !== 'string' || typeof parsed.frontPort !== 'number' || typeof parsed.resolverPort !== 'number' || typeof parsed.caCertPath !== 'string') return undefined;
|
|
383
|
+
return { ...parsed, resolverIp: typeof parsed.resolverIp === 'string' ? parsed.resolverIp : parsed.targetIp } as ReflectManifest;
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
export function clearReflectManifest(root: string, name: string): void {
|
|
387
|
+
const path = reflectManifestPath(root, name);
|
|
388
|
+
if (existsSync(path)) unlinkSync(path);
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
/** Where the CA is mounted inside an attached container, and the env that makes its clients trust it. */
|
|
392
|
+
export const ATTACHED_CA_PATH = '/etc/volter-world/ca.pem';
|
|
393
|
+
export const CA_TRUST_ENV = ['SSL_CERT_FILE', 'REQUESTS_CA_BUNDLE', 'CURL_CA_BUNDLE', 'NODE_EXTRA_CA_CERTS', 'GIT_SSL_CAINFO'] as const;
|
|
394
|
+
|
|
395
|
+
/**
|
|
396
|
+
* A docker compose override that attaches every named service through reflect: its DNS is the resolver
|
|
397
|
+
* (so routed vendor hosts resolve to the front), `host.docker.internal` stays resolvable (a service-level
|
|
398
|
+
* `dns` bypasses Docker's own resolver, which is what served that name), the session CA is mounted and
|
|
399
|
+
* named by the trust variables the common clients read. Requires the resolver on port 53: Docker takes a
|
|
400
|
+
* DNS address, never a port. Compose merges this after the consumer's own files, touching nothing else.
|
|
401
|
+
*/
|
|
402
|
+
export function composeOverrideForReflect(manifest: ReflectManifest, services: string[], worldName: string): string {
|
|
403
|
+
if (manifest.resolverPort !== 53) throw new Error(`reflect attach: containers take a DNS address, not a port — run the resolver on :53 (it is on :${manifest.resolverPort})`);
|
|
404
|
+
if (manifest.frontPort !== 443) throw new Error(`reflect attach: a consumer reaches https on :443 — run the front on :443 (it is on :${manifest.frontPort})`);
|
|
405
|
+
const lines = ['# generated by `volter-world attach --via reflect` — never edit; merged after the consumer\'s own files', 'services:'];
|
|
406
|
+
for (const service of services) {
|
|
407
|
+
lines.push(` ${service}:`);
|
|
408
|
+
lines.push(' dns:', ` - ${manifest.resolverIp}`);
|
|
409
|
+
lines.push(' extra_hosts:', ' - host.docker.internal:host-gateway');
|
|
410
|
+
lines.push(' environment:', ` - VOLTER_WORLD=${worldName}`);
|
|
411
|
+
for (const key of CA_TRUST_ENV) lines.push(` - ${key}=${ATTACHED_CA_PATH}`);
|
|
412
|
+
lines.push(' volumes:', ` - ${manifest.caCertPath}:${ATTACHED_CA_PATH}:ro`);
|
|
413
|
+
}
|
|
414
|
+
return `${lines.join('\n')}\n`;
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
// docker compose's global options that take a value — the override's -f must land after the consumer's own
|
|
418
|
+
// files and before the subcommand, so these are stepped over to find it.
|
|
419
|
+
const COMPOSE_VALUED_OPTIONS = new Set(['-f', '--file', '-p', '--project-name', '--profile', '--env-file', '--project-directory', '--progress', '--parallel', '--ansi']);
|
|
420
|
+
|
|
421
|
+
/** Split `docker [docker flags] compose [compose flags] <subcommand…>`; undefined when it is not a compose command. */
|
|
422
|
+
export function splitDockerComposeArgs(command: string[]): { head: string[]; composeFlags: string[]; tail: string[] } | undefined {
|
|
423
|
+
if (command[0] !== 'docker') return undefined;
|
|
424
|
+
const composeAt = command.indexOf('compose');
|
|
425
|
+
if (composeAt < 0) return undefined;
|
|
426
|
+
const head = command.slice(0, composeAt + 1);
|
|
427
|
+
const composeFlags: string[] = [];
|
|
428
|
+
let i = composeAt + 1;
|
|
429
|
+
while (i < command.length) {
|
|
430
|
+
const arg = command[i]!;
|
|
431
|
+
if (!arg.startsWith('-')) break;
|
|
432
|
+
if (COMPOSE_VALUED_OPTIONS.has(arg)) { composeFlags.push(arg, command[i + 1] ?? ''); i += 2; continue; }
|
|
433
|
+
composeFlags.push(arg); i += 1; // --flag=value or a boolean flag
|
|
434
|
+
}
|
|
435
|
+
return { head, composeFlags, tail: command.slice(i) };
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
/** The same compose command with the override merged last. */
|
|
439
|
+
export function dockerComposeWithOverride(command: string[], overridePath: string): string[] {
|
|
440
|
+
const parts = splitDockerComposeArgs(command);
|
|
441
|
+
if (!parts) return command;
|
|
442
|
+
return [...parts.head, ...parts.composeFlags, '-f', overridePath, ...parts.tail];
|
|
443
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { keepProcessAlive } from '@volter/twin/lifecycle';
|
|
2
|
+
// One inert process keeps a successful World's machine-wide resource reservation live even when
|
|
3
|
+
// every declared service delegates its lifecycle and therefore has no World-owned PID. It is an
|
|
4
|
+
// internal lifecycle detail: recorded in the ordinary pids file, stopped by `world down`, and
|
|
5
|
+
// never presented as a service or backing substrate.
|
|
6
|
+
export {};
|
|
7
|
+
|
|
8
|
+
const finish = (): void => process.exit(0);
|
|
9
|
+
process.once('SIGINT', finish);
|
|
10
|
+
process.once('SIGTERM', finish);
|
|
11
|
+
await keepProcessAlive();
|
package/src/resources.ts
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, statfsSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import { homedir, hostname, totalmem } from 'node:os';
|
|
4
|
+
import { dirname, join, resolve } from 'node:path';
|
|
5
|
+
import { stateDirName, withFileLock } from '@volter/twin';
|
|
6
|
+
import type { WorldResourceRequirements } from './schema.ts';
|
|
7
|
+
|
|
8
|
+
const MIB = 1024 * 1024;
|
|
9
|
+
export const DEFAULT_WORLD_RESOURCES: WorldResourceRequirements = {
|
|
10
|
+
memoryMiB: 16,
|
|
11
|
+
writableStorageMiB: 32,
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
export type WorldResourceClaim = {
|
|
15
|
+
world: string;
|
|
16
|
+
identity: string;
|
|
17
|
+
ownerPid: number;
|
|
18
|
+
holderPids: number[];
|
|
19
|
+
hostname: string;
|
|
20
|
+
claimedAt: string;
|
|
21
|
+
resources: WorldResourceRequirements;
|
|
22
|
+
log: string;
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
/** Claims are machine-wide, not checkout-wide: Worlds rooted in two different repositories still
|
|
26
|
+
* compete for the same physical capacity. This ledger is an opaque implementation detail; each
|
|
27
|
+
* World's human-visible log remains under that World's own state directory. */
|
|
28
|
+
function claimDir(): string {
|
|
29
|
+
return join(homedir(), '.volter', 'world-resource-claims');
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function claimIdentity(root: string, world: string): string {
|
|
33
|
+
return createHash('sha256').update(`${resolve(root)}\0${world}`).digest('hex');
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function claimPath(root: string, world: string): string {
|
|
37
|
+
return join(claimDir(), `${claimIdentity(root, world)}.json`);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function resourceLogPath(root: string, world: string): string {
|
|
41
|
+
return join(resolve(root), stateDirName(), 'worlds', '.resources', `${world}.log`);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function processAlive(pid: number): boolean {
|
|
45
|
+
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
46
|
+
try { process.kill(pid, 0); return true; } catch { return false; }
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function readClaim(path: string): WorldResourceClaim | null {
|
|
50
|
+
try {
|
|
51
|
+
const claim = JSON.parse(readFileSync(path, 'utf8')) as WorldResourceClaim;
|
|
52
|
+
if (!claim || typeof claim.world !== 'string' || !claim.resources) return null;
|
|
53
|
+
return claim;
|
|
54
|
+
} catch {
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function liveClaims(replacingIdentity: string): WorldResourceClaim[] {
|
|
60
|
+
const dir = claimDir();
|
|
61
|
+
if (!existsSync(dir)) return [];
|
|
62
|
+
const claims: WorldResourceClaim[] = [];
|
|
63
|
+
for (const entry of readdirSync(dir)) {
|
|
64
|
+
if (!entry.endsWith('.json')) continue;
|
|
65
|
+
const path = join(dir, entry);
|
|
66
|
+
const claim = readClaim(path);
|
|
67
|
+
const live = claim && claim.identity !== replacingIdentity
|
|
68
|
+
&& (processAlive(claim.ownerPid) || claim.holderPids.some(processAlive));
|
|
69
|
+
if (live) claims.push(claim!);
|
|
70
|
+
else rmSync(path, { force: true });
|
|
71
|
+
}
|
|
72
|
+
return claims;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function writePrivate(path: string, contents: string): void {
|
|
76
|
+
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
|
77
|
+
writeFileSync(path, contents, { mode: 0o600 });
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function capacity(root: string, claims: WorldResourceClaim[]): WorldResourceRequirements {
|
|
81
|
+
const fs = statfsSync(resolve(root));
|
|
82
|
+
const diskFreeMiB = Math.floor((Number(fs.bavail) * Number(fs.bsize)) / MIB);
|
|
83
|
+
const memoryTotalMiB = Math.floor(totalmem() / MIB);
|
|
84
|
+
const memorySafetyMiB = Math.max(256, Math.min(1024, Math.floor(memoryTotalMiB * 0.1)));
|
|
85
|
+
const diskTotalMiB = Math.floor((Number(fs.blocks) * Number(fs.bsize)) / MIB);
|
|
86
|
+
const diskSafetyMiB = Math.max(512, Math.min(2048, Math.floor(diskTotalMiB * 0.05)));
|
|
87
|
+
const reservedMemoryMiB = claims.reduce((sum, claim) => sum + claim.resources.memoryMiB, 0);
|
|
88
|
+
const reservedDiskMiB = claims.reduce((sum, claim) => sum + claim.resources.writableStorageMiB, 0);
|
|
89
|
+
return {
|
|
90
|
+
memoryMiB: Math.max(0, memoryTotalMiB - memorySafetyMiB - reservedMemoryMiB),
|
|
91
|
+
writableStorageMiB: Math.max(0, diskFreeMiB - diskSafetyMiB - reservedDiskMiB),
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export function requestedWorldResources(resources?: WorldResourceRequirements): WorldResourceRequirements {
|
|
96
|
+
return resources ?? DEFAULT_WORLD_RESOURCES;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Serialize capacity admission across every World rooted here. Memory is admitted against the
|
|
100
|
+
* stable machine envelope minus other Worlds' full peak claims and a host safety margin; transient
|
|
101
|
+
* usage by unrelated applications is not part of the declarative World scheduler. Writable
|
|
102
|
+
* storage uses current capacity because allocated bytes cannot be reclaimed like memory pages. */
|
|
103
|
+
export function claimWorldResources(root: string, world: string, requested: WorldResourceRequirements): WorldResourceClaim {
|
|
104
|
+
const dir = claimDir();
|
|
105
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
106
|
+
return withFileLock(join(dir, 'claims.lock'), () => {
|
|
107
|
+
const identity = claimIdentity(root, world);
|
|
108
|
+
const claims = liveClaims(identity);
|
|
109
|
+
const available = capacity(root, claims);
|
|
110
|
+
const shortages: string[] = [];
|
|
111
|
+
if (requested.memoryMiB > available.memoryMiB) shortages.push(`memory requires ${requested.memoryMiB} MiB, ${available.memoryMiB} MiB available`);
|
|
112
|
+
if (requested.writableStorageMiB > available.writableStorageMiB) {
|
|
113
|
+
shortages.push(`writable storage requires ${requested.writableStorageMiB} MiB, ${available.writableStorageMiB} MiB available`);
|
|
114
|
+
}
|
|
115
|
+
const log = resourceLogPath(root, world);
|
|
116
|
+
if (shortages.length > 0) {
|
|
117
|
+
const message = `World "${world}" resource admission refused: ${shortages.join('; ')}. No services were started. Log: ${log}`;
|
|
118
|
+
writePrivate(log, `${new Date().toISOString()} refused ${message}\n`);
|
|
119
|
+
throw new Error(message);
|
|
120
|
+
}
|
|
121
|
+
const claim: WorldResourceClaim = {
|
|
122
|
+
world,
|
|
123
|
+
identity,
|
|
124
|
+
ownerPid: process.pid,
|
|
125
|
+
holderPids: [],
|
|
126
|
+
hostname: hostname(),
|
|
127
|
+
claimedAt: new Date().toISOString(),
|
|
128
|
+
resources: requested,
|
|
129
|
+
log,
|
|
130
|
+
};
|
|
131
|
+
writePrivate(claimPath(root, world), `${JSON.stringify(claim, null, 2)}\n`);
|
|
132
|
+
writePrivate(log, `${claim.claimedAt} admitted memory=${requested.memoryMiB}MiB writableStorage=${requested.writableStorageMiB}MiB\n`);
|
|
133
|
+
return claim;
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export function handoffWorldResourceClaim(root: string, world: string, holderPids: number[]): void {
|
|
138
|
+
const dir = claimDir();
|
|
139
|
+
withFileLock(join(dir, 'claims.lock'), () => {
|
|
140
|
+
const path = claimPath(root, world);
|
|
141
|
+
const claim = readClaim(path);
|
|
142
|
+
if (!claim) return;
|
|
143
|
+
writePrivate(path, `${JSON.stringify({ ...claim, holderPids: [...new Set(holderPids.filter((pid) => pid > 0))] }, null, 2)}\n`);
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
export function recordWorldResourceEvent(root: string, world: string, message: string): void {
|
|
148
|
+
const path = resourceLogPath(root, world);
|
|
149
|
+
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
|
150
|
+
writeFileSync(path, `${new Date().toISOString()} ${message}\n`, { flag: 'a', mode: 0o600 });
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
export function releaseWorldResources(root: string, world: string): void {
|
|
154
|
+
const dir = claimDir();
|
|
155
|
+
if (!existsSync(dir)) return;
|
|
156
|
+
withFileLock(join(dir, 'claims.lock'), () => {
|
|
157
|
+
rmSync(claimPath(root, world), { force: true });
|
|
158
|
+
recordWorldResourceEvent(root, world, 'released');
|
|
159
|
+
});
|
|
160
|
+
}
|