@wpelevator/tug 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -4
- package/dist/agent.d.ts +63 -2
- package/dist/agent.js +804 -75
- package/dist/archive.js +9 -3
- package/dist/artifacts.js +97 -34
- package/dist/build.d.ts +1 -1
- package/dist/build.js +17 -8
- package/dist/cli-build.js +34 -8
- package/dist/cli-config.js +16 -4
- package/dist/cli-install.d.ts +10 -0
- package/dist/cli-install.js +38 -4
- package/dist/cli-live-transport.d.ts +3 -0
- package/dist/cli-live-transport.js +230 -0
- package/dist/cli-live.d.ts +77 -0
- package/dist/cli-live.js +249 -0
- package/dist/cli.js +387 -85
- package/dist/compose.d.ts +46 -7
- package/dist/compose.js +173 -37
- package/dist/control.d.ts +5 -3
- package/dist/control.js +1038 -146
- package/dist/db.d.ts +1 -0
- package/dist/db.js +7 -1
- package/dist/ingress.d.ts +10 -2
- package/dist/ingress.js +57 -16
- package/dist/live.d.ts +80 -0
- package/dist/live.js +340 -0
- package/dist/main.js +3 -1
- package/dist/mesh.js +32 -13
- package/dist/protocol.d.ts +158 -0
- package/dist/protocol.js +16 -2
- package/dist/rollout.js +85 -19
- package/dist/secrets.js +3 -1
- package/dist/templates/control-compose.yml +1 -0
- package/dist/tokens.d.ts +1 -1
- package/dist/tokens.js +35 -9
- package/dist/util.d.ts +7 -0
- package/dist/util.js +73 -16
- package/package.json +8 -4
package/dist/agent.js
CHANGED
|
@@ -7,14 +7,18 @@ import { validateImageArchive, verifyDownload } from "./artifacts.js";
|
|
|
7
7
|
import path from 'node:path';
|
|
8
8
|
import { spawn, spawnSync } from 'node:child_process';
|
|
9
9
|
import net from 'node:net';
|
|
10
|
+
import dns from 'node:dns/promises';
|
|
11
|
+
import { WebSocket } from 'ws';
|
|
10
12
|
import { createHash } from 'node:crypto';
|
|
11
|
-
import { composeForNode, dumpYaml, isRolling, serviceSecretVersions } from "./compose.js";
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
13
|
+
import { addressBoundServices, composeForNode, dumpYaml, isRolling, serviceSecretVersions, } from "./compose.js";
|
|
14
|
+
import { LIVE_MAX_FRAME_BYTES, LIVE_START_TIMEOUT_MS } from "./protocol.js";
|
|
15
|
+
import { LiveOperationEngine } from "./live.js";
|
|
16
|
+
import { loadRolloutState, activeRollingStates, composeBase, hasLegacyContainers, reconcileRolling, retireAll, } from "./rollout.js";
|
|
17
|
+
import { ensureDir, errMessage, isPublicAddress, now, parseDuration } from "./util.js";
|
|
14
18
|
import { openAgentDb } from "./db.js";
|
|
15
19
|
import { createDockerRunner, createHostRunner } from "./runners.js";
|
|
16
20
|
import { CADDY_CONTAINER, reconcileIngress, removeIngress } from "./ingress.js";
|
|
17
|
-
import { WG_INTERFACE, ensureWgKeys, readReportedKey, reconcileMesh, rotateWgKeys, writeReportedKey } from "./mesh.js";
|
|
21
|
+
import { WG_INTERFACE, ensureWgKeys, readReportedKey, reconcileMesh, rotateWgKeys, writeReportedKey, } from "./mesh.js";
|
|
18
22
|
const CONTROL_URL_FROM_ENV = Boolean(process.env.TUG_CONTROL_URL);
|
|
19
23
|
let CONTROL_URL = process.env.TUG_CONTROL_URL || 'http://localhost:8080';
|
|
20
24
|
let TOKEN = process.env.TUG_AGENT_TOKEN;
|
|
@@ -25,7 +29,19 @@ const INTERVAL = Number(process.env.TUG_POLL_INTERVAL_MS || 5000);
|
|
|
25
29
|
// Opt-in WireGuard mesh: needs `wg`/`ip` in the agent image, host networking and NET_ADMIN.
|
|
26
30
|
const MESH = process.env.TUG_MESH === 'true';
|
|
27
31
|
export function defaultContext() {
|
|
28
|
-
return {
|
|
32
|
+
return {
|
|
33
|
+
docker: createDockerRunner(),
|
|
34
|
+
http: fetch,
|
|
35
|
+
nodeName: NODE_NAME,
|
|
36
|
+
stateDir: STATE_DIR,
|
|
37
|
+
apply: APPLY,
|
|
38
|
+
controlUrl: CONTROL_URL,
|
|
39
|
+
token: TOKEN,
|
|
40
|
+
host: createHostRunner(),
|
|
41
|
+
mesh: MESH,
|
|
42
|
+
caddyImage: process.env.TUG_CADDY_IMAGE,
|
|
43
|
+
rolloutPorts: process.env.TUG_ROLLOUT_PORTS,
|
|
44
|
+
};
|
|
29
45
|
}
|
|
30
46
|
const dbsByStateDir = new Map();
|
|
31
47
|
async function agentDb(stateDir) {
|
|
@@ -37,22 +53,417 @@ async function agentDb(stateDir) {
|
|
|
37
53
|
}
|
|
38
54
|
return db;
|
|
39
55
|
}
|
|
40
|
-
function authHeaders(ctx) {
|
|
41
|
-
|
|
56
|
+
function authHeaders(ctx) {
|
|
57
|
+
if (!ctx.token)
|
|
58
|
+
throw new Error('TUG_AGENT_TOKEN required');
|
|
59
|
+
return { Authorization: `Bearer ${ctx.token}` };
|
|
60
|
+
}
|
|
42
61
|
async function api(ctx, route, opts = {}) {
|
|
43
|
-
const res = await ctx.http(`${ctx.controlUrl}${route}`, {
|
|
62
|
+
const res = await ctx.http(`${ctx.controlUrl}${route}`, {
|
|
63
|
+
...opts,
|
|
64
|
+
headers: {
|
|
65
|
+
...authHeaders(ctx),
|
|
66
|
+
...(opts.headers || {}),
|
|
67
|
+
},
|
|
68
|
+
});
|
|
44
69
|
if (!res.ok)
|
|
45
70
|
throw new Error(`${opts.method || 'GET'} ${route}: ${res.status} ${await res.text()}`);
|
|
46
71
|
return res.json();
|
|
47
72
|
}
|
|
73
|
+
/** Keeps one outbound stream open and falls back to a bounded HTTP wait when an intermediary rejects WebSockets. */
|
|
74
|
+
export class AgentWakeChannel {
|
|
75
|
+
ctx;
|
|
76
|
+
createSocket;
|
|
77
|
+
socket;
|
|
78
|
+
cursor = 0;
|
|
79
|
+
pending = false;
|
|
80
|
+
waiter;
|
|
81
|
+
handlers;
|
|
82
|
+
constructor(ctx, createSocket = (url, token) => new WebSocket(url, {
|
|
83
|
+
headers: { Authorization: `Bearer ${token}` },
|
|
84
|
+
maxPayload: LIVE_MAX_FRAME_BYTES,
|
|
85
|
+
}), handlers = {}) {
|
|
86
|
+
this.ctx = ctx;
|
|
87
|
+
this.createSocket = createSocket;
|
|
88
|
+
this.handlers = handlers;
|
|
89
|
+
}
|
|
90
|
+
notify() {
|
|
91
|
+
this.pending = true;
|
|
92
|
+
this.waiter?.();
|
|
93
|
+
}
|
|
94
|
+
async connect() {
|
|
95
|
+
if (this.socket?.readyState === WebSocket.OPEN)
|
|
96
|
+
return true;
|
|
97
|
+
if (!this.ctx.token)
|
|
98
|
+
throw new Error('TUG_AGENT_TOKEN required');
|
|
99
|
+
const url = new URL('/agent/v1/stream', this.ctx.controlUrl);
|
|
100
|
+
url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
|
|
101
|
+
return new Promise((resolve) => {
|
|
102
|
+
let settled = false;
|
|
103
|
+
const socket = this.createSocket(url.toString(), this.ctx.token);
|
|
104
|
+
this.socket = socket;
|
|
105
|
+
const finish = (connected) => {
|
|
106
|
+
if (settled)
|
|
107
|
+
return;
|
|
108
|
+
settled = true;
|
|
109
|
+
clearTimeout(timer);
|
|
110
|
+
resolve(connected);
|
|
111
|
+
};
|
|
112
|
+
const timer = setTimeout(() => {
|
|
113
|
+
socket.terminate();
|
|
114
|
+
finish(false);
|
|
115
|
+
}, 1500);
|
|
116
|
+
socket.on('open', () => finish(true));
|
|
117
|
+
socket.on('message', (data) => {
|
|
118
|
+
if (this.socket !== socket)
|
|
119
|
+
return;
|
|
120
|
+
let message;
|
|
121
|
+
const json = data.toString();
|
|
122
|
+
if (Buffer.byteLength(json) > LIVE_MAX_FRAME_BYTES) {
|
|
123
|
+
this.close();
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
try {
|
|
127
|
+
message = JSON.parse(json);
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
if (!message || typeof message !== 'object')
|
|
133
|
+
return;
|
|
134
|
+
const frame = message;
|
|
135
|
+
if (typeof frame.type === 'string' && frame.type.startsWith('live.')) {
|
|
136
|
+
if (frame.v === 1)
|
|
137
|
+
void Promise.resolve(this.handlers.onLive?.(message)).catch((error) => console.error('agent live frame failed:', errMessage(error)));
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
if ((frame.type !== 'ready' && frame.type !== 'work-available') ||
|
|
141
|
+
!Number.isSafeInteger(frame.cursor) ||
|
|
142
|
+
Number(frame.cursor) < 0)
|
|
143
|
+
return;
|
|
144
|
+
this.cursor = Math.max(this.cursor, Number(frame.cursor));
|
|
145
|
+
if (frame.type === 'work-available')
|
|
146
|
+
this.notify();
|
|
147
|
+
});
|
|
148
|
+
socket.on('close', () => {
|
|
149
|
+
if (this.socket === socket) {
|
|
150
|
+
this.socket = undefined;
|
|
151
|
+
this.handlers.onDisconnect?.();
|
|
152
|
+
}
|
|
153
|
+
if (settled)
|
|
154
|
+
this.waiter?.();
|
|
155
|
+
else
|
|
156
|
+
finish(false);
|
|
157
|
+
});
|
|
158
|
+
socket.on('error', () => {
|
|
159
|
+
if (this.socket === socket) {
|
|
160
|
+
this.socket = undefined;
|
|
161
|
+
this.handlers.onDisconnect?.();
|
|
162
|
+
}
|
|
163
|
+
socket.terminate();
|
|
164
|
+
this.waiter?.();
|
|
165
|
+
finish(false);
|
|
166
|
+
});
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
async waitOnSocket(timeoutMs) {
|
|
170
|
+
if (this.pending) {
|
|
171
|
+
this.pending = false;
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
await new Promise((resolve) => {
|
|
175
|
+
const done = () => {
|
|
176
|
+
clearTimeout(timer);
|
|
177
|
+
if (this.waiter === done)
|
|
178
|
+
this.waiter = undefined;
|
|
179
|
+
resolve();
|
|
180
|
+
};
|
|
181
|
+
this.waiter = done;
|
|
182
|
+
const timer = setTimeout(done, timeoutMs);
|
|
183
|
+
});
|
|
184
|
+
this.pending = false;
|
|
185
|
+
}
|
|
186
|
+
async longPoll(timeoutMs) {
|
|
187
|
+
const wake = await api(this.ctx, `/agent/v1/wake?after=${this.cursor}&timeout=${Math.min(timeoutMs, 30_000)}`);
|
|
188
|
+
this.cursor = Math.max(this.cursor, wake.cursor);
|
|
189
|
+
}
|
|
190
|
+
async wait(timeoutMs) {
|
|
191
|
+
if (await this.connect())
|
|
192
|
+
return this.waitOnSocket(timeoutMs);
|
|
193
|
+
await this.longPoll(timeoutMs);
|
|
194
|
+
}
|
|
195
|
+
send(frame) {
|
|
196
|
+
const socket = this.socket;
|
|
197
|
+
if (!socket || socket.readyState !== WebSocket.OPEN)
|
|
198
|
+
return false;
|
|
199
|
+
if (socket.bufferedAmount > 1048576) {
|
|
200
|
+
this.close();
|
|
201
|
+
socket.terminate();
|
|
202
|
+
return false;
|
|
203
|
+
}
|
|
204
|
+
try {
|
|
205
|
+
socket.send(JSON.stringify(frame));
|
|
206
|
+
return true;
|
|
207
|
+
}
|
|
208
|
+
catch {
|
|
209
|
+
this.close();
|
|
210
|
+
return false;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
close() {
|
|
214
|
+
const socket = this.socket;
|
|
215
|
+
this.socket = undefined;
|
|
216
|
+
this.handlers.onDisconnect?.();
|
|
217
|
+
socket?.close();
|
|
218
|
+
this.waiter?.();
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
/** Resolves a service once before starting a process so rollouts cannot silently retarget an open session. */
|
|
222
|
+
export async function resolveLiveContainer(ctx, request) {
|
|
223
|
+
const name = /^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$/;
|
|
224
|
+
if (!name.test(request.project) ||
|
|
225
|
+
request.project === '..' ||
|
|
226
|
+
!request.service ||
|
|
227
|
+
!name.test(request.service))
|
|
228
|
+
throw new Error('A valid project and service are required');
|
|
229
|
+
const state = await loadRolloutState(path.join(ctx.stateDir, 'projects', request.project));
|
|
230
|
+
const rolling = state.services[request.service];
|
|
231
|
+
if (rolling && !rolling.active)
|
|
232
|
+
throw new Error('Rolling service has no active revision');
|
|
233
|
+
const project = rolling?.active?.project ?? request.project;
|
|
234
|
+
if (!name.test(project))
|
|
235
|
+
throw new Error('Invalid active rollout project');
|
|
236
|
+
const result = await ctx.docker([
|
|
237
|
+
'ps',
|
|
238
|
+
'--all',
|
|
239
|
+
'--no-trunc',
|
|
240
|
+
'--quiet',
|
|
241
|
+
'--filter',
|
|
242
|
+
'label=com.docker.compose.project=' + project,
|
|
243
|
+
'--filter',
|
|
244
|
+
'label=com.docker.compose.service=' + request.service,
|
|
245
|
+
'--filter',
|
|
246
|
+
'label=com.docker.compose.oneoff=False',
|
|
247
|
+
]);
|
|
248
|
+
const ids = result.out.trim().split(/\s+/).filter(Boolean);
|
|
249
|
+
if (ids.length !== 1)
|
|
250
|
+
throw new Error(ids.length
|
|
251
|
+
? 'Service has multiple containers; select an unambiguous service'
|
|
252
|
+
: 'Service has no container on this node');
|
|
253
|
+
if (!/^[a-f0-9]{12,64}$/.test(ids[0]))
|
|
254
|
+
throw new Error('Docker returned an invalid container ID');
|
|
255
|
+
return ids[0];
|
|
256
|
+
}
|
|
257
|
+
export class AgentLiveOperations {
|
|
258
|
+
ctx;
|
|
259
|
+
emit;
|
|
260
|
+
engine;
|
|
261
|
+
sessions = new Map();
|
|
262
|
+
constructor(ctx, emit, options = {}) {
|
|
263
|
+
this.ctx = ctx;
|
|
264
|
+
this.emit = emit;
|
|
265
|
+
this.engine = new LiveOperationEngine({
|
|
266
|
+
...options,
|
|
267
|
+
emit: (event) => {
|
|
268
|
+
if (!this.sessions.has(event.sessionId))
|
|
269
|
+
return;
|
|
270
|
+
if (event.type === 'exit')
|
|
271
|
+
this.sessions.delete(event.sessionId);
|
|
272
|
+
if (event.type === 'output')
|
|
273
|
+
this.emit({
|
|
274
|
+
v: 1,
|
|
275
|
+
type: 'live.output',
|
|
276
|
+
sessionId: event.sessionId,
|
|
277
|
+
stream: event.stream,
|
|
278
|
+
data: event.data,
|
|
279
|
+
});
|
|
280
|
+
else if (event.type === 'result')
|
|
281
|
+
this.emit({
|
|
282
|
+
v: 1,
|
|
283
|
+
type: 'live.result',
|
|
284
|
+
sessionId: event.sessionId,
|
|
285
|
+
result: event.result,
|
|
286
|
+
});
|
|
287
|
+
else
|
|
288
|
+
this.emit({ ...event, v: 1, type: 'live.exit' });
|
|
289
|
+
},
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
async handle(frame) {
|
|
293
|
+
if (frame.v !== 1 ||
|
|
294
|
+
typeof frame.sessionId !== 'string' ||
|
|
295
|
+
!/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,255}$/.test(frame.sessionId))
|
|
296
|
+
return;
|
|
297
|
+
if (frame.type === 'live.open') {
|
|
298
|
+
// An existing open belongs to its original request; replaying it must not start another process.
|
|
299
|
+
if (this.sessions.has(frame.sessionId))
|
|
300
|
+
return;
|
|
301
|
+
const entry = { ready: Promise.resolve(), queuedBytes: 0 };
|
|
302
|
+
this.sessions.set(frame.sessionId, entry);
|
|
303
|
+
entry.ready = this.open({ ...frame.request, sessionId: frame.sessionId }, entry);
|
|
304
|
+
await entry.ready;
|
|
305
|
+
return;
|
|
306
|
+
}
|
|
307
|
+
const entry = this.sessions.get(frame.sessionId);
|
|
308
|
+
if (!entry)
|
|
309
|
+
return;
|
|
310
|
+
if (frame.type === 'live.cancel') {
|
|
311
|
+
this.engine.cancel(frame.sessionId);
|
|
312
|
+
if (this.sessions.delete(frame.sessionId))
|
|
313
|
+
this.emit({
|
|
314
|
+
v: 1,
|
|
315
|
+
type: 'live.exit',
|
|
316
|
+
sessionId: frame.sessionId,
|
|
317
|
+
code: null,
|
|
318
|
+
signal: null,
|
|
319
|
+
cancelled: true,
|
|
320
|
+
});
|
|
321
|
+
return;
|
|
322
|
+
}
|
|
323
|
+
let queued = 0;
|
|
324
|
+
try {
|
|
325
|
+
let input;
|
|
326
|
+
if (frame.type === 'live.stdin') {
|
|
327
|
+
if (typeof frame.data !== 'string' || frame.data.length > 87384)
|
|
328
|
+
throw new Error('Invalid stdin frame');
|
|
329
|
+
input = Buffer.from(frame.data, 'base64');
|
|
330
|
+
if (input.toString('base64') !== frame.data || input.length > 65536)
|
|
331
|
+
throw new Error('Invalid base64 stdin');
|
|
332
|
+
queued = input.length;
|
|
333
|
+
entry.queuedBytes += queued;
|
|
334
|
+
if (entry.queuedBytes > 262144)
|
|
335
|
+
throw new Error('Live stdin queue exceeds size limit');
|
|
336
|
+
}
|
|
337
|
+
await entry.ready;
|
|
338
|
+
if (this.sessions.get(frame.sessionId) !== entry)
|
|
339
|
+
return;
|
|
340
|
+
if (frame.type === 'live.stdin' && input)
|
|
341
|
+
this.engine.stdin(frame.sessionId, input);
|
|
342
|
+
else if (frame.type === 'live.stdin-end')
|
|
343
|
+
this.engine.endStdin(frame.sessionId);
|
|
344
|
+
else if (frame.type === 'live.resize')
|
|
345
|
+
this.engine.resize(frame.sessionId, frame.columns, frame.rows);
|
|
346
|
+
}
|
|
347
|
+
catch (error) {
|
|
348
|
+
this.fail(frame.sessionId, entry, error);
|
|
349
|
+
}
|
|
350
|
+
finally {
|
|
351
|
+
entry.queuedBytes -= queued;
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
dispose() {
|
|
355
|
+
const ids = [...this.sessions.keys()];
|
|
356
|
+
this.engine.dispose();
|
|
357
|
+
for (const id of ids)
|
|
358
|
+
if (this.sessions.delete(id))
|
|
359
|
+
this.emit({
|
|
360
|
+
v: 1,
|
|
361
|
+
type: 'live.exit',
|
|
362
|
+
sessionId: id,
|
|
363
|
+
code: null,
|
|
364
|
+
signal: null,
|
|
365
|
+
cancelled: true,
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
fail(id, entry, error) {
|
|
369
|
+
if (this.sessions.get(id) !== entry)
|
|
370
|
+
return;
|
|
371
|
+
this.sessions.delete(id);
|
|
372
|
+
this.emit({
|
|
373
|
+
v: 1,
|
|
374
|
+
type: 'live.exit',
|
|
375
|
+
sessionId: id,
|
|
376
|
+
code: null,
|
|
377
|
+
signal: null,
|
|
378
|
+
cancelled: false,
|
|
379
|
+
error: errMessage(error),
|
|
380
|
+
});
|
|
381
|
+
this.engine.cancel(id);
|
|
382
|
+
}
|
|
383
|
+
async open(request, entry) {
|
|
384
|
+
try {
|
|
385
|
+
if (this.sessions.size > 16)
|
|
386
|
+
throw new Error('Live session limit reached');
|
|
387
|
+
if (!this.ctx.apply)
|
|
388
|
+
throw new Error('Live operations require an applying agent');
|
|
389
|
+
if (request.node && request.node !== this.ctx.nodeName)
|
|
390
|
+
throw new Error('Live request selected a different node');
|
|
391
|
+
if (typeof request.project !== 'string' ||
|
|
392
|
+
!/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$/.test(request.project))
|
|
393
|
+
throw new Error('Invalid project');
|
|
394
|
+
if (request.operation === 'ps') {
|
|
395
|
+
const rollout = await loadRolloutState(path.join(this.ctx.stateDir, 'projects', request.project));
|
|
396
|
+
if (this.sessions.get(request.sessionId) !== entry)
|
|
397
|
+
return;
|
|
398
|
+
const projects = [
|
|
399
|
+
...new Set([
|
|
400
|
+
request.project,
|
|
401
|
+
...Object.values(rollout.services).flatMap((service) => service.active ? [service.active.project] : []),
|
|
402
|
+
]),
|
|
403
|
+
];
|
|
404
|
+
this.engine.start({ operation: 'ps', sessionId: request.sessionId, projects });
|
|
405
|
+
}
|
|
406
|
+
else {
|
|
407
|
+
if (!['exec', 'logs', 'inspect'].includes(request.operation))
|
|
408
|
+
throw new Error('Invalid live operation');
|
|
409
|
+
let timer;
|
|
410
|
+
let container;
|
|
411
|
+
try {
|
|
412
|
+
container = await Promise.race([
|
|
413
|
+
resolveLiveContainer(this.ctx, request),
|
|
414
|
+
new Promise((_resolve, reject) => {
|
|
415
|
+
timer = setTimeout(() => reject(new Error('Live container resolution timed out')), LIVE_START_TIMEOUT_MS);
|
|
416
|
+
timer.unref();
|
|
417
|
+
}),
|
|
418
|
+
]);
|
|
419
|
+
}
|
|
420
|
+
finally {
|
|
421
|
+
clearTimeout(timer);
|
|
422
|
+
}
|
|
423
|
+
if (this.sessions.get(request.sessionId) !== entry)
|
|
424
|
+
return;
|
|
425
|
+
if (request.operation === 'exec')
|
|
426
|
+
this.engine.start({
|
|
427
|
+
operation: 'exec',
|
|
428
|
+
sessionId: request.sessionId,
|
|
429
|
+
container,
|
|
430
|
+
command: request.command,
|
|
431
|
+
interactive: request.interactive,
|
|
432
|
+
tty: request.tty,
|
|
433
|
+
columns: request.columns,
|
|
434
|
+
rows: request.rows,
|
|
435
|
+
});
|
|
436
|
+
else if (request.operation === 'logs')
|
|
437
|
+
this.engine.start({
|
|
438
|
+
operation: 'logs',
|
|
439
|
+
sessionId: request.sessionId,
|
|
440
|
+
container,
|
|
441
|
+
tail: request.tail,
|
|
442
|
+
});
|
|
443
|
+
else
|
|
444
|
+
this.engine.start({ operation: 'inspect', sessionId: request.sessionId, container });
|
|
445
|
+
}
|
|
446
|
+
if (this.sessions.get(request.sessionId) === entry)
|
|
447
|
+
this.emit({ v: 1, type: 'live.started', sessionId: request.sessionId });
|
|
448
|
+
}
|
|
449
|
+
catch (error) {
|
|
450
|
+
this.fail(request.sessionId, entry, error);
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
}
|
|
48
454
|
async function writeEnv(file, secrets) {
|
|
49
|
-
const body = Object.entries(secrets)
|
|
455
|
+
const body = Object.entries(secrets)
|
|
456
|
+
.map(([k, v]) => `${k}=${String(v).replace(/\n/g, '\\n')}`)
|
|
457
|
+
.join('\n') + '\n';
|
|
50
458
|
await fs.writeFile(file, body, { mode: 0o600 });
|
|
51
459
|
}
|
|
52
460
|
/** `docker compose` prefix for a project; rolling revision projects pass their own Compose file. */
|
|
53
461
|
const composeCmd = (project, cwd, composeFile) => composeFile ? composeBase(project, cwd, composeFile) : ['compose', '-p', project];
|
|
54
462
|
export async function containerIp(ctx, project, service, cwd, composeFile) {
|
|
55
|
-
const ids = (await ctx.docker([...composeCmd(project, cwd, composeFile), 'ps', '-q', service], { cwd })).out
|
|
463
|
+
const ids = (await ctx.docker([...composeCmd(project, cwd, composeFile), 'ps', '-q', service], { cwd })).out
|
|
464
|
+
.trim()
|
|
465
|
+
.split(/\s+/)
|
|
466
|
+
.filter(Boolean);
|
|
56
467
|
if (!ids.length)
|
|
57
468
|
throw new Error(`${service}: no container found`);
|
|
58
469
|
const inspected = JSON.parse((await ctx.docker(['inspect', ids[0]], { cwd })).out)[0];
|
|
@@ -61,8 +472,14 @@ export async function containerIp(ctx, project, service, cwd, composeFile) {
|
|
|
61
472
|
}
|
|
62
473
|
function tcpCheck(host, port, timeoutMs = 2000) {
|
|
63
474
|
return new Promise((resolve, reject) => {
|
|
64
|
-
const s = net.createConnection({ host, port: Number(port), timeout: timeoutMs }, () => {
|
|
65
|
-
|
|
475
|
+
const s = net.createConnection({ host, port: Number(port), timeout: timeoutMs }, () => {
|
|
476
|
+
s.destroy();
|
|
477
|
+
resolve(true);
|
|
478
|
+
});
|
|
479
|
+
s.on('timeout', () => {
|
|
480
|
+
s.destroy();
|
|
481
|
+
reject(new Error(`tcp timeout ${host}:${port}`));
|
|
482
|
+
});
|
|
66
483
|
s.on('error', reject);
|
|
67
484
|
});
|
|
68
485
|
}
|
|
@@ -75,7 +492,9 @@ export async function waitForHealth(ctx, project, service, health, cwd, composeF
|
|
|
75
492
|
for (let i = 0; i < retries; i++) {
|
|
76
493
|
try {
|
|
77
494
|
if (health.type === 'docker') {
|
|
78
|
-
const id = (await ctx.docker([...composeCmd(project, cwd, composeFile), 'ps', '-q', service], { cwd })).out
|
|
495
|
+
const id = (await ctx.docker([...composeCmd(project, cwd, composeFile), 'ps', '-q', service], { cwd })).out
|
|
496
|
+
.trim()
|
|
497
|
+
.split(/\s+/)[0];
|
|
79
498
|
const state = JSON.parse((await ctx.docker(['inspect', id], { cwd })).out)[0]?.State;
|
|
80
499
|
if (!state?.Running)
|
|
81
500
|
throw new Error(`${service}: not running`);
|
|
@@ -87,7 +506,9 @@ export async function waitForHealth(ctx, project, service, health, cwd, composeF
|
|
|
87
506
|
}
|
|
88
507
|
else if (health.type === 'http') {
|
|
89
508
|
const ip = await containerIp(ctx, project, service, cwd, composeFile);
|
|
90
|
-
const res = await ctx.http(`http://${ip}:${health.port || 80}${health.path || '/'}`, {
|
|
509
|
+
const res = await ctx.http(`http://${ip}:${health.port || 80}${health.path || '/'}`, {
|
|
510
|
+
signal: AbortSignal.timeout(parseDuration(health.timeout || '2s')),
|
|
511
|
+
});
|
|
91
512
|
if (!res.ok)
|
|
92
513
|
throw new Error(`${service}: http ${res.status}`);
|
|
93
514
|
}
|
|
@@ -95,7 +516,7 @@ export async function waitForHealth(ctx, project, service, health, cwd, composeF
|
|
|
95
516
|
}
|
|
96
517
|
catch (e) {
|
|
97
518
|
last = e;
|
|
98
|
-
await new Promise(r => setTimeout(r, intervalMs));
|
|
519
|
+
await new Promise((r) => setTimeout(r, intervalMs));
|
|
99
520
|
}
|
|
100
521
|
}
|
|
101
522
|
throw new Error(`${service}: ${describeHealthCheck(health)} failed after ${retries} attempt${retries === 1 ? '' : 's'}: ${describeError(last)}`);
|
|
@@ -114,18 +535,74 @@ function describeHealthCheck(health) {
|
|
|
114
535
|
return 'docker health check';
|
|
115
536
|
}
|
|
116
537
|
export { parseDuration };
|
|
538
|
+
/**
|
|
539
|
+
* Containers of Tug projects (`projects`, plus their rolling revision projects `<project>-…`) that
|
|
540
|
+
* listen on all interfaces or on a public IP, from `docker ps --format '{{.Label "com.docker.compose.project"}}\t{{.Names}}\t{{.Ports}}'`.
|
|
541
|
+
*/
|
|
542
|
+
export function publicListeners(psOutput, projects) {
|
|
543
|
+
const out = [];
|
|
544
|
+
for (const line of psOutput.split('\n')) {
|
|
545
|
+
const [project, container, ports = ''] = line.split('\t');
|
|
546
|
+
if (!project || !projects.some((p) => project === p || project.startsWith(`${p}-`)))
|
|
547
|
+
continue;
|
|
548
|
+
for (const m of ports.matchAll(/(\[[0-9a-fA-F:.]+\]|[0-9a-fA-F:.]+):(\d+(?:-\d+)?)->(\S+?)(?:,|$)/g)) {
|
|
549
|
+
if (isPublicAddress(m[1]))
|
|
550
|
+
out.push({ project, container, listen: `${m[1]}:${m[2]}->${m[3]}` });
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
return out;
|
|
554
|
+
}
|
|
555
|
+
/** `ports.public` diagnostic: a warning, not a failure, since an operator may publish on purpose with an explicit host IP. */
|
|
556
|
+
async function publicPortsCheck(ctx) {
|
|
557
|
+
try {
|
|
558
|
+
const projects = await fs
|
|
559
|
+
.readdir(path.join(ctx.stateDir, 'projects'))
|
|
560
|
+
.catch(() => []);
|
|
561
|
+
const ps = await ctx.docker([
|
|
562
|
+
'ps',
|
|
563
|
+
'--filter',
|
|
564
|
+
'label=com.docker.compose.project',
|
|
565
|
+
'--format',
|
|
566
|
+
'{{.Label "com.docker.compose.project"}}\t{{.Names}}\t{{.Ports}}',
|
|
567
|
+
]);
|
|
568
|
+
const listeners = publicListeners(ps.out, projects);
|
|
569
|
+
const stdout = listeners.map((l) => `${l.project}\t${l.container}\t${l.listen}`).join('\n');
|
|
570
|
+
return listeners.length
|
|
571
|
+
? {
|
|
572
|
+
name: 'ports.public',
|
|
573
|
+
ok: true,
|
|
574
|
+
stdout: stdout.slice(-4000),
|
|
575
|
+
warning: `${listeners.length} published port${listeners.length === 1 ? ' of Tug projects listens' : 's of Tug projects listen'} on a public address; bind them to 127.0.0.1, the overlay IP or a private address unless that is intended`,
|
|
576
|
+
}
|
|
577
|
+
: {
|
|
578
|
+
name: 'ports.public',
|
|
579
|
+
ok: true,
|
|
580
|
+
stdout: 'no Tug project container listens on a public address',
|
|
581
|
+
};
|
|
582
|
+
}
|
|
583
|
+
catch (e) {
|
|
584
|
+
return { name: 'ports.public', ok: false, error: errMessage(e).slice(-4000) };
|
|
585
|
+
}
|
|
586
|
+
}
|
|
117
587
|
export async function executeCommand(command, ctx = defaultContext()) {
|
|
118
|
-
const payload = typeof command.payload_json === 'string'
|
|
588
|
+
const payload = typeof command.payload_json === 'string'
|
|
589
|
+
? JSON.parse(command.payload_json || '{}')
|
|
590
|
+
: command.payload || {};
|
|
119
591
|
if (command.type === 'diagnostics.run') {
|
|
120
592
|
const checks = [];
|
|
121
593
|
for (const [name, args] of [
|
|
122
594
|
['docker.info', ['info']],
|
|
123
595
|
['docker.version', ['version']],
|
|
124
|
-
['docker.compose.version', ['compose', 'version']]
|
|
596
|
+
['docker.compose.version', ['compose', 'version']],
|
|
125
597
|
]) {
|
|
126
598
|
try {
|
|
127
599
|
const result = await ctx.docker([...args]);
|
|
128
|
-
checks.push({
|
|
600
|
+
checks.push({
|
|
601
|
+
name,
|
|
602
|
+
ok: true,
|
|
603
|
+
stdout: result.out.slice(-4000),
|
|
604
|
+
stderr: result.err.slice(-4000),
|
|
605
|
+
});
|
|
129
606
|
}
|
|
130
607
|
catch (e) {
|
|
131
608
|
checks.push({ name, ok: false, error: errMessage(e).slice(-4000) });
|
|
@@ -137,25 +614,42 @@ export async function executeCommand(command, ctx = defaultContext()) {
|
|
|
137
614
|
extra.push(['wireguard.version', () => host('wg', ['--version'])], ['wireguard.interface', () => host('wg', ['show', WG_INTERFACE])]);
|
|
138
615
|
}
|
|
139
616
|
if (await fs.stat(path.join(ctx.stateDir, 'ingress')).then(() => true, () => false)) {
|
|
140
|
-
extra.push([
|
|
617
|
+
extra.push([
|
|
618
|
+
'ingress.caddy',
|
|
619
|
+
() => ctx.docker(['container', 'inspect', '-f', '{{.State.Status}}', CADDY_CONTAINER]),
|
|
620
|
+
]);
|
|
141
621
|
}
|
|
142
622
|
for (const [name, fn] of extra) {
|
|
143
623
|
try {
|
|
144
624
|
const result = await fn();
|
|
145
|
-
checks.push({
|
|
625
|
+
checks.push({
|
|
626
|
+
name,
|
|
627
|
+
ok: true,
|
|
628
|
+
stdout: result.out.slice(-4000),
|
|
629
|
+
stderr: result.err.slice(-4000),
|
|
630
|
+
});
|
|
146
631
|
}
|
|
147
632
|
catch (e) {
|
|
148
633
|
checks.push({ name, ok: false, error: errMessage(e).slice(-4000) });
|
|
149
634
|
}
|
|
150
635
|
}
|
|
151
|
-
|
|
636
|
+
checks.push(await publicPortsCheck(ctx));
|
|
637
|
+
return {
|
|
638
|
+
commandId: command.id,
|
|
639
|
+
state: checks.every((c) => c.ok) ? 'succeeded' : 'failed',
|
|
640
|
+
result: { checks, node: ctx.nodeName },
|
|
641
|
+
};
|
|
152
642
|
}
|
|
153
643
|
if (command.type === 'service.restart') {
|
|
154
644
|
const project = payload.project;
|
|
155
645
|
const service = payload.service;
|
|
156
646
|
const dir = path.join(ctx.stateDir, 'projects', project);
|
|
157
647
|
await ctx.docker(['compose', '-p', project, 'restart', service], { cwd: dir });
|
|
158
|
-
return {
|
|
648
|
+
return {
|
|
649
|
+
commandId: command.id,
|
|
650
|
+
state: 'succeeded',
|
|
651
|
+
result: { project, service, restarted: true },
|
|
652
|
+
};
|
|
159
653
|
}
|
|
160
654
|
if (command.type === 'logs.fetch') {
|
|
161
655
|
const project = payload.project;
|
|
@@ -163,7 +657,11 @@ export async function executeCommand(command, ctx = defaultContext()) {
|
|
|
163
657
|
const tail = String(payload.tail || 200);
|
|
164
658
|
const dir = path.join(ctx.stateDir, 'projects', project);
|
|
165
659
|
const result = await ctx.docker(['compose', '-p', project, 'logs', '--no-color', '--tail', tail, service], { cwd: dir });
|
|
166
|
-
return {
|
|
660
|
+
return {
|
|
661
|
+
commandId: command.id,
|
|
662
|
+
state: 'succeeded',
|
|
663
|
+
result: { stdout: result.out, stderr: result.err },
|
|
664
|
+
};
|
|
167
665
|
}
|
|
168
666
|
if (command.type === 'volume.delete') {
|
|
169
667
|
const project = payload.project;
|
|
@@ -176,23 +674,40 @@ export async function executeCommand(command, ctx = defaultContext()) {
|
|
|
176
674
|
if (!errMessage(e).includes('No such volume'))
|
|
177
675
|
throw e;
|
|
178
676
|
}
|
|
179
|
-
return {
|
|
677
|
+
return {
|
|
678
|
+
commandId: command.id,
|
|
679
|
+
state: 'succeeded',
|
|
680
|
+
result: { project, volume, deleted: true },
|
|
681
|
+
};
|
|
180
682
|
}
|
|
181
683
|
if (command.type === 'wg.rotate-key') {
|
|
182
684
|
if (!ctx.mesh)
|
|
183
685
|
throw new Error('the WireGuard mesh is not enabled on this agent (TUG_MESH=true)');
|
|
184
686
|
const { publicKey } = await rotateWgKeys(ctx.stateDir);
|
|
185
|
-
await api(ctx, '/agent/v1/wireguard/key', {
|
|
687
|
+
await api(ctx, '/agent/v1/wireguard/key', {
|
|
688
|
+
method: 'POST',
|
|
689
|
+
headers: { 'content-type': 'application/json' },
|
|
690
|
+
body: JSON.stringify({ publicKey }),
|
|
691
|
+
});
|
|
186
692
|
await writeReportedKey(ctx.stateDir, publicKey);
|
|
187
693
|
return { commandId: command.id, state: 'succeeded', result: { publicKey } };
|
|
188
694
|
}
|
|
189
|
-
return {
|
|
695
|
+
return {
|
|
696
|
+
commandId: command.id,
|
|
697
|
+
state: 'failed',
|
|
698
|
+
result: { error: `unsupported command ${command.type}` },
|
|
699
|
+
};
|
|
700
|
+
}
|
|
701
|
+
async function managedImageFile(ctx) {
|
|
702
|
+
await ensureDir(ctx.stateDir);
|
|
703
|
+
return path.join(ctx.stateDir, 'managed-images.json');
|
|
190
704
|
}
|
|
191
|
-
async function managedImageFile(ctx) { await ensureDir(ctx.stateDir); return path.join(ctx.stateDir, 'managed-images.json'); }
|
|
192
705
|
async function managedImages(ctx) {
|
|
193
706
|
try {
|
|
194
707
|
const refs = JSON.parse(await fs.readFile(await managedImageFile(ctx), 'utf8'));
|
|
195
|
-
return Array.isArray(refs)
|
|
708
|
+
return Array.isArray(refs)
|
|
709
|
+
? refs.filter((ref) => typeof ref === 'string' && ref.startsWith('tug-build/'))
|
|
710
|
+
: [];
|
|
196
711
|
}
|
|
197
712
|
catch {
|
|
198
713
|
return [];
|
|
@@ -206,7 +721,11 @@ export async function ensureArtifact(ctx, artifact) {
|
|
|
206
721
|
}
|
|
207
722
|
catch { }
|
|
208
723
|
const remember = async (daemonId) => {
|
|
209
|
-
identities[artifact.imageRef] = {
|
|
724
|
+
identities[artifact.imageRef] = {
|
|
725
|
+
daemonId,
|
|
726
|
+
artifactId: artifact.imageId,
|
|
727
|
+
sha256: artifact.sha256,
|
|
728
|
+
};
|
|
210
729
|
await fs.writeFile(identityFile, JSON.stringify(identities), { mode: 0o600 });
|
|
211
730
|
const refs = await managedImages(ctx);
|
|
212
731
|
if (!refs.includes(artifact.imageRef))
|
|
@@ -215,14 +734,19 @@ export async function ensureArtifact(ctx, artifact) {
|
|
|
215
734
|
const inspect = async () => JSON.parse((await ctx.docker(['image', 'inspect', artifact.imageRef])).out)[0];
|
|
216
735
|
try {
|
|
217
736
|
const image = await inspect();
|
|
218
|
-
if ((image?.Id === artifact.imageId ||
|
|
737
|
+
if ((image?.Id === artifact.imageId ||
|
|
738
|
+
(identities[artifact.imageRef]?.daemonId === image?.Id &&
|
|
739
|
+
identities[artifact.imageRef]?.artifactId === artifact.imageId &&
|
|
740
|
+
identities[artifact.imageRef]?.sha256 === artifact.sha256)) &&
|
|
741
|
+
`${image.Os}/${image.Architecture}` === artifact.platform) {
|
|
219
742
|
await remember(image.Id);
|
|
220
743
|
return;
|
|
221
744
|
}
|
|
222
745
|
}
|
|
223
746
|
catch { }
|
|
224
747
|
const host = JSON.parse((await ctx.docker(['info', '--format', '{{json .}}'])).out);
|
|
225
|
-
const arch = { x86_64: 'amd64', aarch64: 'arm64' }[host.Architecture] ||
|
|
748
|
+
const arch = { x86_64: 'amd64', aarch64: 'arm64' }[host.Architecture] ||
|
|
749
|
+
host.Architecture;
|
|
226
750
|
if (host.OSType && host.Architecture && `${host.OSType}/${arch}` !== artifact.platform)
|
|
227
751
|
throw new Error(`image platform ${artifact.platform} does not match node ${host.OSType}/${arch}; build for the target platform`);
|
|
228
752
|
const downloadDir = await fs.mkdtemp(path.join(ctx.stateDir, 'image-download-'));
|
|
@@ -269,7 +793,26 @@ export async function pruneManagedImages(ctx, retained) {
|
|
|
269
793
|
}
|
|
270
794
|
await fs.writeFile(await managedImageFile(ctx), JSON.stringify(remaining), { mode: 0o600 });
|
|
271
795
|
}
|
|
272
|
-
|
|
796
|
+
/**
|
|
797
|
+
* The IP that published ports without a host IP listen on: `127.0.0.1` when the node has no
|
|
798
|
+
* address, the address itself when it is an IP, otherwise what it resolves to on this node.
|
|
799
|
+
*/
|
|
800
|
+
export async function resolveBindAddress(address, lookup = (host) => dns.lookup(host)) {
|
|
801
|
+
if (!address)
|
|
802
|
+
return '127.0.0.1';
|
|
803
|
+
const bare = address.replace(/^\[|\]$/g, '');
|
|
804
|
+
if (net.isIP(bare))
|
|
805
|
+
return bare;
|
|
806
|
+
try {
|
|
807
|
+
return (await lookup(bare)).address;
|
|
808
|
+
}
|
|
809
|
+
catch (e) {
|
|
810
|
+
throw new Error(`cannot resolve node address ${address} to bind published ports: ${errMessage(e)}`, { cause: e });
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
// Last public-port warning per project, so the event is sent when it changes rather than on every poll.
|
|
814
|
+
const publicPortWarnings = new Map();
|
|
815
|
+
export async function reconcile(desiredState, ctx = defaultContext(), endpoints = {}, ports = { address: null, loopback: {} }) {
|
|
273
816
|
const dir = path.join(ctx.stateDir, 'projects', desiredState.project);
|
|
274
817
|
await ensureDir(dir);
|
|
275
818
|
const plannedCompose = composeForNode(desiredState, ctx.nodeName, {});
|
|
@@ -277,29 +820,88 @@ export async function reconcile(desiredState, ctx = defaultContext(), endpoints
|
|
|
277
820
|
if (!serviceNames.length) {
|
|
278
821
|
if (ctx.apply)
|
|
279
822
|
await retireAll(ctx, dir, desiredState).catch(() => { });
|
|
280
|
-
return {
|
|
823
|
+
return {
|
|
824
|
+
project: desiredState.project,
|
|
825
|
+
observedVersion: desiredState.version,
|
|
826
|
+
...(desiredState.revision ? { observedRevision: desiredState.revision } : {}),
|
|
827
|
+
health: 'skipped',
|
|
828
|
+
services: [],
|
|
829
|
+
serviceStates: {},
|
|
830
|
+
};
|
|
281
831
|
}
|
|
282
|
-
const serviceStates = Object.fromEntries(serviceNames.map(service => [
|
|
283
|
-
|
|
832
|
+
const serviceStates = Object.fromEntries(serviceNames.map((service) => [
|
|
833
|
+
service,
|
|
834
|
+
{ rolloutState: 'pending', image: desiredState.services[service]?.image },
|
|
835
|
+
]));
|
|
836
|
+
const emit = (service, rolloutState, message = rolloutState, level = 'info') => api(ctx, '/agent/v1/events', {
|
|
837
|
+
method: 'POST',
|
|
838
|
+
headers: { 'content-type': 'application/json' },
|
|
839
|
+
body: JSON.stringify({
|
|
840
|
+
projectId: desiredState.project,
|
|
841
|
+
level,
|
|
842
|
+
type: `service.${rolloutState}`,
|
|
843
|
+
message,
|
|
844
|
+
metadata: { service, version: desiredState.version, node: ctx.nodeName },
|
|
845
|
+
}),
|
|
846
|
+
}).catch(() => { });
|
|
284
847
|
const sec = await api(ctx, `/agent/v1/secrets?project=${encodeURIComponent(desiredState.project)}&services=${encodeURIComponent(serviceNames.join(','))}`);
|
|
285
848
|
const currentSecretVersions = serviceSecretVersions(desiredState.compose, serviceNames, sec.versions || {});
|
|
286
849
|
const db = await agentDb(ctx.stateDir);
|
|
287
850
|
const previousRow = await db.get('SELECT version, revision, secret_versions_json FROM applied_state WHERE project = ?', desiredState.project);
|
|
288
|
-
const previous = previousRow
|
|
289
|
-
|
|
851
|
+
const previous = previousRow
|
|
852
|
+
? {
|
|
853
|
+
version: previousRow.version,
|
|
854
|
+
revision: previousRow.revision,
|
|
855
|
+
secretVersions: JSON.parse(previousRow.secret_versions_json),
|
|
856
|
+
}
|
|
857
|
+
: {};
|
|
858
|
+
const rolling = serviceNames.filter((service) => isRolling(desiredState.compose.services[service]));
|
|
290
859
|
const rollingSet = new Set(rolling);
|
|
291
|
-
const mainServices = serviceNames.filter(service => !rollingSet.has(service));
|
|
292
|
-
const secretChangedServices = mainServices.filter(service => JSON.stringify(previous.secretVersions?.[service] || {}) !==
|
|
293
|
-
|
|
860
|
+
const mainServices = serviceNames.filter((service) => !rollingSet.has(service));
|
|
861
|
+
const secretChangedServices = mainServices.filter((service) => JSON.stringify(previous.secretVersions?.[service] || {}) !==
|
|
862
|
+
JSON.stringify(currentSecretVersions[service] || {}));
|
|
863
|
+
const sameDesiredVersion = desiredState.revision
|
|
864
|
+
? previous.revision === desiredState.revision
|
|
865
|
+
: previous.version === desiredState.version;
|
|
294
866
|
// Rolling services run in their own revision projects; the main project holds only the recreated ones.
|
|
295
|
-
const
|
|
867
|
+
const loopback = new Set(ports.loopback[desiredState.project] || []);
|
|
868
|
+
// Resolve only when some port actually binds to the node address; a hostname nobody needs must not fail the apply.
|
|
869
|
+
const needsAddress = addressBoundServices(desiredState.compose, serviceNames, loopback).length > 0;
|
|
870
|
+
const bind = {
|
|
871
|
+
address: needsAddress ? await resolveBindAddress(ports.address) : '127.0.0.1',
|
|
872
|
+
loopback,
|
|
873
|
+
};
|
|
874
|
+
const nodeCompose = composeForNode(desiredState, ctx.nodeName, {
|
|
875
|
+
secrets: sec.secrets || {},
|
|
876
|
+
endpoints,
|
|
877
|
+
exclude: rollingSet,
|
|
878
|
+
bind,
|
|
879
|
+
});
|
|
880
|
+
await warnPublicPorts(ctx, desiredState, serviceNames, bind);
|
|
296
881
|
await fs.writeFile(path.join(dir, 'docker-compose.yml'), dumpYaml(nodeCompose));
|
|
297
882
|
await writeEnv(path.join(dir, '.env'), sec.secrets || {});
|
|
298
|
-
const failedStatus = (error) => ({
|
|
883
|
+
const failedStatus = (error) => ({
|
|
884
|
+
project: desiredState.project,
|
|
885
|
+
observedVersion: desiredState.version,
|
|
886
|
+
...(desiredState.revision ? { observedRevision: desiredState.revision } : {}),
|
|
887
|
+
health: 'failed',
|
|
888
|
+
services: serviceNames,
|
|
889
|
+
serviceStates,
|
|
890
|
+
error,
|
|
891
|
+
});
|
|
299
892
|
// Lifecycle events say what changed, not that the agent is still polling. They are sent when the desired revision,
|
|
300
893
|
// this node's rendered configuration or the secrets change, when an apply fails (once) and when it recovers (once).
|
|
301
|
-
const eventFingerprint = createHash('sha256')
|
|
302
|
-
|
|
894
|
+
const eventFingerprint = createHash('sha256')
|
|
895
|
+
.update(JSON.stringify([
|
|
896
|
+
desiredState.revision || desiredState.version,
|
|
897
|
+
dumpYaml(nodeCompose),
|
|
898
|
+
mainServices.map((service) => currentSecretVersions[service] || {}),
|
|
899
|
+
]))
|
|
900
|
+
.digest('hex')
|
|
901
|
+
.slice(0, 16);
|
|
902
|
+
const knownEvents = mainServices.length
|
|
903
|
+
? await db.get('SELECT fingerprint, phase FROM event_state WHERE project = ?', desiredState.project)
|
|
904
|
+
: undefined;
|
|
303
905
|
const announce = knownEvents?.fingerprint !== eventFingerprint;
|
|
304
906
|
const wasFailed = !announce && knownEvents?.phase === 'failed';
|
|
305
907
|
const markEvents = (phase) => db.run('INSERT INTO event_state(project, fingerprint, phase) VALUES(?,?,?) ON CONFLICT(project) DO UPDATE SET fingerprint=excluded.fingerprint, phase=excluded.phase', desiredState.project, eventFingerprint, phase);
|
|
@@ -316,9 +918,19 @@ export async function reconcile(desiredState, ctx = defaultContext(), endpoints
|
|
|
316
918
|
if (artifact)
|
|
317
919
|
await ensureArtifact(ctx, artifact);
|
|
318
920
|
}
|
|
319
|
-
|
|
921
|
+
// Registry images are pulled only for a new desired revision. Pulling on every poll would let a tag that moved
|
|
922
|
+
// upstream (`mariadb:11`) recreate the container without a deploy; `up -d` still pulls images that are missing.
|
|
923
|
+
const pullServices = sameDesiredVersion
|
|
924
|
+
? []
|
|
925
|
+
: mainServices.filter((service) => !desiredState.services[service].artifact);
|
|
320
926
|
if (pullServices.length)
|
|
321
|
-
await ctx.docker([
|
|
927
|
+
await ctx.docker([
|
|
928
|
+
'compose',
|
|
929
|
+
'-p',
|
|
930
|
+
desiredState.project,
|
|
931
|
+
'pull',
|
|
932
|
+
...(pullServices.length === mainServices.length ? [] : pullServices),
|
|
933
|
+
], { cwd: dir });
|
|
322
934
|
for (const service of mainServices) {
|
|
323
935
|
serviceStates[service].rolloutState = 'starting';
|
|
324
936
|
if (announce)
|
|
@@ -326,13 +938,29 @@ export async function reconcile(desiredState, ctx = defaultContext(), endpoints
|
|
|
326
938
|
}
|
|
327
939
|
if (mainServices.length) {
|
|
328
940
|
// Services that just became rolling still run here until their replacement takes over; keep them.
|
|
329
|
-
const keepLegacy = rolling.length > 0 && await hasLegacyContainers(ctx, desiredState.project, rolling);
|
|
330
|
-
await ctx.docker([
|
|
941
|
+
const keepLegacy = rolling.length > 0 && (await hasLegacyContainers(ctx, desiredState.project, rolling));
|
|
942
|
+
await ctx.docker([
|
|
943
|
+
'compose',
|
|
944
|
+
'-p',
|
|
945
|
+
desiredState.project,
|
|
946
|
+
'up',
|
|
947
|
+
'-d',
|
|
948
|
+
...(keepLegacy ? [] : ['--remove-orphans']),
|
|
949
|
+
], { cwd: dir });
|
|
331
950
|
}
|
|
332
951
|
if (sameDesiredVersion && secretChangedServices.length) {
|
|
333
952
|
for (const service of secretChangedServices)
|
|
334
953
|
await emit(service, 'secret_recreate', 'secret version changed; recreating service');
|
|
335
|
-
await ctx.docker([
|
|
954
|
+
await ctx.docker([
|
|
955
|
+
'compose',
|
|
956
|
+
'-p',
|
|
957
|
+
desiredState.project,
|
|
958
|
+
'up',
|
|
959
|
+
'-d',
|
|
960
|
+
'--no-deps',
|
|
961
|
+
'--force-recreate',
|
|
962
|
+
...secretChangedServices,
|
|
963
|
+
], { cwd: dir });
|
|
336
964
|
}
|
|
337
965
|
for (const service of mainServices) {
|
|
338
966
|
serviceStates[service].rolloutState = 'health_checking';
|
|
@@ -363,14 +991,24 @@ export async function reconcile(desiredState, ctx = defaultContext(), endpoints
|
|
|
363
991
|
}
|
|
364
992
|
if (rolling.length) {
|
|
365
993
|
const rolled = await reconcileRolling(ctx, {
|
|
366
|
-
desired: desiredState,
|
|
367
|
-
|
|
994
|
+
desired: desiredState,
|
|
995
|
+
dir,
|
|
996
|
+
services: rolling,
|
|
997
|
+
secretVersions: currentSecretVersions,
|
|
998
|
+
full: composeForNode(desiredState, ctx.nodeName, {
|
|
999
|
+
secrets: sec.secrets || {},
|
|
1000
|
+
endpoints,
|
|
1001
|
+
bind,
|
|
1002
|
+
}),
|
|
368
1003
|
deps: {
|
|
369
1004
|
waitForHealth: (project, service, composeFile) => waitForHealth(ctx, project, service, desiredState.services[service]?.health, dir, composeFile),
|
|
370
|
-
ensureArtifact: async (service) => {
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
1005
|
+
ensureArtifact: async (service) => {
|
|
1006
|
+
const artifact = desiredState.services[service].artifact;
|
|
1007
|
+
if (artifact)
|
|
1008
|
+
await ensureArtifact(ctx, artifact);
|
|
1009
|
+
},
|
|
1010
|
+
emit,
|
|
1011
|
+
},
|
|
374
1012
|
});
|
|
375
1013
|
Object.assign(serviceStates, rolled.serviceStates);
|
|
376
1014
|
const errors = Object.entries(rolled.errors);
|
|
@@ -388,14 +1026,51 @@ export async function reconcile(desiredState, ctx = defaultContext(), endpoints
|
|
|
388
1026
|
VALUES(?,?,?,?,?,?)
|
|
389
1027
|
ON CONFLICT(project) DO UPDATE SET version=excluded.version, revision=excluded.revision, applied_at=excluded.applied_at,
|
|
390
1028
|
service_states_json=excluded.service_states_json, secret_versions_json=excluded.secret_versions_json`, desiredState.project, desiredState.version, desiredState.revision || null, now(), JSON.stringify(serviceStates), JSON.stringify(currentSecretVersions));
|
|
391
|
-
return {
|
|
1029
|
+
return {
|
|
1030
|
+
project: desiredState.project,
|
|
1031
|
+
observedVersion: desiredState.version,
|
|
1032
|
+
...(desiredState.revision ? { observedRevision: desiredState.revision } : {}),
|
|
1033
|
+
health: 'serving',
|
|
1034
|
+
services: serviceNames,
|
|
1035
|
+
serviceStates,
|
|
1036
|
+
};
|
|
1037
|
+
}
|
|
1038
|
+
/** Emits a `ports.public` warning when published ports of this project are bound to a public node address. */
|
|
1039
|
+
async function warnPublicPorts(ctx, desiredState, serviceNames, bind) {
|
|
1040
|
+
const key = `${ctx.stateDir}\0${desiredState.project}`;
|
|
1041
|
+
const exposed = isPublicAddress(bind.address)
|
|
1042
|
+
? addressBoundServices(desiredState.compose, serviceNames, bind.loopback)
|
|
1043
|
+
: [];
|
|
1044
|
+
const message = exposed.length
|
|
1045
|
+
? `published ports of ${exposed.join(', ')} listen on public address ${bind.address}; enable the mesh, give the node a private address, or write the host IP in ports to make the exposure explicit`
|
|
1046
|
+
: '';
|
|
1047
|
+
if ((publicPortWarnings.get(key) ?? '') === message)
|
|
1048
|
+
return;
|
|
1049
|
+
publicPortWarnings.set(key, message);
|
|
1050
|
+
if (!message)
|
|
1051
|
+
return;
|
|
1052
|
+
await api(ctx, '/agent/v1/events', {
|
|
1053
|
+
method: 'POST',
|
|
1054
|
+
headers: { 'content-type': 'application/json' },
|
|
1055
|
+
body: JSON.stringify({
|
|
1056
|
+
projectId: desiredState.project,
|
|
1057
|
+
level: 'warn',
|
|
1058
|
+
type: 'ports.public',
|
|
1059
|
+
message,
|
|
1060
|
+
metadata: { services: exposed, address: bind.address, node: ctx.nodeName },
|
|
1061
|
+
}),
|
|
1062
|
+
}).catch(() => { });
|
|
392
1063
|
}
|
|
393
1064
|
/** Reports this node's WireGuard public key when control has not seen the current one yet. */
|
|
394
1065
|
export async function reportWgKey(ctx) {
|
|
395
1066
|
const { publicKey } = await ensureWgKeys(ctx.stateDir);
|
|
396
|
-
if (await readReportedKey(ctx.stateDir) === publicKey)
|
|
1067
|
+
if ((await readReportedKey(ctx.stateDir)) === publicKey)
|
|
397
1068
|
return;
|
|
398
|
-
await api(ctx, '/agent/v1/wireguard/key', {
|
|
1069
|
+
await api(ctx, '/agent/v1/wireguard/key', {
|
|
1070
|
+
method: 'POST',
|
|
1071
|
+
headers: { 'content-type': 'application/json' },
|
|
1072
|
+
body: JSON.stringify({ publicKey }),
|
|
1073
|
+
});
|
|
399
1074
|
await writeReportedKey(ctx.stateDir, publicKey);
|
|
400
1075
|
}
|
|
401
1076
|
let lastNetworkError = '';
|
|
@@ -423,10 +1098,16 @@ export async function pollOnce(ctx = defaultContext()) {
|
|
|
423
1098
|
const projects = [];
|
|
424
1099
|
for (const desired of work.desiredStates || (work.desiredState ? [work.desiredState] : [])) {
|
|
425
1100
|
try {
|
|
426
|
-
projects.push(await reconcile(desired, ctx, work.serviceEndpoints?.[desired.project]));
|
|
1101
|
+
projects.push(await reconcile(desired, ctx, work.serviceEndpoints?.[desired.project], work.ports));
|
|
427
1102
|
}
|
|
428
1103
|
catch (e) {
|
|
429
|
-
projects.push({
|
|
1104
|
+
projects.push({
|
|
1105
|
+
project: desired.project,
|
|
1106
|
+
observedVersion: desired.version,
|
|
1107
|
+
...(desired.revision ? { observedRevision: desired.revision } : {}),
|
|
1108
|
+
health: 'failed',
|
|
1109
|
+
error: errMessage(e),
|
|
1110
|
+
});
|
|
430
1111
|
}
|
|
431
1112
|
}
|
|
432
1113
|
let ingress;
|
|
@@ -443,15 +1124,23 @@ export async function pollOnce(ctx = defaultContext()) {
|
|
|
443
1124
|
commandResults.push(await executeCommand(command, ctx));
|
|
444
1125
|
}
|
|
445
1126
|
catch (e) {
|
|
446
|
-
commandResults.push({
|
|
1127
|
+
commandResults.push({
|
|
1128
|
+
commandId: command.id,
|
|
1129
|
+
state: 'failed',
|
|
1130
|
+
result: { error: errMessage(e) },
|
|
1131
|
+
});
|
|
447
1132
|
}
|
|
448
1133
|
}
|
|
449
1134
|
const report = { node: ctx.nodeName, projects, commandResults };
|
|
450
1135
|
if (ingress)
|
|
451
1136
|
report.ingress = ingress;
|
|
452
|
-
if (work.retainedImages && projects.every(project => project.health !== 'failed'))
|
|
1137
|
+
if (work.retainedImages && projects.every((project) => project.health !== 'failed'))
|
|
453
1138
|
await pruneManagedImages(ctx, work.retainedImages);
|
|
454
|
-
await api(ctx, '/agent/v1/status', {
|
|
1139
|
+
await api(ctx, '/agent/v1/status', {
|
|
1140
|
+
method: 'POST',
|
|
1141
|
+
headers: { 'content-type': 'application/json' },
|
|
1142
|
+
body: JSON.stringify(report),
|
|
1143
|
+
});
|
|
455
1144
|
return projects;
|
|
456
1145
|
}
|
|
457
1146
|
const CREDS_FILE = () => path.join(STATE_DIR, 'agent.json');
|
|
@@ -483,11 +1172,13 @@ async function readJoinKey() {
|
|
|
483
1172
|
}
|
|
484
1173
|
function dockerLogin(registry, user, password) {
|
|
485
1174
|
return new Promise((resolve, reject) => {
|
|
486
|
-
const p = spawn('docker', ['login', registry, '-u', user, '--password-stdin'], {
|
|
1175
|
+
const p = spawn('docker', ['login', registry, '-u', user, '--password-stdin'], {
|
|
1176
|
+
stdio: ['pipe', 'ignore', 'pipe'],
|
|
1177
|
+
});
|
|
487
1178
|
let err = '';
|
|
488
|
-
p.stderr.on('data', d => err += d);
|
|
1179
|
+
p.stderr.on('data', (d) => (err += d));
|
|
489
1180
|
p.on('error', reject);
|
|
490
|
-
p.on('close', code => code === 0 ? resolve() : reject(new Error(err.trim() || `exit ${code}`)));
|
|
1181
|
+
p.on('close', (code) => code === 0 ? resolve() : reject(new Error(err.trim() || `exit ${code}`)));
|
|
491
1182
|
p.stdin.write(password);
|
|
492
1183
|
p.stdin.end();
|
|
493
1184
|
});
|
|
@@ -502,26 +1193,48 @@ async function enroll(joinKey) {
|
|
|
502
1193
|
}
|
|
503
1194
|
if (!decoded.enrollToken)
|
|
504
1195
|
throw new Error('join key is missing enrollToken');
|
|
505
|
-
const controlUrl =
|
|
1196
|
+
const controlUrl = !CONTROL_URL_FROM_ENV && decoded.controlUrl ? decoded.controlUrl : CONTROL_URL;
|
|
506
1197
|
// node's fetch reads NODE_EXTRA_CA_CERTS only at startup, so apply an embedded CA by
|
|
507
1198
|
// persisting it and re-execing ourselves once with the env var set.
|
|
508
1199
|
if (decoded.ca && !process.env.NODE_EXTRA_CA_CERTS) {
|
|
509
1200
|
await ensureDir(STATE_DIR);
|
|
510
1201
|
const caPath = path.join(STATE_DIR, 'control-ca.pem');
|
|
511
1202
|
await fs.writeFile(caPath, decoded.ca);
|
|
512
|
-
const r = spawnSync(process.execPath, process.argv.slice(1), {
|
|
1203
|
+
const r = spawnSync(process.execPath, process.argv.slice(1), {
|
|
1204
|
+
stdio: 'inherit',
|
|
1205
|
+
env: { ...process.env, NODE_EXTRA_CA_CERTS: caPath },
|
|
1206
|
+
});
|
|
513
1207
|
process.exit(r.status ?? 1);
|
|
514
1208
|
}
|
|
515
1209
|
console.log(`enrolling with ${controlUrl} as ${NODE_NAME}`);
|
|
516
1210
|
const wgPublicKey = MESH ? (await ensureWgKeys(STATE_DIR)).publicKey : undefined;
|
|
517
1211
|
const res = await fetch(`${controlUrl}/agent/v1/enroll`, {
|
|
518
|
-
method: 'POST',
|
|
519
|
-
|
|
1212
|
+
method: 'POST',
|
|
1213
|
+
headers: { 'content-type': 'application/json' },
|
|
1214
|
+
body: JSON.stringify({
|
|
1215
|
+
enrollToken: decoded.enrollToken,
|
|
1216
|
+
name: NODE_NAME,
|
|
1217
|
+
address: process.env.TUG_NODE_ADDRESS,
|
|
1218
|
+
publicAddress: process.env.TUG_NODE_PUBLIC_ADDRESS,
|
|
1219
|
+
wgPublicKey,
|
|
1220
|
+
labels: decoded.labels ||
|
|
1221
|
+
(process.env.TUG_LABELS
|
|
1222
|
+
? process.env.TUG_LABELS.split(',')
|
|
1223
|
+
.map((s) => s.trim())
|
|
1224
|
+
.filter(Boolean)
|
|
1225
|
+
: undefined),
|
|
1226
|
+
}),
|
|
520
1227
|
});
|
|
521
1228
|
if (!res.ok)
|
|
522
1229
|
throw new Error(`enroll failed: ${res.status} ${await res.text()}`);
|
|
523
|
-
const out = await res.json();
|
|
524
|
-
const creds = {
|
|
1230
|
+
const out = (await res.json());
|
|
1231
|
+
const creds = {
|
|
1232
|
+
agentToken: out.agentToken,
|
|
1233
|
+
nodeName: out.node?.name || NODE_NAME,
|
|
1234
|
+
controlUrl,
|
|
1235
|
+
registry: out.registry || null,
|
|
1236
|
+
registryToken: out.registryToken || null,
|
|
1237
|
+
};
|
|
525
1238
|
await saveCreds(creds);
|
|
526
1239
|
if (wgPublicKey)
|
|
527
1240
|
await writeReportedKey(STATE_DIR, wgPublicKey);
|
|
@@ -531,7 +1244,9 @@ async function enroll(joinKey) {
|
|
|
531
1244
|
// Establish durable credentials before polling: explicit env token (SSH/push install) wins,
|
|
532
1245
|
// then credentials persisted on the state volume, then a one-time enroll via the join key.
|
|
533
1246
|
export async function loadOrEnroll() {
|
|
534
|
-
let creds = TOKEN
|
|
1247
|
+
let creds = TOKEN
|
|
1248
|
+
? { agentToken: TOKEN, nodeName: NODE_NAME, controlUrl: CONTROL_URL }
|
|
1249
|
+
: await loadCreds();
|
|
535
1250
|
if (!creds) {
|
|
536
1251
|
const joinKey = await readJoinKey();
|
|
537
1252
|
if (!joinKey)
|
|
@@ -555,14 +1270,28 @@ export async function loadOrEnroll() {
|
|
|
555
1270
|
export async function main() {
|
|
556
1271
|
await loadOrEnroll();
|
|
557
1272
|
console.log(`tug-agent node=${NODE_NAME} control=${CONTROL_URL} apply=${APPLY}`);
|
|
1273
|
+
const ctx = defaultContext();
|
|
1274
|
+
const wake = new AgentWakeChannel(ctx, undefined, {
|
|
1275
|
+
onLive: (frame) => live.handle(frame),
|
|
1276
|
+
onDisconnect: () => live.dispose(),
|
|
1277
|
+
});
|
|
1278
|
+
const live = new AgentLiveOperations(ctx, (frame) => {
|
|
1279
|
+
wake.send(frame);
|
|
1280
|
+
});
|
|
558
1281
|
for (;;) {
|
|
559
1282
|
try {
|
|
560
|
-
await pollOnce();
|
|
1283
|
+
await pollOnce(ctx);
|
|
561
1284
|
}
|
|
562
1285
|
catch (e) {
|
|
563
1286
|
console.error(errMessage(e));
|
|
564
1287
|
}
|
|
565
|
-
|
|
1288
|
+
try {
|
|
1289
|
+
await wake.wait(INTERVAL);
|
|
1290
|
+
}
|
|
1291
|
+
catch (e) {
|
|
1292
|
+
console.error(`agent wake wait failed: ${errMessage(e)}`);
|
|
1293
|
+
await new Promise((r) => setTimeout(r, INTERVAL));
|
|
1294
|
+
}
|
|
566
1295
|
}
|
|
567
1296
|
}
|
|
568
1297
|
if (import.meta.url === `file://${process.argv[1]}`)
|