@wpelevator/tug 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -4
- package/dist/agent.d.ts +63 -2
- package/dist/agent.js +828 -77
- package/dist/archive.js +9 -3
- package/dist/artifacts.js +97 -34
- package/dist/build.d.ts +1 -1
- package/dist/build.js +17 -8
- package/dist/cli-build.js +34 -8
- package/dist/cli-config.js +16 -4
- package/dist/cli-install.d.ts +10 -0
- package/dist/cli-install.js +38 -4
- package/dist/cli-live-transport.d.ts +3 -0
- package/dist/cli-live-transport.js +230 -0
- package/dist/cli-live.d.ts +77 -0
- package/dist/cli-live.js +249 -0
- package/dist/cli.js +396 -85
- package/dist/compose.d.ts +46 -7
- package/dist/compose.js +173 -37
- package/dist/control.d.ts +4 -1
- package/dist/control.js +1103 -148
- package/dist/db.d.ts +7 -0
- package/dist/db.js +13 -1
- package/dist/ingress.d.ts +10 -2
- package/dist/ingress.js +57 -16
- package/dist/live.d.ts +80 -0
- package/dist/live.js +340 -0
- package/dist/main.js +3 -1
- package/dist/mesh.js +32 -13
- package/dist/protocol.d.ts +158 -0
- package/dist/protocol.js +16 -2
- package/dist/rollout.js +85 -19
- package/dist/secrets.js +3 -1
- package/dist/templates/control-compose.yml +1 -0
- package/dist/tokens.d.ts +1 -1
- package/dist/tokens.js +35 -9
- package/dist/util.d.ts +7 -0
- package/dist/util.js +73 -16
- package/package.json +8 -4
package/dist/agent.js
CHANGED
|
@@ -7,13 +7,18 @@ import { validateImageArchive, verifyDownload } from "./artifacts.js";
|
|
|
7
7
|
import path from 'node:path';
|
|
8
8
|
import { spawn, spawnSync } from 'node:child_process';
|
|
9
9
|
import net from 'node:net';
|
|
10
|
-
import
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
10
|
+
import dns from 'node:dns/promises';
|
|
11
|
+
import { WebSocket } from 'ws';
|
|
12
|
+
import { createHash } from 'node:crypto';
|
|
13
|
+
import { addressBoundServices, composeForNode, dumpYaml, isRolling, serviceSecretVersions, } from "./compose.js";
|
|
14
|
+
import { LIVE_MAX_FRAME_BYTES, LIVE_START_TIMEOUT_MS } from "./protocol.js";
|
|
15
|
+
import { LiveOperationEngine } from "./live.js";
|
|
16
|
+
import { loadRolloutState, activeRollingStates, composeBase, hasLegacyContainers, reconcileRolling, retireAll, } from "./rollout.js";
|
|
17
|
+
import { ensureDir, errMessage, isPublicAddress, now, parseDuration } from "./util.js";
|
|
13
18
|
import { openAgentDb } from "./db.js";
|
|
14
19
|
import { createDockerRunner, createHostRunner } from "./runners.js";
|
|
15
20
|
import { CADDY_CONTAINER, reconcileIngress, removeIngress } from "./ingress.js";
|
|
16
|
-
import { WG_INTERFACE, ensureWgKeys, readReportedKey, reconcileMesh, rotateWgKeys, writeReportedKey } from "./mesh.js";
|
|
21
|
+
import { WG_INTERFACE, ensureWgKeys, readReportedKey, reconcileMesh, rotateWgKeys, writeReportedKey, } from "./mesh.js";
|
|
17
22
|
const CONTROL_URL_FROM_ENV = Boolean(process.env.TUG_CONTROL_URL);
|
|
18
23
|
let CONTROL_URL = process.env.TUG_CONTROL_URL || 'http://localhost:8080';
|
|
19
24
|
let TOKEN = process.env.TUG_AGENT_TOKEN;
|
|
@@ -24,7 +29,19 @@ const INTERVAL = Number(process.env.TUG_POLL_INTERVAL_MS || 5000);
|
|
|
24
29
|
// Opt-in WireGuard mesh: needs `wg`/`ip` in the agent image, host networking and NET_ADMIN.
|
|
25
30
|
const MESH = process.env.TUG_MESH === 'true';
|
|
26
31
|
export function defaultContext() {
|
|
27
|
-
return {
|
|
32
|
+
return {
|
|
33
|
+
docker: createDockerRunner(),
|
|
34
|
+
http: fetch,
|
|
35
|
+
nodeName: NODE_NAME,
|
|
36
|
+
stateDir: STATE_DIR,
|
|
37
|
+
apply: APPLY,
|
|
38
|
+
controlUrl: CONTROL_URL,
|
|
39
|
+
token: TOKEN,
|
|
40
|
+
host: createHostRunner(),
|
|
41
|
+
mesh: MESH,
|
|
42
|
+
caddyImage: process.env.TUG_CADDY_IMAGE,
|
|
43
|
+
rolloutPorts: process.env.TUG_ROLLOUT_PORTS,
|
|
44
|
+
};
|
|
28
45
|
}
|
|
29
46
|
const dbsByStateDir = new Map();
|
|
30
47
|
async function agentDb(stateDir) {
|
|
@@ -36,22 +53,417 @@ async function agentDb(stateDir) {
|
|
|
36
53
|
}
|
|
37
54
|
return db;
|
|
38
55
|
}
|
|
39
|
-
function authHeaders(ctx) {
|
|
40
|
-
|
|
56
|
+
function authHeaders(ctx) {
|
|
57
|
+
if (!ctx.token)
|
|
58
|
+
throw new Error('TUG_AGENT_TOKEN required');
|
|
59
|
+
return { Authorization: `Bearer ${ctx.token}` };
|
|
60
|
+
}
|
|
41
61
|
async function api(ctx, route, opts = {}) {
|
|
42
|
-
const res = await ctx.http(`${ctx.controlUrl}${route}`, {
|
|
62
|
+
const res = await ctx.http(`${ctx.controlUrl}${route}`, {
|
|
63
|
+
...opts,
|
|
64
|
+
headers: {
|
|
65
|
+
...authHeaders(ctx),
|
|
66
|
+
...(opts.headers || {}),
|
|
67
|
+
},
|
|
68
|
+
});
|
|
43
69
|
if (!res.ok)
|
|
44
70
|
throw new Error(`${opts.method || 'GET'} ${route}: ${res.status} ${await res.text()}`);
|
|
45
71
|
return res.json();
|
|
46
72
|
}
|
|
73
|
+
/** Keeps one outbound stream open and falls back to a bounded HTTP wait when an intermediary rejects WebSockets. */
|
|
74
|
+
export class AgentWakeChannel {
|
|
75
|
+
ctx;
|
|
76
|
+
createSocket;
|
|
77
|
+
socket;
|
|
78
|
+
cursor = 0;
|
|
79
|
+
pending = false;
|
|
80
|
+
waiter;
|
|
81
|
+
handlers;
|
|
82
|
+
constructor(ctx, createSocket = (url, token) => new WebSocket(url, {
|
|
83
|
+
headers: { Authorization: `Bearer ${token}` },
|
|
84
|
+
maxPayload: LIVE_MAX_FRAME_BYTES,
|
|
85
|
+
}), handlers = {}) {
|
|
86
|
+
this.ctx = ctx;
|
|
87
|
+
this.createSocket = createSocket;
|
|
88
|
+
this.handlers = handlers;
|
|
89
|
+
}
|
|
90
|
+
notify() {
|
|
91
|
+
this.pending = true;
|
|
92
|
+
this.waiter?.();
|
|
93
|
+
}
|
|
94
|
+
async connect() {
|
|
95
|
+
if (this.socket?.readyState === WebSocket.OPEN)
|
|
96
|
+
return true;
|
|
97
|
+
if (!this.ctx.token)
|
|
98
|
+
throw new Error('TUG_AGENT_TOKEN required');
|
|
99
|
+
const url = new URL('/agent/v1/stream', this.ctx.controlUrl);
|
|
100
|
+
url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
|
|
101
|
+
return new Promise((resolve) => {
|
|
102
|
+
let settled = false;
|
|
103
|
+
const socket = this.createSocket(url.toString(), this.ctx.token);
|
|
104
|
+
this.socket = socket;
|
|
105
|
+
const finish = (connected) => {
|
|
106
|
+
if (settled)
|
|
107
|
+
return;
|
|
108
|
+
settled = true;
|
|
109
|
+
clearTimeout(timer);
|
|
110
|
+
resolve(connected);
|
|
111
|
+
};
|
|
112
|
+
const timer = setTimeout(() => {
|
|
113
|
+
socket.terminate();
|
|
114
|
+
finish(false);
|
|
115
|
+
}, 1500);
|
|
116
|
+
socket.on('open', () => finish(true));
|
|
117
|
+
socket.on('message', (data) => {
|
|
118
|
+
if (this.socket !== socket)
|
|
119
|
+
return;
|
|
120
|
+
let message;
|
|
121
|
+
const json = data.toString();
|
|
122
|
+
if (Buffer.byteLength(json) > LIVE_MAX_FRAME_BYTES) {
|
|
123
|
+
this.close();
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
try {
|
|
127
|
+
message = JSON.parse(json);
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
if (!message || typeof message !== 'object')
|
|
133
|
+
return;
|
|
134
|
+
const frame = message;
|
|
135
|
+
if (typeof frame.type === 'string' && frame.type.startsWith('live.')) {
|
|
136
|
+
if (frame.v === 1)
|
|
137
|
+
void Promise.resolve(this.handlers.onLive?.(message)).catch((error) => console.error('agent live frame failed:', errMessage(error)));
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
if ((frame.type !== 'ready' && frame.type !== 'work-available') ||
|
|
141
|
+
!Number.isSafeInteger(frame.cursor) ||
|
|
142
|
+
Number(frame.cursor) < 0)
|
|
143
|
+
return;
|
|
144
|
+
this.cursor = Math.max(this.cursor, Number(frame.cursor));
|
|
145
|
+
if (frame.type === 'work-available')
|
|
146
|
+
this.notify();
|
|
147
|
+
});
|
|
148
|
+
socket.on('close', () => {
|
|
149
|
+
if (this.socket === socket) {
|
|
150
|
+
this.socket = undefined;
|
|
151
|
+
this.handlers.onDisconnect?.();
|
|
152
|
+
}
|
|
153
|
+
if (settled)
|
|
154
|
+
this.waiter?.();
|
|
155
|
+
else
|
|
156
|
+
finish(false);
|
|
157
|
+
});
|
|
158
|
+
socket.on('error', () => {
|
|
159
|
+
if (this.socket === socket) {
|
|
160
|
+
this.socket = undefined;
|
|
161
|
+
this.handlers.onDisconnect?.();
|
|
162
|
+
}
|
|
163
|
+
socket.terminate();
|
|
164
|
+
this.waiter?.();
|
|
165
|
+
finish(false);
|
|
166
|
+
});
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
async waitOnSocket(timeoutMs) {
|
|
170
|
+
if (this.pending) {
|
|
171
|
+
this.pending = false;
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
await new Promise((resolve) => {
|
|
175
|
+
const done = () => {
|
|
176
|
+
clearTimeout(timer);
|
|
177
|
+
if (this.waiter === done)
|
|
178
|
+
this.waiter = undefined;
|
|
179
|
+
resolve();
|
|
180
|
+
};
|
|
181
|
+
this.waiter = done;
|
|
182
|
+
const timer = setTimeout(done, timeoutMs);
|
|
183
|
+
});
|
|
184
|
+
this.pending = false;
|
|
185
|
+
}
|
|
186
|
+
async longPoll(timeoutMs) {
|
|
187
|
+
const wake = await api(this.ctx, `/agent/v1/wake?after=${this.cursor}&timeout=${Math.min(timeoutMs, 30_000)}`);
|
|
188
|
+
this.cursor = Math.max(this.cursor, wake.cursor);
|
|
189
|
+
}
|
|
190
|
+
async wait(timeoutMs) {
|
|
191
|
+
if (await this.connect())
|
|
192
|
+
return this.waitOnSocket(timeoutMs);
|
|
193
|
+
await this.longPoll(timeoutMs);
|
|
194
|
+
}
|
|
195
|
+
send(frame) {
|
|
196
|
+
const socket = this.socket;
|
|
197
|
+
if (!socket || socket.readyState !== WebSocket.OPEN)
|
|
198
|
+
return false;
|
|
199
|
+
if (socket.bufferedAmount > 1048576) {
|
|
200
|
+
this.close();
|
|
201
|
+
socket.terminate();
|
|
202
|
+
return false;
|
|
203
|
+
}
|
|
204
|
+
try {
|
|
205
|
+
socket.send(JSON.stringify(frame));
|
|
206
|
+
return true;
|
|
207
|
+
}
|
|
208
|
+
catch {
|
|
209
|
+
this.close();
|
|
210
|
+
return false;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
close() {
|
|
214
|
+
const socket = this.socket;
|
|
215
|
+
this.socket = undefined;
|
|
216
|
+
this.handlers.onDisconnect?.();
|
|
217
|
+
socket?.close();
|
|
218
|
+
this.waiter?.();
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
/** Resolves a service once before starting a process so rollouts cannot silently retarget an open session. */
|
|
222
|
+
export async function resolveLiveContainer(ctx, request) {
|
|
223
|
+
const name = /^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$/;
|
|
224
|
+
if (!name.test(request.project) ||
|
|
225
|
+
request.project === '..' ||
|
|
226
|
+
!request.service ||
|
|
227
|
+
!name.test(request.service))
|
|
228
|
+
throw new Error('A valid project and service are required');
|
|
229
|
+
const state = await loadRolloutState(path.join(ctx.stateDir, 'projects', request.project));
|
|
230
|
+
const rolling = state.services[request.service];
|
|
231
|
+
if (rolling && !rolling.active)
|
|
232
|
+
throw new Error('Rolling service has no active revision');
|
|
233
|
+
const project = rolling?.active?.project ?? request.project;
|
|
234
|
+
if (!name.test(project))
|
|
235
|
+
throw new Error('Invalid active rollout project');
|
|
236
|
+
const result = await ctx.docker([
|
|
237
|
+
'ps',
|
|
238
|
+
'--all',
|
|
239
|
+
'--no-trunc',
|
|
240
|
+
'--quiet',
|
|
241
|
+
'--filter',
|
|
242
|
+
'label=com.docker.compose.project=' + project,
|
|
243
|
+
'--filter',
|
|
244
|
+
'label=com.docker.compose.service=' + request.service,
|
|
245
|
+
'--filter',
|
|
246
|
+
'label=com.docker.compose.oneoff=False',
|
|
247
|
+
]);
|
|
248
|
+
const ids = result.out.trim().split(/\s+/).filter(Boolean);
|
|
249
|
+
if (ids.length !== 1)
|
|
250
|
+
throw new Error(ids.length
|
|
251
|
+
? 'Service has multiple containers; select an unambiguous service'
|
|
252
|
+
: 'Service has no container on this node');
|
|
253
|
+
if (!/^[a-f0-9]{12,64}$/.test(ids[0]))
|
|
254
|
+
throw new Error('Docker returned an invalid container ID');
|
|
255
|
+
return ids[0];
|
|
256
|
+
}
|
|
257
|
+
export class AgentLiveOperations {
|
|
258
|
+
ctx;
|
|
259
|
+
emit;
|
|
260
|
+
engine;
|
|
261
|
+
sessions = new Map();
|
|
262
|
+
constructor(ctx, emit, options = {}) {
|
|
263
|
+
this.ctx = ctx;
|
|
264
|
+
this.emit = emit;
|
|
265
|
+
this.engine = new LiveOperationEngine({
|
|
266
|
+
...options,
|
|
267
|
+
emit: (event) => {
|
|
268
|
+
if (!this.sessions.has(event.sessionId))
|
|
269
|
+
return;
|
|
270
|
+
if (event.type === 'exit')
|
|
271
|
+
this.sessions.delete(event.sessionId);
|
|
272
|
+
if (event.type === 'output')
|
|
273
|
+
this.emit({
|
|
274
|
+
v: 1,
|
|
275
|
+
type: 'live.output',
|
|
276
|
+
sessionId: event.sessionId,
|
|
277
|
+
stream: event.stream,
|
|
278
|
+
data: event.data,
|
|
279
|
+
});
|
|
280
|
+
else if (event.type === 'result')
|
|
281
|
+
this.emit({
|
|
282
|
+
v: 1,
|
|
283
|
+
type: 'live.result',
|
|
284
|
+
sessionId: event.sessionId,
|
|
285
|
+
result: event.result,
|
|
286
|
+
});
|
|
287
|
+
else
|
|
288
|
+
this.emit({ ...event, v: 1, type: 'live.exit' });
|
|
289
|
+
},
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
async handle(frame) {
|
|
293
|
+
if (frame.v !== 1 ||
|
|
294
|
+
typeof frame.sessionId !== 'string' ||
|
|
295
|
+
!/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,255}$/.test(frame.sessionId))
|
|
296
|
+
return;
|
|
297
|
+
if (frame.type === 'live.open') {
|
|
298
|
+
// An existing open belongs to its original request; replaying it must not start another process.
|
|
299
|
+
if (this.sessions.has(frame.sessionId))
|
|
300
|
+
return;
|
|
301
|
+
const entry = { ready: Promise.resolve(), queuedBytes: 0 };
|
|
302
|
+
this.sessions.set(frame.sessionId, entry);
|
|
303
|
+
entry.ready = this.open({ ...frame.request, sessionId: frame.sessionId }, entry);
|
|
304
|
+
await entry.ready;
|
|
305
|
+
return;
|
|
306
|
+
}
|
|
307
|
+
const entry = this.sessions.get(frame.sessionId);
|
|
308
|
+
if (!entry)
|
|
309
|
+
return;
|
|
310
|
+
if (frame.type === 'live.cancel') {
|
|
311
|
+
this.engine.cancel(frame.sessionId);
|
|
312
|
+
if (this.sessions.delete(frame.sessionId))
|
|
313
|
+
this.emit({
|
|
314
|
+
v: 1,
|
|
315
|
+
type: 'live.exit',
|
|
316
|
+
sessionId: frame.sessionId,
|
|
317
|
+
code: null,
|
|
318
|
+
signal: null,
|
|
319
|
+
cancelled: true,
|
|
320
|
+
});
|
|
321
|
+
return;
|
|
322
|
+
}
|
|
323
|
+
let queued = 0;
|
|
324
|
+
try {
|
|
325
|
+
let input;
|
|
326
|
+
if (frame.type === 'live.stdin') {
|
|
327
|
+
if (typeof frame.data !== 'string' || frame.data.length > 87384)
|
|
328
|
+
throw new Error('Invalid stdin frame');
|
|
329
|
+
input = Buffer.from(frame.data, 'base64');
|
|
330
|
+
if (input.toString('base64') !== frame.data || input.length > 65536)
|
|
331
|
+
throw new Error('Invalid base64 stdin');
|
|
332
|
+
queued = input.length;
|
|
333
|
+
entry.queuedBytes += queued;
|
|
334
|
+
if (entry.queuedBytes > 262144)
|
|
335
|
+
throw new Error('Live stdin queue exceeds size limit');
|
|
336
|
+
}
|
|
337
|
+
await entry.ready;
|
|
338
|
+
if (this.sessions.get(frame.sessionId) !== entry)
|
|
339
|
+
return;
|
|
340
|
+
if (frame.type === 'live.stdin' && input)
|
|
341
|
+
this.engine.stdin(frame.sessionId, input);
|
|
342
|
+
else if (frame.type === 'live.stdin-end')
|
|
343
|
+
this.engine.endStdin(frame.sessionId);
|
|
344
|
+
else if (frame.type === 'live.resize')
|
|
345
|
+
this.engine.resize(frame.sessionId, frame.columns, frame.rows);
|
|
346
|
+
}
|
|
347
|
+
catch (error) {
|
|
348
|
+
this.fail(frame.sessionId, entry, error);
|
|
349
|
+
}
|
|
350
|
+
finally {
|
|
351
|
+
entry.queuedBytes -= queued;
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
dispose() {
|
|
355
|
+
const ids = [...this.sessions.keys()];
|
|
356
|
+
this.engine.dispose();
|
|
357
|
+
for (const id of ids)
|
|
358
|
+
if (this.sessions.delete(id))
|
|
359
|
+
this.emit({
|
|
360
|
+
v: 1,
|
|
361
|
+
type: 'live.exit',
|
|
362
|
+
sessionId: id,
|
|
363
|
+
code: null,
|
|
364
|
+
signal: null,
|
|
365
|
+
cancelled: true,
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
fail(id, entry, error) {
|
|
369
|
+
if (this.sessions.get(id) !== entry)
|
|
370
|
+
return;
|
|
371
|
+
this.sessions.delete(id);
|
|
372
|
+
this.emit({
|
|
373
|
+
v: 1,
|
|
374
|
+
type: 'live.exit',
|
|
375
|
+
sessionId: id,
|
|
376
|
+
code: null,
|
|
377
|
+
signal: null,
|
|
378
|
+
cancelled: false,
|
|
379
|
+
error: errMessage(error),
|
|
380
|
+
});
|
|
381
|
+
this.engine.cancel(id);
|
|
382
|
+
}
|
|
383
|
+
async open(request, entry) {
|
|
384
|
+
try {
|
|
385
|
+
if (this.sessions.size > 16)
|
|
386
|
+
throw new Error('Live session limit reached');
|
|
387
|
+
if (!this.ctx.apply)
|
|
388
|
+
throw new Error('Live operations require an applying agent');
|
|
389
|
+
if (request.node && request.node !== this.ctx.nodeName)
|
|
390
|
+
throw new Error('Live request selected a different node');
|
|
391
|
+
if (typeof request.project !== 'string' ||
|
|
392
|
+
!/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$/.test(request.project))
|
|
393
|
+
throw new Error('Invalid project');
|
|
394
|
+
if (request.operation === 'ps') {
|
|
395
|
+
const rollout = await loadRolloutState(path.join(this.ctx.stateDir, 'projects', request.project));
|
|
396
|
+
if (this.sessions.get(request.sessionId) !== entry)
|
|
397
|
+
return;
|
|
398
|
+
const projects = [
|
|
399
|
+
...new Set([
|
|
400
|
+
request.project,
|
|
401
|
+
...Object.values(rollout.services).flatMap((service) => service.active ? [service.active.project] : []),
|
|
402
|
+
]),
|
|
403
|
+
];
|
|
404
|
+
this.engine.start({ operation: 'ps', sessionId: request.sessionId, projects });
|
|
405
|
+
}
|
|
406
|
+
else {
|
|
407
|
+
if (!['exec', 'logs', 'inspect'].includes(request.operation))
|
|
408
|
+
throw new Error('Invalid live operation');
|
|
409
|
+
let timer;
|
|
410
|
+
let container;
|
|
411
|
+
try {
|
|
412
|
+
container = await Promise.race([
|
|
413
|
+
resolveLiveContainer(this.ctx, request),
|
|
414
|
+
new Promise((_resolve, reject) => {
|
|
415
|
+
timer = setTimeout(() => reject(new Error('Live container resolution timed out')), LIVE_START_TIMEOUT_MS);
|
|
416
|
+
timer.unref();
|
|
417
|
+
}),
|
|
418
|
+
]);
|
|
419
|
+
}
|
|
420
|
+
finally {
|
|
421
|
+
clearTimeout(timer);
|
|
422
|
+
}
|
|
423
|
+
if (this.sessions.get(request.sessionId) !== entry)
|
|
424
|
+
return;
|
|
425
|
+
if (request.operation === 'exec')
|
|
426
|
+
this.engine.start({
|
|
427
|
+
operation: 'exec',
|
|
428
|
+
sessionId: request.sessionId,
|
|
429
|
+
container,
|
|
430
|
+
command: request.command,
|
|
431
|
+
interactive: request.interactive,
|
|
432
|
+
tty: request.tty,
|
|
433
|
+
columns: request.columns,
|
|
434
|
+
rows: request.rows,
|
|
435
|
+
});
|
|
436
|
+
else if (request.operation === 'logs')
|
|
437
|
+
this.engine.start({
|
|
438
|
+
operation: 'logs',
|
|
439
|
+
sessionId: request.sessionId,
|
|
440
|
+
container,
|
|
441
|
+
tail: request.tail,
|
|
442
|
+
});
|
|
443
|
+
else
|
|
444
|
+
this.engine.start({ operation: 'inspect', sessionId: request.sessionId, container });
|
|
445
|
+
}
|
|
446
|
+
if (this.sessions.get(request.sessionId) === entry)
|
|
447
|
+
this.emit({ v: 1, type: 'live.started', sessionId: request.sessionId });
|
|
448
|
+
}
|
|
449
|
+
catch (error) {
|
|
450
|
+
this.fail(request.sessionId, entry, error);
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
}
|
|
47
454
|
async function writeEnv(file, secrets) {
|
|
48
|
-
const body = Object.entries(secrets)
|
|
455
|
+
const body = Object.entries(secrets)
|
|
456
|
+
.map(([k, v]) => `${k}=${String(v).replace(/\n/g, '\\n')}`)
|
|
457
|
+
.join('\n') + '\n';
|
|
49
458
|
await fs.writeFile(file, body, { mode: 0o600 });
|
|
50
459
|
}
|
|
51
460
|
/** `docker compose` prefix for a project; rolling revision projects pass their own Compose file. */
|
|
52
461
|
const composeCmd = (project, cwd, composeFile) => composeFile ? composeBase(project, cwd, composeFile) : ['compose', '-p', project];
|
|
53
462
|
export async function containerIp(ctx, project, service, cwd, composeFile) {
|
|
54
|
-
const ids = (await ctx.docker([...composeCmd(project, cwd, composeFile), 'ps', '-q', service], { cwd })).out
|
|
463
|
+
const ids = (await ctx.docker([...composeCmd(project, cwd, composeFile), 'ps', '-q', service], { cwd })).out
|
|
464
|
+
.trim()
|
|
465
|
+
.split(/\s+/)
|
|
466
|
+
.filter(Boolean);
|
|
55
467
|
if (!ids.length)
|
|
56
468
|
throw new Error(`${service}: no container found`);
|
|
57
469
|
const inspected = JSON.parse((await ctx.docker(['inspect', ids[0]], { cwd })).out)[0];
|
|
@@ -60,8 +472,14 @@ export async function containerIp(ctx, project, service, cwd, composeFile) {
|
|
|
60
472
|
}
|
|
61
473
|
function tcpCheck(host, port, timeoutMs = 2000) {
|
|
62
474
|
return new Promise((resolve, reject) => {
|
|
63
|
-
const s = net.createConnection({ host, port: Number(port), timeout: timeoutMs }, () => {
|
|
64
|
-
|
|
475
|
+
const s = net.createConnection({ host, port: Number(port), timeout: timeoutMs }, () => {
|
|
476
|
+
s.destroy();
|
|
477
|
+
resolve(true);
|
|
478
|
+
});
|
|
479
|
+
s.on('timeout', () => {
|
|
480
|
+
s.destroy();
|
|
481
|
+
reject(new Error(`tcp timeout ${host}:${port}`));
|
|
482
|
+
});
|
|
65
483
|
s.on('error', reject);
|
|
66
484
|
});
|
|
67
485
|
}
|
|
@@ -74,7 +492,9 @@ export async function waitForHealth(ctx, project, service, health, cwd, composeF
|
|
|
74
492
|
for (let i = 0; i < retries; i++) {
|
|
75
493
|
try {
|
|
76
494
|
if (health.type === 'docker') {
|
|
77
|
-
const id = (await ctx.docker([...composeCmd(project, cwd, composeFile), 'ps', '-q', service], { cwd })).out
|
|
495
|
+
const id = (await ctx.docker([...composeCmd(project, cwd, composeFile), 'ps', '-q', service], { cwd })).out
|
|
496
|
+
.trim()
|
|
497
|
+
.split(/\s+/)[0];
|
|
78
498
|
const state = JSON.parse((await ctx.docker(['inspect', id], { cwd })).out)[0]?.State;
|
|
79
499
|
if (!state?.Running)
|
|
80
500
|
throw new Error(`${service}: not running`);
|
|
@@ -86,7 +506,9 @@ export async function waitForHealth(ctx, project, service, health, cwd, composeF
|
|
|
86
506
|
}
|
|
87
507
|
else if (health.type === 'http') {
|
|
88
508
|
const ip = await containerIp(ctx, project, service, cwd, composeFile);
|
|
89
|
-
const res = await ctx.http(`http://${ip}:${health.port || 80}${health.path || '/'}`, {
|
|
509
|
+
const res = await ctx.http(`http://${ip}:${health.port || 80}${health.path || '/'}`, {
|
|
510
|
+
signal: AbortSignal.timeout(parseDuration(health.timeout || '2s')),
|
|
511
|
+
});
|
|
90
512
|
if (!res.ok)
|
|
91
513
|
throw new Error(`${service}: http ${res.status}`);
|
|
92
514
|
}
|
|
@@ -94,7 +516,7 @@ export async function waitForHealth(ctx, project, service, health, cwd, composeF
|
|
|
94
516
|
}
|
|
95
517
|
catch (e) {
|
|
96
518
|
last = e;
|
|
97
|
-
await new Promise(r => setTimeout(r, intervalMs));
|
|
519
|
+
await new Promise((r) => setTimeout(r, intervalMs));
|
|
98
520
|
}
|
|
99
521
|
}
|
|
100
522
|
throw new Error(`${service}: ${describeHealthCheck(health)} failed after ${retries} attempt${retries === 1 ? '' : 's'}: ${describeError(last)}`);
|
|
@@ -113,18 +535,74 @@ function describeHealthCheck(health) {
|
|
|
113
535
|
return 'docker health check';
|
|
114
536
|
}
|
|
115
537
|
export { parseDuration };
|
|
538
|
+
/**
|
|
539
|
+
* Containers of Tug projects (`projects`, plus their rolling revision projects `<project>-…`) that
|
|
540
|
+
* listen on all interfaces or on a public IP, from `docker ps --format '{{.Label "com.docker.compose.project"}}\t{{.Names}}\t{{.Ports}}'`.
|
|
541
|
+
*/
|
|
542
|
+
export function publicListeners(psOutput, projects) {
|
|
543
|
+
const out = [];
|
|
544
|
+
for (const line of psOutput.split('\n')) {
|
|
545
|
+
const [project, container, ports = ''] = line.split('\t');
|
|
546
|
+
if (!project || !projects.some((p) => project === p || project.startsWith(`${p}-`)))
|
|
547
|
+
continue;
|
|
548
|
+
for (const m of ports.matchAll(/(\[[0-9a-fA-F:.]+\]|[0-9a-fA-F:.]+):(\d+(?:-\d+)?)->(\S+?)(?:,|$)/g)) {
|
|
549
|
+
if (isPublicAddress(m[1]))
|
|
550
|
+
out.push({ project, container, listen: `${m[1]}:${m[2]}->${m[3]}` });
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
return out;
|
|
554
|
+
}
|
|
555
|
+
/** `ports.public` diagnostic: a warning, not a failure, since an operator may publish on purpose with an explicit host IP. */
|
|
556
|
+
async function publicPortsCheck(ctx) {
|
|
557
|
+
try {
|
|
558
|
+
const projects = await fs
|
|
559
|
+
.readdir(path.join(ctx.stateDir, 'projects'))
|
|
560
|
+
.catch(() => []);
|
|
561
|
+
const ps = await ctx.docker([
|
|
562
|
+
'ps',
|
|
563
|
+
'--filter',
|
|
564
|
+
'label=com.docker.compose.project',
|
|
565
|
+
'--format',
|
|
566
|
+
'{{.Label "com.docker.compose.project"}}\t{{.Names}}\t{{.Ports}}',
|
|
567
|
+
]);
|
|
568
|
+
const listeners = publicListeners(ps.out, projects);
|
|
569
|
+
const stdout = listeners.map((l) => `${l.project}\t${l.container}\t${l.listen}`).join('\n');
|
|
570
|
+
return listeners.length
|
|
571
|
+
? {
|
|
572
|
+
name: 'ports.public',
|
|
573
|
+
ok: true,
|
|
574
|
+
stdout: stdout.slice(-4000),
|
|
575
|
+
warning: `${listeners.length} published port${listeners.length === 1 ? ' of Tug projects listens' : 's of Tug projects listen'} on a public address; bind them to 127.0.0.1, the overlay IP or a private address unless that is intended`,
|
|
576
|
+
}
|
|
577
|
+
: {
|
|
578
|
+
name: 'ports.public',
|
|
579
|
+
ok: true,
|
|
580
|
+
stdout: 'no Tug project container listens on a public address',
|
|
581
|
+
};
|
|
582
|
+
}
|
|
583
|
+
catch (e) {
|
|
584
|
+
return { name: 'ports.public', ok: false, error: errMessage(e).slice(-4000) };
|
|
585
|
+
}
|
|
586
|
+
}
|
|
116
587
|
export async function executeCommand(command, ctx = defaultContext()) {
|
|
117
|
-
const payload = typeof command.payload_json === 'string'
|
|
588
|
+
const payload = typeof command.payload_json === 'string'
|
|
589
|
+
? JSON.parse(command.payload_json || '{}')
|
|
590
|
+
: command.payload || {};
|
|
118
591
|
if (command.type === 'diagnostics.run') {
|
|
119
592
|
const checks = [];
|
|
120
593
|
for (const [name, args] of [
|
|
121
594
|
['docker.info', ['info']],
|
|
122
595
|
['docker.version', ['version']],
|
|
123
|
-
['docker.compose.version', ['compose', 'version']]
|
|
596
|
+
['docker.compose.version', ['compose', 'version']],
|
|
124
597
|
]) {
|
|
125
598
|
try {
|
|
126
599
|
const result = await ctx.docker([...args]);
|
|
127
|
-
checks.push({
|
|
600
|
+
checks.push({
|
|
601
|
+
name,
|
|
602
|
+
ok: true,
|
|
603
|
+
stdout: result.out.slice(-4000),
|
|
604
|
+
stderr: result.err.slice(-4000),
|
|
605
|
+
});
|
|
128
606
|
}
|
|
129
607
|
catch (e) {
|
|
130
608
|
checks.push({ name, ok: false, error: errMessage(e).slice(-4000) });
|
|
@@ -136,25 +614,42 @@ export async function executeCommand(command, ctx = defaultContext()) {
|
|
|
136
614
|
extra.push(['wireguard.version', () => host('wg', ['--version'])], ['wireguard.interface', () => host('wg', ['show', WG_INTERFACE])]);
|
|
137
615
|
}
|
|
138
616
|
if (await fs.stat(path.join(ctx.stateDir, 'ingress')).then(() => true, () => false)) {
|
|
139
|
-
extra.push([
|
|
617
|
+
extra.push([
|
|
618
|
+
'ingress.caddy',
|
|
619
|
+
() => ctx.docker(['container', 'inspect', '-f', '{{.State.Status}}', CADDY_CONTAINER]),
|
|
620
|
+
]);
|
|
140
621
|
}
|
|
141
622
|
for (const [name, fn] of extra) {
|
|
142
623
|
try {
|
|
143
624
|
const result = await fn();
|
|
144
|
-
checks.push({
|
|
625
|
+
checks.push({
|
|
626
|
+
name,
|
|
627
|
+
ok: true,
|
|
628
|
+
stdout: result.out.slice(-4000),
|
|
629
|
+
stderr: result.err.slice(-4000),
|
|
630
|
+
});
|
|
145
631
|
}
|
|
146
632
|
catch (e) {
|
|
147
633
|
checks.push({ name, ok: false, error: errMessage(e).slice(-4000) });
|
|
148
634
|
}
|
|
149
635
|
}
|
|
150
|
-
|
|
636
|
+
checks.push(await publicPortsCheck(ctx));
|
|
637
|
+
return {
|
|
638
|
+
commandId: command.id,
|
|
639
|
+
state: checks.every((c) => c.ok) ? 'succeeded' : 'failed',
|
|
640
|
+
result: { checks, node: ctx.nodeName },
|
|
641
|
+
};
|
|
151
642
|
}
|
|
152
643
|
if (command.type === 'service.restart') {
|
|
153
644
|
const project = payload.project;
|
|
154
645
|
const service = payload.service;
|
|
155
646
|
const dir = path.join(ctx.stateDir, 'projects', project);
|
|
156
647
|
await ctx.docker(['compose', '-p', project, 'restart', service], { cwd: dir });
|
|
157
|
-
return {
|
|
648
|
+
return {
|
|
649
|
+
commandId: command.id,
|
|
650
|
+
state: 'succeeded',
|
|
651
|
+
result: { project, service, restarted: true },
|
|
652
|
+
};
|
|
158
653
|
}
|
|
159
654
|
if (command.type === 'logs.fetch') {
|
|
160
655
|
const project = payload.project;
|
|
@@ -162,7 +657,11 @@ export async function executeCommand(command, ctx = defaultContext()) {
|
|
|
162
657
|
const tail = String(payload.tail || 200);
|
|
163
658
|
const dir = path.join(ctx.stateDir, 'projects', project);
|
|
164
659
|
const result = await ctx.docker(['compose', '-p', project, 'logs', '--no-color', '--tail', tail, service], { cwd: dir });
|
|
165
|
-
return {
|
|
660
|
+
return {
|
|
661
|
+
commandId: command.id,
|
|
662
|
+
state: 'succeeded',
|
|
663
|
+
result: { stdout: result.out, stderr: result.err },
|
|
664
|
+
};
|
|
166
665
|
}
|
|
167
666
|
if (command.type === 'volume.delete') {
|
|
168
667
|
const project = payload.project;
|
|
@@ -175,23 +674,40 @@ export async function executeCommand(command, ctx = defaultContext()) {
|
|
|
175
674
|
if (!errMessage(e).includes('No such volume'))
|
|
176
675
|
throw e;
|
|
177
676
|
}
|
|
178
|
-
return {
|
|
677
|
+
return {
|
|
678
|
+
commandId: command.id,
|
|
679
|
+
state: 'succeeded',
|
|
680
|
+
result: { project, volume, deleted: true },
|
|
681
|
+
};
|
|
179
682
|
}
|
|
180
683
|
if (command.type === 'wg.rotate-key') {
|
|
181
684
|
if (!ctx.mesh)
|
|
182
685
|
throw new Error('the WireGuard mesh is not enabled on this agent (TUG_MESH=true)');
|
|
183
686
|
const { publicKey } = await rotateWgKeys(ctx.stateDir);
|
|
184
|
-
await api(ctx, '/agent/v1/wireguard/key', {
|
|
687
|
+
await api(ctx, '/agent/v1/wireguard/key', {
|
|
688
|
+
method: 'POST',
|
|
689
|
+
headers: { 'content-type': 'application/json' },
|
|
690
|
+
body: JSON.stringify({ publicKey }),
|
|
691
|
+
});
|
|
185
692
|
await writeReportedKey(ctx.stateDir, publicKey);
|
|
186
693
|
return { commandId: command.id, state: 'succeeded', result: { publicKey } };
|
|
187
694
|
}
|
|
188
|
-
return {
|
|
695
|
+
return {
|
|
696
|
+
commandId: command.id,
|
|
697
|
+
state: 'failed',
|
|
698
|
+
result: { error: `unsupported command ${command.type}` },
|
|
699
|
+
};
|
|
700
|
+
}
|
|
701
|
+
async function managedImageFile(ctx) {
|
|
702
|
+
await ensureDir(ctx.stateDir);
|
|
703
|
+
return path.join(ctx.stateDir, 'managed-images.json');
|
|
189
704
|
}
|
|
190
|
-
async function managedImageFile(ctx) { await ensureDir(ctx.stateDir); return path.join(ctx.stateDir, 'managed-images.json'); }
|
|
191
705
|
async function managedImages(ctx) {
|
|
192
706
|
try {
|
|
193
707
|
const refs = JSON.parse(await fs.readFile(await managedImageFile(ctx), 'utf8'));
|
|
194
|
-
return Array.isArray(refs)
|
|
708
|
+
return Array.isArray(refs)
|
|
709
|
+
? refs.filter((ref) => typeof ref === 'string' && ref.startsWith('tug-build/'))
|
|
710
|
+
: [];
|
|
195
711
|
}
|
|
196
712
|
catch {
|
|
197
713
|
return [];
|
|
@@ -205,7 +721,11 @@ export async function ensureArtifact(ctx, artifact) {
|
|
|
205
721
|
}
|
|
206
722
|
catch { }
|
|
207
723
|
const remember = async (daemonId) => {
|
|
208
|
-
identities[artifact.imageRef] = {
|
|
724
|
+
identities[artifact.imageRef] = {
|
|
725
|
+
daemonId,
|
|
726
|
+
artifactId: artifact.imageId,
|
|
727
|
+
sha256: artifact.sha256,
|
|
728
|
+
};
|
|
209
729
|
await fs.writeFile(identityFile, JSON.stringify(identities), { mode: 0o600 });
|
|
210
730
|
const refs = await managedImages(ctx);
|
|
211
731
|
if (!refs.includes(artifact.imageRef))
|
|
@@ -214,14 +734,19 @@ export async function ensureArtifact(ctx, artifact) {
|
|
|
214
734
|
const inspect = async () => JSON.parse((await ctx.docker(['image', 'inspect', artifact.imageRef])).out)[0];
|
|
215
735
|
try {
|
|
216
736
|
const image = await inspect();
|
|
217
|
-
if ((image?.Id === artifact.imageId ||
|
|
737
|
+
if ((image?.Id === artifact.imageId ||
|
|
738
|
+
(identities[artifact.imageRef]?.daemonId === image?.Id &&
|
|
739
|
+
identities[artifact.imageRef]?.artifactId === artifact.imageId &&
|
|
740
|
+
identities[artifact.imageRef]?.sha256 === artifact.sha256)) &&
|
|
741
|
+
`${image.Os}/${image.Architecture}` === artifact.platform) {
|
|
218
742
|
await remember(image.Id);
|
|
219
743
|
return;
|
|
220
744
|
}
|
|
221
745
|
}
|
|
222
746
|
catch { }
|
|
223
747
|
const host = JSON.parse((await ctx.docker(['info', '--format', '{{json .}}'])).out);
|
|
224
|
-
const arch = { x86_64: 'amd64', aarch64: 'arm64' }[host.Architecture] ||
|
|
748
|
+
const arch = { x86_64: 'amd64', aarch64: 'arm64' }[host.Architecture] ||
|
|
749
|
+
host.Architecture;
|
|
225
750
|
if (host.OSType && host.Architecture && `${host.OSType}/${arch}` !== artifact.platform)
|
|
226
751
|
throw new Error(`image platform ${artifact.platform} does not match node ${host.OSType}/${arch}; build for the target platform`);
|
|
227
752
|
const downloadDir = await fs.mkdtemp(path.join(ctx.stateDir, 'image-download-'));
|
|
@@ -268,7 +793,26 @@ export async function pruneManagedImages(ctx, retained) {
|
|
|
268
793
|
}
|
|
269
794
|
await fs.writeFile(await managedImageFile(ctx), JSON.stringify(remaining), { mode: 0o600 });
|
|
270
795
|
}
|
|
271
|
-
|
|
796
|
+
/**
|
|
797
|
+
* The IP that published ports without a host IP listen on: `127.0.0.1` when the node has no
|
|
798
|
+
* address, the address itself when it is an IP, otherwise what it resolves to on this node.
|
|
799
|
+
*/
|
|
800
|
+
export async function resolveBindAddress(address, lookup = (host) => dns.lookup(host)) {
|
|
801
|
+
if (!address)
|
|
802
|
+
return '127.0.0.1';
|
|
803
|
+
const bare = address.replace(/^\[|\]$/g, '');
|
|
804
|
+
if (net.isIP(bare))
|
|
805
|
+
return bare;
|
|
806
|
+
try {
|
|
807
|
+
return (await lookup(bare)).address;
|
|
808
|
+
}
|
|
809
|
+
catch (e) {
|
|
810
|
+
throw new Error(`cannot resolve node address ${address} to bind published ports: ${errMessage(e)}`, { cause: e });
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
// Last public-port warning per project, so the event is sent when it changes rather than on every poll.
|
|
814
|
+
const publicPortWarnings = new Map();
|
|
815
|
+
export async function reconcile(desiredState, ctx = defaultContext(), endpoints = {}, ports = { address: null, loopback: {} }) {
|
|
272
816
|
const dir = path.join(ctx.stateDir, 'projects', desiredState.project);
|
|
273
817
|
await ensureDir(dir);
|
|
274
818
|
const plannedCompose = composeForNode(desiredState, ctx.nodeName, {});
|
|
@@ -276,63 +820,169 @@ export async function reconcile(desiredState, ctx = defaultContext(), endpoints
|
|
|
276
820
|
if (!serviceNames.length) {
|
|
277
821
|
if (ctx.apply)
|
|
278
822
|
await retireAll(ctx, dir, desiredState).catch(() => { });
|
|
279
|
-
return {
|
|
823
|
+
return {
|
|
824
|
+
project: desiredState.project,
|
|
825
|
+
observedVersion: desiredState.version,
|
|
826
|
+
...(desiredState.revision ? { observedRevision: desiredState.revision } : {}),
|
|
827
|
+
health: 'skipped',
|
|
828
|
+
services: [],
|
|
829
|
+
serviceStates: {},
|
|
830
|
+
};
|
|
280
831
|
}
|
|
281
|
-
const serviceStates = Object.fromEntries(serviceNames.map(service => [
|
|
282
|
-
|
|
832
|
+
const serviceStates = Object.fromEntries(serviceNames.map((service) => [
|
|
833
|
+
service,
|
|
834
|
+
{ rolloutState: 'pending', image: desiredState.services[service]?.image },
|
|
835
|
+
]));
|
|
836
|
+
const emit = (service, rolloutState, message = rolloutState, level = 'info') => api(ctx, '/agent/v1/events', {
|
|
837
|
+
method: 'POST',
|
|
838
|
+
headers: { 'content-type': 'application/json' },
|
|
839
|
+
body: JSON.stringify({
|
|
840
|
+
projectId: desiredState.project,
|
|
841
|
+
level,
|
|
842
|
+
type: `service.${rolloutState}`,
|
|
843
|
+
message,
|
|
844
|
+
metadata: { service, version: desiredState.version, node: ctx.nodeName },
|
|
845
|
+
}),
|
|
846
|
+
}).catch(() => { });
|
|
283
847
|
const sec = await api(ctx, `/agent/v1/secrets?project=${encodeURIComponent(desiredState.project)}&services=${encodeURIComponent(serviceNames.join(','))}`);
|
|
284
848
|
const currentSecretVersions = serviceSecretVersions(desiredState.compose, serviceNames, sec.versions || {});
|
|
285
849
|
const db = await agentDb(ctx.stateDir);
|
|
286
850
|
const previousRow = await db.get('SELECT version, revision, secret_versions_json FROM applied_state WHERE project = ?', desiredState.project);
|
|
287
|
-
const previous = previousRow
|
|
288
|
-
|
|
851
|
+
const previous = previousRow
|
|
852
|
+
? {
|
|
853
|
+
version: previousRow.version,
|
|
854
|
+
revision: previousRow.revision,
|
|
855
|
+
secretVersions: JSON.parse(previousRow.secret_versions_json),
|
|
856
|
+
}
|
|
857
|
+
: {};
|
|
858
|
+
const rolling = serviceNames.filter((service) => isRolling(desiredState.compose.services[service]));
|
|
289
859
|
const rollingSet = new Set(rolling);
|
|
290
|
-
const mainServices = serviceNames.filter(service => !rollingSet.has(service));
|
|
291
|
-
const secretChangedServices = mainServices.filter(service => JSON.stringify(previous.secretVersions?.[service] || {}) !==
|
|
292
|
-
|
|
860
|
+
const mainServices = serviceNames.filter((service) => !rollingSet.has(service));
|
|
861
|
+
const secretChangedServices = mainServices.filter((service) => JSON.stringify(previous.secretVersions?.[service] || {}) !==
|
|
862
|
+
JSON.stringify(currentSecretVersions[service] || {}));
|
|
863
|
+
const sameDesiredVersion = desiredState.revision
|
|
864
|
+
? previous.revision === desiredState.revision
|
|
865
|
+
: previous.version === desiredState.version;
|
|
293
866
|
// Rolling services run in their own revision projects; the main project holds only the recreated ones.
|
|
294
|
-
const
|
|
867
|
+
const loopback = new Set(ports.loopback[desiredState.project] || []);
|
|
868
|
+
// Resolve only when some port actually binds to the node address; a hostname nobody needs must not fail the apply.
|
|
869
|
+
const needsAddress = addressBoundServices(desiredState.compose, serviceNames, loopback).length > 0;
|
|
870
|
+
const bind = {
|
|
871
|
+
address: needsAddress ? await resolveBindAddress(ports.address) : '127.0.0.1',
|
|
872
|
+
loopback,
|
|
873
|
+
};
|
|
874
|
+
const nodeCompose = composeForNode(desiredState, ctx.nodeName, {
|
|
875
|
+
secrets: sec.secrets || {},
|
|
876
|
+
endpoints,
|
|
877
|
+
exclude: rollingSet,
|
|
878
|
+
bind,
|
|
879
|
+
});
|
|
880
|
+
await warnPublicPorts(ctx, desiredState, serviceNames, bind);
|
|
295
881
|
await fs.writeFile(path.join(dir, 'docker-compose.yml'), dumpYaml(nodeCompose));
|
|
296
882
|
await writeEnv(path.join(dir, '.env'), sec.secrets || {});
|
|
297
|
-
const failedStatus = (error) => ({
|
|
883
|
+
const failedStatus = (error) => ({
|
|
884
|
+
project: desiredState.project,
|
|
885
|
+
observedVersion: desiredState.version,
|
|
886
|
+
...(desiredState.revision ? { observedRevision: desiredState.revision } : {}),
|
|
887
|
+
health: 'failed',
|
|
888
|
+
services: serviceNames,
|
|
889
|
+
serviceStates,
|
|
890
|
+
error,
|
|
891
|
+
});
|
|
892
|
+
// Lifecycle events say what changed, not that the agent is still polling. They are sent when the desired revision,
|
|
893
|
+
// this node's rendered configuration or the secrets change, when an apply fails (once) and when it recovers (once).
|
|
894
|
+
const eventFingerprint = createHash('sha256')
|
|
895
|
+
.update(JSON.stringify([
|
|
896
|
+
desiredState.revision || desiredState.version,
|
|
897
|
+
dumpYaml(nodeCompose),
|
|
898
|
+
mainServices.map((service) => currentSecretVersions[service] || {}),
|
|
899
|
+
]))
|
|
900
|
+
.digest('hex')
|
|
901
|
+
.slice(0, 16);
|
|
902
|
+
const knownEvents = mainServices.length
|
|
903
|
+
? await db.get('SELECT fingerprint, phase FROM event_state WHERE project = ?', desiredState.project)
|
|
904
|
+
: undefined;
|
|
905
|
+
const announce = knownEvents?.fingerprint !== eventFingerprint;
|
|
906
|
+
const wasFailed = !announce && knownEvents?.phase === 'failed';
|
|
907
|
+
const markEvents = (phase) => db.run('INSERT INTO event_state(project, fingerprint, phase) VALUES(?,?,?) ON CONFLICT(project) DO UPDATE SET fingerprint=excluded.fingerprint, phase=excluded.phase', desiredState.project, eventFingerprint, phase);
|
|
908
|
+
let failing;
|
|
298
909
|
if (ctx.apply) {
|
|
299
910
|
try {
|
|
300
911
|
for (const service of mainServices) {
|
|
301
912
|
serviceStates[service].rolloutState = 'pulling';
|
|
302
|
-
|
|
913
|
+
if (announce)
|
|
914
|
+
await emit(service, 'pulling');
|
|
303
915
|
}
|
|
304
916
|
for (const service of mainServices) {
|
|
305
917
|
const artifact = desiredState.services[service].artifact;
|
|
306
918
|
if (artifact)
|
|
307
919
|
await ensureArtifact(ctx, artifact);
|
|
308
920
|
}
|
|
309
|
-
|
|
921
|
+
// Registry images are pulled only for a new desired revision. Pulling on every poll would let a tag that moved
|
|
922
|
+
// upstream (`mariadb:11`) recreate the container without a deploy; `up -d` still pulls images that are missing.
|
|
923
|
+
const pullServices = sameDesiredVersion
|
|
924
|
+
? []
|
|
925
|
+
: mainServices.filter((service) => !desiredState.services[service].artifact);
|
|
310
926
|
if (pullServices.length)
|
|
311
|
-
await ctx.docker([
|
|
927
|
+
await ctx.docker([
|
|
928
|
+
'compose',
|
|
929
|
+
'-p',
|
|
930
|
+
desiredState.project,
|
|
931
|
+
'pull',
|
|
932
|
+
...(pullServices.length === mainServices.length ? [] : pullServices),
|
|
933
|
+
], { cwd: dir });
|
|
312
934
|
for (const service of mainServices) {
|
|
313
935
|
serviceStates[service].rolloutState = 'starting';
|
|
314
|
-
|
|
936
|
+
if (announce)
|
|
937
|
+
await emit(service, 'starting');
|
|
315
938
|
}
|
|
316
939
|
if (mainServices.length) {
|
|
317
940
|
// Services that just became rolling still run here until their replacement takes over; keep them.
|
|
318
|
-
const keepLegacy = rolling.length > 0 && await hasLegacyContainers(ctx, desiredState.project, rolling);
|
|
319
|
-
await ctx.docker([
|
|
941
|
+
const keepLegacy = rolling.length > 0 && (await hasLegacyContainers(ctx, desiredState.project, rolling));
|
|
942
|
+
await ctx.docker([
|
|
943
|
+
'compose',
|
|
944
|
+
'-p',
|
|
945
|
+
desiredState.project,
|
|
946
|
+
'up',
|
|
947
|
+
'-d',
|
|
948
|
+
...(keepLegacy ? [] : ['--remove-orphans']),
|
|
949
|
+
], { cwd: dir });
|
|
320
950
|
}
|
|
321
951
|
if (sameDesiredVersion && secretChangedServices.length) {
|
|
322
952
|
for (const service of secretChangedServices)
|
|
323
953
|
await emit(service, 'secret_recreate', 'secret version changed; recreating service');
|
|
324
|
-
await ctx.docker([
|
|
954
|
+
await ctx.docker([
|
|
955
|
+
'compose',
|
|
956
|
+
'-p',
|
|
957
|
+
desiredState.project,
|
|
958
|
+
'up',
|
|
959
|
+
'-d',
|
|
960
|
+
'--no-deps',
|
|
961
|
+
'--force-recreate',
|
|
962
|
+
...secretChangedServices,
|
|
963
|
+
], { cwd: dir });
|
|
325
964
|
}
|
|
326
965
|
for (const service of mainServices) {
|
|
327
966
|
serviceStates[service].rolloutState = 'health_checking';
|
|
328
|
-
|
|
967
|
+
if (announce)
|
|
968
|
+
await emit(service, 'health_checking');
|
|
969
|
+
failing = service;
|
|
329
970
|
await waitForHealth(ctx, desiredState.project, service, desiredState.services[service]?.health, dir);
|
|
971
|
+
failing = undefined;
|
|
330
972
|
serviceStates[service].rolloutState = 'serving';
|
|
331
973
|
serviceStates[service].healthyAt = now();
|
|
332
|
-
|
|
974
|
+
if (announce || wasFailed)
|
|
975
|
+
await emit(service, 'serving', wasFailed ? 'serving again after a failure' : 'serving');
|
|
333
976
|
}
|
|
977
|
+
if (mainServices.length && (announce || wasFailed))
|
|
978
|
+
await markEvents('serving');
|
|
334
979
|
}
|
|
335
980
|
catch (e) {
|
|
981
|
+
if (mainServices.length && !wasFailed) {
|
|
982
|
+
// Announce a failure once; retries of the same failure stay quiet until it recovers or the inputs change.
|
|
983
|
+
await emit(failing ?? mainServices.join(', '), 'failed', errMessage(e), 'error');
|
|
984
|
+
await markEvents('failed');
|
|
985
|
+
}
|
|
336
986
|
// Rolling services are untouched by a failure here, so keep reporting their current revision as serving.
|
|
337
987
|
if (!rolling.length)
|
|
338
988
|
throw e;
|
|
@@ -341,14 +991,24 @@ export async function reconcile(desiredState, ctx = defaultContext(), endpoints
|
|
|
341
991
|
}
|
|
342
992
|
if (rolling.length) {
|
|
343
993
|
const rolled = await reconcileRolling(ctx, {
|
|
344
|
-
desired: desiredState,
|
|
345
|
-
|
|
994
|
+
desired: desiredState,
|
|
995
|
+
dir,
|
|
996
|
+
services: rolling,
|
|
997
|
+
secretVersions: currentSecretVersions,
|
|
998
|
+
full: composeForNode(desiredState, ctx.nodeName, {
|
|
999
|
+
secrets: sec.secrets || {},
|
|
1000
|
+
endpoints,
|
|
1001
|
+
bind,
|
|
1002
|
+
}),
|
|
346
1003
|
deps: {
|
|
347
1004
|
waitForHealth: (project, service, composeFile) => waitForHealth(ctx, project, service, desiredState.services[service]?.health, dir, composeFile),
|
|
348
|
-
ensureArtifact: async (service) => {
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
1005
|
+
ensureArtifact: async (service) => {
|
|
1006
|
+
const artifact = desiredState.services[service].artifact;
|
|
1007
|
+
if (artifact)
|
|
1008
|
+
await ensureArtifact(ctx, artifact);
|
|
1009
|
+
},
|
|
1010
|
+
emit,
|
|
1011
|
+
},
|
|
352
1012
|
});
|
|
353
1013
|
Object.assign(serviceStates, rolled.serviceStates);
|
|
354
1014
|
const errors = Object.entries(rolled.errors);
|
|
@@ -366,14 +1026,51 @@ export async function reconcile(desiredState, ctx = defaultContext(), endpoints
|
|
|
366
1026
|
VALUES(?,?,?,?,?,?)
|
|
367
1027
|
ON CONFLICT(project) DO UPDATE SET version=excluded.version, revision=excluded.revision, applied_at=excluded.applied_at,
|
|
368
1028
|
service_states_json=excluded.service_states_json, secret_versions_json=excluded.secret_versions_json`, desiredState.project, desiredState.version, desiredState.revision || null, now(), JSON.stringify(serviceStates), JSON.stringify(currentSecretVersions));
|
|
369
|
-
return {
|
|
1029
|
+
return {
|
|
1030
|
+
project: desiredState.project,
|
|
1031
|
+
observedVersion: desiredState.version,
|
|
1032
|
+
...(desiredState.revision ? { observedRevision: desiredState.revision } : {}),
|
|
1033
|
+
health: 'serving',
|
|
1034
|
+
services: serviceNames,
|
|
1035
|
+
serviceStates,
|
|
1036
|
+
};
|
|
1037
|
+
}
|
|
1038
|
+
/** Emits a `ports.public` warning when published ports of this project are bound to a public node address. */
|
|
1039
|
+
async function warnPublicPorts(ctx, desiredState, serviceNames, bind) {
|
|
1040
|
+
const key = `${ctx.stateDir}\0${desiredState.project}`;
|
|
1041
|
+
const exposed = isPublicAddress(bind.address)
|
|
1042
|
+
? addressBoundServices(desiredState.compose, serviceNames, bind.loopback)
|
|
1043
|
+
: [];
|
|
1044
|
+
const message = exposed.length
|
|
1045
|
+
? `published ports of ${exposed.join(', ')} listen on public address ${bind.address}; enable the mesh, give the node a private address, or write the host IP in ports to make the exposure explicit`
|
|
1046
|
+
: '';
|
|
1047
|
+
if ((publicPortWarnings.get(key) ?? '') === message)
|
|
1048
|
+
return;
|
|
1049
|
+
publicPortWarnings.set(key, message);
|
|
1050
|
+
if (!message)
|
|
1051
|
+
return;
|
|
1052
|
+
await api(ctx, '/agent/v1/events', {
|
|
1053
|
+
method: 'POST',
|
|
1054
|
+
headers: { 'content-type': 'application/json' },
|
|
1055
|
+
body: JSON.stringify({
|
|
1056
|
+
projectId: desiredState.project,
|
|
1057
|
+
level: 'warn',
|
|
1058
|
+
type: 'ports.public',
|
|
1059
|
+
message,
|
|
1060
|
+
metadata: { services: exposed, address: bind.address, node: ctx.nodeName },
|
|
1061
|
+
}),
|
|
1062
|
+
}).catch(() => { });
|
|
370
1063
|
}
|
|
371
1064
|
/** Reports this node's WireGuard public key when control has not seen the current one yet. */
|
|
372
1065
|
export async function reportWgKey(ctx) {
|
|
373
1066
|
const { publicKey } = await ensureWgKeys(ctx.stateDir);
|
|
374
|
-
if (await readReportedKey(ctx.stateDir) === publicKey)
|
|
1067
|
+
if ((await readReportedKey(ctx.stateDir)) === publicKey)
|
|
375
1068
|
return;
|
|
376
|
-
await api(ctx, '/agent/v1/wireguard/key', {
|
|
1069
|
+
await api(ctx, '/agent/v1/wireguard/key', {
|
|
1070
|
+
method: 'POST',
|
|
1071
|
+
headers: { 'content-type': 'application/json' },
|
|
1072
|
+
body: JSON.stringify({ publicKey }),
|
|
1073
|
+
});
|
|
377
1074
|
await writeReportedKey(ctx.stateDir, publicKey);
|
|
378
1075
|
}
|
|
379
1076
|
let lastNetworkError = '';
|
|
@@ -401,10 +1098,16 @@ export async function pollOnce(ctx = defaultContext()) {
|
|
|
401
1098
|
const projects = [];
|
|
402
1099
|
for (const desired of work.desiredStates || (work.desiredState ? [work.desiredState] : [])) {
|
|
403
1100
|
try {
|
|
404
|
-
projects.push(await reconcile(desired, ctx, work.serviceEndpoints?.[desired.project]));
|
|
1101
|
+
projects.push(await reconcile(desired, ctx, work.serviceEndpoints?.[desired.project], work.ports));
|
|
405
1102
|
}
|
|
406
1103
|
catch (e) {
|
|
407
|
-
projects.push({
|
|
1104
|
+
projects.push({
|
|
1105
|
+
project: desired.project,
|
|
1106
|
+
observedVersion: desired.version,
|
|
1107
|
+
...(desired.revision ? { observedRevision: desired.revision } : {}),
|
|
1108
|
+
health: 'failed',
|
|
1109
|
+
error: errMessage(e),
|
|
1110
|
+
});
|
|
408
1111
|
}
|
|
409
1112
|
}
|
|
410
1113
|
let ingress;
|
|
@@ -421,15 +1124,23 @@ export async function pollOnce(ctx = defaultContext()) {
|
|
|
421
1124
|
commandResults.push(await executeCommand(command, ctx));
|
|
422
1125
|
}
|
|
423
1126
|
catch (e) {
|
|
424
|
-
commandResults.push({
|
|
1127
|
+
commandResults.push({
|
|
1128
|
+
commandId: command.id,
|
|
1129
|
+
state: 'failed',
|
|
1130
|
+
result: { error: errMessage(e) },
|
|
1131
|
+
});
|
|
425
1132
|
}
|
|
426
1133
|
}
|
|
427
1134
|
const report = { node: ctx.nodeName, projects, commandResults };
|
|
428
1135
|
if (ingress)
|
|
429
1136
|
report.ingress = ingress;
|
|
430
|
-
if (work.retainedImages && projects.every(project => project.health !== 'failed'))
|
|
1137
|
+
if (work.retainedImages && projects.every((project) => project.health !== 'failed'))
|
|
431
1138
|
await pruneManagedImages(ctx, work.retainedImages);
|
|
432
|
-
await api(ctx, '/agent/v1/status', {
|
|
1139
|
+
await api(ctx, '/agent/v1/status', {
|
|
1140
|
+
method: 'POST',
|
|
1141
|
+
headers: { 'content-type': 'application/json' },
|
|
1142
|
+
body: JSON.stringify(report),
|
|
1143
|
+
});
|
|
433
1144
|
return projects;
|
|
434
1145
|
}
|
|
435
1146
|
const CREDS_FILE = () => path.join(STATE_DIR, 'agent.json');
|
|
@@ -461,11 +1172,13 @@ async function readJoinKey() {
|
|
|
461
1172
|
}
|
|
462
1173
|
function dockerLogin(registry, user, password) {
|
|
463
1174
|
return new Promise((resolve, reject) => {
|
|
464
|
-
const p = spawn('docker', ['login', registry, '-u', user, '--password-stdin'], {
|
|
1175
|
+
const p = spawn('docker', ['login', registry, '-u', user, '--password-stdin'], {
|
|
1176
|
+
stdio: ['pipe', 'ignore', 'pipe'],
|
|
1177
|
+
});
|
|
465
1178
|
let err = '';
|
|
466
|
-
p.stderr.on('data', d => err += d);
|
|
1179
|
+
p.stderr.on('data', (d) => (err += d));
|
|
467
1180
|
p.on('error', reject);
|
|
468
|
-
p.on('close', code => code === 0 ? resolve() : reject(new Error(err.trim() || `exit ${code}`)));
|
|
1181
|
+
p.on('close', (code) => code === 0 ? resolve() : reject(new Error(err.trim() || `exit ${code}`)));
|
|
469
1182
|
p.stdin.write(password);
|
|
470
1183
|
p.stdin.end();
|
|
471
1184
|
});
|
|
@@ -480,26 +1193,48 @@ async function enroll(joinKey) {
|
|
|
480
1193
|
}
|
|
481
1194
|
if (!decoded.enrollToken)
|
|
482
1195
|
throw new Error('join key is missing enrollToken');
|
|
483
|
-
const controlUrl =
|
|
1196
|
+
const controlUrl = !CONTROL_URL_FROM_ENV && decoded.controlUrl ? decoded.controlUrl : CONTROL_URL;
|
|
484
1197
|
// node's fetch reads NODE_EXTRA_CA_CERTS only at startup, so apply an embedded CA by
|
|
485
1198
|
// persisting it and re-execing ourselves once with the env var set.
|
|
486
1199
|
if (decoded.ca && !process.env.NODE_EXTRA_CA_CERTS) {
|
|
487
1200
|
await ensureDir(STATE_DIR);
|
|
488
1201
|
const caPath = path.join(STATE_DIR, 'control-ca.pem');
|
|
489
1202
|
await fs.writeFile(caPath, decoded.ca);
|
|
490
|
-
const r = spawnSync(process.execPath, process.argv.slice(1), {
|
|
1203
|
+
const r = spawnSync(process.execPath, process.argv.slice(1), {
|
|
1204
|
+
stdio: 'inherit',
|
|
1205
|
+
env: { ...process.env, NODE_EXTRA_CA_CERTS: caPath },
|
|
1206
|
+
});
|
|
491
1207
|
process.exit(r.status ?? 1);
|
|
492
1208
|
}
|
|
493
1209
|
console.log(`enrolling with ${controlUrl} as ${NODE_NAME}`);
|
|
494
1210
|
const wgPublicKey = MESH ? (await ensureWgKeys(STATE_DIR)).publicKey : undefined;
|
|
495
1211
|
const res = await fetch(`${controlUrl}/agent/v1/enroll`, {
|
|
496
|
-
method: 'POST',
|
|
497
|
-
|
|
1212
|
+
method: 'POST',
|
|
1213
|
+
headers: { 'content-type': 'application/json' },
|
|
1214
|
+
body: JSON.stringify({
|
|
1215
|
+
enrollToken: decoded.enrollToken,
|
|
1216
|
+
name: NODE_NAME,
|
|
1217
|
+
address: process.env.TUG_NODE_ADDRESS,
|
|
1218
|
+
publicAddress: process.env.TUG_NODE_PUBLIC_ADDRESS,
|
|
1219
|
+
wgPublicKey,
|
|
1220
|
+
labels: decoded.labels ||
|
|
1221
|
+
(process.env.TUG_LABELS
|
|
1222
|
+
? process.env.TUG_LABELS.split(',')
|
|
1223
|
+
.map((s) => s.trim())
|
|
1224
|
+
.filter(Boolean)
|
|
1225
|
+
: undefined),
|
|
1226
|
+
}),
|
|
498
1227
|
});
|
|
499
1228
|
if (!res.ok)
|
|
500
1229
|
throw new Error(`enroll failed: ${res.status} ${await res.text()}`);
|
|
501
|
-
const out = await res.json();
|
|
502
|
-
const creds = {
|
|
1230
|
+
const out = (await res.json());
|
|
1231
|
+
const creds = {
|
|
1232
|
+
agentToken: out.agentToken,
|
|
1233
|
+
nodeName: out.node?.name || NODE_NAME,
|
|
1234
|
+
controlUrl,
|
|
1235
|
+
registry: out.registry || null,
|
|
1236
|
+
registryToken: out.registryToken || null,
|
|
1237
|
+
};
|
|
503
1238
|
await saveCreds(creds);
|
|
504
1239
|
if (wgPublicKey)
|
|
505
1240
|
await writeReportedKey(STATE_DIR, wgPublicKey);
|
|
@@ -509,7 +1244,9 @@ async function enroll(joinKey) {
|
|
|
509
1244
|
// Establish durable credentials before polling: explicit env token (SSH/push install) wins,
|
|
510
1245
|
// then credentials persisted on the state volume, then a one-time enroll via the join key.
|
|
511
1246
|
export async function loadOrEnroll() {
|
|
512
|
-
let creds = TOKEN
|
|
1247
|
+
let creds = TOKEN
|
|
1248
|
+
? { agentToken: TOKEN, nodeName: NODE_NAME, controlUrl: CONTROL_URL }
|
|
1249
|
+
: await loadCreds();
|
|
513
1250
|
if (!creds) {
|
|
514
1251
|
const joinKey = await readJoinKey();
|
|
515
1252
|
if (!joinKey)
|
|
@@ -533,14 +1270,28 @@ export async function loadOrEnroll() {
|
|
|
533
1270
|
export async function main() {
|
|
534
1271
|
await loadOrEnroll();
|
|
535
1272
|
console.log(`tug-agent node=${NODE_NAME} control=${CONTROL_URL} apply=${APPLY}`);
|
|
1273
|
+
const ctx = defaultContext();
|
|
1274
|
+
const wake = new AgentWakeChannel(ctx, undefined, {
|
|
1275
|
+
onLive: (frame) => live.handle(frame),
|
|
1276
|
+
onDisconnect: () => live.dispose(),
|
|
1277
|
+
});
|
|
1278
|
+
const live = new AgentLiveOperations(ctx, (frame) => {
|
|
1279
|
+
wake.send(frame);
|
|
1280
|
+
});
|
|
536
1281
|
for (;;) {
|
|
537
1282
|
try {
|
|
538
|
-
await pollOnce();
|
|
1283
|
+
await pollOnce(ctx);
|
|
539
1284
|
}
|
|
540
1285
|
catch (e) {
|
|
541
1286
|
console.error(errMessage(e));
|
|
542
1287
|
}
|
|
543
|
-
|
|
1288
|
+
try {
|
|
1289
|
+
await wake.wait(INTERVAL);
|
|
1290
|
+
}
|
|
1291
|
+
catch (e) {
|
|
1292
|
+
console.error(`agent wake wait failed: ${errMessage(e)}`);
|
|
1293
|
+
await new Promise((r) => setTimeout(r, INTERVAL));
|
|
1294
|
+
}
|
|
544
1295
|
}
|
|
545
1296
|
}
|
|
546
1297
|
if (import.meta.url === `file://${process.argv[1]}`)
|