@wpelevator/tug 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -8
- package/dist/agent.d.ts +9 -12
- package/dist/agent.js +232 -302
- package/dist/build.d.ts +2 -3
- package/dist/build.js +1 -3
- package/dist/cli-build.d.ts +2 -2
- package/dist/cli-build.js +3 -4
- package/dist/cli.js +26 -41
- package/dist/compose.d.ts +7 -21
- package/dist/compose.js +120 -150
- package/dist/container-spec.d.ts +56 -0
- package/dist/container-spec.js +359 -0
- package/dist/control.d.ts +4 -2
- package/dist/control.js +350 -365
- package/dist/db-driver.d.ts +20 -0
- package/dist/db-driver.js +11 -0
- package/dist/db-node.d.ts +3 -0
- package/dist/db-node.js +60 -0
- package/dist/db.d.ts +9 -6
- package/dist/db.js +40 -26
- package/dist/docker-api.d.ts +232 -0
- package/dist/docker-api.js +428 -0
- package/dist/ingress.d.ts +6 -6
- package/dist/ingress.js +56 -42
- package/dist/live.d.ts +10 -7
- package/dist/live.js +130 -194
- package/dist/normalize.d.ts +20 -0
- package/dist/normalize.js +75 -0
- package/dist/protocol.d.ts +5 -5
- package/dist/reconciler.d.ts +89 -0
- package/dist/reconciler.js +362 -0
- package/dist/registry.d.ts +17 -0
- package/dist/registry.js +133 -0
- package/dist/runners.d.ts +1 -6
- package/dist/runners.js +1 -5
- package/dist/templates/control-compose.yml +0 -4
- package/dist/templates/install-docker.sh +12 -7
- package/dist/templates/node-install.sh +0 -1
- package/dist/tokens.d.ts +7 -7
- package/dist/tokens.js +24 -21
- package/package.json +1 -3
- package/dist/rollout.d.ts +0 -72
- package/dist/rollout.js +0 -385
package/dist/control.js
CHANGED
|
@@ -1,22 +1,23 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import express from 'express';
|
|
3
|
+
import { resolveImage, parseRegistryCredentials, imageLocation, REGISTRY_AUTH_SECRET, } from "./registry.js";
|
|
3
4
|
import multer from 'multer';
|
|
4
5
|
import fs from 'node:fs/promises';
|
|
5
6
|
import { createReadStream } from 'node:fs';
|
|
6
7
|
import { pipeline } from 'node:stream/promises';
|
|
7
8
|
import { saveImageArchive, inspectImage, validateImageArchive, hashArchive } from "./artifacts.js";
|
|
8
|
-
import sqlite3 from 'sqlite3';
|
|
9
|
-
import { open } from 'sqlite';
|
|
10
9
|
import { scanArchive } from "./archive.js";
|
|
11
10
|
import https from 'node:https';
|
|
12
11
|
import http from 'node:http';
|
|
13
12
|
import path from 'node:path';
|
|
14
13
|
import { WebSocketServer, WebSocket } from 'ws';
|
|
15
14
|
import * as tar from 'tar';
|
|
16
|
-
import { openDb } from "./db.js";
|
|
15
|
+
import { openDb, openSnapshotDb } from "./db.js";
|
|
16
|
+
import { isUniqueViolation } from "./db-driver.js";
|
|
17
17
|
import { encrypt, decrypt, loadOrCreateMasterKey } from "./secrets.js";
|
|
18
18
|
import { buildDesiredState, loopbackIngressServices, serviceSecretNames, validateCompose, } from "./compose.js";
|
|
19
|
-
import {
|
|
19
|
+
import { buildImage, unpackContext, run as runCommand } from "./build.js";
|
|
20
|
+
import { sentinelNames } from "./normalize.js";
|
|
20
21
|
import { ensureDir, errMessage, hashToken, id, now, token } from "./util.js";
|
|
21
22
|
import { DEFAULT_MESH_CIDR, DEFAULT_WG_PORT, WG_INTERFACE, WG_KEEPALIVE, allocateOverlayIp, isWgKey, parseCidr, wgEndpoint, } from "./mesh.js";
|
|
22
23
|
import { buildIngressWork, resolveServiceEndpoints } from "./ingress.js";
|
|
@@ -393,7 +394,8 @@ const ACTIVE_STATES_SQL = ACTIVE_DEPLOYMENT_STATES.map((s) => `'${s}'`).join(','
|
|
|
393
394
|
const PORT = Number(process.env.PORT || 8080);
|
|
394
395
|
export async function createApp(options = {}) {
|
|
395
396
|
const images = {
|
|
396
|
-
|
|
397
|
+
resolve: resolveImage,
|
|
398
|
+
build: buildImage,
|
|
397
399
|
save: saveImageArchive,
|
|
398
400
|
inspect: inspectImage,
|
|
399
401
|
validate: validateImageArchive,
|
|
@@ -410,9 +412,6 @@ export async function createApp(options = {}) {
|
|
|
410
412
|
});
|
|
411
413
|
const dbFile = path.join(DATA_DIR, 'tug.db');
|
|
412
414
|
const masterKeyFile = path.join(DATA_DIR, 'master.key');
|
|
413
|
-
const REGISTRY = process.env.TUG_REGISTRY?.trim() || '';
|
|
414
|
-
const REGISTRY_AUTH_PATH = process.env.TUG_REGISTRY_AUTH_PATH || null;
|
|
415
|
-
const REGISTRY_CONTAINER = process.env.TUG_REGISTRY_CONTAINER || null;
|
|
416
415
|
// Networking layers are opt-in; with all of them off the work response is unchanged.
|
|
417
416
|
const MESH = process.env.TUG_MESH === 'true';
|
|
418
417
|
const MESH_CIDR = process.env.TUG_MESH_CIDR || DEFAULT_MESH_CIDR;
|
|
@@ -421,56 +420,6 @@ export async function createApp(options = {}) {
|
|
|
421
420
|
const ACME_EMAIL = process.env.TUG_ACME_EMAIL || undefined;
|
|
422
421
|
if (MESH)
|
|
423
422
|
parseCidr(MESH_CIDR);
|
|
424
|
-
let htpasswdQueue = Promise.resolve();
|
|
425
|
-
function withHtpasswdLock(fn) {
|
|
426
|
-
const next = htpasswdQueue.catch(() => { }).then(fn);
|
|
427
|
-
htpasswdQueue = next.then(() => { }, () => { });
|
|
428
|
-
return next;
|
|
429
|
-
}
|
|
430
|
-
async function reloadRegistry() {
|
|
431
|
-
if (!REGISTRY_CONTAINER)
|
|
432
|
-
return;
|
|
433
|
-
await runCommand('docker', ['restart', REGISTRY_CONTAINER]).catch((e) => console.error(`registry reload failed: ${e.message}`));
|
|
434
|
-
}
|
|
435
|
-
async function upsertRegistryCredential(name, password) {
|
|
436
|
-
if (!REGISTRY || !REGISTRY_AUTH_PATH)
|
|
437
|
-
return;
|
|
438
|
-
await withHtpasswdLock(async () => {
|
|
439
|
-
const { out } = await runCommand('htpasswd', ['-nbB', name, password]);
|
|
440
|
-
const line = out.trim();
|
|
441
|
-
let existing = '';
|
|
442
|
-
try {
|
|
443
|
-
existing = await fs.readFile(REGISTRY_AUTH_PATH, 'utf8');
|
|
444
|
-
}
|
|
445
|
-
catch { }
|
|
446
|
-
const lines = existing
|
|
447
|
-
.split('\n')
|
|
448
|
-
.filter(Boolean)
|
|
449
|
-
.filter((l) => !l.startsWith(`${name}:`));
|
|
450
|
-
lines.push(line);
|
|
451
|
-
await ensureDir(path.dirname(REGISTRY_AUTH_PATH));
|
|
452
|
-
await fs.writeFile(REGISTRY_AUTH_PATH, `${lines.join('\n')}\n`);
|
|
453
|
-
});
|
|
454
|
-
await reloadRegistry();
|
|
455
|
-
}
|
|
456
|
-
async function removeRegistryCredential(name) {
|
|
457
|
-
if (!REGISTRY_AUTH_PATH)
|
|
458
|
-
return;
|
|
459
|
-
await withHtpasswdLock(async () => {
|
|
460
|
-
let existing = '';
|
|
461
|
-
try {
|
|
462
|
-
existing = await fs.readFile(REGISTRY_AUTH_PATH, 'utf8');
|
|
463
|
-
}
|
|
464
|
-
catch { }
|
|
465
|
-
const lines = existing
|
|
466
|
-
.split('\n')
|
|
467
|
-
.filter(Boolean)
|
|
468
|
-
.filter((l) => !l.startsWith(`${name}:`));
|
|
469
|
-
await ensureDir(path.dirname(REGISTRY_AUTH_PATH));
|
|
470
|
-
await fs.writeFile(REGISTRY_AUTH_PATH, lines.length ? `${lines.join('\n')}\n` : '');
|
|
471
|
-
});
|
|
472
|
-
await reloadRegistry();
|
|
473
|
-
}
|
|
474
423
|
const db = await (options.openDatabase || openDb)(dbFile);
|
|
475
424
|
const wakeHub = new WakeHub();
|
|
476
425
|
// Events are an operator-facing log, not a record of truth, so keep them bounded. `0` turns a limit off; anything that is
|
|
@@ -489,32 +438,37 @@ export async function createApp(options = {}) {
|
|
|
489
438
|
const EVENT_MAX_ROWS = envCount('TUG_EVENT_MAX_ROWS', 100_000);
|
|
490
439
|
const eventPruneIntervalMs = options.eventPruneIntervalMs ?? 60 * 60 * 1000;
|
|
491
440
|
let lastEventPrune = Date.now();
|
|
492
|
-
|
|
441
|
+
function pruneEvents() {
|
|
493
442
|
lastEventPrune = Date.now();
|
|
494
443
|
let removed = 0;
|
|
495
444
|
if (EVENT_RETENTION_DAYS > 0)
|
|
496
445
|
removed +=
|
|
497
|
-
|
|
446
|
+
db.run('DELETE FROM events WHERE created_at < ?', new Date(Date.now() - EVENT_RETENTION_DAYS * 86_400_000).toISOString()).changes ?? 0;
|
|
498
447
|
if (EVENT_MAX_ROWS > 0)
|
|
499
448
|
removed +=
|
|
500
|
-
|
|
449
|
+
db.run('DELETE FROM events WHERE id IN (SELECT id FROM events ORDER BY created_at DESC, id DESC LIMIT -1 OFFSET ?)', EVENT_MAX_ROWS).changes ?? 0;
|
|
501
450
|
if (removed)
|
|
502
451
|
console.log(`removed ${removed} old events (kept ${EVENT_RETENTION_DAYS || 'any'} days, at most ${EVENT_MAX_ROWS || 'unlimited'} rows)`);
|
|
503
452
|
return removed;
|
|
504
453
|
}
|
|
505
|
-
|
|
454
|
+
try {
|
|
455
|
+
pruneEvents();
|
|
456
|
+
}
|
|
457
|
+
catch (e) {
|
|
458
|
+
console.error(`event pruning failed: ${errMessage(e)}`);
|
|
459
|
+
}
|
|
506
460
|
const masterKey = await loadOrCreateMasterKey(masterKeyFile);
|
|
507
|
-
const operator =
|
|
461
|
+
const operator = db.get('SELECT value FROM meta WHERE key=?', 'operator_token_hash');
|
|
508
462
|
if (!operator) {
|
|
509
463
|
const t = process.env.TUG_OPERATOR_TOKEN || token('tug_op');
|
|
510
|
-
|
|
464
|
+
db.run('INSERT INTO meta(key,value) VALUES(?,?)', 'operator_token_hash', hashToken(t));
|
|
511
465
|
await fs
|
|
512
466
|
.writeFile(path.join(DATA_DIR, 'operator.token'), `${t}\n`, { mode: 0o600 })
|
|
513
467
|
.catch(() => { });
|
|
514
468
|
console.log(`operator token: ${t}`);
|
|
515
469
|
}
|
|
516
470
|
const app = express();
|
|
517
|
-
const authenticateAgent =
|
|
471
|
+
const authenticateAgent = (raw) => raw
|
|
518
472
|
? db.get('SELECT * FROM nodes WHERE agent_token_hash=?', hashToken(raw))
|
|
519
473
|
: undefined;
|
|
520
474
|
let restartRequired = false;
|
|
@@ -578,13 +532,21 @@ export async function createApp(options = {}) {
|
|
|
578
532
|
apiRoutes.set(key, role);
|
|
579
533
|
route(method, url, [guard(role, resolve ?? undefined), ...handlers]);
|
|
580
534
|
};
|
|
581
|
-
const deploymentProject =
|
|
582
|
-
|
|
583
|
-
|
|
535
|
+
const deploymentProject = (req) => db.get('SELECT project_id FROM deployments WHERE id=?', req.params.id)
|
|
536
|
+
?.project_id;
|
|
537
|
+
const commandProject = (req) => {
|
|
538
|
+
const row = db.get('SELECT project_id FROM commands WHERE id=?', req.params.id);
|
|
584
539
|
return row ? row.project_id : undefined;
|
|
585
540
|
};
|
|
586
541
|
/** Audit trail entry for a control action; bookkeeping failures are logged, never returned to the caller. */
|
|
587
|
-
const audit = (actor, event) =>
|
|
542
|
+
const audit = (actor, event) => {
|
|
543
|
+
try {
|
|
544
|
+
recordEvent(db, { ...event, actor: actor.name });
|
|
545
|
+
}
|
|
546
|
+
catch (e) {
|
|
547
|
+
console.error(`audit event failed: ${errMessage(e)}`);
|
|
548
|
+
}
|
|
549
|
+
};
|
|
588
550
|
const projectScope = (actor, column) => actor.projects
|
|
589
551
|
? {
|
|
590
552
|
where: `${column} IN (${actor.projects.map(() => '?').join(',') || "''"})`,
|
|
@@ -593,22 +555,22 @@ export async function createApp(options = {}) {
|
|
|
593
555
|
: { where: '1=1', args: [] };
|
|
594
556
|
const authAgent = async (req, res, next) => {
|
|
595
557
|
const raw = (req.headers.authorization || '').replace(/^Bearer /, '');
|
|
596
|
-
const node =
|
|
558
|
+
const node = authenticateAgent(raw);
|
|
597
559
|
if (!node)
|
|
598
560
|
return res.status(401).json({ error: 'unauthorized' });
|
|
599
561
|
req.node = node;
|
|
600
562
|
next();
|
|
601
563
|
};
|
|
602
|
-
const wakeAll =
|
|
603
|
-
for (const node of
|
|
564
|
+
const wakeAll = (project) => {
|
|
565
|
+
for (const node of db.all('SELECT id FROM nodes'))
|
|
604
566
|
wakeHub.wake(node.id, project);
|
|
605
567
|
};
|
|
606
568
|
let endpointSnapshot = '';
|
|
607
|
-
|
|
608
|
-
const project =
|
|
569
|
+
function updateDeploymentConvergence(projectId) {
|
|
570
|
+
const project = db.get('SELECT * FROM projects WHERE id=?', projectId);
|
|
609
571
|
if (!project?.current_deployment_id)
|
|
610
572
|
return;
|
|
611
|
-
const ds =
|
|
573
|
+
const ds = db.get('SELECT * FROM desired_states WHERE deployment_id=?', project.current_deployment_id);
|
|
612
574
|
if (!ds)
|
|
613
575
|
return;
|
|
614
576
|
const state = JSON.parse(ds.state_json);
|
|
@@ -618,7 +580,7 @@ export async function createApp(options = {}) {
|
|
|
618
580
|
required.add(node);
|
|
619
581
|
if (!required.size)
|
|
620
582
|
return;
|
|
621
|
-
const statuses =
|
|
583
|
+
const statuses = db.all('SELECT node_id,observed_version,health,status_json FROM node_statuses WHERE project_id=?', projectId);
|
|
622
584
|
const byNode = new Map(statuses.map((s) => [s.node_id, s]));
|
|
623
585
|
const reportsRevision = (node) => state.revision
|
|
624
586
|
? JSON.parse(byNode.get(node)?.status_json || '{}').observedRevision === state.revision
|
|
@@ -627,7 +589,7 @@ export async function createApp(options = {}) {
|
|
|
627
589
|
.filter((node) => reportsRevision(node) && byNode.get(node)?.health === 'failed')
|
|
628
590
|
.map((node) => byNode.get(node));
|
|
629
591
|
if (failedNodes.length) {
|
|
630
|
-
const marked =
|
|
592
|
+
const marked = db.run('UPDATE deployments SET state=?,error_message=?,updated_at=? WHERE id=? AND state=?', 'failed', deploymentFailureMessage(failedNodes), now(), project.current_deployment_id, 'rolling_out');
|
|
631
593
|
if (marked.changes === 1)
|
|
632
594
|
scheduleAutoRollback(projectId, project.current_deployment_id, state);
|
|
633
595
|
}
|
|
@@ -636,29 +598,29 @@ export async function createApp(options = {}) {
|
|
|
636
598
|
: byNode.get(node)?.observed_version === state.version) &&
|
|
637
599
|
['serving', 'skipped'].includes(byNode.get(node)?.health ?? ''));
|
|
638
600
|
if (ok)
|
|
639
|
-
|
|
601
|
+
db.run('UPDATE deployments SET state=?,updated_at=? WHERE id=? AND state=?', 'succeeded', now(), project.current_deployment_id, 'rolling_out');
|
|
640
602
|
}
|
|
641
603
|
/** Events control writes when it rolls back on its own; they are attributed to a reserved system actor. */
|
|
642
|
-
|
|
643
|
-
|
|
604
|
+
function recordAutoRollbackEvent(projectId, deploymentId, type, message, metadata = {}) {
|
|
605
|
+
db.run('INSERT INTO events(id,project_id,deployment_id,node_id,level,type,message,metadata_json,created_at,actor) VALUES(?,?,?,?,?,?,?,?,?,?)', id('evt'), projectId, deploymentId, null, 'warn', type, message, JSON.stringify(metadata), now(), AUTO_ROLLBACK_ACTOR);
|
|
644
606
|
}
|
|
645
607
|
/**
|
|
646
608
|
* Creates a rollback deployment that copies the most recent earlier deployment's desired state under a
|
|
647
609
|
* new revision. Shared by `POST /api/v1/rollback` and automatic rollback; the caller holds the writer queue.
|
|
648
610
|
* `auto` only considers `succeeded` deployments and requires `failedDeploymentId` to still be current.
|
|
649
611
|
*/
|
|
650
|
-
|
|
612
|
+
function createRollback(project, opts) {
|
|
651
613
|
let lock;
|
|
652
614
|
try {
|
|
653
|
-
lock =
|
|
654
|
-
const active =
|
|
615
|
+
lock = acquireProjectLock(project, opts.auto ? 'auto-rollback' : 'rollback');
|
|
616
|
+
const active = db.get(`SELECT id,state FROM deployments WHERE project_id=? AND state IN (${ACTIVE_STATES_SQL}) ORDER BY created_at DESC LIMIT 1`, project);
|
|
655
617
|
if (active && !opts.force)
|
|
656
618
|
throw httpError(409, `deployment ${active.id} is already ${active.state}; retry with force to rollback anyway`);
|
|
657
|
-
const current =
|
|
619
|
+
const current = db.get('SELECT current_deployment_id FROM projects WHERE id=?', project);
|
|
658
620
|
if (opts.failedDeploymentId && current?.current_deployment_id !== opts.failedDeploymentId)
|
|
659
621
|
throw httpError(409, `deployment ${opts.failedDeploymentId} is no longer the current deployment`);
|
|
660
622
|
const states = opts.auto ? `'succeeded'` : `'succeeded','rolling_out'`;
|
|
661
|
-
const previousDs =
|
|
623
|
+
const previousDs = db.get(`SELECT ds.*, d.version AS deployment_version FROM desired_states ds JOIN deployments d ON d.id=ds.deployment_id
|
|
662
624
|
WHERE ds.project_id=? AND ds.deployment_id != ? AND d.state IN (${states}) ORDER BY ds.created_at DESC LIMIT 1`, project, current?.current_deployment_id || '');
|
|
663
625
|
if (!previousDs)
|
|
664
626
|
throw httpError(404, 'no previous deployment found');
|
|
@@ -666,17 +628,17 @@ export async function createApp(options = {}) {
|
|
|
666
628
|
const depId = id('dep');
|
|
667
629
|
const previousState = JSON.parse(previousDs.state_json);
|
|
668
630
|
previousState.revision = depId;
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
631
|
+
db.run('INSERT INTO deployments(id,project_id,version,state,created_at,updated_at,actor) VALUES(?,?,?,?,?,?,?)', depId, project, `${ROLLBACK_VERSION_PREFIX}${previousState.version}-${Date.now()}`, 'rolling_out', ts, ts, opts.actor);
|
|
632
|
+
db.run('INSERT INTO desired_states(id,project_id,deployment_id,version,state_json,created_at) VALUES(?,?,?,?,?,?)', id('ds'), project, depId, previousState.version, JSON.stringify(previousState), ts);
|
|
633
|
+
db.run('UPDATE projects SET current_deployment_id=?,updated_at=? WHERE id=?', depId, now(), project);
|
|
634
|
+
wakeAll(project);
|
|
673
635
|
return {
|
|
674
|
-
deployment:
|
|
636
|
+
deployment: db.get('SELECT * FROM deployments WHERE id=?', depId),
|
|
675
637
|
desiredState: previousState,
|
|
676
638
|
};
|
|
677
639
|
}
|
|
678
640
|
finally {
|
|
679
|
-
|
|
641
|
+
releaseLock(lock);
|
|
680
642
|
}
|
|
681
643
|
}
|
|
682
644
|
/**
|
|
@@ -689,9 +651,9 @@ export async function createApp(options = {}) {
|
|
|
689
651
|
if (state.compose['x-tug']?.rollback !== 'auto')
|
|
690
652
|
return;
|
|
691
653
|
void (async () => {
|
|
692
|
-
const failed =
|
|
654
|
+
const failed = db.get('SELECT * FROM deployments WHERE id=?', deploymentId);
|
|
693
655
|
if (failed?.version.startsWith(ROLLBACK_VERSION_PREFIX)) {
|
|
694
|
-
|
|
656
|
+
recordAutoRollbackEvent(projectId, deploymentId, 'deployment.auto_rollback_skipped', `rollback deployment ${failed.version} failed; not rolling back a rollback`);
|
|
695
657
|
return;
|
|
696
658
|
}
|
|
697
659
|
try {
|
|
@@ -700,27 +662,32 @@ export async function createApp(options = {}) {
|
|
|
700
662
|
auto: true,
|
|
701
663
|
failedDeploymentId: deploymentId,
|
|
702
664
|
}));
|
|
703
|
-
|
|
665
|
+
recordAutoRollbackEvent(projectId, out.deployment.id, 'deployment.auto_rollback', `deployment ${deploymentId} (${failed?.version}) reported failed health; rolled back to ${out.desiredState.version} as deployment ${out.deployment.id}`, {
|
|
704
666
|
failedDeploymentId: deploymentId,
|
|
705
667
|
rollbackDeploymentId: out.deployment.id,
|
|
706
668
|
version: out.desiredState.version,
|
|
707
669
|
});
|
|
708
670
|
}
|
|
709
671
|
catch (e) {
|
|
710
|
-
|
|
672
|
+
try {
|
|
673
|
+
recordAutoRollbackEvent(projectId, deploymentId, 'deployment.auto_rollback_skipped', `deployment ${deploymentId} failed but automatic rollback was not possible: ${errMessage(e)}`);
|
|
674
|
+
}
|
|
675
|
+
catch {
|
|
676
|
+
/* Best-effort audit event. */
|
|
677
|
+
}
|
|
711
678
|
}
|
|
712
679
|
})().catch((e) => console.error('auto rollback failed:', errMessage(e)));
|
|
713
680
|
}
|
|
714
|
-
|
|
681
|
+
function currentDesired(projectId) {
|
|
715
682
|
return db.get('SELECT ds.* FROM desired_states ds JOIN projects p ON p.current_deployment_id=ds.deployment_id WHERE p.id=?', projectId);
|
|
716
683
|
}
|
|
717
684
|
const live = new LiveHub(async (actor, request) => {
|
|
718
685
|
assertProject(actor, request.project);
|
|
719
|
-
const row =
|
|
686
|
+
const row = currentDesired(request.project);
|
|
720
687
|
if (!row)
|
|
721
688
|
throw httpError(404, missingProject(request.project));
|
|
722
689
|
const desired = JSON.parse(row.state_json);
|
|
723
|
-
const nodes =
|
|
690
|
+
const nodes = db.all('SELECT * FROM nodes');
|
|
724
691
|
const eligible = new Set();
|
|
725
692
|
const services = request.operation === 'ps'
|
|
726
693
|
? Object.values(desired.services)
|
|
@@ -749,38 +716,44 @@ export async function createApp(options = {}) {
|
|
|
749
716
|
api('get', '/api/v1/live', null, (_req, res) => {
|
|
750
717
|
res.status(426).json({ error: 'WebSocket upgrade required' });
|
|
751
718
|
});
|
|
752
|
-
|
|
719
|
+
function acquireProjectLock(project, operation, ttlMs = Number(process.env.TUG_LOCK_TTL_MS || 10 * 60 * 1000)) {
|
|
753
720
|
const name = `project:${project}`;
|
|
754
721
|
const ts = now();
|
|
755
|
-
|
|
722
|
+
db.run('DELETE FROM locks WHERE expires_at < ?', ts);
|
|
756
723
|
const owner = `${operation}:${id('lock')}`;
|
|
757
724
|
try {
|
|
758
|
-
|
|
725
|
+
db.run('INSERT INTO locks(name,owner,expires_at,created_at) VALUES(?,?,?,?)', name, owner, new Date(Date.now() + ttlMs).toISOString(), ts);
|
|
759
726
|
}
|
|
760
727
|
catch (e) {
|
|
761
|
-
if (
|
|
728
|
+
if (!isUniqueViolation(e))
|
|
762
729
|
throw e;
|
|
763
|
-
const existing =
|
|
730
|
+
const existing = db.get('SELECT * FROM locks WHERE name=?', name);
|
|
764
731
|
const err = new Error(`project ${project} is locked by ${existing?.owner} until ${existing?.expires_at}`);
|
|
765
732
|
err.status = 409;
|
|
766
733
|
err.lock = existing;
|
|
767
734
|
throw err;
|
|
768
735
|
}
|
|
769
736
|
const heartbeat = setInterval(() => {
|
|
770
|
-
|
|
771
|
-
.run('UPDATE locks SET expires_at=? WHERE name=? AND owner=?', new Date(Date.now() + ttlMs).toISOString(), name, owner)
|
|
772
|
-
|
|
737
|
+
try {
|
|
738
|
+
db.run('UPDATE locks SET expires_at=? WHERE name=? AND owner=?', new Date(Date.now() + ttlMs).toISOString(), name, owner);
|
|
739
|
+
}
|
|
740
|
+
catch {
|
|
741
|
+
/* Best-effort bookkeeping. */
|
|
742
|
+
}
|
|
773
743
|
}, Math.max(10, Math.floor(ttlMs / 3)));
|
|
774
744
|
heartbeat.unref();
|
|
775
745
|
return { name, owner, heartbeat };
|
|
776
746
|
}
|
|
777
|
-
|
|
747
|
+
function releaseLock(lock) {
|
|
778
748
|
if (lock?.heartbeat)
|
|
779
749
|
clearInterval(lock.heartbeat);
|
|
780
750
|
if (lock)
|
|
781
|
-
|
|
782
|
-
.run('DELETE FROM locks WHERE name=? AND owner=?', lock.name, lock.owner)
|
|
783
|
-
|
|
751
|
+
try {
|
|
752
|
+
db.run('DELETE FROM locks WHERE name=? AND owner=?', lock.name, lock.owner);
|
|
753
|
+
}
|
|
754
|
+
catch {
|
|
755
|
+
/* Best-effort bookkeeping. */
|
|
756
|
+
}
|
|
784
757
|
}
|
|
785
758
|
function sendError(res, e) {
|
|
786
759
|
const err = e;
|
|
@@ -795,13 +768,13 @@ export async function createApp(options = {}) {
|
|
|
795
768
|
if (!project || !service || !Number.isInteger(replicas) || replicas < 1)
|
|
796
769
|
throw httpError(400, 'project, service and a positive integer replicas are required');
|
|
797
770
|
assertProject(req.actor, project);
|
|
798
|
-
const dsRow =
|
|
771
|
+
const dsRow = currentDesired(project);
|
|
799
772
|
if (!dsRow)
|
|
800
773
|
throw httpError(404, `project ${project} has no desired state`);
|
|
801
774
|
const state = JSON.parse(dsRow.state_json);
|
|
802
775
|
if (!state.services?.[service])
|
|
803
776
|
throw httpError(404, `service ${service} not found in project ${project}`);
|
|
804
|
-
const nodeRows =
|
|
777
|
+
const nodeRows = db.all('SELECT name,labels_json,last_seen_at FROM nodes ORDER BY created_at,name');
|
|
805
778
|
const nodeInfos = nodeRows
|
|
806
779
|
.filter((row) => !isStale(row))
|
|
807
780
|
.map((row) => ({
|
|
@@ -826,8 +799,8 @@ export async function createApp(options = {}) {
|
|
|
826
799
|
};
|
|
827
800
|
state.services[service].replicas = selected.length;
|
|
828
801
|
state.revision = id('rev');
|
|
829
|
-
|
|
830
|
-
|
|
802
|
+
db.run('UPDATE desired_states SET state_json=? WHERE id=?', JSON.stringify(state), dsRow.id);
|
|
803
|
+
wakeAll(project);
|
|
831
804
|
res.json({
|
|
832
805
|
ok: true,
|
|
833
806
|
project,
|
|
@@ -840,59 +813,59 @@ export async function createApp(options = {}) {
|
|
|
840
813
|
api('post', '/api/v1/projects', (req) => String(req.body?.name || 'site'), async (req, res) => {
|
|
841
814
|
const name = req.body.name || 'site';
|
|
842
815
|
const ts = now();
|
|
843
|
-
|
|
844
|
-
res.json(
|
|
816
|
+
db.run('INSERT OR IGNORE INTO projects(id,name,created_at,updated_at) VALUES(?,?,?,?)', name, name, ts, ts);
|
|
817
|
+
res.json(db.get('SELECT * FROM projects WHERE id=?', name));
|
|
845
818
|
});
|
|
846
|
-
api('get', '/api/v1/projects/:project', (req) => req.params.project, async (req, res) => res.json(
|
|
819
|
+
api('get', '/api/v1/projects/:project', (req) => req.params.project, async (req, res) => res.json(db.get('SELECT * FROM projects WHERE id=?', req.params.project)));
|
|
847
820
|
/** A node whose agent has been seen before but not within TUG_STALE_SECONDS; never-seen (just registered) nodes stay schedulable. */
|
|
848
821
|
const isStale = (row) => !!row.last_seen_at &&
|
|
849
822
|
Date.parse(row.last_seen_at) < Date.now() - Number(process.env.TUG_STALE_SECONDS || 60) * 1000;
|
|
850
823
|
const NODE_INFO_SQL = 'SELECT id,name,address,public_address,overlay_ip,labels_json,status,created_at,updated_at FROM nodes WHERE id=?';
|
|
851
824
|
/** Allocates the node's stable overlay IP on first registration (retrying if a concurrent register took it). */
|
|
852
|
-
|
|
825
|
+
function ensureOverlayIp(nodeId) {
|
|
853
826
|
if (!MESH)
|
|
854
827
|
return;
|
|
855
828
|
for (let attempt = 0; attempt < 5; attempt++) {
|
|
856
|
-
const row =
|
|
829
|
+
const row = db.get('SELECT overlay_ip FROM nodes WHERE id=?', nodeId);
|
|
857
830
|
if (row?.overlay_ip)
|
|
858
831
|
return;
|
|
859
|
-
const used =
|
|
832
|
+
const used = db
|
|
833
|
+
.all('SELECT overlay_ip FROM nodes WHERE overlay_ip IS NOT NULL')
|
|
834
|
+
.map((r) => r.overlay_ip);
|
|
860
835
|
try {
|
|
861
|
-
|
|
836
|
+
db.run('UPDATE nodes SET overlay_ip=? WHERE id=? AND overlay_ip IS NULL', allocateOverlayIp(MESH_CIDR, used), nodeId);
|
|
862
837
|
return;
|
|
863
838
|
}
|
|
864
839
|
catch (e) {
|
|
865
|
-
if (
|
|
840
|
+
if (!isUniqueViolation(e))
|
|
866
841
|
throw e;
|
|
867
842
|
}
|
|
868
843
|
}
|
|
869
844
|
throw new Error('could not allocate an overlay address');
|
|
870
845
|
}
|
|
871
846
|
/** Stores (or refreshes) the node's WireGuard peer entry; the private key never reaches control. */
|
|
872
|
-
|
|
847
|
+
function syncWgPeer(nodeId, publicKey) {
|
|
873
848
|
if (!MESH)
|
|
874
849
|
return;
|
|
875
|
-
const node =
|
|
876
|
-
const existing =
|
|
850
|
+
const node = db.get('SELECT * FROM nodes WHERE id=?', nodeId);
|
|
851
|
+
const existing = db.get('SELECT * FROM wg_peers WHERE node_id=?', nodeId);
|
|
877
852
|
const key = publicKey ?? existing?.public_key;
|
|
878
853
|
if (!node?.overlay_ip || !key)
|
|
879
854
|
return;
|
|
880
|
-
|
|
855
|
+
db.run(`INSERT INTO wg_peers(node_id,public_key,overlay_ip,endpoint,allowed_ips,updated_at) VALUES(?,?,?,?,?,?)
|
|
881
856
|
ON CONFLICT(node_id) DO UPDATE SET public_key=excluded.public_key,overlay_ip=excluded.overlay_ip,endpoint=excluded.endpoint,allowed_ips=excluded.allowed_ips,updated_at=excluded.updated_at`, nodeId, key, node.overlay_ip, wgEndpoint(node.public_address, WG_PORT) ?? null, `${node.overlay_ip}/32`, now());
|
|
882
857
|
}
|
|
883
858
|
async function registerNode({ name, address, publicAddress, wgPublicKey, labels, }) {
|
|
884
859
|
if (wgPublicKey !== undefined && !isWgKey(wgPublicKey))
|
|
885
860
|
throw httpError(400, 'wgPublicKey must be a base64 WireGuard public key');
|
|
886
861
|
const agentToken = token('tug_agent');
|
|
887
|
-
const registryToken = token('tug_reg');
|
|
888
862
|
const ts = now();
|
|
889
|
-
|
|
890
|
-
VALUES(
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
const row = await db.get(NODE_INFO_SQL, name);
|
|
863
|
+
db.run(`INSERT INTO nodes(id,name,address,public_address,status,labels_json,agent_token_hash,created_at,updated_at)
|
|
864
|
+
VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET name=excluded.name,address=excluded.address,public_address=excluded.public_address,labels_json=excluded.labels_json,agent_token_hash=excluded.agent_token_hash,updated_at=excluded.updated_at`, name, name, address || null, publicAddress || null, 'registered', JSON.stringify(Array.isArray(labels) ? labels : []), hashToken(agentToken), ts, ts);
|
|
865
|
+
ensureOverlayIp(name);
|
|
866
|
+
syncWgPeer(name, wgPublicKey);
|
|
867
|
+
wakeAll();
|
|
868
|
+
const row = db.get(NODE_INFO_SQL, name);
|
|
896
869
|
if (!row)
|
|
897
870
|
throw httpError(500, 'node not found after registration');
|
|
898
871
|
const node = {
|
|
@@ -906,7 +879,7 @@ export async function createApp(options = {}) {
|
|
|
906
879
|
created_at: row.created_at,
|
|
907
880
|
updated_at: row.updated_at,
|
|
908
881
|
};
|
|
909
|
-
return { node, agentToken
|
|
882
|
+
return { node, agentToken };
|
|
910
883
|
}
|
|
911
884
|
api('post', '/api/v1/nodes/register', null, async (req, res) => {
|
|
912
885
|
const name = req.body.name;
|
|
@@ -933,18 +906,18 @@ export async function createApp(options = {}) {
|
|
|
933
906
|
const uses = Math.max(1, Number(req.body.uses ?? 1));
|
|
934
907
|
const enrollToken = token('tug_enroll');
|
|
935
908
|
const expiresAt = new Date(Date.now() + ttlSeconds * 1000).toISOString();
|
|
936
|
-
|
|
937
|
-
res.json({ enrollToken, name, usesRemaining: uses, expiresAt
|
|
909
|
+
db.run('INSERT INTO enroll_tokens(token_hash,name,uses_remaining,expires_at,created_at) VALUES(?,?,?,?,?)', hashToken(enrollToken), name, uses, expiresAt, now());
|
|
910
|
+
res.json({ enrollToken, name, usesRemaining: uses, expiresAt });
|
|
938
911
|
});
|
|
939
912
|
post('/agent/v1/enroll', async (req, res) => {
|
|
940
913
|
const body = req.body;
|
|
941
914
|
const raw = String(body.enrollToken || (req.headers.authorization || '').replace(/^Bearer /, '')).trim();
|
|
942
915
|
if (!raw)
|
|
943
916
|
return res.status(400).json({ error: 'enrollToken required' });
|
|
944
|
-
const row =
|
|
917
|
+
const row = db.get('SELECT * FROM enroll_tokens WHERE token_hash=?', hashToken(raw));
|
|
945
918
|
if (!row || row.uses_remaining < 1 || row.expires_at < now()) {
|
|
946
919
|
if (row)
|
|
947
|
-
|
|
920
|
+
db.run('DELETE FROM enroll_tokens WHERE token_hash=?', hashToken(raw));
|
|
948
921
|
return res.status(401).json({ error: 'invalid or expired enroll token' });
|
|
949
922
|
}
|
|
950
923
|
const name = row.name || body.name;
|
|
@@ -967,14 +940,14 @@ export async function createApp(options = {}) {
|
|
|
967
940
|
}
|
|
968
941
|
const remaining = row.uses_remaining - 1;
|
|
969
942
|
if (remaining < 1)
|
|
970
|
-
|
|
943
|
+
db.run('DELETE FROM enroll_tokens WHERE token_hash=?', hashToken(raw));
|
|
971
944
|
else
|
|
972
|
-
|
|
973
|
-
const enrolled = { ...result
|
|
945
|
+
db.run('UPDATE enroll_tokens SET uses_remaining=? WHERE token_hash=?', remaining, hashToken(raw));
|
|
946
|
+
const enrolled = { ...result };
|
|
974
947
|
res.json(enrolled);
|
|
975
948
|
});
|
|
976
949
|
api('get', '/api/v1/nodes', null, async (req, res) => {
|
|
977
|
-
const rows =
|
|
950
|
+
const rows = db.all('SELECT id,name,address,public_address,overlay_ip,ingress,ingress_status_json,status,labels_json,last_seen_at,created_at,updated_at FROM nodes ORDER BY name');
|
|
978
951
|
res.json({
|
|
979
952
|
nodes: rows.map((row) => {
|
|
980
953
|
const { labels_json, ...rest } = row;
|
|
@@ -982,30 +955,13 @@ export async function createApp(options = {}) {
|
|
|
982
955
|
}),
|
|
983
956
|
});
|
|
984
957
|
});
|
|
985
|
-
|
|
986
|
-
const node = await db.get('SELECT id,name FROM nodes WHERE name=? OR id=?', req.params.name, req.params.name);
|
|
987
|
-
if (!node)
|
|
988
|
-
return res.status(404).json({ error: 'node not found' });
|
|
989
|
-
const registryToken = token('tug_reg');
|
|
990
|
-
await db.run('UPDATE nodes SET registry_token_hash=?,updated_at=? WHERE id=?', hashToken(registryToken), now(), node.id);
|
|
991
|
-
await upsertRegistryCredential(node.name, registryToken);
|
|
992
|
-
res.json({ ok: true, node: node.name, registryToken });
|
|
993
|
-
});
|
|
994
|
-
api('post', '/api/v1/nodes/:name/revoke-registry-token', null, async (req, res) => {
|
|
995
|
-
const node = await db.get('SELECT id,name FROM nodes WHERE name=? OR id=?', req.params.name, req.params.name);
|
|
996
|
-
if (!node)
|
|
997
|
-
return res.status(404).json({ error: 'node not found' });
|
|
998
|
-
await db.run('UPDATE nodes SET registry_token_hash=NULL,updated_at=? WHERE id=?', now(), node.id);
|
|
999
|
-
await removeRegistryCredential(node.name);
|
|
1000
|
-
res.json({ ok: true, node: node.name, revoked: true });
|
|
1001
|
-
});
|
|
1002
|
-
async function findNode(nameOrId) {
|
|
958
|
+
function findNode(nameOrId) {
|
|
1003
959
|
return db.get('SELECT id,name FROM nodes WHERE name=? OR id=?', nameOrId, nameOrId);
|
|
1004
960
|
}
|
|
1005
|
-
|
|
961
|
+
function queueCommand(nodeId, projectId, type, payload = {}) {
|
|
1006
962
|
const commandId = id('cmd');
|
|
1007
963
|
const ts = now();
|
|
1008
|
-
|
|
964
|
+
db.run('INSERT INTO commands(id,node_id,project_id,type,state,payload_json,attempts,run_after,deadline,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)', commandId, nodeId, projectId, type, 'pending', JSON.stringify(payload), 0, ts, null, ts, ts);
|
|
1009
965
|
wakeHub.wake(nodeId, projectId ?? undefined);
|
|
1010
966
|
return commandId;
|
|
1011
967
|
}
|
|
@@ -1014,19 +970,19 @@ export async function createApp(options = {}) {
|
|
|
1014
970
|
api('post', '/api/v1/nodes/:name/rotate-wg-key', null, async (req, res) => {
|
|
1015
971
|
if (!MESH)
|
|
1016
972
|
return res.status(409).json({ error: 'the WireGuard mesh is not enabled (TUG_MESH=true)' });
|
|
1017
|
-
const node =
|
|
973
|
+
const node = findNode(req.params.name);
|
|
1018
974
|
if (!node)
|
|
1019
975
|
return res.status(404).json({ error: 'node not found' });
|
|
1020
|
-
res.json({ commandId:
|
|
976
|
+
res.json({ commandId: queueCommand(node.id, null, 'wg.rotate-key'), nodeId: node.id });
|
|
1021
977
|
});
|
|
1022
978
|
// Removes a retired node: its token, WireGuard peer, statuses and queued commands. Refused while a current deployment still places
|
|
1023
979
|
// something on it (redeploy without it first) or while its agent is still reporting, unless force=true.
|
|
1024
980
|
api('delete', '/api/v1/nodes/:name', null, async (req, res) => {
|
|
1025
|
-
const node =
|
|
981
|
+
const node = db.get('SELECT id,name,last_seen_at FROM nodes WHERE name=? OR id=?', req.params.name, req.params.name);
|
|
1026
982
|
if (!node)
|
|
1027
983
|
return res.status(404).json({ error: 'node not found' });
|
|
1028
984
|
const referenced = [];
|
|
1029
|
-
for (const state of
|
|
985
|
+
for (const state of currentDesiredStates()) {
|
|
1030
986
|
for (const [name, svc] of Object.entries(state.services || {}))
|
|
1031
987
|
if (svc.placement?.nodes?.includes(node.name))
|
|
1032
988
|
referenced.push(`${state.project}/${name}`);
|
|
@@ -1042,24 +998,24 @@ export async function createApp(options = {}) {
|
|
|
1042
998
|
return res.status(409).json({
|
|
1043
999
|
error: `node ${node.name} is still reporting; stop its agent first or pass force=true`,
|
|
1044
1000
|
});
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1001
|
+
db.run('DELETE FROM commands WHERE node_id=?', node.id);
|
|
1002
|
+
db.run('DELETE FROM node_statuses WHERE node_id=?', node.id);
|
|
1003
|
+
db.run('DELETE FROM wg_peers WHERE node_id=?', node.id);
|
|
1004
|
+
db.run('DELETE FROM nodes WHERE id=?', node.id);
|
|
1005
|
+
audit(req.actor, {
|
|
1050
1006
|
type: 'node.removed',
|
|
1051
1007
|
message: `node ${node.name} removed`,
|
|
1052
1008
|
metadata: { node: node.name },
|
|
1053
1009
|
});
|
|
1054
|
-
|
|
1010
|
+
wakeAll();
|
|
1055
1011
|
res.json({ ok: true, node: node.name });
|
|
1056
1012
|
});
|
|
1057
1013
|
api('post', '/api/v1/nodes/:name/ingress', null, async (req, res) => {
|
|
1058
|
-
const node =
|
|
1014
|
+
const node = findNode(req.params.name);
|
|
1059
1015
|
if (!node)
|
|
1060
1016
|
return res.status(404).json({ error: 'node not found' });
|
|
1061
1017
|
const enabled = req.body.enabled !== false;
|
|
1062
|
-
|
|
1018
|
+
db.run('UPDATE nodes SET ingress=?,updated_at=? WHERE id=?', enabled ? 1 : 0, now(), node.id);
|
|
1063
1019
|
wakeHub.wake(node.id);
|
|
1064
1020
|
res.json({ ok: true, node: node.name, ingress: enabled });
|
|
1065
1021
|
});
|
|
@@ -1070,30 +1026,18 @@ export async function createApp(options = {}) {
|
|
|
1070
1026
|
if (!isWgKey(publicKey))
|
|
1071
1027
|
return res.status(400).json({ error: 'publicKey must be a base64 WireGuard public key' });
|
|
1072
1028
|
try {
|
|
1073
|
-
|
|
1074
|
-
|
|
1029
|
+
syncWgPeer(req.node.id, publicKey);
|
|
1030
|
+
wakeAll();
|
|
1075
1031
|
res.json({ ok: true });
|
|
1076
1032
|
}
|
|
1077
1033
|
catch (e) {
|
|
1078
|
-
sendError(res,
|
|
1079
|
-
? httpError(409, 'public key already registered to another node')
|
|
1080
|
-
: e);
|
|
1034
|
+
sendError(res, isUniqueViolation(e) ? httpError(409, 'public key already registered to another node') : e);
|
|
1081
1035
|
}
|
|
1082
1036
|
});
|
|
1083
|
-
post('/api/v1/registry/auth', async (req, res) => {
|
|
1084
|
-
const username = req.body.username;
|
|
1085
|
-
const password = req.body.password;
|
|
1086
|
-
const node = username && password
|
|
1087
|
-
? await db.get('SELECT id,name FROM nodes WHERE (name=? OR id=?) AND registry_token_hash IS NOT NULL AND registry_token_hash=?', username, username, hashToken(password))
|
|
1088
|
-
: null;
|
|
1089
|
-
if (!node)
|
|
1090
|
-
return res.status(401).json({ ok: false, error: 'invalid registry credentials' });
|
|
1091
|
-
res.json({ ok: true, node: node.name });
|
|
1092
|
-
});
|
|
1093
1037
|
api('post', '/api/v1/tokens', null, async (req, res) => {
|
|
1094
1038
|
const request = parseTokenRequest(req.body);
|
|
1095
|
-
const created =
|
|
1096
|
-
|
|
1039
|
+
const created = createToken(db, request, req.actor.name);
|
|
1040
|
+
audit(req.actor, {
|
|
1097
1041
|
type: 'token.create',
|
|
1098
1042
|
message: `token ${created.name} (${created.role}${created.projects ? `, projects: ${created.projects.join(', ')}` : ''}) created by ${req.actor.name}`,
|
|
1099
1043
|
metadata: {
|
|
@@ -1105,11 +1049,11 @@ export async function createApp(options = {}) {
|
|
|
1105
1049
|
});
|
|
1106
1050
|
res.json(created);
|
|
1107
1051
|
});
|
|
1108
|
-
api('get', '/api/v1/tokens', null, async (_req, res) => res.json({ tokens:
|
|
1052
|
+
api('get', '/api/v1/tokens', null, async (_req, res) => res.json({ tokens: listTokens(db) }));
|
|
1109
1053
|
api('delete', '/api/v1/tokens/:name', null, async (req, res) => {
|
|
1110
|
-
if (!
|
|
1054
|
+
if (!revokeToken(db, req.params.name))
|
|
1111
1055
|
return res.status(404).json({ error: 'token not found' });
|
|
1112
|
-
|
|
1056
|
+
audit(req.actor, {
|
|
1113
1057
|
type: 'token.revoke',
|
|
1114
1058
|
message: `token ${req.params.name} revoked by ${req.actor.name}`,
|
|
1115
1059
|
metadata: { name: req.params.name },
|
|
@@ -1122,13 +1066,13 @@ export async function createApp(options = {}) {
|
|
|
1122
1066
|
return res.status(400).json({ error: 'name and value required' });
|
|
1123
1067
|
let lock;
|
|
1124
1068
|
try {
|
|
1125
|
-
lock =
|
|
1126
|
-
const old =
|
|
1069
|
+
lock = acquireProjectLock(project, 'secret.set');
|
|
1070
|
+
const old = db.get('SELECT version FROM secrets WHERE project_id=? AND name=?', project, name);
|
|
1127
1071
|
const ts = now();
|
|
1128
|
-
|
|
1072
|
+
db.run(`INSERT INTO secrets(id,project_id,name,encrypted_value,version,created_at,updated_at) VALUES(?,?,?,?,?,?,?)
|
|
1129
1073
|
ON CONFLICT(project_id,name) DO UPDATE SET encrypted_value=excluded.encrypted_value,version=excluded.version,updated_at=excluded.updated_at`, id('sec'), project, name, encrypt(value, masterKey), (old?.version || 0) + 1, ts, ts);
|
|
1130
|
-
|
|
1131
|
-
|
|
1074
|
+
wakeAll(project);
|
|
1075
|
+
audit(req.actor, {
|
|
1132
1076
|
type: 'secret.set',
|
|
1133
1077
|
project,
|
|
1134
1078
|
message: `secret ${name} set to version ${(old?.version || 0) + 1} by ${req.actor.name}`,
|
|
@@ -1140,18 +1084,18 @@ export async function createApp(options = {}) {
|
|
|
1140
1084
|
sendError(res, e);
|
|
1141
1085
|
}
|
|
1142
1086
|
finally {
|
|
1143
|
-
|
|
1087
|
+
releaseLock(lock);
|
|
1144
1088
|
}
|
|
1145
1089
|
});
|
|
1146
1090
|
api('get', '/api/v1/secrets', queryProject, async (req, res) => res.json({
|
|
1147
|
-
secrets:
|
|
1091
|
+
secrets: db.all('SELECT project_id,name,version,updated_at FROM secrets WHERE project_id=? ORDER BY name', String(req.query.project || 'site')),
|
|
1148
1092
|
}));
|
|
1149
1093
|
api('delete', '/api/v1/secrets/:name', queryProject, async (req, res) => {
|
|
1150
1094
|
const project = String(req.query.project || 'site');
|
|
1151
|
-
const removed =
|
|
1095
|
+
const removed = db.run('DELETE FROM secrets WHERE project_id=? AND name=?', project, req.params.name);
|
|
1152
1096
|
if (removed.changes) {
|
|
1153
|
-
|
|
1154
|
-
|
|
1097
|
+
wakeAll(project);
|
|
1098
|
+
audit(req.actor, {
|
|
1155
1099
|
type: 'secret.delete',
|
|
1156
1100
|
project,
|
|
1157
1101
|
message: `secret ${req.params.name} deleted by ${req.actor.name}`,
|
|
@@ -1162,33 +1106,35 @@ export async function createApp(options = {}) {
|
|
|
1162
1106
|
});
|
|
1163
1107
|
const artifactDir = path.join(DATA_DIR, 'images');
|
|
1164
1108
|
await ensureDir(artifactDir);
|
|
1165
|
-
|
|
1109
|
+
function retainedStates() {
|
|
1166
1110
|
// All published history is retained. This conservative superset protects failed/lagging versions too.
|
|
1167
|
-
return
|
|
1111
|
+
return db
|
|
1112
|
+
.all('SELECT * FROM desired_states')
|
|
1113
|
+
.map((row) => JSON.parse(row.state_json));
|
|
1168
1114
|
}
|
|
1169
1115
|
async function collectArtifacts() {
|
|
1170
|
-
const states =
|
|
1116
|
+
const states = retainedStates();
|
|
1171
1117
|
const refs = new Set(states.flatMap((state) => Object.values(state.services).flatMap((service) => service.artifact ? [service.artifact.buildId] : [])));
|
|
1172
1118
|
const imageRefs = new Set(states.flatMap((state) => Object.values(state.services).map((service) => service.image)));
|
|
1173
|
-
const rows =
|
|
1119
|
+
const rows = db.all('SELECT * FROM builds');
|
|
1174
1120
|
for (const row of rows) {
|
|
1175
1121
|
if (refs.has(row.id) || imageRefs.has(row.image_ref))
|
|
1176
1122
|
continue;
|
|
1177
|
-
const active =
|
|
1123
|
+
const active = db.get("SELECT 1 FROM deployments WHERE id=? AND state IN ('uploading','building')", row.deployment_id);
|
|
1178
1124
|
if (active)
|
|
1179
1125
|
continue;
|
|
1180
1126
|
await fs.rm(path.join(DATA_DIR, row.artifact_path || path.join('images', `${row.id}.tar.gz`)), { force: true });
|
|
1181
1127
|
await images.run('docker', ['image', 'rm', row.image_ref]).catch(() => { });
|
|
1182
|
-
|
|
1128
|
+
db.run('DELETE FROM builds WHERE id=?', row.id);
|
|
1183
1129
|
}
|
|
1184
|
-
const remaining =
|
|
1130
|
+
const remaining = db.all('SELECT id FROM builds');
|
|
1185
1131
|
const owned = new Set(remaining.flatMap((row) => [`${row.id}.tar.gz`, `${row.id}.tar.gz.tmp`]));
|
|
1186
1132
|
for (const name of await fs.readdir(artifactDir))
|
|
1187
1133
|
if (!owned.has(name))
|
|
1188
1134
|
await fs.rm(path.join(artifactDir, name), { force: true });
|
|
1189
1135
|
}
|
|
1190
1136
|
// Previous-process builds cannot finish after a restart; clear their unpublished files.
|
|
1191
|
-
|
|
1137
|
+
db.run("UPDATE deployments SET state='build_failed',error_message='control restarted during build' WHERE state IN ('uploading','building')");
|
|
1192
1138
|
await collectArtifacts();
|
|
1193
1139
|
const receiveDeployment = upload.any();
|
|
1194
1140
|
api('post', '/api/v1/deployments', null, (req, res) => {
|
|
@@ -1229,7 +1175,7 @@ export async function createApp(options = {}) {
|
|
|
1229
1175
|
throw httpError(400, 'invalid project or version');
|
|
1230
1176
|
if (version.startsWith(ROLLBACK_VERSION_PREFIX))
|
|
1231
1177
|
throw httpError(400, `version cannot start with "${ROLLBACK_VERSION_PREFIX}" (reserved for rollback deployments)`);
|
|
1232
|
-
const nodeRows =
|
|
1178
|
+
const nodeRows = db.all('SELECT name,labels_json,last_seen_at,created_at FROM nodes ORDER BY created_at,name');
|
|
1233
1179
|
// Stale nodes (agent not seen recently) are not scheduled onto, so a dead host cannot hold a rollout open.
|
|
1234
1180
|
const nodeInfos = nodeRows
|
|
1235
1181
|
.filter((row) => !isStale(row))
|
|
@@ -1269,7 +1215,7 @@ export async function createApp(options = {}) {
|
|
|
1269
1215
|
xe.address ||
|
|
1270
1216
|
!xe.placement?.nodes?.[0])
|
|
1271
1217
|
continue;
|
|
1272
|
-
const exporter =
|
|
1218
|
+
const exporter = db.get('SELECT address,overlay_ip FROM nodes WHERE name=?', xe.placement.nodes[0]);
|
|
1273
1219
|
const addr = (MESH && exporter?.overlay_ip) || exporter?.address;
|
|
1274
1220
|
if (addr)
|
|
1275
1221
|
xe.address = addr;
|
|
@@ -1280,6 +1226,19 @@ export async function createApp(options = {}) {
|
|
|
1280
1226
|
catch (e) {
|
|
1281
1227
|
throw httpError(400, errMessage(e));
|
|
1282
1228
|
}
|
|
1229
|
+
// The CLI interpolates control secrets as sentinels; a reference to a secret that is gone would deploy an empty value.
|
|
1230
|
+
const referenced = sentinelNames(compose);
|
|
1231
|
+
if (referenced.includes(REGISTRY_AUTH_SECRET))
|
|
1232
|
+
throw httpError(400, REGISTRY_AUTH_SECRET +
|
|
1233
|
+
' is reserved for registry authentication and cannot be used in service definitions');
|
|
1234
|
+
if (referenced.length) {
|
|
1235
|
+
const known = new Set(db
|
|
1236
|
+
.all('SELECT name FROM secrets WHERE project_id=?', project)
|
|
1237
|
+
.map((row) => row.name));
|
|
1238
|
+
const missing = referenced.filter((name) => !known.has(name));
|
|
1239
|
+
if (missing.length)
|
|
1240
|
+
throw httpError(400, `secrets not found in project ${project}: ${missing.join(', ')}; create them with tug secret-set`);
|
|
1241
|
+
}
|
|
1283
1242
|
const builtServices = Object.entries(compose.services)
|
|
1284
1243
|
.filter(([, service]) => service.build)
|
|
1285
1244
|
.map(([name]) => name);
|
|
@@ -1311,30 +1270,29 @@ export async function createApp(options = {}) {
|
|
|
1311
1270
|
}
|
|
1312
1271
|
if (files.length !== (local ? local.length : context ? 1 : 0))
|
|
1313
1272
|
throw httpError(400, 'unexpected deployment upload');
|
|
1314
|
-
if (
|
|
1315
|
-
throw httpError(400, 'build services require a source context or localImages upload
|
|
1273
|
+
if (builtServices.length && !context && !local)
|
|
1274
|
+
throw httpError(400, 'build services require a source context or localImages upload');
|
|
1316
1275
|
for (const [svcName, svc] of Object.entries(compose.services)) {
|
|
1317
1276
|
const ing = svc['x-tug']?.ingress;
|
|
1318
1277
|
if (ing?.tls !== 'custom')
|
|
1319
1278
|
continue;
|
|
1320
1279
|
for (const secret of [ing.cert, ing.key])
|
|
1321
|
-
if (!
|
|
1280
|
+
if (!db.get('SELECT 1 FROM secrets WHERE project_id=? AND name=?', project, secret ?? null))
|
|
1322
1281
|
throw httpError(400, `${svcName}: custom TLS secret ${secret} not found in project ${project}; set it with secret-set first`);
|
|
1323
1282
|
}
|
|
1324
|
-
lock =
|
|
1325
|
-
const active =
|
|
1283
|
+
lock = acquireProjectLock(project, 'deploy');
|
|
1284
|
+
const active = db.get(`SELECT * FROM deployments WHERE project_id=? AND state IN (${ACTIVE_STATES_SQL}) LIMIT 1`, project);
|
|
1326
1285
|
if (active)
|
|
1327
1286
|
throw httpError(409, `deployment ${active.id} is already ${active.state}`);
|
|
1328
1287
|
depId = id('dep');
|
|
1329
1288
|
const ts = now();
|
|
1330
|
-
|
|
1331
|
-
|
|
1289
|
+
db.run('INSERT OR IGNORE INTO projects(id,name,created_at,updated_at) VALUES(?,?,?,?)', project, project, ts, ts);
|
|
1290
|
+
db.run('INSERT INTO deployments(id,project_id,version,state,created_at,updated_at,actor) VALUES(?,?,?,?,?,?,?)', depId, project, version, 'building', ts, ts, req.actor.name);
|
|
1332
1291
|
const desired = buildDesiredState({
|
|
1333
1292
|
project,
|
|
1334
1293
|
version,
|
|
1335
1294
|
revision: depId,
|
|
1336
1295
|
compose,
|
|
1337
|
-
registry: REGISTRY,
|
|
1338
1296
|
defaultNode: nodes[0]?.name || null,
|
|
1339
1297
|
});
|
|
1340
1298
|
if (context) {
|
|
@@ -1342,14 +1300,10 @@ export async function createApp(options = {}) {
|
|
|
1342
1300
|
await unpackContext(context.path, contextDir);
|
|
1343
1301
|
}
|
|
1344
1302
|
for (const service of builtServices) {
|
|
1345
|
-
if (!contextDir && !local) {
|
|
1346
|
-
desired.services[service].image = `${REGISTRY}/${project}/${service}:${version}`;
|
|
1347
|
-
continue;
|
|
1348
|
-
} // Explicit registry/no-build-upload means an existing pushed image.
|
|
1349
1303
|
const spec = desired.services[service];
|
|
1350
1304
|
const buildId = id('build');
|
|
1351
1305
|
const imageRef = spec.image;
|
|
1352
|
-
|
|
1306
|
+
db.run('INSERT INTO builds(id,project_id,deployment_id,service,image_ref,state,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?)', buildId, project, depId, service, imageRef, 'building', ts, ts);
|
|
1353
1307
|
let info;
|
|
1354
1308
|
let importedClientRef;
|
|
1355
1309
|
const relative = path.join('images', `${buildId}.tar.gz`);
|
|
@@ -1383,21 +1337,12 @@ export async function createApp(options = {}) {
|
|
|
1383
1337
|
service,
|
|
1384
1338
|
buildSpec: compose.services[service].build,
|
|
1385
1339
|
imageRef,
|
|
1386
|
-
push: false,
|
|
1387
1340
|
});
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
}
|
|
1394
|
-
else {
|
|
1395
|
-
info = await images.save(imageRef, temporary);
|
|
1396
|
-
await fs.rename(temporary, final);
|
|
1397
|
-
const artifact = { buildId, imageRef, ...info };
|
|
1398
|
-
spec.artifact = artifact;
|
|
1399
|
-
await db.run('UPDATE builds SET state=?,image_id=?,platform=?,sha256=?,size_bytes=?,artifact_path=?,updated_at=? WHERE id=?', 'complete', info.imageId, info.platform, info.sha256, info.sizeBytes, relative, now(), buildId);
|
|
1400
|
-
}
|
|
1341
|
+
info = await images.save(imageRef, temporary);
|
|
1342
|
+
await fs.rename(temporary, final);
|
|
1343
|
+
const artifact = { buildId, imageRef, ...info };
|
|
1344
|
+
spec.artifact = artifact;
|
|
1345
|
+
db.run('UPDATE builds SET state=?,image_id=?,platform=?,sha256=?,size_bytes=?,artifact_path=?,updated_at=? WHERE id=?', 'complete', info.imageId, info.platform, info.sha256, info.sizeBytes, relative, now(), buildId);
|
|
1401
1346
|
}
|
|
1402
1347
|
finally {
|
|
1403
1348
|
if (importedClientRef)
|
|
@@ -1405,43 +1350,61 @@ export async function createApp(options = {}) {
|
|
|
1405
1350
|
await fs.rm(temporary, { force: true });
|
|
1406
1351
|
}
|
|
1407
1352
|
}
|
|
1353
|
+
const registrySecret = db.get('SELECT * FROM secrets WHERE project_id=? AND name=?', project, REGISTRY_AUTH_SECRET);
|
|
1354
|
+
const credentials = parseRegistryCredentials(registrySecret ? decrypt(registrySecret.encrypted_value, masterKey) : undefined);
|
|
1355
|
+
const resolved = new Map();
|
|
1356
|
+
for (const [name, spec] of Object.entries(desired.services)) {
|
|
1357
|
+
if (spec.artifact)
|
|
1358
|
+
continue;
|
|
1359
|
+
const image = spec.image;
|
|
1360
|
+
if (!resolved.has(image))
|
|
1361
|
+
resolved.set(image, await images.resolve(image, credentials));
|
|
1362
|
+
spec.image = resolved.get(image);
|
|
1363
|
+
desired.compose.services[name].image = spec.image;
|
|
1364
|
+
}
|
|
1408
1365
|
const buildLock = lock;
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
const
|
|
1366
|
+
const publishingId = depId;
|
|
1367
|
+
await withPublication(() => {
|
|
1368
|
+
const owned = db.get('SELECT 1 FROM locks WHERE name=? AND owner=? AND expires_at>?', buildLock.name, buildLock.owner, now());
|
|
1369
|
+
const pending = db.get('SELECT 1 FROM deployments WHERE id=? AND state=?', publishingId, 'building');
|
|
1412
1370
|
if (!owned || !pending)
|
|
1413
1371
|
throw httpError(409, 'deployment was cancelled or its build lease was lost');
|
|
1414
|
-
|
|
1372
|
+
db.exec('BEGIN IMMEDIATE');
|
|
1415
1373
|
try {
|
|
1416
|
-
const published =
|
|
1374
|
+
const published = db.run("UPDATE deployments SET state='rolling_out',updated_at=? WHERE id=? AND state='building'", now(), publishingId);
|
|
1417
1375
|
if (published.changes !== 1 ||
|
|
1418
|
-
!
|
|
1376
|
+
!db.get('SELECT 1 FROM locks WHERE name=? AND owner=? AND expires_at>?', buildLock.name, buildLock.owner, now()))
|
|
1419
1377
|
throw httpError(409, 'deployment was cancelled or its build lease was lost');
|
|
1420
|
-
|
|
1378
|
+
db.run('INSERT INTO desired_states(id,project_id,deployment_id,version,state_json,created_at) VALUES(?,?,?,?,?,?)', id('ds'), project, publishingId, version, JSON.stringify(desired), ts);
|
|
1421
1379
|
for (const [volumeName, volumeSpec] of Object.entries(compose.volumes || {})) {
|
|
1422
1380
|
const lp = volumeSpec?.['x-tug'] || {};
|
|
1423
1381
|
const nodeName = lp.placement?.nodes?.[0] || null;
|
|
1424
1382
|
const node = nodeName
|
|
1425
|
-
?
|
|
1383
|
+
? db.get('SELECT id FROM nodes WHERE name=? OR id=?', nodeName, nodeName)
|
|
1426
1384
|
: null;
|
|
1427
|
-
|
|
1385
|
+
db.run(`INSERT INTO volumes(id,project_id,name,type,driver,node_id,config_json,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(project_id,name) DO UPDATE SET type=excluded.type,driver=excluded.driver,node_id=excluded.node_id,config_json=excluded.config_json,updated_at=excluded.updated_at`, id('vol'), project, volumeName, lp.type || 'local', lp.driver || null, node?.id || null, JSON.stringify(lp), ts, ts);
|
|
1428
1386
|
}
|
|
1429
|
-
|
|
1430
|
-
|
|
1387
|
+
db.run('UPDATE projects SET current_deployment_id=?,updated_at=? WHERE id=?', publishingId, now(), project);
|
|
1388
|
+
db.exec('COMMIT');
|
|
1431
1389
|
}
|
|
1432
1390
|
catch (e) {
|
|
1433
|
-
|
|
1391
|
+
try {
|
|
1392
|
+
db.exec('ROLLBACK');
|
|
1393
|
+
}
|
|
1394
|
+
catch {
|
|
1395
|
+
/* Preserve the publication error. */
|
|
1396
|
+
}
|
|
1434
1397
|
throw e;
|
|
1435
1398
|
}
|
|
1436
1399
|
});
|
|
1437
|
-
|
|
1400
|
+
audit(req.actor, {
|
|
1438
1401
|
type: 'deploy',
|
|
1439
1402
|
project,
|
|
1440
1403
|
deploymentId: depId,
|
|
1441
1404
|
message: `deployment ${depId} (${version}) started by ${req.actor.name}`,
|
|
1442
1405
|
metadata: { version },
|
|
1443
1406
|
});
|
|
1444
|
-
|
|
1407
|
+
wakeAll(project);
|
|
1445
1408
|
responseBody = {
|
|
1446
1409
|
deployment: { id: depId, project, version, state: 'rolling_out', actor: req.actor.name },
|
|
1447
1410
|
desiredState: desired,
|
|
@@ -1449,13 +1412,16 @@ export async function createApp(options = {}) {
|
|
|
1449
1412
|
}
|
|
1450
1413
|
catch (e) {
|
|
1451
1414
|
if (depId)
|
|
1452
|
-
|
|
1453
|
-
.run("UPDATE deployments SET state='build_failed',error_message=?,updated_at=? WHERE id=? AND state='building'", errMessage(e), now(), depId)
|
|
1454
|
-
|
|
1415
|
+
try {
|
|
1416
|
+
db.run("UPDATE deployments SET state='build_failed',error_message=?,updated_at=? WHERE id=? AND state='building'", errMessage(e), now(), depId);
|
|
1417
|
+
}
|
|
1418
|
+
catch {
|
|
1419
|
+
/* Best-effort bookkeeping. */
|
|
1420
|
+
}
|
|
1455
1421
|
responseError = e;
|
|
1456
1422
|
}
|
|
1457
1423
|
finally {
|
|
1458
|
-
|
|
1424
|
+
releaseLock(lock);
|
|
1459
1425
|
await Promise.all(files.map((file) => fs.rm(file.path, { force: true }).catch(() => { })));
|
|
1460
1426
|
if (contextDir)
|
|
1461
1427
|
await fs.rm(contextDir, { recursive: true, force: true });
|
|
@@ -1467,12 +1433,12 @@ export async function createApp(options = {}) {
|
|
|
1467
1433
|
res.json(responseBody);
|
|
1468
1434
|
}
|
|
1469
1435
|
get('/agent/v1/images/:buildId', authAgent, async (req, res) => {
|
|
1470
|
-
const states =
|
|
1436
|
+
const states = retainedStates();
|
|
1471
1437
|
const allowed = states.some((state) => Object.values(state.services).some((service) => service.artifact?.buildId === req.params.buildId &&
|
|
1472
1438
|
(!service.placement.nodes?.length || service.placement.nodes.includes(req.node.name))));
|
|
1473
1439
|
if (!allowed)
|
|
1474
1440
|
return res.status(404).json({ error: 'image not found' });
|
|
1475
|
-
const row =
|
|
1441
|
+
const row = db.get('SELECT * FROM builds WHERE id=? AND state=?', req.params.buildId, 'complete');
|
|
1476
1442
|
if (!row?.artifact_path)
|
|
1477
1443
|
return res.status(404).json({ error: 'image not found' });
|
|
1478
1444
|
try {
|
|
@@ -1490,17 +1456,17 @@ export async function createApp(options = {}) {
|
|
|
1490
1456
|
res.status(404).json({ error: 'image artifact unavailable' });
|
|
1491
1457
|
}
|
|
1492
1458
|
});
|
|
1493
|
-
api('get', '/api/v1/deployments/:id', deploymentProject, async (req, res) => res.json(
|
|
1494
|
-
api('post', '/api/v1/deployments/:id/cancel', deploymentProject, async (req, res) => withPublication(
|
|
1495
|
-
const row =
|
|
1459
|
+
api('get', '/api/v1/deployments/:id', deploymentProject, async (req, res) => res.json(db.get('SELECT * FROM deployments WHERE id=?', req.params.id)));
|
|
1460
|
+
api('post', '/api/v1/deployments/:id/cancel', deploymentProject, async (req, res) => withPublication(() => {
|
|
1461
|
+
const row = db.get('SELECT * FROM deployments WHERE id=?', req.params.id);
|
|
1496
1462
|
if (!row)
|
|
1497
1463
|
return res.status(404).json({ error: 'deployment not found' });
|
|
1498
1464
|
if (!ACTIVE_DEPLOYMENT_STATES.includes(row.state))
|
|
1499
1465
|
return res.status(409).json({ error: `deployment is ${row.state}` });
|
|
1500
|
-
const result =
|
|
1466
|
+
const result = db.run(`UPDATE deployments SET state=?,updated_at=? WHERE id=? AND state IN (${ACTIVE_STATES_SQL})`, 'cancelled', now(), req.params.id);
|
|
1501
1467
|
if (result.changes !== 1)
|
|
1502
1468
|
return res.status(409).json({ error: 'deployment is no longer active' });
|
|
1503
|
-
|
|
1469
|
+
audit(req.actor, {
|
|
1504
1470
|
type: 'cancel',
|
|
1505
1471
|
project: row.project_id,
|
|
1506
1472
|
deploymentId: row.id,
|
|
@@ -1511,11 +1477,11 @@ export async function createApp(options = {}) {
|
|
|
1511
1477
|
api('post', '/api/v1/rollback', bodyProject, async (req, res) => withWriter(async () => {
|
|
1512
1478
|
const project = req.body.project || 'site';
|
|
1513
1479
|
try {
|
|
1514
|
-
const out =
|
|
1480
|
+
const out = createRollback(project, {
|
|
1515
1481
|
actor: req.actor.name,
|
|
1516
1482
|
force: Boolean(req.body.force),
|
|
1517
1483
|
});
|
|
1518
|
-
|
|
1484
|
+
audit(req.actor, {
|
|
1519
1485
|
type: 'rollback',
|
|
1520
1486
|
project,
|
|
1521
1487
|
deploymentId: out.deployment.id,
|
|
@@ -1529,15 +1495,15 @@ export async function createApp(options = {}) {
|
|
|
1529
1495
|
}
|
|
1530
1496
|
}));
|
|
1531
1497
|
api('get', '/api/v1/volumes', queryProject, async (req, res) => res.json({
|
|
1532
|
-
volumes:
|
|
1498
|
+
volumes: db.all('SELECT * FROM volumes WHERE project_id=? ORDER BY name', String(req.query.project || 'site')),
|
|
1533
1499
|
}));
|
|
1534
1500
|
api('delete', '/api/v1/volumes/:name', queryProject, async (req, res) => {
|
|
1535
1501
|
const project = String(req.query.project || 'site');
|
|
1536
1502
|
const deleteData = String(req.query.deleteData || 'false') === 'true';
|
|
1537
1503
|
let lock;
|
|
1538
1504
|
try {
|
|
1539
|
-
lock = deleteData ?
|
|
1540
|
-
const volume =
|
|
1505
|
+
lock = deleteData ? acquireProjectLock(project, 'volume.delete') : null;
|
|
1506
|
+
const volume = db.get('SELECT * FROM volumes WHERE project_id=? AND name=?', project, req.params.name);
|
|
1541
1507
|
if (!volume)
|
|
1542
1508
|
return res.status(404).json({ error: 'volume not found' });
|
|
1543
1509
|
const config = JSON.parse(volume.config_json || '{}');
|
|
@@ -1545,7 +1511,7 @@ export async function createApp(options = {}) {
|
|
|
1545
1511
|
return res
|
|
1546
1512
|
.status(409)
|
|
1547
1513
|
.json({ error: 'refusing to delete persistent volume without deleteData=true' });
|
|
1548
|
-
const ds =
|
|
1514
|
+
const ds = currentDesired(project);
|
|
1549
1515
|
if (ds) {
|
|
1550
1516
|
const state = JSON.parse(ds.state_json);
|
|
1551
1517
|
const inUse = Object.entries(state.compose?.services || {})
|
|
@@ -1558,19 +1524,19 @@ export async function createApp(options = {}) {
|
|
|
1558
1524
|
}
|
|
1559
1525
|
let commandId = null;
|
|
1560
1526
|
if (deleteData && volume.node_id) {
|
|
1561
|
-
commandId =
|
|
1527
|
+
commandId = queueCommand(volume.node_id, project, 'volume.delete', {
|
|
1562
1528
|
project,
|
|
1563
1529
|
volume: req.params.name,
|
|
1564
1530
|
});
|
|
1565
1531
|
}
|
|
1566
|
-
|
|
1532
|
+
db.run('DELETE FROM volumes WHERE project_id=? AND name=?', project, req.params.name);
|
|
1567
1533
|
res.json({ ok: true, commandId, deletedMetadata: true, deleteData });
|
|
1568
1534
|
}
|
|
1569
1535
|
catch (e) {
|
|
1570
1536
|
sendError(res, e);
|
|
1571
1537
|
}
|
|
1572
1538
|
finally {
|
|
1573
|
-
|
|
1539
|
+
releaseLock(lock);
|
|
1574
1540
|
}
|
|
1575
1541
|
});
|
|
1576
1542
|
api('post', '/api/v1/control/backups/run', null, async (req, res) => withWriter(async () => {
|
|
@@ -1581,8 +1547,8 @@ export async function createApp(options = {}) {
|
|
|
1581
1547
|
const staging = path.join(backupDir, `${backupId}-staging`);
|
|
1582
1548
|
await fs.rm(staging, { recursive: true, force: true });
|
|
1583
1549
|
await ensureDir(staging);
|
|
1584
|
-
const snapshotPath = path.join(staging, 'tug.db')
|
|
1585
|
-
|
|
1550
|
+
const snapshotPath = path.join(staging, 'tug.db');
|
|
1551
|
+
db.run('VACUUM INTO ?', snapshotPath);
|
|
1586
1552
|
await fs.copyFile(masterKeyFile, path.join(staging, 'master.key'));
|
|
1587
1553
|
await fs.cp(artifactDir, path.join(staging, 'images'), { recursive: true });
|
|
1588
1554
|
await fs.writeFile(path.join(staging, 'metadata.json'), JSON.stringify({ backupId, createdAt: ts, type: 'control' }, null, 2));
|
|
@@ -1594,7 +1560,7 @@ export async function createApp(options = {}) {
|
|
|
1594
1560
|
'images',
|
|
1595
1561
|
]);
|
|
1596
1562
|
await fs.rm(staging, { recursive: true, force: true });
|
|
1597
|
-
|
|
1563
|
+
db.run('INSERT INTO backups(id,project_id,target,node_id,state,artifact_ref,metadata_json,created_at,completed_at) VALUES(?,?,?,?,?,?,?,?,?)', backupId, '_control', 'control-plane', null, 'completed', artifactRef, JSON.stringify({ type: 'control' }), ts, now());
|
|
1598
1564
|
res.json({ backupId, artifactRef, state: 'completed' });
|
|
1599
1565
|
}));
|
|
1600
1566
|
api('post', '/api/v1/control/restore', null, async (req, res) => withWriter(async () => {
|
|
@@ -1620,25 +1586,21 @@ export async function createApp(options = {}) {
|
|
|
1620
1586
|
const key = (await fs.readFile(path.join(restoreDir, 'master.key'), 'utf8')).trim();
|
|
1621
1587
|
if (!/^[A-Za-z0-9+/]{43}=$/.test(key))
|
|
1622
1588
|
throw new Error('invalid backup master key');
|
|
1623
|
-
snapshot =
|
|
1624
|
-
|
|
1625
|
-
driver: sqlite3.Database,
|
|
1626
|
-
mode: sqlite3.OPEN_READONLY,
|
|
1627
|
-
});
|
|
1628
|
-
const integrity = await snapshot.get('PRAGMA integrity_check');
|
|
1589
|
+
snapshot = openSnapshotDb(path.join(restoreDir, 'tug.db'));
|
|
1590
|
+
const integrity = snapshot.get('PRAGMA integrity_check');
|
|
1629
1591
|
if (integrity?.integrity_check !== 'ok')
|
|
1630
1592
|
throw new Error('backup database failed integrity check');
|
|
1631
|
-
if (!
|
|
1593
|
+
if (!snapshot.get("SELECT 1 FROM meta WHERE key='operator_token_hash'"))
|
|
1632
1594
|
throw new Error('backup database has no operator identity');
|
|
1633
|
-
for (const secret of
|
|
1595
|
+
for (const secret of snapshot.all('SELECT * FROM secrets'))
|
|
1634
1596
|
decrypt(secret.encrypted_value, key);
|
|
1635
|
-
const states =
|
|
1597
|
+
const states = snapshot.all('SELECT * FROM desired_states');
|
|
1636
1598
|
for (const stateRow of states) {
|
|
1637
1599
|
const state = JSON.parse(stateRow.state_json);
|
|
1638
1600
|
for (const service of Object.values(state.services)) {
|
|
1639
1601
|
if (!service.artifact)
|
|
1640
1602
|
continue;
|
|
1641
|
-
const build =
|
|
1603
|
+
const build = snapshot.get('SELECT * FROM builds WHERE id=? AND state=?', service.artifact.buildId, 'complete');
|
|
1642
1604
|
if (!build?.artifact_path ||
|
|
1643
1605
|
!/^images\/build_[a-zA-Z0-9_-]+\.tar\.gz$/.test(build.artifact_path))
|
|
1644
1606
|
throw new Error('backup contains an unsafe or missing image reference');
|
|
@@ -1650,12 +1612,12 @@ export async function createApp(options = {}) {
|
|
|
1650
1612
|
throw new Error('backup image archive checksum mismatch');
|
|
1651
1613
|
}
|
|
1652
1614
|
}
|
|
1653
|
-
|
|
1615
|
+
snapshot.close();
|
|
1654
1616
|
snapshot = undefined;
|
|
1655
1617
|
await ensureDir(path.join(restoreDir, 'images'));
|
|
1656
1618
|
// Validation completes before changing any live file. Stop DB-dependent work until restart.
|
|
1657
1619
|
restartRequired = true;
|
|
1658
|
-
|
|
1620
|
+
db.close();
|
|
1659
1621
|
await fs.rm(`${dbFile}-wal`, { force: true });
|
|
1660
1622
|
await fs.rm(`${dbFile}-shm`, { force: true });
|
|
1661
1623
|
await fs.cp(path.join(restoreDir, 'images'), artifactDir, { recursive: true });
|
|
@@ -1664,15 +1626,19 @@ export async function createApp(options = {}) {
|
|
|
1664
1626
|
res.json({ ok: true, restored: true, restartRequired: true });
|
|
1665
1627
|
}
|
|
1666
1628
|
catch (e) {
|
|
1667
|
-
|
|
1668
|
-
|
|
1629
|
+
try {
|
|
1630
|
+
snapshot?.close();
|
|
1631
|
+
}
|
|
1632
|
+
catch {
|
|
1633
|
+
/* Preserve the restore error. */
|
|
1634
|
+
}
|
|
1669
1635
|
res.status(restartRequired ? 500 : 400).json({ error: errMessage(e), restartRequired });
|
|
1670
1636
|
}
|
|
1671
1637
|
}));
|
|
1672
|
-
|
|
1638
|
+
function commandForService({ project, service, type, payload, }) {
|
|
1673
1639
|
if (!service)
|
|
1674
1640
|
throw httpError(400, 'service required');
|
|
1675
|
-
const ds =
|
|
1641
|
+
const ds = currentDesired(project);
|
|
1676
1642
|
if (!ds)
|
|
1677
1643
|
throw httpError(404, missingProject(project));
|
|
1678
1644
|
const state = JSON.parse(ds.state_json);
|
|
@@ -1682,22 +1648,22 @@ export async function createApp(options = {}) {
|
|
|
1682
1648
|
const nodeName = svc.placement?.nodes?.[0];
|
|
1683
1649
|
if (!nodeName)
|
|
1684
1650
|
throw httpError(400, 'service has no explicit node placement');
|
|
1685
|
-
const node =
|
|
1651
|
+
const node = db.get('SELECT id FROM nodes WHERE name=? OR id=?', nodeName, nodeName);
|
|
1686
1652
|
if (!node)
|
|
1687
1653
|
throw httpError(404, 'node not found');
|
|
1688
|
-
const cmdId =
|
|
1654
|
+
const cmdId = queueCommand(node.id, project, type, { project, service, ...payload });
|
|
1689
1655
|
return { commandId: cmdId, nodeId: node.id };
|
|
1690
1656
|
}
|
|
1691
1657
|
api('post', '/api/v1/restart', bodyProject, async (req, res) => {
|
|
1692
1658
|
try {
|
|
1693
1659
|
const project = req.body.project || 'site';
|
|
1694
|
-
const queued =
|
|
1660
|
+
const queued = commandForService({
|
|
1695
1661
|
project,
|
|
1696
1662
|
service: req.body.service,
|
|
1697
1663
|
type: 'service.restart',
|
|
1698
1664
|
payload: {},
|
|
1699
1665
|
});
|
|
1700
|
-
|
|
1666
|
+
audit(req.actor, {
|
|
1701
1667
|
type: 'restart',
|
|
1702
1668
|
project,
|
|
1703
1669
|
message: `service ${req.body.service} restart requested by ${req.actor.name}`,
|
|
@@ -1715,7 +1681,7 @@ export async function createApp(options = {}) {
|
|
|
1715
1681
|
const tail = Number(req.body.tail || 200);
|
|
1716
1682
|
if (!service)
|
|
1717
1683
|
return res.status(400).json({ error: 'service required' });
|
|
1718
|
-
const ds =
|
|
1684
|
+
const ds = currentDesired(project);
|
|
1719
1685
|
if (!ds)
|
|
1720
1686
|
return res.status(404).json({ error: missingProject(project) });
|
|
1721
1687
|
const state = JSON.parse(ds.state_json);
|
|
@@ -1725,25 +1691,25 @@ export async function createApp(options = {}) {
|
|
|
1725
1691
|
const nodeName = svc.placement?.nodes?.[0];
|
|
1726
1692
|
if (!nodeName)
|
|
1727
1693
|
return res.status(400).json({ error: 'service has no explicit node placement' });
|
|
1728
|
-
const node =
|
|
1694
|
+
const node = db.get('SELECT id FROM nodes WHERE name=? OR id=?', nodeName, nodeName);
|
|
1729
1695
|
if (!node)
|
|
1730
1696
|
return res.status(404).json({ error: 'node not found' });
|
|
1731
|
-
const cmdId =
|
|
1697
|
+
const cmdId = queueCommand(node.id, project, 'logs.fetch', { project, service, tail });
|
|
1732
1698
|
res.json({ commandId: cmdId, nodeId: node.id });
|
|
1733
1699
|
});
|
|
1734
|
-
api('get', '/api/v1/commands/:id', commandProject, async (req, res) => res.json(
|
|
1700
|
+
api('get', '/api/v1/commands/:id', commandProject, async (req, res) => res.json(db.get('SELECT * FROM commands WHERE id=?', req.params.id)));
|
|
1735
1701
|
api('get', '/api/v1/status', null, async (req, res) => {
|
|
1736
1702
|
const staleSeconds = Number(process.env.TUG_STALE_SECONDS || 60);
|
|
1737
1703
|
const cutoff = new Date(Date.now() - staleSeconds * 1000).toISOString();
|
|
1738
|
-
|
|
1704
|
+
db.run("UPDATE nodes SET status='stale',updated_at=? WHERE last_seen_at IS NOT NULL AND last_seen_at < ?", now(), cutoff);
|
|
1739
1705
|
const projects = projectScope(req.actor, 'id');
|
|
1740
1706
|
const deployments = projectScope(req.actor, 'project_id');
|
|
1741
1707
|
const statuses = projectScope(req.actor, 'project_id');
|
|
1742
1708
|
res.json({
|
|
1743
|
-
projects:
|
|
1744
|
-
deployments:
|
|
1745
|
-
nodes:
|
|
1746
|
-
statuses:
|
|
1709
|
+
projects: db.all(`SELECT * FROM projects WHERE ${projects.where}`, ...projects.args),
|
|
1710
|
+
deployments: db.all(`SELECT * FROM deployments WHERE ${deployments.where} ORDER BY created_at DESC LIMIT 20`, ...deployments.args),
|
|
1711
|
+
nodes: db.all('SELECT id,name,status,last_seen_at FROM nodes'),
|
|
1712
|
+
statuses: db.all(`SELECT * FROM node_statuses WHERE ${statuses.where}`, ...statuses.args),
|
|
1747
1713
|
});
|
|
1748
1714
|
});
|
|
1749
1715
|
api('get', '/api/v1/events', null, async (req, res) => {
|
|
@@ -1753,8 +1719,8 @@ export async function createApp(options = {}) {
|
|
|
1753
1719
|
const limit = Math.min(Number(req.query.limit || 100), 500);
|
|
1754
1720
|
const scope = projectScope(req.actor, 'project_id'); // scoped tokens never see node-level or other projects' events
|
|
1755
1721
|
const events = project
|
|
1756
|
-
?
|
|
1757
|
-
:
|
|
1722
|
+
? db.all('SELECT * FROM events WHERE project_id=? ORDER BY created_at DESC LIMIT ?', project, limit)
|
|
1723
|
+
: db.all(`SELECT * FROM events WHERE ${scope.where} ORDER BY created_at DESC LIMIT ?`, ...scope.args, limit);
|
|
1758
1724
|
res.json({ events });
|
|
1759
1725
|
});
|
|
1760
1726
|
api('get', '/api/v1/diagnostics', null, async (req, res) => {
|
|
@@ -1768,10 +1734,10 @@ export async function createApp(options = {}) {
|
|
|
1768
1734
|
}
|
|
1769
1735
|
}
|
|
1770
1736
|
await check('sqlite', async () => ({
|
|
1771
|
-
value:
|
|
1737
|
+
value: db.get('SELECT COUNT(*) AS count FROM projects').count,
|
|
1772
1738
|
}));
|
|
1773
1739
|
await check('operator_token', async () => ({
|
|
1774
|
-
configured: Boolean(
|
|
1740
|
+
configured: Boolean(db.get('SELECT value FROM meta WHERE key=?', 'operator_token_hash')),
|
|
1775
1741
|
}));
|
|
1776
1742
|
await check('master_key', async () => ({ configured: Boolean(masterKey) }));
|
|
1777
1743
|
await check('docker', async () => ({
|
|
@@ -1782,14 +1748,14 @@ export async function createApp(options = {}) {
|
|
|
1782
1748
|
}));
|
|
1783
1749
|
if (MESH) {
|
|
1784
1750
|
await check('mesh', async () => {
|
|
1785
|
-
const nodes =
|
|
1751
|
+
const nodes = db.all('SELECT n.name, n.overlay_ip, EXISTS(SELECT 1 FROM wg_peers w WHERE w.node_id=n.id) AS has_key FROM nodes n');
|
|
1786
1752
|
const missing = nodes.filter((n) => !n.has_key).map((n) => n.name);
|
|
1787
1753
|
if (missing.length)
|
|
1788
1754
|
throw new Error(`nodes without a reported WireGuard key (is TUG_MESH=true on their agents?): ${missing.join(', ')}`);
|
|
1789
1755
|
return { cidr: MESH_CIDR, port: WG_PORT, nodes: nodes.length };
|
|
1790
1756
|
});
|
|
1791
1757
|
}
|
|
1792
|
-
const ingressNodes =
|
|
1758
|
+
const ingressNodes = db.all('SELECT name,ingress_status_json FROM nodes WHERE ingress=1');
|
|
1793
1759
|
if (ingressNodes.length) {
|
|
1794
1760
|
await check('ingress', async () => {
|
|
1795
1761
|
const bad = ingressNodes
|
|
@@ -1801,45 +1767,47 @@ export async function createApp(options = {}) {
|
|
|
1801
1767
|
return { nodes: ingressNodes.map((n) => n.name) };
|
|
1802
1768
|
});
|
|
1803
1769
|
}
|
|
1804
|
-
res.json({ ok: checks.every((c) => c.ok),
|
|
1770
|
+
res.json({ ok: checks.every((c) => c.ok), checks });
|
|
1805
1771
|
});
|
|
1806
1772
|
api('post', '/api/v1/diagnostics/nodes', null, async (req, res) => {
|
|
1807
1773
|
const nodes = req.body.node
|
|
1808
|
-
?
|
|
1809
|
-
:
|
|
1774
|
+
? db.all('SELECT id FROM nodes WHERE id=? OR name=?', req.body.node, req.body.node)
|
|
1775
|
+
: db.all('SELECT id FROM nodes');
|
|
1810
1776
|
if (req.body.node && !nodes.length)
|
|
1811
1777
|
return res.status(404).json({ error: `node ${req.body.node} not found` });
|
|
1812
1778
|
const commands = [];
|
|
1813
1779
|
for (const node of nodes) {
|
|
1814
|
-
const cmdId =
|
|
1780
|
+
const cmdId = queueCommand(node.id, null, 'diagnostics.run');
|
|
1815
1781
|
commands.push({ commandId: cmdId, nodeId: node.id });
|
|
1816
1782
|
}
|
|
1817
1783
|
res.json({ commands });
|
|
1818
1784
|
});
|
|
1819
|
-
|
|
1820
|
-
const rows =
|
|
1785
|
+
function currentDesiredStates() {
|
|
1786
|
+
const rows = db.all(`SELECT ds.* FROM desired_states ds JOIN projects p ON p.current_deployment_id=ds.deployment_id ORDER BY ds.created_at DESC`);
|
|
1821
1787
|
return rows.map((s) => JSON.parse(s.state_json));
|
|
1822
1788
|
}
|
|
1823
|
-
|
|
1789
|
+
function endpointsFor(states) {
|
|
1824
1790
|
const staleBefore = new Date(Date.now() - Number(process.env.TUG_STALE_SECONDS || 60) * 1000).toISOString();
|
|
1825
|
-
const nodes =
|
|
1791
|
+
const nodes = db.all('SELECT n.*, EXISTS(SELECT 1 FROM wg_peers w WHERE w.node_id=n.id) AS has_key FROM nodes n');
|
|
1826
1792
|
const views = nodes.map((n) => ({
|
|
1827
1793
|
name: n.name,
|
|
1828
1794
|
overlayIp: MESH && n.has_key ? n.overlay_ip : null,
|
|
1829
1795
|
address: n.address,
|
|
1830
1796
|
lastSeenAt: n.last_seen_at,
|
|
1831
1797
|
}));
|
|
1832
|
-
const statuses =
|
|
1798
|
+
const statuses = db
|
|
1799
|
+
.all('SELECT node_id,project_id,status_json FROM node_statuses')
|
|
1800
|
+
.map((r) => ({
|
|
1833
1801
|
node: nodes.find((n) => n.id === r.node_id)?.name ?? r.node_id,
|
|
1834
1802
|
project: r.project_id,
|
|
1835
1803
|
status: JSON.parse(r.status_json),
|
|
1836
1804
|
}));
|
|
1837
1805
|
return resolveServiceEndpoints(states, views, statuses, staleBefore);
|
|
1838
1806
|
}
|
|
1839
|
-
|
|
1807
|
+
function networkFor(node) {
|
|
1840
1808
|
if (!MESH || !node.overlay_ip)
|
|
1841
1809
|
return undefined;
|
|
1842
|
-
const peers =
|
|
1810
|
+
const peers = db.all('SELECT w.*, n.name FROM wg_peers w JOIN nodes n ON n.id=w.node_id WHERE w.node_id != ? ORDER BY n.name', node.id);
|
|
1843
1811
|
return {
|
|
1844
1812
|
interface: WG_INTERFACE,
|
|
1845
1813
|
overlayIp: node.overlay_ip,
|
|
@@ -1855,17 +1823,17 @@ export async function createApp(options = {}) {
|
|
|
1855
1823
|
};
|
|
1856
1824
|
}
|
|
1857
1825
|
/** Names of all nodes and of the ingress nodes, for port binding and upstream decisions. */
|
|
1858
|
-
|
|
1859
|
-
const rows =
|
|
1826
|
+
function nodeTopology() {
|
|
1827
|
+
const rows = db.all('SELECT name,ingress FROM nodes ORDER BY name');
|
|
1860
1828
|
return {
|
|
1861
1829
|
nodes: rows.map((r) => r.name),
|
|
1862
1830
|
ingressNodes: rows.filter((r) => r.ingress).map((r) => r.name),
|
|
1863
1831
|
};
|
|
1864
1832
|
}
|
|
1865
1833
|
/** Where this node's published ports listen: the address other nodes and the ingress reach it on, or loopback. */
|
|
1866
|
-
|
|
1867
|
-
const hasKey = MESH && node.overlay_ip &&
|
|
1868
|
-
const { nodes, ingressNodes } =
|
|
1834
|
+
function portBindingFor(node, states) {
|
|
1835
|
+
const hasKey = MESH && node.overlay_ip && db.get('SELECT 1 FROM wg_peers WHERE node_id=?', node.id);
|
|
1836
|
+
const { nodes, ingressNodes } = nodeTopology();
|
|
1869
1837
|
const loopback = {};
|
|
1870
1838
|
for (const state of states) {
|
|
1871
1839
|
const services = loopbackIngressServices(state, node.name, ingressNodes, nodes);
|
|
@@ -1874,7 +1842,7 @@ export async function createApp(options = {}) {
|
|
|
1874
1842
|
}
|
|
1875
1843
|
return { address: (hasKey ? node.overlay_ip : node.address) || null, loopback };
|
|
1876
1844
|
}
|
|
1877
|
-
|
|
1845
|
+
function ingressFor(node, states) {
|
|
1878
1846
|
const secrets = new Map();
|
|
1879
1847
|
for (const state of states) {
|
|
1880
1848
|
for (const svc of Object.values(state.compose.services || {})) {
|
|
@@ -1882,7 +1850,7 @@ export async function createApp(options = {}) {
|
|
|
1882
1850
|
if (ing?.tls !== 'custom')
|
|
1883
1851
|
continue;
|
|
1884
1852
|
for (const name of [ing.cert, ing.key]) {
|
|
1885
|
-
const row =
|
|
1853
|
+
const row = db.get('SELECT encrypted_value FROM secrets WHERE project_id=? AND name=?', state.project, name ?? null);
|
|
1886
1854
|
if (row)
|
|
1887
1855
|
secrets.set(`${state.project}/${name}`, decrypt(row.encrypted_value, masterKey));
|
|
1888
1856
|
}
|
|
@@ -1890,9 +1858,9 @@ export async function createApp(options = {}) {
|
|
|
1890
1858
|
}
|
|
1891
1859
|
return buildIngressWork({
|
|
1892
1860
|
states,
|
|
1893
|
-
endpoints:
|
|
1861
|
+
endpoints: endpointsFor(states),
|
|
1894
1862
|
node: node.name,
|
|
1895
|
-
...
|
|
1863
|
+
...nodeTopology(),
|
|
1896
1864
|
secrets,
|
|
1897
1865
|
acmeEmail: ACME_EMAIL,
|
|
1898
1866
|
});
|
|
@@ -1900,19 +1868,19 @@ export async function createApp(options = {}) {
|
|
|
1900
1868
|
get('/agent/v1/wake', authAgent, async (req, res) => {
|
|
1901
1869
|
const after = Math.max(0, Number.parseInt(String(req.query.after ?? '0'), 10) || 0);
|
|
1902
1870
|
const timeoutMs = Math.min(30_000, Math.max(0, Number.parseInt(String(req.query.timeout ?? '25000'), 10) || 0));
|
|
1903
|
-
|
|
1871
|
+
db.run('UPDATE nodes SET last_seen_at=?,status=? WHERE id=?', now(), 'online', req.node.id);
|
|
1904
1872
|
res.json(await wakeHub.wait(req.node.id, after, timeoutMs));
|
|
1905
1873
|
});
|
|
1906
1874
|
get('/agent/v1/work', authAgent, async (req, res) => {
|
|
1907
|
-
|
|
1908
|
-
const node =
|
|
1909
|
-
const allStates =
|
|
1875
|
+
db.run('UPDATE nodes SET last_seen_at=?,status=? WHERE id=?', now(), 'online', req.node.id);
|
|
1876
|
+
const node = db.get('SELECT * FROM nodes WHERE id=?', req.node.id);
|
|
1877
|
+
const allStates = currentDesiredStates();
|
|
1910
1878
|
const desiredStates = allStates.filter((st) => Object.values(st.services).some((svc) => (svc.placement?.nodes || []).includes(node.name) || !svc.placement?.nodes?.length));
|
|
1911
|
-
const commands =
|
|
1879
|
+
const commands = db.all('SELECT * FROM commands WHERE node_id=? AND state=?', node.id, 'pending');
|
|
1912
1880
|
if (commands.length)
|
|
1913
|
-
|
|
1881
|
+
db.run(`UPDATE commands SET state='running',attempts=attempts+1,updated_at=? WHERE node_id=? AND state='pending'`, now(), node.id);
|
|
1914
1882
|
const retainedImages = [
|
|
1915
|
-
...new Set(
|
|
1883
|
+
...new Set(retainedStates().flatMap((state) => Object.values(state.services)
|
|
1916
1884
|
.filter((service) => service.artifact &&
|
|
1917
1885
|
(!service.placement.nodes?.length || service.placement.nodes.includes(node.name)))
|
|
1918
1886
|
.map((service) => service.image))),
|
|
@@ -1924,64 +1892,81 @@ export async function createApp(options = {}) {
|
|
|
1924
1892
|
commands,
|
|
1925
1893
|
retainedImages,
|
|
1926
1894
|
};
|
|
1927
|
-
const network =
|
|
1895
|
+
const network = networkFor(node);
|
|
1928
1896
|
if (network)
|
|
1929
1897
|
work.network = network;
|
|
1930
|
-
work.ports =
|
|
1898
|
+
work.ports = portBindingFor(node, desiredStates);
|
|
1931
1899
|
if (SERVICE_DNS) {
|
|
1932
|
-
const all =
|
|
1900
|
+
const all = endpointsFor(allStates);
|
|
1933
1901
|
work.serviceEndpoints = Object.fromEntries(desiredStates.map((st) => [st.project, all[st.project] || {}]));
|
|
1934
1902
|
}
|
|
1935
1903
|
if (node.ingress)
|
|
1936
|
-
work.ingress =
|
|
1904
|
+
work.ingress = ingressFor(node, allStates);
|
|
1937
1905
|
res.json(work);
|
|
1938
1906
|
});
|
|
1939
1907
|
post('/agent/v1/status', authAgent, async (req, res) => {
|
|
1940
1908
|
const ts = now();
|
|
1941
1909
|
const report = req.body;
|
|
1942
1910
|
for (const st of report.projects || []) {
|
|
1943
|
-
|
|
1944
|
-
|
|
1911
|
+
db.run(`INSERT INTO node_statuses(id,node_id,project_id,observed_version,status_json,health,updated_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(node_id,project_id) DO UPDATE SET observed_version=excluded.observed_version,status_json=excluded.status_json,health=excluded.health,updated_at=excluded.updated_at`, id('ns'), req.node.id, st.project, st.observedVersion, JSON.stringify(st), st.health || 'unknown', ts);
|
|
1912
|
+
updateDeploymentConvergence(st.project);
|
|
1945
1913
|
}
|
|
1946
1914
|
for (const cr of (report.commandResults || [])) {
|
|
1947
1915
|
const result = cr.result || {};
|
|
1948
|
-
|
|
1916
|
+
db.run('UPDATE commands SET state=?,result_json=?,updated_at=? WHERE id=? AND node_id=?', cr.state || 'succeeded', JSON.stringify(result), ts, cr.commandId ?? null, req.node.id);
|
|
1949
1917
|
}
|
|
1950
1918
|
if (report.ingress)
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
const nextEndpointSnapshot = JSON.stringify(
|
|
1919
|
+
db.run('UPDATE nodes SET ingress_status_json=? WHERE id=?', JSON.stringify({ ...report.ingress, reportedAt: ts }), req.node.id);
|
|
1920
|
+
db.run('UPDATE nodes SET last_seen_at=?,status=? WHERE id=?', ts, 'online', req.node.id);
|
|
1921
|
+
const nextEndpointSnapshot = JSON.stringify(endpointsFor(currentDesiredStates()));
|
|
1954
1922
|
if (nextEndpointSnapshot !== endpointSnapshot) {
|
|
1955
1923
|
endpointSnapshot = nextEndpointSnapshot;
|
|
1956
|
-
|
|
1924
|
+
wakeAll();
|
|
1957
1925
|
}
|
|
1958
1926
|
res.json({ ok: true });
|
|
1959
1927
|
});
|
|
1960
1928
|
post('/agent/v1/events', authAgent, async (req, res) => {
|
|
1961
1929
|
const evs = req.body.events || [req.body];
|
|
1962
1930
|
for (const ev of evs)
|
|
1963
|
-
|
|
1931
|
+
db.run('INSERT INTO events(id,project_id,deployment_id,node_id,level,type,message,metadata_json,created_at) VALUES(?,?,?,?,?,?,?,?,?)', id('evt'), ev.projectId || null, ev.deploymentId || null, req.node.id, ev.level || 'info', ev.type || 'event', ev.message || '', JSON.stringify(ev.metadata || {}), now());
|
|
1964
1932
|
if (Date.now() - lastEventPrune >= eventPruneIntervalMs)
|
|
1965
|
-
|
|
1933
|
+
try {
|
|
1934
|
+
pruneEvents();
|
|
1935
|
+
}
|
|
1936
|
+
catch (e) {
|
|
1937
|
+
console.error(`event pruning failed: ${errMessage(e)}`);
|
|
1938
|
+
}
|
|
1966
1939
|
res.json({ ok: true });
|
|
1967
1940
|
});
|
|
1968
1941
|
get('/agent/v1/secrets', authAgent, async (req, res) => {
|
|
1969
1942
|
const project = String(req.query.project || 'site');
|
|
1970
|
-
let rows =
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1943
|
+
let rows = db.all('SELECT name,encrypted_value,version FROM secrets WHERE project_id=?', project);
|
|
1944
|
+
const ds = currentDesired(project);
|
|
1945
|
+
const desired = ds ? JSON.parse(ds.state_json) : undefined;
|
|
1946
|
+
const requested = req.query.services
|
|
1947
|
+
? String(req.query.services)
|
|
1974
1948
|
.split(',')
|
|
1975
1949
|
.map((s) => s.trim())
|
|
1976
|
-
.filter(Boolean)
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1950
|
+
.filter(Boolean)
|
|
1951
|
+
: Object.keys(desired?.services || {});
|
|
1952
|
+
const names = requested.filter((name) => desired?.services[name]?.placement.nodes?.includes(req.node.name));
|
|
1953
|
+
const allowed = serviceSecretNames(desired?.compose, names);
|
|
1954
|
+
const registrySecret = rows.find((row) => row.name === REGISTRY_AUTH_SECRET);
|
|
1955
|
+
const registryAuth = {};
|
|
1956
|
+
const credentials = parseRegistryCredentials(registrySecret ? decrypt(registrySecret.encrypted_value, masterKey) : undefined);
|
|
1957
|
+
for (const name of names) {
|
|
1958
|
+
const spec = desired.services[name];
|
|
1959
|
+
if (!spec.artifact && spec.image) {
|
|
1960
|
+
const host = imageLocation(spec.image).host;
|
|
1961
|
+
if (credentials[host])
|
|
1962
|
+
registryAuth[host] = credentials[host];
|
|
1963
|
+
}
|
|
1981
1964
|
}
|
|
1965
|
+
rows = rows.filter((row) => allowed.has(row.name) && row.name !== REGISTRY_AUTH_SECRET);
|
|
1982
1966
|
const secrets = {
|
|
1983
1967
|
secrets: Object.fromEntries(rows.map((r) => [r.name, decrypt(r.encrypted_value, masterKey)])),
|
|
1984
1968
|
versions: Object.fromEntries(rows.map((r) => [r.name, r.version])),
|
|
1969
|
+
...(Object.keys(registryAuth).length ? { registryAuth } : {}),
|
|
1985
1970
|
};
|
|
1986
1971
|
res.json(secrets);
|
|
1987
1972
|
});
|
|
@@ -2010,7 +1995,7 @@ function attachAgentStream(app, server) {
|
|
|
2010
1995
|
const raw = String(request.headers.authorization || '').replace(/^Bearer /, '');
|
|
2011
1996
|
void (async () => {
|
|
2012
1997
|
if (pathname === '/api/v1/live') {
|
|
2013
|
-
const actor =
|
|
1998
|
+
const actor = runtime.authenticateOperator(raw);
|
|
2014
1999
|
if (!actor) {
|
|
2015
2000
|
reject(401, 'Unauthorized');
|
|
2016
2001
|
return;
|
|
@@ -2022,7 +2007,7 @@ function attachAgentStream(app, server) {
|
|
|
2022
2007
|
wss.handleUpgrade(request, socket, head, (ws) => runtime.live.operator(actor, ws));
|
|
2023
2008
|
}
|
|
2024
2009
|
else {
|
|
2025
|
-
const node =
|
|
2010
|
+
const node = runtime.authenticate(raw);
|
|
2026
2011
|
if (!node) {
|
|
2027
2012
|
reject(401, 'Unauthorized');
|
|
2028
2013
|
return;
|