@wpelevator/tug 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -8
- package/dist/agent.d.ts +0 -2
- package/dist/agent.js +8 -30
- package/dist/build.d.ts +2 -3
- package/dist/build.js +1 -3
- package/dist/cli-build.d.ts +2 -2
- package/dist/cli-build.js +3 -4
- package/dist/cli.js +2 -33
- package/dist/compose.d.ts +1 -2
- package/dist/compose.js +2 -4
- package/dist/control.d.ts +4 -2
- package/dist/control.js +342 -366
- package/dist/db-driver.d.ts +20 -0
- package/dist/db-driver.js +11 -0
- package/dist/db-node.d.ts +3 -0
- package/dist/db-node.js +60 -0
- package/dist/db.d.ts +9 -6
- package/dist/db.js +40 -26
- package/dist/docker-api.js +15 -3
- package/dist/protocol.d.ts +5 -5
- package/dist/registry.d.ts +17 -0
- package/dist/registry.js +133 -0
- package/dist/templates/control-compose.yml +0 -4
- package/dist/templates/node-install.sh +0 -1
- package/dist/tokens.d.ts +7 -7
- package/dist/tokens.js +24 -21
- package/package.json +1 -3
package/dist/control.js
CHANGED
|
@@ -1,22 +1,22 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import express from 'express';
|
|
3
|
+
import { resolveImage, parseRegistryCredentials, imageLocation, REGISTRY_AUTH_SECRET, } from "./registry.js";
|
|
3
4
|
import multer from 'multer';
|
|
4
5
|
import fs from 'node:fs/promises';
|
|
5
6
|
import { createReadStream } from 'node:fs';
|
|
6
7
|
import { pipeline } from 'node:stream/promises';
|
|
7
8
|
import { saveImageArchive, inspectImage, validateImageArchive, hashArchive } from "./artifacts.js";
|
|
8
|
-
import sqlite3 from 'sqlite3';
|
|
9
|
-
import { open } from 'sqlite';
|
|
10
9
|
import { scanArchive } from "./archive.js";
|
|
11
10
|
import https from 'node:https';
|
|
12
11
|
import http from 'node:http';
|
|
13
12
|
import path from 'node:path';
|
|
14
13
|
import { WebSocketServer, WebSocket } from 'ws';
|
|
15
14
|
import * as tar from 'tar';
|
|
16
|
-
import { openDb } from "./db.js";
|
|
15
|
+
import { openDb, openSnapshotDb } from "./db.js";
|
|
16
|
+
import { isUniqueViolation } from "./db-driver.js";
|
|
17
17
|
import { encrypt, decrypt, loadOrCreateMasterKey } from "./secrets.js";
|
|
18
18
|
import { buildDesiredState, loopbackIngressServices, serviceSecretNames, validateCompose, } from "./compose.js";
|
|
19
|
-
import {
|
|
19
|
+
import { buildImage, unpackContext, run as runCommand } from "./build.js";
|
|
20
20
|
import { sentinelNames } from "./normalize.js";
|
|
21
21
|
import { ensureDir, errMessage, hashToken, id, now, token } from "./util.js";
|
|
22
22
|
import { DEFAULT_MESH_CIDR, DEFAULT_WG_PORT, WG_INTERFACE, WG_KEEPALIVE, allocateOverlayIp, isWgKey, parseCidr, wgEndpoint, } from "./mesh.js";
|
|
@@ -394,7 +394,8 @@ const ACTIVE_STATES_SQL = ACTIVE_DEPLOYMENT_STATES.map((s) => `'${s}'`).join(','
|
|
|
394
394
|
const PORT = Number(process.env.PORT || 8080);
|
|
395
395
|
export async function createApp(options = {}) {
|
|
396
396
|
const images = {
|
|
397
|
-
|
|
397
|
+
resolve: resolveImage,
|
|
398
|
+
build: buildImage,
|
|
398
399
|
save: saveImageArchive,
|
|
399
400
|
inspect: inspectImage,
|
|
400
401
|
validate: validateImageArchive,
|
|
@@ -411,9 +412,6 @@ export async function createApp(options = {}) {
|
|
|
411
412
|
});
|
|
412
413
|
const dbFile = path.join(DATA_DIR, 'tug.db');
|
|
413
414
|
const masterKeyFile = path.join(DATA_DIR, 'master.key');
|
|
414
|
-
const REGISTRY = process.env.TUG_REGISTRY?.trim() || '';
|
|
415
|
-
const REGISTRY_AUTH_PATH = process.env.TUG_REGISTRY_AUTH_PATH || null;
|
|
416
|
-
const REGISTRY_CONTAINER = process.env.TUG_REGISTRY_CONTAINER || null;
|
|
417
415
|
// Networking layers are opt-in; with all of them off the work response is unchanged.
|
|
418
416
|
const MESH = process.env.TUG_MESH === 'true';
|
|
419
417
|
const MESH_CIDR = process.env.TUG_MESH_CIDR || DEFAULT_MESH_CIDR;
|
|
@@ -422,56 +420,6 @@ export async function createApp(options = {}) {
|
|
|
422
420
|
const ACME_EMAIL = process.env.TUG_ACME_EMAIL || undefined;
|
|
423
421
|
if (MESH)
|
|
424
422
|
parseCidr(MESH_CIDR);
|
|
425
|
-
let htpasswdQueue = Promise.resolve();
|
|
426
|
-
function withHtpasswdLock(fn) {
|
|
427
|
-
const next = htpasswdQueue.catch(() => { }).then(fn);
|
|
428
|
-
htpasswdQueue = next.then(() => { }, () => { });
|
|
429
|
-
return next;
|
|
430
|
-
}
|
|
431
|
-
async function reloadRegistry() {
|
|
432
|
-
if (!REGISTRY_CONTAINER)
|
|
433
|
-
return;
|
|
434
|
-
await runCommand('docker', ['restart', REGISTRY_CONTAINER]).catch((e) => console.error(`registry reload failed: ${e.message}`));
|
|
435
|
-
}
|
|
436
|
-
async function upsertRegistryCredential(name, password) {
|
|
437
|
-
if (!REGISTRY || !REGISTRY_AUTH_PATH)
|
|
438
|
-
return;
|
|
439
|
-
await withHtpasswdLock(async () => {
|
|
440
|
-
const { out } = await runCommand('htpasswd', ['-nbB', name, password]);
|
|
441
|
-
const line = out.trim();
|
|
442
|
-
let existing = '';
|
|
443
|
-
try {
|
|
444
|
-
existing = await fs.readFile(REGISTRY_AUTH_PATH, 'utf8');
|
|
445
|
-
}
|
|
446
|
-
catch { }
|
|
447
|
-
const lines = existing
|
|
448
|
-
.split('\n')
|
|
449
|
-
.filter(Boolean)
|
|
450
|
-
.filter((l) => !l.startsWith(`${name}:`));
|
|
451
|
-
lines.push(line);
|
|
452
|
-
await ensureDir(path.dirname(REGISTRY_AUTH_PATH));
|
|
453
|
-
await fs.writeFile(REGISTRY_AUTH_PATH, `${lines.join('\n')}\n`);
|
|
454
|
-
});
|
|
455
|
-
await reloadRegistry();
|
|
456
|
-
}
|
|
457
|
-
async function removeRegistryCredential(name) {
|
|
458
|
-
if (!REGISTRY_AUTH_PATH)
|
|
459
|
-
return;
|
|
460
|
-
await withHtpasswdLock(async () => {
|
|
461
|
-
let existing = '';
|
|
462
|
-
try {
|
|
463
|
-
existing = await fs.readFile(REGISTRY_AUTH_PATH, 'utf8');
|
|
464
|
-
}
|
|
465
|
-
catch { }
|
|
466
|
-
const lines = existing
|
|
467
|
-
.split('\n')
|
|
468
|
-
.filter(Boolean)
|
|
469
|
-
.filter((l) => !l.startsWith(`${name}:`));
|
|
470
|
-
await ensureDir(path.dirname(REGISTRY_AUTH_PATH));
|
|
471
|
-
await fs.writeFile(REGISTRY_AUTH_PATH, lines.length ? `${lines.join('\n')}\n` : '');
|
|
472
|
-
});
|
|
473
|
-
await reloadRegistry();
|
|
474
|
-
}
|
|
475
423
|
const db = await (options.openDatabase || openDb)(dbFile);
|
|
476
424
|
const wakeHub = new WakeHub();
|
|
477
425
|
// Events are an operator-facing log, not a record of truth, so keep them bounded. `0` turns a limit off; anything that is
|
|
@@ -490,32 +438,37 @@ export async function createApp(options = {}) {
|
|
|
490
438
|
const EVENT_MAX_ROWS = envCount('TUG_EVENT_MAX_ROWS', 100_000);
|
|
491
439
|
const eventPruneIntervalMs = options.eventPruneIntervalMs ?? 60 * 60 * 1000;
|
|
492
440
|
let lastEventPrune = Date.now();
|
|
493
|
-
|
|
441
|
+
function pruneEvents() {
|
|
494
442
|
lastEventPrune = Date.now();
|
|
495
443
|
let removed = 0;
|
|
496
444
|
if (EVENT_RETENTION_DAYS > 0)
|
|
497
445
|
removed +=
|
|
498
|
-
|
|
446
|
+
db.run('DELETE FROM events WHERE created_at < ?', new Date(Date.now() - EVENT_RETENTION_DAYS * 86_400_000).toISOString()).changes ?? 0;
|
|
499
447
|
if (EVENT_MAX_ROWS > 0)
|
|
500
448
|
removed +=
|
|
501
|
-
|
|
449
|
+
db.run('DELETE FROM events WHERE id IN (SELECT id FROM events ORDER BY created_at DESC, id DESC LIMIT -1 OFFSET ?)', EVENT_MAX_ROWS).changes ?? 0;
|
|
502
450
|
if (removed)
|
|
503
451
|
console.log(`removed ${removed} old events (kept ${EVENT_RETENTION_DAYS || 'any'} days, at most ${EVENT_MAX_ROWS || 'unlimited'} rows)`);
|
|
504
452
|
return removed;
|
|
505
453
|
}
|
|
506
|
-
|
|
454
|
+
try {
|
|
455
|
+
pruneEvents();
|
|
456
|
+
}
|
|
457
|
+
catch (e) {
|
|
458
|
+
console.error(`event pruning failed: ${errMessage(e)}`);
|
|
459
|
+
}
|
|
507
460
|
const masterKey = await loadOrCreateMasterKey(masterKeyFile);
|
|
508
|
-
const operator =
|
|
461
|
+
const operator = db.get('SELECT value FROM meta WHERE key=?', 'operator_token_hash');
|
|
509
462
|
if (!operator) {
|
|
510
463
|
const t = process.env.TUG_OPERATOR_TOKEN || token('tug_op');
|
|
511
|
-
|
|
464
|
+
db.run('INSERT INTO meta(key,value) VALUES(?,?)', 'operator_token_hash', hashToken(t));
|
|
512
465
|
await fs
|
|
513
466
|
.writeFile(path.join(DATA_DIR, 'operator.token'), `${t}\n`, { mode: 0o600 })
|
|
514
467
|
.catch(() => { });
|
|
515
468
|
console.log(`operator token: ${t}`);
|
|
516
469
|
}
|
|
517
470
|
const app = express();
|
|
518
|
-
const authenticateAgent =
|
|
471
|
+
const authenticateAgent = (raw) => raw
|
|
519
472
|
? db.get('SELECT * FROM nodes WHERE agent_token_hash=?', hashToken(raw))
|
|
520
473
|
: undefined;
|
|
521
474
|
let restartRequired = false;
|
|
@@ -579,13 +532,21 @@ export async function createApp(options = {}) {
|
|
|
579
532
|
apiRoutes.set(key, role);
|
|
580
533
|
route(method, url, [guard(role, resolve ?? undefined), ...handlers]);
|
|
581
534
|
};
|
|
582
|
-
const deploymentProject =
|
|
583
|
-
|
|
584
|
-
|
|
535
|
+
const deploymentProject = (req) => db.get('SELECT project_id FROM deployments WHERE id=?', req.params.id)
|
|
536
|
+
?.project_id;
|
|
537
|
+
const commandProject = (req) => {
|
|
538
|
+
const row = db.get('SELECT project_id FROM commands WHERE id=?', req.params.id);
|
|
585
539
|
return row ? row.project_id : undefined;
|
|
586
540
|
};
|
|
587
541
|
/** Audit trail entry for a control action; bookkeeping failures are logged, never returned to the caller. */
|
|
588
|
-
const audit = (actor, event) =>
|
|
542
|
+
const audit = (actor, event) => {
|
|
543
|
+
try {
|
|
544
|
+
recordEvent(db, { ...event, actor: actor.name });
|
|
545
|
+
}
|
|
546
|
+
catch (e) {
|
|
547
|
+
console.error(`audit event failed: ${errMessage(e)}`);
|
|
548
|
+
}
|
|
549
|
+
};
|
|
589
550
|
const projectScope = (actor, column) => actor.projects
|
|
590
551
|
? {
|
|
591
552
|
where: `${column} IN (${actor.projects.map(() => '?').join(',') || "''"})`,
|
|
@@ -594,22 +555,22 @@ export async function createApp(options = {}) {
|
|
|
594
555
|
: { where: '1=1', args: [] };
|
|
595
556
|
const authAgent = async (req, res, next) => {
|
|
596
557
|
const raw = (req.headers.authorization || '').replace(/^Bearer /, '');
|
|
597
|
-
const node =
|
|
558
|
+
const node = authenticateAgent(raw);
|
|
598
559
|
if (!node)
|
|
599
560
|
return res.status(401).json({ error: 'unauthorized' });
|
|
600
561
|
req.node = node;
|
|
601
562
|
next();
|
|
602
563
|
};
|
|
603
|
-
const wakeAll =
|
|
604
|
-
for (const node of
|
|
564
|
+
const wakeAll = (project) => {
|
|
565
|
+
for (const node of db.all('SELECT id FROM nodes'))
|
|
605
566
|
wakeHub.wake(node.id, project);
|
|
606
567
|
};
|
|
607
568
|
let endpointSnapshot = '';
|
|
608
|
-
|
|
609
|
-
const project =
|
|
569
|
+
function updateDeploymentConvergence(projectId) {
|
|
570
|
+
const project = db.get('SELECT * FROM projects WHERE id=?', projectId);
|
|
610
571
|
if (!project?.current_deployment_id)
|
|
611
572
|
return;
|
|
612
|
-
const ds =
|
|
573
|
+
const ds = db.get('SELECT * FROM desired_states WHERE deployment_id=?', project.current_deployment_id);
|
|
613
574
|
if (!ds)
|
|
614
575
|
return;
|
|
615
576
|
const state = JSON.parse(ds.state_json);
|
|
@@ -619,7 +580,7 @@ export async function createApp(options = {}) {
|
|
|
619
580
|
required.add(node);
|
|
620
581
|
if (!required.size)
|
|
621
582
|
return;
|
|
622
|
-
const statuses =
|
|
583
|
+
const statuses = db.all('SELECT node_id,observed_version,health,status_json FROM node_statuses WHERE project_id=?', projectId);
|
|
623
584
|
const byNode = new Map(statuses.map((s) => [s.node_id, s]));
|
|
624
585
|
const reportsRevision = (node) => state.revision
|
|
625
586
|
? JSON.parse(byNode.get(node)?.status_json || '{}').observedRevision === state.revision
|
|
@@ -628,7 +589,7 @@ export async function createApp(options = {}) {
|
|
|
628
589
|
.filter((node) => reportsRevision(node) && byNode.get(node)?.health === 'failed')
|
|
629
590
|
.map((node) => byNode.get(node));
|
|
630
591
|
if (failedNodes.length) {
|
|
631
|
-
const marked =
|
|
592
|
+
const marked = db.run('UPDATE deployments SET state=?,error_message=?,updated_at=? WHERE id=? AND state=?', 'failed', deploymentFailureMessage(failedNodes), now(), project.current_deployment_id, 'rolling_out');
|
|
632
593
|
if (marked.changes === 1)
|
|
633
594
|
scheduleAutoRollback(projectId, project.current_deployment_id, state);
|
|
634
595
|
}
|
|
@@ -637,29 +598,29 @@ export async function createApp(options = {}) {
|
|
|
637
598
|
: byNode.get(node)?.observed_version === state.version) &&
|
|
638
599
|
['serving', 'skipped'].includes(byNode.get(node)?.health ?? ''));
|
|
639
600
|
if (ok)
|
|
640
|
-
|
|
601
|
+
db.run('UPDATE deployments SET state=?,updated_at=? WHERE id=? AND state=?', 'succeeded', now(), project.current_deployment_id, 'rolling_out');
|
|
641
602
|
}
|
|
642
603
|
/** Events control writes when it rolls back on its own; they are attributed to a reserved system actor. */
|
|
643
|
-
|
|
644
|
-
|
|
604
|
+
function recordAutoRollbackEvent(projectId, deploymentId, type, message, metadata = {}) {
|
|
605
|
+
db.run('INSERT INTO events(id,project_id,deployment_id,node_id,level,type,message,metadata_json,created_at,actor) VALUES(?,?,?,?,?,?,?,?,?,?)', id('evt'), projectId, deploymentId, null, 'warn', type, message, JSON.stringify(metadata), now(), AUTO_ROLLBACK_ACTOR);
|
|
645
606
|
}
|
|
646
607
|
/**
|
|
647
608
|
* Creates a rollback deployment that copies the most recent earlier deployment's desired state under a
|
|
648
609
|
* new revision. Shared by `POST /api/v1/rollback` and automatic rollback; the caller holds the writer queue.
|
|
649
610
|
* `auto` only considers `succeeded` deployments and requires `failedDeploymentId` to still be current.
|
|
650
611
|
*/
|
|
651
|
-
|
|
612
|
+
function createRollback(project, opts) {
|
|
652
613
|
let lock;
|
|
653
614
|
try {
|
|
654
|
-
lock =
|
|
655
|
-
const active =
|
|
615
|
+
lock = acquireProjectLock(project, opts.auto ? 'auto-rollback' : 'rollback');
|
|
616
|
+
const active = db.get(`SELECT id,state FROM deployments WHERE project_id=? AND state IN (${ACTIVE_STATES_SQL}) ORDER BY created_at DESC LIMIT 1`, project);
|
|
656
617
|
if (active && !opts.force)
|
|
657
618
|
throw httpError(409, `deployment ${active.id} is already ${active.state}; retry with force to rollback anyway`);
|
|
658
|
-
const current =
|
|
619
|
+
const current = db.get('SELECT current_deployment_id FROM projects WHERE id=?', project);
|
|
659
620
|
if (opts.failedDeploymentId && current?.current_deployment_id !== opts.failedDeploymentId)
|
|
660
621
|
throw httpError(409, `deployment ${opts.failedDeploymentId} is no longer the current deployment`);
|
|
661
622
|
const states = opts.auto ? `'succeeded'` : `'succeeded','rolling_out'`;
|
|
662
|
-
const previousDs =
|
|
623
|
+
const previousDs = db.get(`SELECT ds.*, d.version AS deployment_version FROM desired_states ds JOIN deployments d ON d.id=ds.deployment_id
|
|
663
624
|
WHERE ds.project_id=? AND ds.deployment_id != ? AND d.state IN (${states}) ORDER BY ds.created_at DESC LIMIT 1`, project, current?.current_deployment_id || '');
|
|
664
625
|
if (!previousDs)
|
|
665
626
|
throw httpError(404, 'no previous deployment found');
|
|
@@ -667,17 +628,17 @@ export async function createApp(options = {}) {
|
|
|
667
628
|
const depId = id('dep');
|
|
668
629
|
const previousState = JSON.parse(previousDs.state_json);
|
|
669
630
|
previousState.revision = depId;
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
631
|
+
db.run('INSERT INTO deployments(id,project_id,version,state,created_at,updated_at,actor) VALUES(?,?,?,?,?,?,?)', depId, project, `${ROLLBACK_VERSION_PREFIX}${previousState.version}-${Date.now()}`, 'rolling_out', ts, ts, opts.actor);
|
|
632
|
+
db.run('INSERT INTO desired_states(id,project_id,deployment_id,version,state_json,created_at) VALUES(?,?,?,?,?,?)', id('ds'), project, depId, previousState.version, JSON.stringify(previousState), ts);
|
|
633
|
+
db.run('UPDATE projects SET current_deployment_id=?,updated_at=? WHERE id=?', depId, now(), project);
|
|
634
|
+
wakeAll(project);
|
|
674
635
|
return {
|
|
675
|
-
deployment:
|
|
636
|
+
deployment: db.get('SELECT * FROM deployments WHERE id=?', depId),
|
|
676
637
|
desiredState: previousState,
|
|
677
638
|
};
|
|
678
639
|
}
|
|
679
640
|
finally {
|
|
680
|
-
|
|
641
|
+
releaseLock(lock);
|
|
681
642
|
}
|
|
682
643
|
}
|
|
683
644
|
/**
|
|
@@ -690,9 +651,9 @@ export async function createApp(options = {}) {
|
|
|
690
651
|
if (state.compose['x-tug']?.rollback !== 'auto')
|
|
691
652
|
return;
|
|
692
653
|
void (async () => {
|
|
693
|
-
const failed =
|
|
654
|
+
const failed = db.get('SELECT * FROM deployments WHERE id=?', deploymentId);
|
|
694
655
|
if (failed?.version.startsWith(ROLLBACK_VERSION_PREFIX)) {
|
|
695
|
-
|
|
656
|
+
recordAutoRollbackEvent(projectId, deploymentId, 'deployment.auto_rollback_skipped', `rollback deployment ${failed.version} failed; not rolling back a rollback`);
|
|
696
657
|
return;
|
|
697
658
|
}
|
|
698
659
|
try {
|
|
@@ -701,27 +662,32 @@ export async function createApp(options = {}) {
|
|
|
701
662
|
auto: true,
|
|
702
663
|
failedDeploymentId: deploymentId,
|
|
703
664
|
}));
|
|
704
|
-
|
|
665
|
+
recordAutoRollbackEvent(projectId, out.deployment.id, 'deployment.auto_rollback', `deployment ${deploymentId} (${failed?.version}) reported failed health; rolled back to ${out.desiredState.version} as deployment ${out.deployment.id}`, {
|
|
705
666
|
failedDeploymentId: deploymentId,
|
|
706
667
|
rollbackDeploymentId: out.deployment.id,
|
|
707
668
|
version: out.desiredState.version,
|
|
708
669
|
});
|
|
709
670
|
}
|
|
710
671
|
catch (e) {
|
|
711
|
-
|
|
672
|
+
try {
|
|
673
|
+
recordAutoRollbackEvent(projectId, deploymentId, 'deployment.auto_rollback_skipped', `deployment ${deploymentId} failed but automatic rollback was not possible: ${errMessage(e)}`);
|
|
674
|
+
}
|
|
675
|
+
catch {
|
|
676
|
+
/* Best-effort audit event. */
|
|
677
|
+
}
|
|
712
678
|
}
|
|
713
679
|
})().catch((e) => console.error('auto rollback failed:', errMessage(e)));
|
|
714
680
|
}
|
|
715
|
-
|
|
681
|
+
function currentDesired(projectId) {
|
|
716
682
|
return db.get('SELECT ds.* FROM desired_states ds JOIN projects p ON p.current_deployment_id=ds.deployment_id WHERE p.id=?', projectId);
|
|
717
683
|
}
|
|
718
684
|
const live = new LiveHub(async (actor, request) => {
|
|
719
685
|
assertProject(actor, request.project);
|
|
720
|
-
const row =
|
|
686
|
+
const row = currentDesired(request.project);
|
|
721
687
|
if (!row)
|
|
722
688
|
throw httpError(404, missingProject(request.project));
|
|
723
689
|
const desired = JSON.parse(row.state_json);
|
|
724
|
-
const nodes =
|
|
690
|
+
const nodes = db.all('SELECT * FROM nodes');
|
|
725
691
|
const eligible = new Set();
|
|
726
692
|
const services = request.operation === 'ps'
|
|
727
693
|
? Object.values(desired.services)
|
|
@@ -750,38 +716,44 @@ export async function createApp(options = {}) {
|
|
|
750
716
|
api('get', '/api/v1/live', null, (_req, res) => {
|
|
751
717
|
res.status(426).json({ error: 'WebSocket upgrade required' });
|
|
752
718
|
});
|
|
753
|
-
|
|
719
|
+
function acquireProjectLock(project, operation, ttlMs = Number(process.env.TUG_LOCK_TTL_MS || 10 * 60 * 1000)) {
|
|
754
720
|
const name = `project:${project}`;
|
|
755
721
|
const ts = now();
|
|
756
|
-
|
|
722
|
+
db.run('DELETE FROM locks WHERE expires_at < ?', ts);
|
|
757
723
|
const owner = `${operation}:${id('lock')}`;
|
|
758
724
|
try {
|
|
759
|
-
|
|
725
|
+
db.run('INSERT INTO locks(name,owner,expires_at,created_at) VALUES(?,?,?,?)', name, owner, new Date(Date.now() + ttlMs).toISOString(), ts);
|
|
760
726
|
}
|
|
761
727
|
catch (e) {
|
|
762
|
-
if (
|
|
728
|
+
if (!isUniqueViolation(e))
|
|
763
729
|
throw e;
|
|
764
|
-
const existing =
|
|
730
|
+
const existing = db.get('SELECT * FROM locks WHERE name=?', name);
|
|
765
731
|
const err = new Error(`project ${project} is locked by ${existing?.owner} until ${existing?.expires_at}`);
|
|
766
732
|
err.status = 409;
|
|
767
733
|
err.lock = existing;
|
|
768
734
|
throw err;
|
|
769
735
|
}
|
|
770
736
|
const heartbeat = setInterval(() => {
|
|
771
|
-
|
|
772
|
-
.run('UPDATE locks SET expires_at=? WHERE name=? AND owner=?', new Date(Date.now() + ttlMs).toISOString(), name, owner)
|
|
773
|
-
|
|
737
|
+
try {
|
|
738
|
+
db.run('UPDATE locks SET expires_at=? WHERE name=? AND owner=?', new Date(Date.now() + ttlMs).toISOString(), name, owner);
|
|
739
|
+
}
|
|
740
|
+
catch {
|
|
741
|
+
/* Best-effort bookkeeping. */
|
|
742
|
+
}
|
|
774
743
|
}, Math.max(10, Math.floor(ttlMs / 3)));
|
|
775
744
|
heartbeat.unref();
|
|
776
745
|
return { name, owner, heartbeat };
|
|
777
746
|
}
|
|
778
|
-
|
|
747
|
+
function releaseLock(lock) {
|
|
779
748
|
if (lock?.heartbeat)
|
|
780
749
|
clearInterval(lock.heartbeat);
|
|
781
750
|
if (lock)
|
|
782
|
-
|
|
783
|
-
.run('DELETE FROM locks WHERE name=? AND owner=?', lock.name, lock.owner)
|
|
784
|
-
|
|
751
|
+
try {
|
|
752
|
+
db.run('DELETE FROM locks WHERE name=? AND owner=?', lock.name, lock.owner);
|
|
753
|
+
}
|
|
754
|
+
catch {
|
|
755
|
+
/* Best-effort bookkeeping. */
|
|
756
|
+
}
|
|
785
757
|
}
|
|
786
758
|
function sendError(res, e) {
|
|
787
759
|
const err = e;
|
|
@@ -796,13 +768,13 @@ export async function createApp(options = {}) {
|
|
|
796
768
|
if (!project || !service || !Number.isInteger(replicas) || replicas < 1)
|
|
797
769
|
throw httpError(400, 'project, service and a positive integer replicas are required');
|
|
798
770
|
assertProject(req.actor, project);
|
|
799
|
-
const dsRow =
|
|
771
|
+
const dsRow = currentDesired(project);
|
|
800
772
|
if (!dsRow)
|
|
801
773
|
throw httpError(404, `project ${project} has no desired state`);
|
|
802
774
|
const state = JSON.parse(dsRow.state_json);
|
|
803
775
|
if (!state.services?.[service])
|
|
804
776
|
throw httpError(404, `service ${service} not found in project ${project}`);
|
|
805
|
-
const nodeRows =
|
|
777
|
+
const nodeRows = db.all('SELECT name,labels_json,last_seen_at FROM nodes ORDER BY created_at,name');
|
|
806
778
|
const nodeInfos = nodeRows
|
|
807
779
|
.filter((row) => !isStale(row))
|
|
808
780
|
.map((row) => ({
|
|
@@ -827,8 +799,8 @@ export async function createApp(options = {}) {
|
|
|
827
799
|
};
|
|
828
800
|
state.services[service].replicas = selected.length;
|
|
829
801
|
state.revision = id('rev');
|
|
830
|
-
|
|
831
|
-
|
|
802
|
+
db.run('UPDATE desired_states SET state_json=? WHERE id=?', JSON.stringify(state), dsRow.id);
|
|
803
|
+
wakeAll(project);
|
|
832
804
|
res.json({
|
|
833
805
|
ok: true,
|
|
834
806
|
project,
|
|
@@ -841,59 +813,59 @@ export async function createApp(options = {}) {
|
|
|
841
813
|
api('post', '/api/v1/projects', (req) => String(req.body?.name || 'site'), async (req, res) => {
|
|
842
814
|
const name = req.body.name || 'site';
|
|
843
815
|
const ts = now();
|
|
844
|
-
|
|
845
|
-
res.json(
|
|
816
|
+
db.run('INSERT OR IGNORE INTO projects(id,name,created_at,updated_at) VALUES(?,?,?,?)', name, name, ts, ts);
|
|
817
|
+
res.json(db.get('SELECT * FROM projects WHERE id=?', name));
|
|
846
818
|
});
|
|
847
|
-
api('get', '/api/v1/projects/:project', (req) => req.params.project, async (req, res) => res.json(
|
|
819
|
+
api('get', '/api/v1/projects/:project', (req) => req.params.project, async (req, res) => res.json(db.get('SELECT * FROM projects WHERE id=?', req.params.project)));
|
|
848
820
|
/** A node whose agent has been seen before but not within TUG_STALE_SECONDS; never-seen (just registered) nodes stay schedulable. */
|
|
849
821
|
const isStale = (row) => !!row.last_seen_at &&
|
|
850
822
|
Date.parse(row.last_seen_at) < Date.now() - Number(process.env.TUG_STALE_SECONDS || 60) * 1000;
|
|
851
823
|
const NODE_INFO_SQL = 'SELECT id,name,address,public_address,overlay_ip,labels_json,status,created_at,updated_at FROM nodes WHERE id=?';
|
|
852
824
|
/** Allocates the node's stable overlay IP on first registration (retrying if a concurrent register took it). */
|
|
853
|
-
|
|
825
|
+
function ensureOverlayIp(nodeId) {
|
|
854
826
|
if (!MESH)
|
|
855
827
|
return;
|
|
856
828
|
for (let attempt = 0; attempt < 5; attempt++) {
|
|
857
|
-
const row =
|
|
829
|
+
const row = db.get('SELECT overlay_ip FROM nodes WHERE id=?', nodeId);
|
|
858
830
|
if (row?.overlay_ip)
|
|
859
831
|
return;
|
|
860
|
-
const used =
|
|
832
|
+
const used = db
|
|
833
|
+
.all('SELECT overlay_ip FROM nodes WHERE overlay_ip IS NOT NULL')
|
|
834
|
+
.map((r) => r.overlay_ip);
|
|
861
835
|
try {
|
|
862
|
-
|
|
836
|
+
db.run('UPDATE nodes SET overlay_ip=? WHERE id=? AND overlay_ip IS NULL', allocateOverlayIp(MESH_CIDR, used), nodeId);
|
|
863
837
|
return;
|
|
864
838
|
}
|
|
865
839
|
catch (e) {
|
|
866
|
-
if (
|
|
840
|
+
if (!isUniqueViolation(e))
|
|
867
841
|
throw e;
|
|
868
842
|
}
|
|
869
843
|
}
|
|
870
844
|
throw new Error('could not allocate an overlay address');
|
|
871
845
|
}
|
|
872
846
|
/** Stores (or refreshes) the node's WireGuard peer entry; the private key never reaches control. */
|
|
873
|
-
|
|
847
|
+
function syncWgPeer(nodeId, publicKey) {
|
|
874
848
|
if (!MESH)
|
|
875
849
|
return;
|
|
876
|
-
const node =
|
|
877
|
-
const existing =
|
|
850
|
+
const node = db.get('SELECT * FROM nodes WHERE id=?', nodeId);
|
|
851
|
+
const existing = db.get('SELECT * FROM wg_peers WHERE node_id=?', nodeId);
|
|
878
852
|
const key = publicKey ?? existing?.public_key;
|
|
879
853
|
if (!node?.overlay_ip || !key)
|
|
880
854
|
return;
|
|
881
|
-
|
|
855
|
+
db.run(`INSERT INTO wg_peers(node_id,public_key,overlay_ip,endpoint,allowed_ips,updated_at) VALUES(?,?,?,?,?,?)
|
|
882
856
|
ON CONFLICT(node_id) DO UPDATE SET public_key=excluded.public_key,overlay_ip=excluded.overlay_ip,endpoint=excluded.endpoint,allowed_ips=excluded.allowed_ips,updated_at=excluded.updated_at`, nodeId, key, node.overlay_ip, wgEndpoint(node.public_address, WG_PORT) ?? null, `${node.overlay_ip}/32`, now());
|
|
883
857
|
}
|
|
884
858
|
async function registerNode({ name, address, publicAddress, wgPublicKey, labels, }) {
|
|
885
859
|
if (wgPublicKey !== undefined && !isWgKey(wgPublicKey))
|
|
886
860
|
throw httpError(400, 'wgPublicKey must be a base64 WireGuard public key');
|
|
887
861
|
const agentToken = token('tug_agent');
|
|
888
|
-
const registryToken = token('tug_reg');
|
|
889
862
|
const ts = now();
|
|
890
|
-
|
|
891
|
-
VALUES(
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
const row = await db.get(NODE_INFO_SQL, name);
|
|
863
|
+
db.run(`INSERT INTO nodes(id,name,address,public_address,status,labels_json,agent_token_hash,created_at,updated_at)
|
|
864
|
+
VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET name=excluded.name,address=excluded.address,public_address=excluded.public_address,labels_json=excluded.labels_json,agent_token_hash=excluded.agent_token_hash,updated_at=excluded.updated_at`, name, name, address || null, publicAddress || null, 'registered', JSON.stringify(Array.isArray(labels) ? labels : []), hashToken(agentToken), ts, ts);
|
|
865
|
+
ensureOverlayIp(name);
|
|
866
|
+
syncWgPeer(name, wgPublicKey);
|
|
867
|
+
wakeAll();
|
|
868
|
+
const row = db.get(NODE_INFO_SQL, name);
|
|
897
869
|
if (!row)
|
|
898
870
|
throw httpError(500, 'node not found after registration');
|
|
899
871
|
const node = {
|
|
@@ -907,7 +879,7 @@ export async function createApp(options = {}) {
|
|
|
907
879
|
created_at: row.created_at,
|
|
908
880
|
updated_at: row.updated_at,
|
|
909
881
|
};
|
|
910
|
-
return { node, agentToken
|
|
882
|
+
return { node, agentToken };
|
|
911
883
|
}
|
|
912
884
|
api('post', '/api/v1/nodes/register', null, async (req, res) => {
|
|
913
885
|
const name = req.body.name;
|
|
@@ -934,18 +906,18 @@ export async function createApp(options = {}) {
|
|
|
934
906
|
const uses = Math.max(1, Number(req.body.uses ?? 1));
|
|
935
907
|
const enrollToken = token('tug_enroll');
|
|
936
908
|
const expiresAt = new Date(Date.now() + ttlSeconds * 1000).toISOString();
|
|
937
|
-
|
|
938
|
-
res.json({ enrollToken, name, usesRemaining: uses, expiresAt
|
|
909
|
+
db.run('INSERT INTO enroll_tokens(token_hash,name,uses_remaining,expires_at,created_at) VALUES(?,?,?,?,?)', hashToken(enrollToken), name, uses, expiresAt, now());
|
|
910
|
+
res.json({ enrollToken, name, usesRemaining: uses, expiresAt });
|
|
939
911
|
});
|
|
940
912
|
post('/agent/v1/enroll', async (req, res) => {
|
|
941
913
|
const body = req.body;
|
|
942
914
|
const raw = String(body.enrollToken || (req.headers.authorization || '').replace(/^Bearer /, '')).trim();
|
|
943
915
|
if (!raw)
|
|
944
916
|
return res.status(400).json({ error: 'enrollToken required' });
|
|
945
|
-
const row =
|
|
917
|
+
const row = db.get('SELECT * FROM enroll_tokens WHERE token_hash=?', hashToken(raw));
|
|
946
918
|
if (!row || row.uses_remaining < 1 || row.expires_at < now()) {
|
|
947
919
|
if (row)
|
|
948
|
-
|
|
920
|
+
db.run('DELETE FROM enroll_tokens WHERE token_hash=?', hashToken(raw));
|
|
949
921
|
return res.status(401).json({ error: 'invalid or expired enroll token' });
|
|
950
922
|
}
|
|
951
923
|
const name = row.name || body.name;
|
|
@@ -968,14 +940,14 @@ export async function createApp(options = {}) {
|
|
|
968
940
|
}
|
|
969
941
|
const remaining = row.uses_remaining - 1;
|
|
970
942
|
if (remaining < 1)
|
|
971
|
-
|
|
943
|
+
db.run('DELETE FROM enroll_tokens WHERE token_hash=?', hashToken(raw));
|
|
972
944
|
else
|
|
973
|
-
|
|
974
|
-
const enrolled = { ...result
|
|
945
|
+
db.run('UPDATE enroll_tokens SET uses_remaining=? WHERE token_hash=?', remaining, hashToken(raw));
|
|
946
|
+
const enrolled = { ...result };
|
|
975
947
|
res.json(enrolled);
|
|
976
948
|
});
|
|
977
949
|
api('get', '/api/v1/nodes', null, async (req, res) => {
|
|
978
|
-
const rows =
|
|
950
|
+
const rows = db.all('SELECT id,name,address,public_address,overlay_ip,ingress,ingress_status_json,status,labels_json,last_seen_at,created_at,updated_at FROM nodes ORDER BY name');
|
|
979
951
|
res.json({
|
|
980
952
|
nodes: rows.map((row) => {
|
|
981
953
|
const { labels_json, ...rest } = row;
|
|
@@ -983,30 +955,13 @@ export async function createApp(options = {}) {
|
|
|
983
955
|
}),
|
|
984
956
|
});
|
|
985
957
|
});
|
|
986
|
-
|
|
987
|
-
const node = await db.get('SELECT id,name FROM nodes WHERE name=? OR id=?', req.params.name, req.params.name);
|
|
988
|
-
if (!node)
|
|
989
|
-
return res.status(404).json({ error: 'node not found' });
|
|
990
|
-
const registryToken = token('tug_reg');
|
|
991
|
-
await db.run('UPDATE nodes SET registry_token_hash=?,updated_at=? WHERE id=?', hashToken(registryToken), now(), node.id);
|
|
992
|
-
await upsertRegistryCredential(node.name, registryToken);
|
|
993
|
-
res.json({ ok: true, node: node.name, registryToken });
|
|
994
|
-
});
|
|
995
|
-
api('post', '/api/v1/nodes/:name/revoke-registry-token', null, async (req, res) => {
|
|
996
|
-
const node = await db.get('SELECT id,name FROM nodes WHERE name=? OR id=?', req.params.name, req.params.name);
|
|
997
|
-
if (!node)
|
|
998
|
-
return res.status(404).json({ error: 'node not found' });
|
|
999
|
-
await db.run('UPDATE nodes SET registry_token_hash=NULL,updated_at=? WHERE id=?', now(), node.id);
|
|
1000
|
-
await removeRegistryCredential(node.name);
|
|
1001
|
-
res.json({ ok: true, node: node.name, revoked: true });
|
|
1002
|
-
});
|
|
1003
|
-
async function findNode(nameOrId) {
|
|
958
|
+
function findNode(nameOrId) {
|
|
1004
959
|
return db.get('SELECT id,name FROM nodes WHERE name=? OR id=?', nameOrId, nameOrId);
|
|
1005
960
|
}
|
|
1006
|
-
|
|
961
|
+
function queueCommand(nodeId, projectId, type, payload = {}) {
|
|
1007
962
|
const commandId = id('cmd');
|
|
1008
963
|
const ts = now();
|
|
1009
|
-
|
|
964
|
+
db.run('INSERT INTO commands(id,node_id,project_id,type,state,payload_json,attempts,run_after,deadline,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)', commandId, nodeId, projectId, type, 'pending', JSON.stringify(payload), 0, ts, null, ts, ts);
|
|
1010
965
|
wakeHub.wake(nodeId, projectId ?? undefined);
|
|
1011
966
|
return commandId;
|
|
1012
967
|
}
|
|
@@ -1015,19 +970,19 @@ export async function createApp(options = {}) {
|
|
|
1015
970
|
api('post', '/api/v1/nodes/:name/rotate-wg-key', null, async (req, res) => {
|
|
1016
971
|
if (!MESH)
|
|
1017
972
|
return res.status(409).json({ error: 'the WireGuard mesh is not enabled (TUG_MESH=true)' });
|
|
1018
|
-
const node =
|
|
973
|
+
const node = findNode(req.params.name);
|
|
1019
974
|
if (!node)
|
|
1020
975
|
return res.status(404).json({ error: 'node not found' });
|
|
1021
|
-
res.json({ commandId:
|
|
976
|
+
res.json({ commandId: queueCommand(node.id, null, 'wg.rotate-key'), nodeId: node.id });
|
|
1022
977
|
});
|
|
1023
978
|
// Removes a retired node: its token, WireGuard peer, statuses and queued commands. Refused while a current deployment still places
|
|
1024
979
|
// something on it (redeploy without it first) or while its agent is still reporting, unless force=true.
|
|
1025
980
|
api('delete', '/api/v1/nodes/:name', null, async (req, res) => {
|
|
1026
|
-
const node =
|
|
981
|
+
const node = db.get('SELECT id,name,last_seen_at FROM nodes WHERE name=? OR id=?', req.params.name, req.params.name);
|
|
1027
982
|
if (!node)
|
|
1028
983
|
return res.status(404).json({ error: 'node not found' });
|
|
1029
984
|
const referenced = [];
|
|
1030
|
-
for (const state of
|
|
985
|
+
for (const state of currentDesiredStates()) {
|
|
1031
986
|
for (const [name, svc] of Object.entries(state.services || {}))
|
|
1032
987
|
if (svc.placement?.nodes?.includes(node.name))
|
|
1033
988
|
referenced.push(`${state.project}/${name}`);
|
|
@@ -1043,24 +998,24 @@ export async function createApp(options = {}) {
|
|
|
1043
998
|
return res.status(409).json({
|
|
1044
999
|
error: `node ${node.name} is still reporting; stop its agent first or pass force=true`,
|
|
1045
1000
|
});
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1001
|
+
db.run('DELETE FROM commands WHERE node_id=?', node.id);
|
|
1002
|
+
db.run('DELETE FROM node_statuses WHERE node_id=?', node.id);
|
|
1003
|
+
db.run('DELETE FROM wg_peers WHERE node_id=?', node.id);
|
|
1004
|
+
db.run('DELETE FROM nodes WHERE id=?', node.id);
|
|
1005
|
+
audit(req.actor, {
|
|
1051
1006
|
type: 'node.removed',
|
|
1052
1007
|
message: `node ${node.name} removed`,
|
|
1053
1008
|
metadata: { node: node.name },
|
|
1054
1009
|
});
|
|
1055
|
-
|
|
1010
|
+
wakeAll();
|
|
1056
1011
|
res.json({ ok: true, node: node.name });
|
|
1057
1012
|
});
|
|
1058
1013
|
api('post', '/api/v1/nodes/:name/ingress', null, async (req, res) => {
|
|
1059
|
-
const node =
|
|
1014
|
+
const node = findNode(req.params.name);
|
|
1060
1015
|
if (!node)
|
|
1061
1016
|
return res.status(404).json({ error: 'node not found' });
|
|
1062
1017
|
const enabled = req.body.enabled !== false;
|
|
1063
|
-
|
|
1018
|
+
db.run('UPDATE nodes SET ingress=?,updated_at=? WHERE id=?', enabled ? 1 : 0, now(), node.id);
|
|
1064
1019
|
wakeHub.wake(node.id);
|
|
1065
1020
|
res.json({ ok: true, node: node.name, ingress: enabled });
|
|
1066
1021
|
});
|
|
@@ -1071,30 +1026,18 @@ export async function createApp(options = {}) {
|
|
|
1071
1026
|
if (!isWgKey(publicKey))
|
|
1072
1027
|
return res.status(400).json({ error: 'publicKey must be a base64 WireGuard public key' });
|
|
1073
1028
|
try {
|
|
1074
|
-
|
|
1075
|
-
|
|
1029
|
+
syncWgPeer(req.node.id, publicKey);
|
|
1030
|
+
wakeAll();
|
|
1076
1031
|
res.json({ ok: true });
|
|
1077
1032
|
}
|
|
1078
1033
|
catch (e) {
|
|
1079
|
-
sendError(res,
|
|
1080
|
-
? httpError(409, 'public key already registered to another node')
|
|
1081
|
-
: e);
|
|
1034
|
+
sendError(res, isUniqueViolation(e) ? httpError(409, 'public key already registered to another node') : e);
|
|
1082
1035
|
}
|
|
1083
1036
|
});
|
|
1084
|
-
post('/api/v1/registry/auth', async (req, res) => {
|
|
1085
|
-
const username = req.body.username;
|
|
1086
|
-
const password = req.body.password;
|
|
1087
|
-
const node = username && password
|
|
1088
|
-
? await db.get('SELECT id,name FROM nodes WHERE (name=? OR id=?) AND registry_token_hash IS NOT NULL AND registry_token_hash=?', username, username, hashToken(password))
|
|
1089
|
-
: null;
|
|
1090
|
-
if (!node)
|
|
1091
|
-
return res.status(401).json({ ok: false, error: 'invalid registry credentials' });
|
|
1092
|
-
res.json({ ok: true, node: node.name });
|
|
1093
|
-
});
|
|
1094
1037
|
api('post', '/api/v1/tokens', null, async (req, res) => {
|
|
1095
1038
|
const request = parseTokenRequest(req.body);
|
|
1096
|
-
const created =
|
|
1097
|
-
|
|
1039
|
+
const created = createToken(db, request, req.actor.name);
|
|
1040
|
+
audit(req.actor, {
|
|
1098
1041
|
type: 'token.create',
|
|
1099
1042
|
message: `token ${created.name} (${created.role}${created.projects ? `, projects: ${created.projects.join(', ')}` : ''}) created by ${req.actor.name}`,
|
|
1100
1043
|
metadata: {
|
|
@@ -1106,11 +1049,11 @@ export async function createApp(options = {}) {
|
|
|
1106
1049
|
});
|
|
1107
1050
|
res.json(created);
|
|
1108
1051
|
});
|
|
1109
|
-
api('get', '/api/v1/tokens', null, async (_req, res) => res.json({ tokens:
|
|
1052
|
+
api('get', '/api/v1/tokens', null, async (_req, res) => res.json({ tokens: listTokens(db) }));
|
|
1110
1053
|
api('delete', '/api/v1/tokens/:name', null, async (req, res) => {
|
|
1111
|
-
if (!
|
|
1054
|
+
if (!revokeToken(db, req.params.name))
|
|
1112
1055
|
return res.status(404).json({ error: 'token not found' });
|
|
1113
|
-
|
|
1056
|
+
audit(req.actor, {
|
|
1114
1057
|
type: 'token.revoke',
|
|
1115
1058
|
message: `token ${req.params.name} revoked by ${req.actor.name}`,
|
|
1116
1059
|
metadata: { name: req.params.name },
|
|
@@ -1123,13 +1066,13 @@ export async function createApp(options = {}) {
|
|
|
1123
1066
|
return res.status(400).json({ error: 'name and value required' });
|
|
1124
1067
|
let lock;
|
|
1125
1068
|
try {
|
|
1126
|
-
lock =
|
|
1127
|
-
const old =
|
|
1069
|
+
lock = acquireProjectLock(project, 'secret.set');
|
|
1070
|
+
const old = db.get('SELECT version FROM secrets WHERE project_id=? AND name=?', project, name);
|
|
1128
1071
|
const ts = now();
|
|
1129
|
-
|
|
1072
|
+
db.run(`INSERT INTO secrets(id,project_id,name,encrypted_value,version,created_at,updated_at) VALUES(?,?,?,?,?,?,?)
|
|
1130
1073
|
ON CONFLICT(project_id,name) DO UPDATE SET encrypted_value=excluded.encrypted_value,version=excluded.version,updated_at=excluded.updated_at`, id('sec'), project, name, encrypt(value, masterKey), (old?.version || 0) + 1, ts, ts);
|
|
1131
|
-
|
|
1132
|
-
|
|
1074
|
+
wakeAll(project);
|
|
1075
|
+
audit(req.actor, {
|
|
1133
1076
|
type: 'secret.set',
|
|
1134
1077
|
project,
|
|
1135
1078
|
message: `secret ${name} set to version ${(old?.version || 0) + 1} by ${req.actor.name}`,
|
|
@@ -1141,18 +1084,18 @@ export async function createApp(options = {}) {
|
|
|
1141
1084
|
sendError(res, e);
|
|
1142
1085
|
}
|
|
1143
1086
|
finally {
|
|
1144
|
-
|
|
1087
|
+
releaseLock(lock);
|
|
1145
1088
|
}
|
|
1146
1089
|
});
|
|
1147
1090
|
api('get', '/api/v1/secrets', queryProject, async (req, res) => res.json({
|
|
1148
|
-
secrets:
|
|
1091
|
+
secrets: db.all('SELECT project_id,name,version,updated_at FROM secrets WHERE project_id=? ORDER BY name', String(req.query.project || 'site')),
|
|
1149
1092
|
}));
|
|
1150
1093
|
api('delete', '/api/v1/secrets/:name', queryProject, async (req, res) => {
|
|
1151
1094
|
const project = String(req.query.project || 'site');
|
|
1152
|
-
const removed =
|
|
1095
|
+
const removed = db.run('DELETE FROM secrets WHERE project_id=? AND name=?', project, req.params.name);
|
|
1153
1096
|
if (removed.changes) {
|
|
1154
|
-
|
|
1155
|
-
|
|
1097
|
+
wakeAll(project);
|
|
1098
|
+
audit(req.actor, {
|
|
1156
1099
|
type: 'secret.delete',
|
|
1157
1100
|
project,
|
|
1158
1101
|
message: `secret ${req.params.name} deleted by ${req.actor.name}`,
|
|
@@ -1163,33 +1106,35 @@ export async function createApp(options = {}) {
|
|
|
1163
1106
|
});
|
|
1164
1107
|
const artifactDir = path.join(DATA_DIR, 'images');
|
|
1165
1108
|
await ensureDir(artifactDir);
|
|
1166
|
-
|
|
1109
|
+
function retainedStates() {
|
|
1167
1110
|
// All published history is retained. This conservative superset protects failed/lagging versions too.
|
|
1168
|
-
return
|
|
1111
|
+
return db
|
|
1112
|
+
.all('SELECT * FROM desired_states')
|
|
1113
|
+
.map((row) => JSON.parse(row.state_json));
|
|
1169
1114
|
}
|
|
1170
1115
|
async function collectArtifacts() {
|
|
1171
|
-
const states =
|
|
1116
|
+
const states = retainedStates();
|
|
1172
1117
|
const refs = new Set(states.flatMap((state) => Object.values(state.services).flatMap((service) => service.artifact ? [service.artifact.buildId] : [])));
|
|
1173
1118
|
const imageRefs = new Set(states.flatMap((state) => Object.values(state.services).map((service) => service.image)));
|
|
1174
|
-
const rows =
|
|
1119
|
+
const rows = db.all('SELECT * FROM builds');
|
|
1175
1120
|
for (const row of rows) {
|
|
1176
1121
|
if (refs.has(row.id) || imageRefs.has(row.image_ref))
|
|
1177
1122
|
continue;
|
|
1178
|
-
const active =
|
|
1123
|
+
const active = db.get("SELECT 1 FROM deployments WHERE id=? AND state IN ('uploading','building')", row.deployment_id);
|
|
1179
1124
|
if (active)
|
|
1180
1125
|
continue;
|
|
1181
1126
|
await fs.rm(path.join(DATA_DIR, row.artifact_path || path.join('images', `${row.id}.tar.gz`)), { force: true });
|
|
1182
1127
|
await images.run('docker', ['image', 'rm', row.image_ref]).catch(() => { });
|
|
1183
|
-
|
|
1128
|
+
db.run('DELETE FROM builds WHERE id=?', row.id);
|
|
1184
1129
|
}
|
|
1185
|
-
const remaining =
|
|
1130
|
+
const remaining = db.all('SELECT id FROM builds');
|
|
1186
1131
|
const owned = new Set(remaining.flatMap((row) => [`${row.id}.tar.gz`, `${row.id}.tar.gz.tmp`]));
|
|
1187
1132
|
for (const name of await fs.readdir(artifactDir))
|
|
1188
1133
|
if (!owned.has(name))
|
|
1189
1134
|
await fs.rm(path.join(artifactDir, name), { force: true });
|
|
1190
1135
|
}
|
|
1191
1136
|
// Previous-process builds cannot finish after a restart; clear their unpublished files.
|
|
1192
|
-
|
|
1137
|
+
db.run("UPDATE deployments SET state='build_failed',error_message='control restarted during build' WHERE state IN ('uploading','building')");
|
|
1193
1138
|
await collectArtifacts();
|
|
1194
1139
|
const receiveDeployment = upload.any();
|
|
1195
1140
|
api('post', '/api/v1/deployments', null, (req, res) => {
|
|
@@ -1230,7 +1175,7 @@ export async function createApp(options = {}) {
|
|
|
1230
1175
|
throw httpError(400, 'invalid project or version');
|
|
1231
1176
|
if (version.startsWith(ROLLBACK_VERSION_PREFIX))
|
|
1232
1177
|
throw httpError(400, `version cannot start with "${ROLLBACK_VERSION_PREFIX}" (reserved for rollback deployments)`);
|
|
1233
|
-
const nodeRows =
|
|
1178
|
+
const nodeRows = db.all('SELECT name,labels_json,last_seen_at,created_at FROM nodes ORDER BY created_at,name');
|
|
1234
1179
|
// Stale nodes (agent not seen recently) are not scheduled onto, so a dead host cannot hold a rollout open.
|
|
1235
1180
|
const nodeInfos = nodeRows
|
|
1236
1181
|
.filter((row) => !isStale(row))
|
|
@@ -1270,7 +1215,7 @@ export async function createApp(options = {}) {
|
|
|
1270
1215
|
xe.address ||
|
|
1271
1216
|
!xe.placement?.nodes?.[0])
|
|
1272
1217
|
continue;
|
|
1273
|
-
const exporter =
|
|
1218
|
+
const exporter = db.get('SELECT address,overlay_ip FROM nodes WHERE name=?', xe.placement.nodes[0]);
|
|
1274
1219
|
const addr = (MESH && exporter?.overlay_ip) || exporter?.address;
|
|
1275
1220
|
if (addr)
|
|
1276
1221
|
xe.address = addr;
|
|
@@ -1283,8 +1228,13 @@ export async function createApp(options = {}) {
|
|
|
1283
1228
|
}
|
|
1284
1229
|
// The CLI interpolates control secrets as sentinels; a reference to a secret that is gone would deploy an empty value.
|
|
1285
1230
|
const referenced = sentinelNames(compose);
|
|
1231
|
+
if (referenced.includes(REGISTRY_AUTH_SECRET))
|
|
1232
|
+
throw httpError(400, REGISTRY_AUTH_SECRET +
|
|
1233
|
+
' is reserved for registry authentication and cannot be used in service definitions');
|
|
1286
1234
|
if (referenced.length) {
|
|
1287
|
-
const known = new Set(
|
|
1235
|
+
const known = new Set(db
|
|
1236
|
+
.all('SELECT name FROM secrets WHERE project_id=?', project)
|
|
1237
|
+
.map((row) => row.name));
|
|
1288
1238
|
const missing = referenced.filter((name) => !known.has(name));
|
|
1289
1239
|
if (missing.length)
|
|
1290
1240
|
throw httpError(400, `secrets not found in project ${project}: ${missing.join(', ')}; create them with tug secret-set`);
|
|
@@ -1320,30 +1270,29 @@ export async function createApp(options = {}) {
|
|
|
1320
1270
|
}
|
|
1321
1271
|
if (files.length !== (local ? local.length : context ? 1 : 0))
|
|
1322
1272
|
throw httpError(400, 'unexpected deployment upload');
|
|
1323
|
-
if (
|
|
1324
|
-
throw httpError(400, 'build services require a source context or localImages upload
|
|
1273
|
+
if (builtServices.length && !context && !local)
|
|
1274
|
+
throw httpError(400, 'build services require a source context or localImages upload');
|
|
1325
1275
|
for (const [svcName, svc] of Object.entries(compose.services)) {
|
|
1326
1276
|
const ing = svc['x-tug']?.ingress;
|
|
1327
1277
|
if (ing?.tls !== 'custom')
|
|
1328
1278
|
continue;
|
|
1329
1279
|
for (const secret of [ing.cert, ing.key])
|
|
1330
|
-
if (!
|
|
1280
|
+
if (!db.get('SELECT 1 FROM secrets WHERE project_id=? AND name=?', project, secret ?? null))
|
|
1331
1281
|
throw httpError(400, `${svcName}: custom TLS secret ${secret} not found in project ${project}; set it with secret-set first`);
|
|
1332
1282
|
}
|
|
1333
|
-
lock =
|
|
1334
|
-
const active =
|
|
1283
|
+
lock = acquireProjectLock(project, 'deploy');
|
|
1284
|
+
const active = db.get(`SELECT * FROM deployments WHERE project_id=? AND state IN (${ACTIVE_STATES_SQL}) LIMIT 1`, project);
|
|
1335
1285
|
if (active)
|
|
1336
1286
|
throw httpError(409, `deployment ${active.id} is already ${active.state}`);
|
|
1337
1287
|
depId = id('dep');
|
|
1338
1288
|
const ts = now();
|
|
1339
|
-
|
|
1340
|
-
|
|
1289
|
+
db.run('INSERT OR IGNORE INTO projects(id,name,created_at,updated_at) VALUES(?,?,?,?)', project, project, ts, ts);
|
|
1290
|
+
db.run('INSERT INTO deployments(id,project_id,version,state,created_at,updated_at,actor) VALUES(?,?,?,?,?,?,?)', depId, project, version, 'building', ts, ts, req.actor.name);
|
|
1341
1291
|
const desired = buildDesiredState({
|
|
1342
1292
|
project,
|
|
1343
1293
|
version,
|
|
1344
1294
|
revision: depId,
|
|
1345
1295
|
compose,
|
|
1346
|
-
registry: REGISTRY,
|
|
1347
1296
|
defaultNode: nodes[0]?.name || null,
|
|
1348
1297
|
});
|
|
1349
1298
|
if (context) {
|
|
@@ -1351,14 +1300,10 @@ export async function createApp(options = {}) {
|
|
|
1351
1300
|
await unpackContext(context.path, contextDir);
|
|
1352
1301
|
}
|
|
1353
1302
|
for (const service of builtServices) {
|
|
1354
|
-
if (!contextDir && !local) {
|
|
1355
|
-
desired.services[service].image = `${REGISTRY}/${project}/${service}:${version}`;
|
|
1356
|
-
continue;
|
|
1357
|
-
} // Explicit registry/no-build-upload means an existing pushed image.
|
|
1358
1303
|
const spec = desired.services[service];
|
|
1359
1304
|
const buildId = id('build');
|
|
1360
1305
|
const imageRef = spec.image;
|
|
1361
|
-
|
|
1306
|
+
db.run('INSERT INTO builds(id,project_id,deployment_id,service,image_ref,state,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?)', buildId, project, depId, service, imageRef, 'building', ts, ts);
|
|
1362
1307
|
let info;
|
|
1363
1308
|
let importedClientRef;
|
|
1364
1309
|
const relative = path.join('images', `${buildId}.tar.gz`);
|
|
@@ -1392,21 +1337,12 @@ export async function createApp(options = {}) {
|
|
|
1392
1337
|
service,
|
|
1393
1338
|
buildSpec: compose.services[service].build,
|
|
1394
1339
|
imageRef,
|
|
1395
|
-
push: false,
|
|
1396
1340
|
});
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
}
|
|
1403
|
-
else {
|
|
1404
|
-
info = await images.save(imageRef, temporary);
|
|
1405
|
-
await fs.rename(temporary, final);
|
|
1406
|
-
const artifact = { buildId, imageRef, ...info };
|
|
1407
|
-
spec.artifact = artifact;
|
|
1408
|
-
await db.run('UPDATE builds SET state=?,image_id=?,platform=?,sha256=?,size_bytes=?,artifact_path=?,updated_at=? WHERE id=?', 'complete', info.imageId, info.platform, info.sha256, info.sizeBytes, relative, now(), buildId);
|
|
1409
|
-
}
|
|
1341
|
+
info = await images.save(imageRef, temporary);
|
|
1342
|
+
await fs.rename(temporary, final);
|
|
1343
|
+
const artifact = { buildId, imageRef, ...info };
|
|
1344
|
+
spec.artifact = artifact;
|
|
1345
|
+
db.run('UPDATE builds SET state=?,image_id=?,platform=?,sha256=?,size_bytes=?,artifact_path=?,updated_at=? WHERE id=?', 'complete', info.imageId, info.platform, info.sha256, info.sizeBytes, relative, now(), buildId);
|
|
1410
1346
|
}
|
|
1411
1347
|
finally {
|
|
1412
1348
|
if (importedClientRef)
|
|
@@ -1414,43 +1350,61 @@ export async function createApp(options = {}) {
|
|
|
1414
1350
|
await fs.rm(temporary, { force: true });
|
|
1415
1351
|
}
|
|
1416
1352
|
}
|
|
1353
|
+
const registrySecret = db.get('SELECT * FROM secrets WHERE project_id=? AND name=?', project, REGISTRY_AUTH_SECRET);
|
|
1354
|
+
const credentials = parseRegistryCredentials(registrySecret ? decrypt(registrySecret.encrypted_value, masterKey) : undefined);
|
|
1355
|
+
const resolved = new Map();
|
|
1356
|
+
for (const [name, spec] of Object.entries(desired.services)) {
|
|
1357
|
+
if (spec.artifact)
|
|
1358
|
+
continue;
|
|
1359
|
+
const image = spec.image;
|
|
1360
|
+
if (!resolved.has(image))
|
|
1361
|
+
resolved.set(image, await images.resolve(image, credentials));
|
|
1362
|
+
spec.image = resolved.get(image);
|
|
1363
|
+
desired.compose.services[name].image = spec.image;
|
|
1364
|
+
}
|
|
1417
1365
|
const buildLock = lock;
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
const
|
|
1366
|
+
const publishingId = depId;
|
|
1367
|
+
await withPublication(() => {
|
|
1368
|
+
const owned = db.get('SELECT 1 FROM locks WHERE name=? AND owner=? AND expires_at>?', buildLock.name, buildLock.owner, now());
|
|
1369
|
+
const pending = db.get('SELECT 1 FROM deployments WHERE id=? AND state=?', publishingId, 'building');
|
|
1421
1370
|
if (!owned || !pending)
|
|
1422
1371
|
throw httpError(409, 'deployment was cancelled or its build lease was lost');
|
|
1423
|
-
|
|
1372
|
+
db.exec('BEGIN IMMEDIATE');
|
|
1424
1373
|
try {
|
|
1425
|
-
const published =
|
|
1374
|
+
const published = db.run("UPDATE deployments SET state='rolling_out',updated_at=? WHERE id=? AND state='building'", now(), publishingId);
|
|
1426
1375
|
if (published.changes !== 1 ||
|
|
1427
|
-
!
|
|
1376
|
+
!db.get('SELECT 1 FROM locks WHERE name=? AND owner=? AND expires_at>?', buildLock.name, buildLock.owner, now()))
|
|
1428
1377
|
throw httpError(409, 'deployment was cancelled or its build lease was lost');
|
|
1429
|
-
|
|
1378
|
+
db.run('INSERT INTO desired_states(id,project_id,deployment_id,version,state_json,created_at) VALUES(?,?,?,?,?,?)', id('ds'), project, publishingId, version, JSON.stringify(desired), ts);
|
|
1430
1379
|
for (const [volumeName, volumeSpec] of Object.entries(compose.volumes || {})) {
|
|
1431
1380
|
const lp = volumeSpec?.['x-tug'] || {};
|
|
1432
1381
|
const nodeName = lp.placement?.nodes?.[0] || null;
|
|
1433
1382
|
const node = nodeName
|
|
1434
|
-
?
|
|
1383
|
+
? db.get('SELECT id FROM nodes WHERE name=? OR id=?', nodeName, nodeName)
|
|
1435
1384
|
: null;
|
|
1436
|
-
|
|
1385
|
+
db.run(`INSERT INTO volumes(id,project_id,name,type,driver,node_id,config_json,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(project_id,name) DO UPDATE SET type=excluded.type,driver=excluded.driver,node_id=excluded.node_id,config_json=excluded.config_json,updated_at=excluded.updated_at`, id('vol'), project, volumeName, lp.type || 'local', lp.driver || null, node?.id || null, JSON.stringify(lp), ts, ts);
|
|
1437
1386
|
}
|
|
1438
|
-
|
|
1439
|
-
|
|
1387
|
+
db.run('UPDATE projects SET current_deployment_id=?,updated_at=? WHERE id=?', publishingId, now(), project);
|
|
1388
|
+
db.exec('COMMIT');
|
|
1440
1389
|
}
|
|
1441
1390
|
catch (e) {
|
|
1442
|
-
|
|
1391
|
+
try {
|
|
1392
|
+
db.exec('ROLLBACK');
|
|
1393
|
+
}
|
|
1394
|
+
catch {
|
|
1395
|
+
/* Preserve the publication error. */
|
|
1396
|
+
}
|
|
1443
1397
|
throw e;
|
|
1444
1398
|
}
|
|
1445
1399
|
});
|
|
1446
|
-
|
|
1400
|
+
audit(req.actor, {
|
|
1447
1401
|
type: 'deploy',
|
|
1448
1402
|
project,
|
|
1449
1403
|
deploymentId: depId,
|
|
1450
1404
|
message: `deployment ${depId} (${version}) started by ${req.actor.name}`,
|
|
1451
1405
|
metadata: { version },
|
|
1452
1406
|
});
|
|
1453
|
-
|
|
1407
|
+
wakeAll(project);
|
|
1454
1408
|
responseBody = {
|
|
1455
1409
|
deployment: { id: depId, project, version, state: 'rolling_out', actor: req.actor.name },
|
|
1456
1410
|
desiredState: desired,
|
|
@@ -1458,13 +1412,16 @@ export async function createApp(options = {}) {
|
|
|
1458
1412
|
}
|
|
1459
1413
|
catch (e) {
|
|
1460
1414
|
if (depId)
|
|
1461
|
-
|
|
1462
|
-
.run("UPDATE deployments SET state='build_failed',error_message=?,updated_at=? WHERE id=? AND state='building'", errMessage(e), now(), depId)
|
|
1463
|
-
|
|
1415
|
+
try {
|
|
1416
|
+
db.run("UPDATE deployments SET state='build_failed',error_message=?,updated_at=? WHERE id=? AND state='building'", errMessage(e), now(), depId);
|
|
1417
|
+
}
|
|
1418
|
+
catch {
|
|
1419
|
+
/* Best-effort bookkeeping. */
|
|
1420
|
+
}
|
|
1464
1421
|
responseError = e;
|
|
1465
1422
|
}
|
|
1466
1423
|
finally {
|
|
1467
|
-
|
|
1424
|
+
releaseLock(lock);
|
|
1468
1425
|
await Promise.all(files.map((file) => fs.rm(file.path, { force: true }).catch(() => { })));
|
|
1469
1426
|
if (contextDir)
|
|
1470
1427
|
await fs.rm(contextDir, { recursive: true, force: true });
|
|
@@ -1476,12 +1433,12 @@ export async function createApp(options = {}) {
|
|
|
1476
1433
|
res.json(responseBody);
|
|
1477
1434
|
}
|
|
1478
1435
|
get('/agent/v1/images/:buildId', authAgent, async (req, res) => {
|
|
1479
|
-
const states =
|
|
1436
|
+
const states = retainedStates();
|
|
1480
1437
|
const allowed = states.some((state) => Object.values(state.services).some((service) => service.artifact?.buildId === req.params.buildId &&
|
|
1481
1438
|
(!service.placement.nodes?.length || service.placement.nodes.includes(req.node.name))));
|
|
1482
1439
|
if (!allowed)
|
|
1483
1440
|
return res.status(404).json({ error: 'image not found' });
|
|
1484
|
-
const row =
|
|
1441
|
+
const row = db.get('SELECT * FROM builds WHERE id=? AND state=?', req.params.buildId, 'complete');
|
|
1485
1442
|
if (!row?.artifact_path)
|
|
1486
1443
|
return res.status(404).json({ error: 'image not found' });
|
|
1487
1444
|
try {
|
|
@@ -1499,17 +1456,17 @@ export async function createApp(options = {}) {
|
|
|
1499
1456
|
res.status(404).json({ error: 'image artifact unavailable' });
|
|
1500
1457
|
}
|
|
1501
1458
|
});
|
|
1502
|
-
api('get', '/api/v1/deployments/:id', deploymentProject, async (req, res) => res.json(
|
|
1503
|
-
api('post', '/api/v1/deployments/:id/cancel', deploymentProject, async (req, res) => withPublication(
|
|
1504
|
-
const row =
|
|
1459
|
+
api('get', '/api/v1/deployments/:id', deploymentProject, async (req, res) => res.json(db.get('SELECT * FROM deployments WHERE id=?', req.params.id)));
|
|
1460
|
+
api('post', '/api/v1/deployments/:id/cancel', deploymentProject, async (req, res) => withPublication(() => {
|
|
1461
|
+
const row = db.get('SELECT * FROM deployments WHERE id=?', req.params.id);
|
|
1505
1462
|
if (!row)
|
|
1506
1463
|
return res.status(404).json({ error: 'deployment not found' });
|
|
1507
1464
|
if (!ACTIVE_DEPLOYMENT_STATES.includes(row.state))
|
|
1508
1465
|
return res.status(409).json({ error: `deployment is ${row.state}` });
|
|
1509
|
-
const result =
|
|
1466
|
+
const result = db.run(`UPDATE deployments SET state=?,updated_at=? WHERE id=? AND state IN (${ACTIVE_STATES_SQL})`, 'cancelled', now(), req.params.id);
|
|
1510
1467
|
if (result.changes !== 1)
|
|
1511
1468
|
return res.status(409).json({ error: 'deployment is no longer active' });
|
|
1512
|
-
|
|
1469
|
+
audit(req.actor, {
|
|
1513
1470
|
type: 'cancel',
|
|
1514
1471
|
project: row.project_id,
|
|
1515
1472
|
deploymentId: row.id,
|
|
@@ -1520,11 +1477,11 @@ export async function createApp(options = {}) {
|
|
|
1520
1477
|
api('post', '/api/v1/rollback', bodyProject, async (req, res) => withWriter(async () => {
|
|
1521
1478
|
const project = req.body.project || 'site';
|
|
1522
1479
|
try {
|
|
1523
|
-
const out =
|
|
1480
|
+
const out = createRollback(project, {
|
|
1524
1481
|
actor: req.actor.name,
|
|
1525
1482
|
force: Boolean(req.body.force),
|
|
1526
1483
|
});
|
|
1527
|
-
|
|
1484
|
+
audit(req.actor, {
|
|
1528
1485
|
type: 'rollback',
|
|
1529
1486
|
project,
|
|
1530
1487
|
deploymentId: out.deployment.id,
|
|
@@ -1538,15 +1495,15 @@ export async function createApp(options = {}) {
|
|
|
1538
1495
|
}
|
|
1539
1496
|
}));
|
|
1540
1497
|
api('get', '/api/v1/volumes', queryProject, async (req, res) => res.json({
|
|
1541
|
-
volumes:
|
|
1498
|
+
volumes: db.all('SELECT * FROM volumes WHERE project_id=? ORDER BY name', String(req.query.project || 'site')),
|
|
1542
1499
|
}));
|
|
1543
1500
|
api('delete', '/api/v1/volumes/:name', queryProject, async (req, res) => {
|
|
1544
1501
|
const project = String(req.query.project || 'site');
|
|
1545
1502
|
const deleteData = String(req.query.deleteData || 'false') === 'true';
|
|
1546
1503
|
let lock;
|
|
1547
1504
|
try {
|
|
1548
|
-
lock = deleteData ?
|
|
1549
|
-
const volume =
|
|
1505
|
+
lock = deleteData ? acquireProjectLock(project, 'volume.delete') : null;
|
|
1506
|
+
const volume = db.get('SELECT * FROM volumes WHERE project_id=? AND name=?', project, req.params.name);
|
|
1550
1507
|
if (!volume)
|
|
1551
1508
|
return res.status(404).json({ error: 'volume not found' });
|
|
1552
1509
|
const config = JSON.parse(volume.config_json || '{}');
|
|
@@ -1554,7 +1511,7 @@ export async function createApp(options = {}) {
|
|
|
1554
1511
|
return res
|
|
1555
1512
|
.status(409)
|
|
1556
1513
|
.json({ error: 'refusing to delete persistent volume without deleteData=true' });
|
|
1557
|
-
const ds =
|
|
1514
|
+
const ds = currentDesired(project);
|
|
1558
1515
|
if (ds) {
|
|
1559
1516
|
const state = JSON.parse(ds.state_json);
|
|
1560
1517
|
const inUse = Object.entries(state.compose?.services || {})
|
|
@@ -1567,19 +1524,19 @@ export async function createApp(options = {}) {
|
|
|
1567
1524
|
}
|
|
1568
1525
|
let commandId = null;
|
|
1569
1526
|
if (deleteData && volume.node_id) {
|
|
1570
|
-
commandId =
|
|
1527
|
+
commandId = queueCommand(volume.node_id, project, 'volume.delete', {
|
|
1571
1528
|
project,
|
|
1572
1529
|
volume: req.params.name,
|
|
1573
1530
|
});
|
|
1574
1531
|
}
|
|
1575
|
-
|
|
1532
|
+
db.run('DELETE FROM volumes WHERE project_id=? AND name=?', project, req.params.name);
|
|
1576
1533
|
res.json({ ok: true, commandId, deletedMetadata: true, deleteData });
|
|
1577
1534
|
}
|
|
1578
1535
|
catch (e) {
|
|
1579
1536
|
sendError(res, e);
|
|
1580
1537
|
}
|
|
1581
1538
|
finally {
|
|
1582
|
-
|
|
1539
|
+
releaseLock(lock);
|
|
1583
1540
|
}
|
|
1584
1541
|
});
|
|
1585
1542
|
api('post', '/api/v1/control/backups/run', null, async (req, res) => withWriter(async () => {
|
|
@@ -1590,8 +1547,8 @@ export async function createApp(options = {}) {
|
|
|
1590
1547
|
const staging = path.join(backupDir, `${backupId}-staging`);
|
|
1591
1548
|
await fs.rm(staging, { recursive: true, force: true });
|
|
1592
1549
|
await ensureDir(staging);
|
|
1593
|
-
const snapshotPath = path.join(staging, 'tug.db')
|
|
1594
|
-
|
|
1550
|
+
const snapshotPath = path.join(staging, 'tug.db');
|
|
1551
|
+
db.run('VACUUM INTO ?', snapshotPath);
|
|
1595
1552
|
await fs.copyFile(masterKeyFile, path.join(staging, 'master.key'));
|
|
1596
1553
|
await fs.cp(artifactDir, path.join(staging, 'images'), { recursive: true });
|
|
1597
1554
|
await fs.writeFile(path.join(staging, 'metadata.json'), JSON.stringify({ backupId, createdAt: ts, type: 'control' }, null, 2));
|
|
@@ -1603,7 +1560,7 @@ export async function createApp(options = {}) {
|
|
|
1603
1560
|
'images',
|
|
1604
1561
|
]);
|
|
1605
1562
|
await fs.rm(staging, { recursive: true, force: true });
|
|
1606
|
-
|
|
1563
|
+
db.run('INSERT INTO backups(id,project_id,target,node_id,state,artifact_ref,metadata_json,created_at,completed_at) VALUES(?,?,?,?,?,?,?,?,?)', backupId, '_control', 'control-plane', null, 'completed', artifactRef, JSON.stringify({ type: 'control' }), ts, now());
|
|
1607
1564
|
res.json({ backupId, artifactRef, state: 'completed' });
|
|
1608
1565
|
}));
|
|
1609
1566
|
api('post', '/api/v1/control/restore', null, async (req, res) => withWriter(async () => {
|
|
@@ -1629,25 +1586,21 @@ export async function createApp(options = {}) {
|
|
|
1629
1586
|
const key = (await fs.readFile(path.join(restoreDir, 'master.key'), 'utf8')).trim();
|
|
1630
1587
|
if (!/^[A-Za-z0-9+/]{43}=$/.test(key))
|
|
1631
1588
|
throw new Error('invalid backup master key');
|
|
1632
|
-
snapshot =
|
|
1633
|
-
|
|
1634
|
-
driver: sqlite3.Database,
|
|
1635
|
-
mode: sqlite3.OPEN_READONLY,
|
|
1636
|
-
});
|
|
1637
|
-
const integrity = await snapshot.get('PRAGMA integrity_check');
|
|
1589
|
+
snapshot = openSnapshotDb(path.join(restoreDir, 'tug.db'));
|
|
1590
|
+
const integrity = snapshot.get('PRAGMA integrity_check');
|
|
1638
1591
|
if (integrity?.integrity_check !== 'ok')
|
|
1639
1592
|
throw new Error('backup database failed integrity check');
|
|
1640
|
-
if (!
|
|
1593
|
+
if (!snapshot.get("SELECT 1 FROM meta WHERE key='operator_token_hash'"))
|
|
1641
1594
|
throw new Error('backup database has no operator identity');
|
|
1642
|
-
for (const secret of
|
|
1595
|
+
for (const secret of snapshot.all('SELECT * FROM secrets'))
|
|
1643
1596
|
decrypt(secret.encrypted_value, key);
|
|
1644
|
-
const states =
|
|
1597
|
+
const states = snapshot.all('SELECT * FROM desired_states');
|
|
1645
1598
|
for (const stateRow of states) {
|
|
1646
1599
|
const state = JSON.parse(stateRow.state_json);
|
|
1647
1600
|
for (const service of Object.values(state.services)) {
|
|
1648
1601
|
if (!service.artifact)
|
|
1649
1602
|
continue;
|
|
1650
|
-
const build =
|
|
1603
|
+
const build = snapshot.get('SELECT * FROM builds WHERE id=? AND state=?', service.artifact.buildId, 'complete');
|
|
1651
1604
|
if (!build?.artifact_path ||
|
|
1652
1605
|
!/^images\/build_[a-zA-Z0-9_-]+\.tar\.gz$/.test(build.artifact_path))
|
|
1653
1606
|
throw new Error('backup contains an unsafe or missing image reference');
|
|
@@ -1659,12 +1612,12 @@ export async function createApp(options = {}) {
|
|
|
1659
1612
|
throw new Error('backup image archive checksum mismatch');
|
|
1660
1613
|
}
|
|
1661
1614
|
}
|
|
1662
|
-
|
|
1615
|
+
snapshot.close();
|
|
1663
1616
|
snapshot = undefined;
|
|
1664
1617
|
await ensureDir(path.join(restoreDir, 'images'));
|
|
1665
1618
|
// Validation completes before changing any live file. Stop DB-dependent work until restart.
|
|
1666
1619
|
restartRequired = true;
|
|
1667
|
-
|
|
1620
|
+
db.close();
|
|
1668
1621
|
await fs.rm(`${dbFile}-wal`, { force: true });
|
|
1669
1622
|
await fs.rm(`${dbFile}-shm`, { force: true });
|
|
1670
1623
|
await fs.cp(path.join(restoreDir, 'images'), artifactDir, { recursive: true });
|
|
@@ -1673,15 +1626,19 @@ export async function createApp(options = {}) {
|
|
|
1673
1626
|
res.json({ ok: true, restored: true, restartRequired: true });
|
|
1674
1627
|
}
|
|
1675
1628
|
catch (e) {
|
|
1676
|
-
|
|
1677
|
-
|
|
1629
|
+
try {
|
|
1630
|
+
snapshot?.close();
|
|
1631
|
+
}
|
|
1632
|
+
catch {
|
|
1633
|
+
/* Preserve the restore error. */
|
|
1634
|
+
}
|
|
1678
1635
|
res.status(restartRequired ? 500 : 400).json({ error: errMessage(e), restartRequired });
|
|
1679
1636
|
}
|
|
1680
1637
|
}));
|
|
1681
|
-
|
|
1638
|
+
function commandForService({ project, service, type, payload, }) {
|
|
1682
1639
|
if (!service)
|
|
1683
1640
|
throw httpError(400, 'service required');
|
|
1684
|
-
const ds =
|
|
1641
|
+
const ds = currentDesired(project);
|
|
1685
1642
|
if (!ds)
|
|
1686
1643
|
throw httpError(404, missingProject(project));
|
|
1687
1644
|
const state = JSON.parse(ds.state_json);
|
|
@@ -1691,22 +1648,22 @@ export async function createApp(options = {}) {
|
|
|
1691
1648
|
const nodeName = svc.placement?.nodes?.[0];
|
|
1692
1649
|
if (!nodeName)
|
|
1693
1650
|
throw httpError(400, 'service has no explicit node placement');
|
|
1694
|
-
const node =
|
|
1651
|
+
const node = db.get('SELECT id FROM nodes WHERE name=? OR id=?', nodeName, nodeName);
|
|
1695
1652
|
if (!node)
|
|
1696
1653
|
throw httpError(404, 'node not found');
|
|
1697
|
-
const cmdId =
|
|
1654
|
+
const cmdId = queueCommand(node.id, project, type, { project, service, ...payload });
|
|
1698
1655
|
return { commandId: cmdId, nodeId: node.id };
|
|
1699
1656
|
}
|
|
1700
1657
|
api('post', '/api/v1/restart', bodyProject, async (req, res) => {
|
|
1701
1658
|
try {
|
|
1702
1659
|
const project = req.body.project || 'site';
|
|
1703
|
-
const queued =
|
|
1660
|
+
const queued = commandForService({
|
|
1704
1661
|
project,
|
|
1705
1662
|
service: req.body.service,
|
|
1706
1663
|
type: 'service.restart',
|
|
1707
1664
|
payload: {},
|
|
1708
1665
|
});
|
|
1709
|
-
|
|
1666
|
+
audit(req.actor, {
|
|
1710
1667
|
type: 'restart',
|
|
1711
1668
|
project,
|
|
1712
1669
|
message: `service ${req.body.service} restart requested by ${req.actor.name}`,
|
|
@@ -1724,7 +1681,7 @@ export async function createApp(options = {}) {
|
|
|
1724
1681
|
const tail = Number(req.body.tail || 200);
|
|
1725
1682
|
if (!service)
|
|
1726
1683
|
return res.status(400).json({ error: 'service required' });
|
|
1727
|
-
const ds =
|
|
1684
|
+
const ds = currentDesired(project);
|
|
1728
1685
|
if (!ds)
|
|
1729
1686
|
return res.status(404).json({ error: missingProject(project) });
|
|
1730
1687
|
const state = JSON.parse(ds.state_json);
|
|
@@ -1734,25 +1691,25 @@ export async function createApp(options = {}) {
|
|
|
1734
1691
|
const nodeName = svc.placement?.nodes?.[0];
|
|
1735
1692
|
if (!nodeName)
|
|
1736
1693
|
return res.status(400).json({ error: 'service has no explicit node placement' });
|
|
1737
|
-
const node =
|
|
1694
|
+
const node = db.get('SELECT id FROM nodes WHERE name=? OR id=?', nodeName, nodeName);
|
|
1738
1695
|
if (!node)
|
|
1739
1696
|
return res.status(404).json({ error: 'node not found' });
|
|
1740
|
-
const cmdId =
|
|
1697
|
+
const cmdId = queueCommand(node.id, project, 'logs.fetch', { project, service, tail });
|
|
1741
1698
|
res.json({ commandId: cmdId, nodeId: node.id });
|
|
1742
1699
|
});
|
|
1743
|
-
api('get', '/api/v1/commands/:id', commandProject, async (req, res) => res.json(
|
|
1700
|
+
api('get', '/api/v1/commands/:id', commandProject, async (req, res) => res.json(db.get('SELECT * FROM commands WHERE id=?', req.params.id)));
|
|
1744
1701
|
api('get', '/api/v1/status', null, async (req, res) => {
|
|
1745
1702
|
const staleSeconds = Number(process.env.TUG_STALE_SECONDS || 60);
|
|
1746
1703
|
const cutoff = new Date(Date.now() - staleSeconds * 1000).toISOString();
|
|
1747
|
-
|
|
1704
|
+
db.run("UPDATE nodes SET status='stale',updated_at=? WHERE last_seen_at IS NOT NULL AND last_seen_at < ?", now(), cutoff);
|
|
1748
1705
|
const projects = projectScope(req.actor, 'id');
|
|
1749
1706
|
const deployments = projectScope(req.actor, 'project_id');
|
|
1750
1707
|
const statuses = projectScope(req.actor, 'project_id');
|
|
1751
1708
|
res.json({
|
|
1752
|
-
projects:
|
|
1753
|
-
deployments:
|
|
1754
|
-
nodes:
|
|
1755
|
-
statuses:
|
|
1709
|
+
projects: db.all(`SELECT * FROM projects WHERE ${projects.where}`, ...projects.args),
|
|
1710
|
+
deployments: db.all(`SELECT * FROM deployments WHERE ${deployments.where} ORDER BY created_at DESC LIMIT 20`, ...deployments.args),
|
|
1711
|
+
nodes: db.all('SELECT id,name,status,last_seen_at FROM nodes'),
|
|
1712
|
+
statuses: db.all(`SELECT * FROM node_statuses WHERE ${statuses.where}`, ...statuses.args),
|
|
1756
1713
|
});
|
|
1757
1714
|
});
|
|
1758
1715
|
api('get', '/api/v1/events', null, async (req, res) => {
|
|
@@ -1762,8 +1719,8 @@ export async function createApp(options = {}) {
|
|
|
1762
1719
|
const limit = Math.min(Number(req.query.limit || 100), 500);
|
|
1763
1720
|
const scope = projectScope(req.actor, 'project_id'); // scoped tokens never see node-level or other projects' events
|
|
1764
1721
|
const events = project
|
|
1765
|
-
?
|
|
1766
|
-
:
|
|
1722
|
+
? db.all('SELECT * FROM events WHERE project_id=? ORDER BY created_at DESC LIMIT ?', project, limit)
|
|
1723
|
+
: db.all(`SELECT * FROM events WHERE ${scope.where} ORDER BY created_at DESC LIMIT ?`, ...scope.args, limit);
|
|
1767
1724
|
res.json({ events });
|
|
1768
1725
|
});
|
|
1769
1726
|
api('get', '/api/v1/diagnostics', null, async (req, res) => {
|
|
@@ -1777,10 +1734,10 @@ export async function createApp(options = {}) {
|
|
|
1777
1734
|
}
|
|
1778
1735
|
}
|
|
1779
1736
|
await check('sqlite', async () => ({
|
|
1780
|
-
value:
|
|
1737
|
+
value: db.get('SELECT COUNT(*) AS count FROM projects').count,
|
|
1781
1738
|
}));
|
|
1782
1739
|
await check('operator_token', async () => ({
|
|
1783
|
-
configured: Boolean(
|
|
1740
|
+
configured: Boolean(db.get('SELECT value FROM meta WHERE key=?', 'operator_token_hash')),
|
|
1784
1741
|
}));
|
|
1785
1742
|
await check('master_key', async () => ({ configured: Boolean(masterKey) }));
|
|
1786
1743
|
await check('docker', async () => ({
|
|
@@ -1791,14 +1748,14 @@ export async function createApp(options = {}) {
|
|
|
1791
1748
|
}));
|
|
1792
1749
|
if (MESH) {
|
|
1793
1750
|
await check('mesh', async () => {
|
|
1794
|
-
const nodes =
|
|
1751
|
+
const nodes = db.all('SELECT n.name, n.overlay_ip, EXISTS(SELECT 1 FROM wg_peers w WHERE w.node_id=n.id) AS has_key FROM nodes n');
|
|
1795
1752
|
const missing = nodes.filter((n) => !n.has_key).map((n) => n.name);
|
|
1796
1753
|
if (missing.length)
|
|
1797
1754
|
throw new Error(`nodes without a reported WireGuard key (is TUG_MESH=true on their agents?): ${missing.join(', ')}`);
|
|
1798
1755
|
return { cidr: MESH_CIDR, port: WG_PORT, nodes: nodes.length };
|
|
1799
1756
|
});
|
|
1800
1757
|
}
|
|
1801
|
-
const ingressNodes =
|
|
1758
|
+
const ingressNodes = db.all('SELECT name,ingress_status_json FROM nodes WHERE ingress=1');
|
|
1802
1759
|
if (ingressNodes.length) {
|
|
1803
1760
|
await check('ingress', async () => {
|
|
1804
1761
|
const bad = ingressNodes
|
|
@@ -1810,45 +1767,47 @@ export async function createApp(options = {}) {
|
|
|
1810
1767
|
return { nodes: ingressNodes.map((n) => n.name) };
|
|
1811
1768
|
});
|
|
1812
1769
|
}
|
|
1813
|
-
res.json({ ok: checks.every((c) => c.ok),
|
|
1770
|
+
res.json({ ok: checks.every((c) => c.ok), checks });
|
|
1814
1771
|
});
|
|
1815
1772
|
api('post', '/api/v1/diagnostics/nodes', null, async (req, res) => {
|
|
1816
1773
|
const nodes = req.body.node
|
|
1817
|
-
?
|
|
1818
|
-
:
|
|
1774
|
+
? db.all('SELECT id FROM nodes WHERE id=? OR name=?', req.body.node, req.body.node)
|
|
1775
|
+
: db.all('SELECT id FROM nodes');
|
|
1819
1776
|
if (req.body.node && !nodes.length)
|
|
1820
1777
|
return res.status(404).json({ error: `node ${req.body.node} not found` });
|
|
1821
1778
|
const commands = [];
|
|
1822
1779
|
for (const node of nodes) {
|
|
1823
|
-
const cmdId =
|
|
1780
|
+
const cmdId = queueCommand(node.id, null, 'diagnostics.run');
|
|
1824
1781
|
commands.push({ commandId: cmdId, nodeId: node.id });
|
|
1825
1782
|
}
|
|
1826
1783
|
res.json({ commands });
|
|
1827
1784
|
});
|
|
1828
|
-
|
|
1829
|
-
const rows =
|
|
1785
|
+
function currentDesiredStates() {
|
|
1786
|
+
const rows = db.all(`SELECT ds.* FROM desired_states ds JOIN projects p ON p.current_deployment_id=ds.deployment_id ORDER BY ds.created_at DESC`);
|
|
1830
1787
|
return rows.map((s) => JSON.parse(s.state_json));
|
|
1831
1788
|
}
|
|
1832
|
-
|
|
1789
|
+
function endpointsFor(states) {
|
|
1833
1790
|
const staleBefore = new Date(Date.now() - Number(process.env.TUG_STALE_SECONDS || 60) * 1000).toISOString();
|
|
1834
|
-
const nodes =
|
|
1791
|
+
const nodes = db.all('SELECT n.*, EXISTS(SELECT 1 FROM wg_peers w WHERE w.node_id=n.id) AS has_key FROM nodes n');
|
|
1835
1792
|
const views = nodes.map((n) => ({
|
|
1836
1793
|
name: n.name,
|
|
1837
1794
|
overlayIp: MESH && n.has_key ? n.overlay_ip : null,
|
|
1838
1795
|
address: n.address,
|
|
1839
1796
|
lastSeenAt: n.last_seen_at,
|
|
1840
1797
|
}));
|
|
1841
|
-
const statuses =
|
|
1798
|
+
const statuses = db
|
|
1799
|
+
.all('SELECT node_id,project_id,status_json FROM node_statuses')
|
|
1800
|
+
.map((r) => ({
|
|
1842
1801
|
node: nodes.find((n) => n.id === r.node_id)?.name ?? r.node_id,
|
|
1843
1802
|
project: r.project_id,
|
|
1844
1803
|
status: JSON.parse(r.status_json),
|
|
1845
1804
|
}));
|
|
1846
1805
|
return resolveServiceEndpoints(states, views, statuses, staleBefore);
|
|
1847
1806
|
}
|
|
1848
|
-
|
|
1807
|
+
function networkFor(node) {
|
|
1849
1808
|
if (!MESH || !node.overlay_ip)
|
|
1850
1809
|
return undefined;
|
|
1851
|
-
const peers =
|
|
1810
|
+
const peers = db.all('SELECT w.*, n.name FROM wg_peers w JOIN nodes n ON n.id=w.node_id WHERE w.node_id != ? ORDER BY n.name', node.id);
|
|
1852
1811
|
return {
|
|
1853
1812
|
interface: WG_INTERFACE,
|
|
1854
1813
|
overlayIp: node.overlay_ip,
|
|
@@ -1864,17 +1823,17 @@ export async function createApp(options = {}) {
|
|
|
1864
1823
|
};
|
|
1865
1824
|
}
|
|
1866
1825
|
/** Names of all nodes and of the ingress nodes, for port binding and upstream decisions. */
|
|
1867
|
-
|
|
1868
|
-
const rows =
|
|
1826
|
+
function nodeTopology() {
|
|
1827
|
+
const rows = db.all('SELECT name,ingress FROM nodes ORDER BY name');
|
|
1869
1828
|
return {
|
|
1870
1829
|
nodes: rows.map((r) => r.name),
|
|
1871
1830
|
ingressNodes: rows.filter((r) => r.ingress).map((r) => r.name),
|
|
1872
1831
|
};
|
|
1873
1832
|
}
|
|
1874
1833
|
/** Where this node's published ports listen: the address other nodes and the ingress reach it on, or loopback. */
|
|
1875
|
-
|
|
1876
|
-
const hasKey = MESH && node.overlay_ip &&
|
|
1877
|
-
const { nodes, ingressNodes } =
|
|
1834
|
+
function portBindingFor(node, states) {
|
|
1835
|
+
const hasKey = MESH && node.overlay_ip && db.get('SELECT 1 FROM wg_peers WHERE node_id=?', node.id);
|
|
1836
|
+
const { nodes, ingressNodes } = nodeTopology();
|
|
1878
1837
|
const loopback = {};
|
|
1879
1838
|
for (const state of states) {
|
|
1880
1839
|
const services = loopbackIngressServices(state, node.name, ingressNodes, nodes);
|
|
@@ -1883,7 +1842,7 @@ export async function createApp(options = {}) {
|
|
|
1883
1842
|
}
|
|
1884
1843
|
return { address: (hasKey ? node.overlay_ip : node.address) || null, loopback };
|
|
1885
1844
|
}
|
|
1886
|
-
|
|
1845
|
+
function ingressFor(node, states) {
|
|
1887
1846
|
const secrets = new Map();
|
|
1888
1847
|
for (const state of states) {
|
|
1889
1848
|
for (const svc of Object.values(state.compose.services || {})) {
|
|
@@ -1891,7 +1850,7 @@ export async function createApp(options = {}) {
|
|
|
1891
1850
|
if (ing?.tls !== 'custom')
|
|
1892
1851
|
continue;
|
|
1893
1852
|
for (const name of [ing.cert, ing.key]) {
|
|
1894
|
-
const row =
|
|
1853
|
+
const row = db.get('SELECT encrypted_value FROM secrets WHERE project_id=? AND name=?', state.project, name ?? null);
|
|
1895
1854
|
if (row)
|
|
1896
1855
|
secrets.set(`${state.project}/${name}`, decrypt(row.encrypted_value, masterKey));
|
|
1897
1856
|
}
|
|
@@ -1899,9 +1858,9 @@ export async function createApp(options = {}) {
|
|
|
1899
1858
|
}
|
|
1900
1859
|
return buildIngressWork({
|
|
1901
1860
|
states,
|
|
1902
|
-
endpoints:
|
|
1861
|
+
endpoints: endpointsFor(states),
|
|
1903
1862
|
node: node.name,
|
|
1904
|
-
...
|
|
1863
|
+
...nodeTopology(),
|
|
1905
1864
|
secrets,
|
|
1906
1865
|
acmeEmail: ACME_EMAIL,
|
|
1907
1866
|
});
|
|
@@ -1909,19 +1868,19 @@ export async function createApp(options = {}) {
|
|
|
1909
1868
|
get('/agent/v1/wake', authAgent, async (req, res) => {
|
|
1910
1869
|
const after = Math.max(0, Number.parseInt(String(req.query.after ?? '0'), 10) || 0);
|
|
1911
1870
|
const timeoutMs = Math.min(30_000, Math.max(0, Number.parseInt(String(req.query.timeout ?? '25000'), 10) || 0));
|
|
1912
|
-
|
|
1871
|
+
db.run('UPDATE nodes SET last_seen_at=?,status=? WHERE id=?', now(), 'online', req.node.id);
|
|
1913
1872
|
res.json(await wakeHub.wait(req.node.id, after, timeoutMs));
|
|
1914
1873
|
});
|
|
1915
1874
|
get('/agent/v1/work', authAgent, async (req, res) => {
|
|
1916
|
-
|
|
1917
|
-
const node =
|
|
1918
|
-
const allStates =
|
|
1875
|
+
db.run('UPDATE nodes SET last_seen_at=?,status=? WHERE id=?', now(), 'online', req.node.id);
|
|
1876
|
+
const node = db.get('SELECT * FROM nodes WHERE id=?', req.node.id);
|
|
1877
|
+
const allStates = currentDesiredStates();
|
|
1919
1878
|
const desiredStates = allStates.filter((st) => Object.values(st.services).some((svc) => (svc.placement?.nodes || []).includes(node.name) || !svc.placement?.nodes?.length));
|
|
1920
|
-
const commands =
|
|
1879
|
+
const commands = db.all('SELECT * FROM commands WHERE node_id=? AND state=?', node.id, 'pending');
|
|
1921
1880
|
if (commands.length)
|
|
1922
|
-
|
|
1881
|
+
db.run(`UPDATE commands SET state='running',attempts=attempts+1,updated_at=? WHERE node_id=? AND state='pending'`, now(), node.id);
|
|
1923
1882
|
const retainedImages = [
|
|
1924
|
-
...new Set(
|
|
1883
|
+
...new Set(retainedStates().flatMap((state) => Object.values(state.services)
|
|
1925
1884
|
.filter((service) => service.artifact &&
|
|
1926
1885
|
(!service.placement.nodes?.length || service.placement.nodes.includes(node.name)))
|
|
1927
1886
|
.map((service) => service.image))),
|
|
@@ -1933,64 +1892,81 @@ export async function createApp(options = {}) {
|
|
|
1933
1892
|
commands,
|
|
1934
1893
|
retainedImages,
|
|
1935
1894
|
};
|
|
1936
|
-
const network =
|
|
1895
|
+
const network = networkFor(node);
|
|
1937
1896
|
if (network)
|
|
1938
1897
|
work.network = network;
|
|
1939
|
-
work.ports =
|
|
1898
|
+
work.ports = portBindingFor(node, desiredStates);
|
|
1940
1899
|
if (SERVICE_DNS) {
|
|
1941
|
-
const all =
|
|
1900
|
+
const all = endpointsFor(allStates);
|
|
1942
1901
|
work.serviceEndpoints = Object.fromEntries(desiredStates.map((st) => [st.project, all[st.project] || {}]));
|
|
1943
1902
|
}
|
|
1944
1903
|
if (node.ingress)
|
|
1945
|
-
work.ingress =
|
|
1904
|
+
work.ingress = ingressFor(node, allStates);
|
|
1946
1905
|
res.json(work);
|
|
1947
1906
|
});
|
|
1948
1907
|
post('/agent/v1/status', authAgent, async (req, res) => {
|
|
1949
1908
|
const ts = now();
|
|
1950
1909
|
const report = req.body;
|
|
1951
1910
|
for (const st of report.projects || []) {
|
|
1952
|
-
|
|
1953
|
-
|
|
1911
|
+
db.run(`INSERT INTO node_statuses(id,node_id,project_id,observed_version,status_json,health,updated_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(node_id,project_id) DO UPDATE SET observed_version=excluded.observed_version,status_json=excluded.status_json,health=excluded.health,updated_at=excluded.updated_at`, id('ns'), req.node.id, st.project, st.observedVersion, JSON.stringify(st), st.health || 'unknown', ts);
|
|
1912
|
+
updateDeploymentConvergence(st.project);
|
|
1954
1913
|
}
|
|
1955
1914
|
for (const cr of (report.commandResults || [])) {
|
|
1956
1915
|
const result = cr.result || {};
|
|
1957
|
-
|
|
1916
|
+
db.run('UPDATE commands SET state=?,result_json=?,updated_at=? WHERE id=? AND node_id=?', cr.state || 'succeeded', JSON.stringify(result), ts, cr.commandId ?? null, req.node.id);
|
|
1958
1917
|
}
|
|
1959
1918
|
if (report.ingress)
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
const nextEndpointSnapshot = JSON.stringify(
|
|
1919
|
+
db.run('UPDATE nodes SET ingress_status_json=? WHERE id=?', JSON.stringify({ ...report.ingress, reportedAt: ts }), req.node.id);
|
|
1920
|
+
db.run('UPDATE nodes SET last_seen_at=?,status=? WHERE id=?', ts, 'online', req.node.id);
|
|
1921
|
+
const nextEndpointSnapshot = JSON.stringify(endpointsFor(currentDesiredStates()));
|
|
1963
1922
|
if (nextEndpointSnapshot !== endpointSnapshot) {
|
|
1964
1923
|
endpointSnapshot = nextEndpointSnapshot;
|
|
1965
|
-
|
|
1924
|
+
wakeAll();
|
|
1966
1925
|
}
|
|
1967
1926
|
res.json({ ok: true });
|
|
1968
1927
|
});
|
|
1969
1928
|
post('/agent/v1/events', authAgent, async (req, res) => {
|
|
1970
1929
|
const evs = req.body.events || [req.body];
|
|
1971
1930
|
for (const ev of evs)
|
|
1972
|
-
|
|
1931
|
+
db.run('INSERT INTO events(id,project_id,deployment_id,node_id,level,type,message,metadata_json,created_at) VALUES(?,?,?,?,?,?,?,?,?)', id('evt'), ev.projectId || null, ev.deploymentId || null, req.node.id, ev.level || 'info', ev.type || 'event', ev.message || '', JSON.stringify(ev.metadata || {}), now());
|
|
1973
1932
|
if (Date.now() - lastEventPrune >= eventPruneIntervalMs)
|
|
1974
|
-
|
|
1933
|
+
try {
|
|
1934
|
+
pruneEvents();
|
|
1935
|
+
}
|
|
1936
|
+
catch (e) {
|
|
1937
|
+
console.error(`event pruning failed: ${errMessage(e)}`);
|
|
1938
|
+
}
|
|
1975
1939
|
res.json({ ok: true });
|
|
1976
1940
|
});
|
|
1977
1941
|
get('/agent/v1/secrets', authAgent, async (req, res) => {
|
|
1978
1942
|
const project = String(req.query.project || 'site');
|
|
1979
|
-
let rows =
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1943
|
+
let rows = db.all('SELECT name,encrypted_value,version FROM secrets WHERE project_id=?', project);
|
|
1944
|
+
const ds = currentDesired(project);
|
|
1945
|
+
const desired = ds ? JSON.parse(ds.state_json) : undefined;
|
|
1946
|
+
const requested = req.query.services
|
|
1947
|
+
? String(req.query.services)
|
|
1983
1948
|
.split(',')
|
|
1984
1949
|
.map((s) => s.trim())
|
|
1985
|
-
.filter(Boolean)
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1950
|
+
.filter(Boolean)
|
|
1951
|
+
: Object.keys(desired?.services || {});
|
|
1952
|
+
const names = requested.filter((name) => desired?.services[name]?.placement.nodes?.includes(req.node.name));
|
|
1953
|
+
const allowed = serviceSecretNames(desired?.compose, names);
|
|
1954
|
+
const registrySecret = rows.find((row) => row.name === REGISTRY_AUTH_SECRET);
|
|
1955
|
+
const registryAuth = {};
|
|
1956
|
+
const credentials = parseRegistryCredentials(registrySecret ? decrypt(registrySecret.encrypted_value, masterKey) : undefined);
|
|
1957
|
+
for (const name of names) {
|
|
1958
|
+
const spec = desired.services[name];
|
|
1959
|
+
if (!spec.artifact && spec.image) {
|
|
1960
|
+
const host = imageLocation(spec.image).host;
|
|
1961
|
+
if (credentials[host])
|
|
1962
|
+
registryAuth[host] = credentials[host];
|
|
1963
|
+
}
|
|
1990
1964
|
}
|
|
1965
|
+
rows = rows.filter((row) => allowed.has(row.name) && row.name !== REGISTRY_AUTH_SECRET);
|
|
1991
1966
|
const secrets = {
|
|
1992
1967
|
secrets: Object.fromEntries(rows.map((r) => [r.name, decrypt(r.encrypted_value, masterKey)])),
|
|
1993
1968
|
versions: Object.fromEntries(rows.map((r) => [r.name, r.version])),
|
|
1969
|
+
...(Object.keys(registryAuth).length ? { registryAuth } : {}),
|
|
1994
1970
|
};
|
|
1995
1971
|
res.json(secrets);
|
|
1996
1972
|
});
|
|
@@ -2019,7 +1995,7 @@ function attachAgentStream(app, server) {
|
|
|
2019
1995
|
const raw = String(request.headers.authorization || '').replace(/^Bearer /, '');
|
|
2020
1996
|
void (async () => {
|
|
2021
1997
|
if (pathname === '/api/v1/live') {
|
|
2022
|
-
const actor =
|
|
1998
|
+
const actor = runtime.authenticateOperator(raw);
|
|
2023
1999
|
if (!actor) {
|
|
2024
2000
|
reject(401, 'Unauthorized');
|
|
2025
2001
|
return;
|
|
@@ -2031,7 +2007,7 @@ function attachAgentStream(app, server) {
|
|
|
2031
2007
|
wss.handleUpgrade(request, socket, head, (ws) => runtime.live.operator(actor, ws));
|
|
2032
2008
|
}
|
|
2033
2009
|
else {
|
|
2034
|
-
const node =
|
|
2010
|
+
const node = runtime.authenticate(raw);
|
|
2035
2011
|
if (!node) {
|
|
2036
2012
|
reject(401, 'Unauthorized');
|
|
2037
2013
|
return;
|