@andrian.yablonskyy/thub-coordinator 1.0.37 → 1.0.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/thub-admin.js +3 -5
- package/config.json +1 -6
- package/package.json +2 -2
- package/public/css/thub.css +129 -0
- package/public/js/config-import.js +37 -33
- package/public/js/copy-to-clipboard.js +21 -18
- package/public/js/help.js +93 -0
- package/public/js/list-search.js +51 -0
- package/public/js/live.js +287 -0
- package/public/js/log-viewer.js +289 -45
- package/public/js/remove-resource.js +5 -3
- package/public/js/resource-card.js +14 -13
- package/public/js/tooltips.js +3 -1
- package/scripts/install-default-config.js +1 -1
- package/scripts/install-target.js +2 -2
- package/src/api/admin.js +1 -1
- package/src/api/agent.js +4 -10
- package/src/api/resource.js +6 -11
- package/src/api/sse.js +13 -10
- package/src/config.js +16 -9
- package/src/db/migrations/022_add_sessions.sql +10 -0
- package/src/db/migrations/023_drop_artifacts.sql +4 -0
- package/src/dev/virtual.js +1 -1
- package/src/server.js +55 -21
- package/src/services/cleanup.js +2 -2
- package/src/services/jobs.js +33 -19
- package/src/services/list-prefs.js +4 -0
- package/src/services/live.js +124 -0
- package/src/services/logs.js +12 -14
- package/src/services/retention.js +5 -14
- package/src/services/search.js +59 -0
- package/src/services/session-store.js +127 -0
- package/src/web/routes.js +205 -34
- package/test/cleanup.test.js +1 -5
- package/test/list-prefs.test.js +6 -0
- package/test/live.test.js +153 -0
- package/test/logs.test.js +150 -0
- package/test/scheduler.test.js +10 -12
- package/test/search.test.js +97 -0
- package/test/session.test.js +132 -0
- package/test/views.test.js +30 -0
- package/views/admin/agents.pug +28 -10
- package/views/groups/list.pug +14 -1
- package/views/help/_agent-cli.pug +103 -0
- package/views/help/_agent-setup.pug +72 -0
- package/views/help/_ci.pug +44 -0
- package/views/help/_client-machines.pug +87 -0
- package/views/help/_client-setup.pug +122 -0
- package/views/help/_coordinator.pug +137 -0
- package/views/help/_docker.pug +124 -0
- package/views/help/_env.pug +127 -0
- package/views/help/_git.pug +130 -0
- package/views/help/_mixins.pug +24 -0
- package/views/help/_overview.pug +118 -0
- package/views/help/_quick-start.pug +37 -0
- package/views/help/_troubleshooting.pug +44 -0
- package/views/help/index.pug +67 -0
- package/views/index.pug +3 -2
- package/views/jobs/list.pug +8 -5
- package/views/jobs/show.pug +0 -10
- package/views/layout.pug +24 -1
- package/views/mixins/list-controls.pug +36 -0
- package/views/mixins/log-viewer.pug +39 -3
- package/views/mixins/resource.pug +1 -1
- package/views/resources/list.pug +7 -4
- package/src/services/artifacts.js +0 -149
|
@@ -21,18 +21,19 @@
|
|
|
21
21
|
}
|
|
22
22
|
}
|
|
23
23
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
24
|
+
// Delegated, so rows and cards swapped in by live updates (live.js) work.
|
|
25
|
+
document.addEventListener('click', (e) => {
|
|
26
|
+
const el = e.target.closest('[data-resource-card]');
|
|
27
|
+
// The Resources table's own action buttons stay plain actions.
|
|
28
|
+
if (el && !e.target.closest('form, button, a, input')){
|
|
29
|
+
open(el);
|
|
30
|
+
}
|
|
31
|
+
});
|
|
32
|
+
document.addEventListener('keydown', (e) => {
|
|
33
|
+
const el = e.target.closest?.('[data-resource-card]');
|
|
34
|
+
if (el && (e.key === 'Enter' || e.key === ' ') && e.target === el){
|
|
35
|
+
e.preventDefault();
|
|
36
|
+
open(el);
|
|
37
|
+
}
|
|
37
38
|
});
|
|
38
39
|
})();
|
package/public/js/tooltips.js
CHANGED
|
@@ -13,6 +13,8 @@
|
|
|
13
13
|
|
|
14
14
|
'use strict';
|
|
15
15
|
|
|
16
|
+
// Delegated (one instance per element, created on first hover/focus), so
|
|
17
|
+
// content swapped in by live updates (public/js/live.js) gets tooltips too.
|
|
16
18
|
(function (){
|
|
17
|
-
document.
|
|
19
|
+
new bootstrap.Tooltip(document.body, { selector: '[data-bs-toggle="tooltip"]' });
|
|
18
20
|
})();
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @file scripts/install-default-config.js
|
|
3
3
|
* @description npm postinstall: on a real global install, creates the Coordinator's directory layout
|
|
4
|
-
* (~/.config/thub, ~/.thub and its
|
|
4
|
+
* (~/.config/thub, ~/.thub and its avatars/work subdirs) and, if it
|
|
5
5
|
* doesn't already exist, ~/.config/thub/coordinator.json with every default option and
|
|
6
6
|
* a freshly generated sessionSecret (README §12, §13)
|
|
7
7
|
*
|
|
@@ -52,7 +52,7 @@ function resolveTargetUser(){
|
|
|
52
52
|
}
|
|
53
53
|
|
|
54
54
|
// Default layout (README §13): config under ~/.config/thub, runtime data
|
|
55
|
-
// (SQLite DB,
|
|
55
|
+
// (SQLite DB, avatars, work files) under ~/.thub.
|
|
56
56
|
// An existing coordinator.json's own dataDir wins over the default, so a
|
|
57
57
|
// re-install prepares (and the systemd unit allows writes to) the
|
|
58
58
|
// directory the Coordinator will really use.
|
|
@@ -69,7 +69,7 @@ function targetPaths(user){
|
|
|
69
69
|
// Written by the dashboard's "Update app" button, watched by
|
|
70
70
|
// thub-coordinator-update.path (README §10.2).
|
|
71
71
|
updateRequestFile: path.join(dataDir, 'update-request.json'),
|
|
72
|
-
dataSubdirs: ['
|
|
72
|
+
dataSubdirs: ['avatars', 'work'].map((d) => path.join(dataDir, d))
|
|
73
73
|
};
|
|
74
74
|
}
|
|
75
75
|
|
package/src/api/admin.js
CHANGED
|
@@ -134,7 +134,7 @@ function createAdminRouter({ services }){
|
|
|
134
134
|
res.json({ canceled });
|
|
135
135
|
});
|
|
136
136
|
|
|
137
|
-
// Permanently deletes finished jobs and their logs
|
|
137
|
+
// Permanently deletes finished jobs and their logs (§13.1).
|
|
138
138
|
// Database cleanup (§9): optional `before` (ISO 8601; default now) —
|
|
139
139
|
// finished jobs that ended before it, older history, then VACUUM.
|
|
140
140
|
router.post('/jobs/clean-history', (req, res) => {
|
package/src/api/agent.js
CHANGED
|
@@ -80,19 +80,13 @@ function createAgentRouter({ services, config }){
|
|
|
80
80
|
});
|
|
81
81
|
|
|
82
82
|
router.get('/jobs/:id/logs/stream', auth, (req, res) => {
|
|
83
|
-
attachJobStream(req, res, { jobId: req.params.id, services
|
|
83
|
+
attachJobStream(req, res, { jobId: req.params.id, services });
|
|
84
84
|
});
|
|
85
85
|
|
|
86
|
+
// No artifacts are stored any more (README §9); an Agent older than that
|
|
87
|
+
// still asks after a finished job's `thub status`, so answer "none".
|
|
86
88
|
router.get('/jobs/:id/artifacts', auth, (req, res) => {
|
|
87
|
-
|
|
88
|
-
id: a.id,
|
|
89
|
-
name: a.name,
|
|
90
|
-
size: a.size,
|
|
91
|
-
sha256: a.sha256,
|
|
92
|
-
contentType: a.content_type,
|
|
93
|
-
url: services.artifacts.signedUrl(a)
|
|
94
|
-
}));
|
|
95
|
-
res.json({ artifacts });
|
|
89
|
+
res.json({ artifacts: [] });
|
|
96
90
|
});
|
|
97
91
|
|
|
98
92
|
router.get('/resources', auth, (req, res) => {
|
package/src/api/resource.js
CHANGED
|
@@ -14,12 +14,9 @@
|
|
|
14
14
|
'use strict';
|
|
15
15
|
|
|
16
16
|
const express = require('express'),
|
|
17
|
-
multer = require('multer'),
|
|
18
17
|
{ requireRole, requireJoinKey, appVersion } = require('../auth'),
|
|
19
18
|
{ JOB_STATES } = require('@andrian.yablonskyy/thub-common');
|
|
20
19
|
|
|
21
|
-
const upload = multer({ dest: require('node:os').tmpdir() });
|
|
22
|
-
|
|
23
20
|
function requireOwnResource(req, res, next){
|
|
24
21
|
if (req.resource.id !== req.params.id){
|
|
25
22
|
return res.status(403).json({ error: 'Token does not match resource' });
|
|
@@ -211,14 +208,12 @@ function createResourceRouter({ services, config }){
|
|
|
211
208
|
}
|
|
212
209
|
});
|
|
213
210
|
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
next(err);
|
|
221
|
-
}
|
|
211
|
+
// Artifacts are no longer stored on the Coordinator (README §9). A Client
|
|
212
|
+
// older than that still uploads its results after each job and would end
|
|
213
|
+
// the job as ERROR on a 404, so accept the upload and discard it unread.
|
|
214
|
+
router.post('/jobs/:id/artifacts', auth, requireOwnJob(services), (req, res) => {
|
|
215
|
+
req.resume();
|
|
216
|
+
req.on('end', () => res.status(201).json({ artifacts: [] }));
|
|
222
217
|
});
|
|
223
218
|
|
|
224
219
|
router.post('/jobs/:id/result', auth, requireOwnJob(services), (req, res, next) => {
|
package/src/api/sse.js
CHANGED
|
@@ -15,11 +15,12 @@
|
|
|
15
15
|
|
|
16
16
|
const { TERMINAL_JOB_STATES, JOB_STATES } = require('@andrian.yablonskyy/thub-common');
|
|
17
17
|
|
|
18
|
-
const WAITING_CHECK_MS = 15_000
|
|
18
|
+
const WAITING_CHECK_MS = 15_000,
|
|
19
|
+
REPLAY_PAGE = 2000;
|
|
19
20
|
|
|
20
21
|
// §6.4: log / state / end events, resumable via Last-Event-ID (§7: "the
|
|
21
22
|
// Agent reconnects with exponential backoff and resumes from the last seq").
|
|
22
|
-
function attachJobStream(req, res, { jobId, services
|
|
23
|
+
function attachJobStream(req, res, { jobId, services }){
|
|
23
24
|
const { jobs, logs, registry, bus } = services,
|
|
24
25
|
|
|
25
26
|
job = jobs.get(jobId);
|
|
@@ -47,17 +48,19 @@ function attachJobStream(req, res, { jobId, services, config }){
|
|
|
47
48
|
res.write(`data: ${JSON.stringify(data)}\n\n`);
|
|
48
49
|
}
|
|
49
50
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
51
|
+
// Replay everything after `afterSeq`, in pages (listSince stops at 500 a
|
|
52
|
+
// call — replaying one page used to skip lines 501.. of a longer log). All
|
|
53
|
+
// synchronous, so no live line can slip in between replay and subscribing.
|
|
54
|
+
for (let seq = afterSeq, page; (page = logs.listSince(jobId, seq, REPLAY_PAGE)).length;){
|
|
55
|
+
for (const line of page){
|
|
56
|
+
writeEvent('log', { ts: line.ts, stream: line.stream, line: line.line }, line.seq);
|
|
57
|
+
}
|
|
58
|
+
seq = page.at(-1).seq;
|
|
56
59
|
}
|
|
57
60
|
|
|
58
61
|
const current = jobs.get(jobId);
|
|
59
62
|
if (TERMINAL_JOB_STATES.has(current.state)){
|
|
60
|
-
writeEvent('end', { state: current.state
|
|
63
|
+
writeEvent('end', { state: current.state });
|
|
61
64
|
return res.end();
|
|
62
65
|
}
|
|
63
66
|
|
|
@@ -97,7 +100,7 @@ function attachJobStream(req, res, { jobId, services, config }){
|
|
|
97
100
|
if (evt.jobId !== jobId){
|
|
98
101
|
return;
|
|
99
102
|
}
|
|
100
|
-
writeEvent('end', { state: evt.state
|
|
103
|
+
writeEvent('end', { state: evt.state });
|
|
101
104
|
cleanup();
|
|
102
105
|
res.end();
|
|
103
106
|
};
|
package/src/config.js
CHANGED
|
@@ -28,6 +28,12 @@ const DEFAULTS = {
|
|
|
28
28
|
trustProxy: 'loopback',
|
|
29
29
|
dataDir: path.join(process.cwd(), '.data'),
|
|
30
30
|
sessionSecret: 'dev-only-change-me',
|
|
31
|
+
session: {
|
|
32
|
+
// The dashboard session cookie's Secure flag (server.js cookieSecure):
|
|
33
|
+
// "auto" = Secure whenever publicUrl is https:// (else per request:
|
|
34
|
+
// Secure only on a connection that is itself HTTPS); true / false force it.
|
|
35
|
+
secureCookie: 'auto'
|
|
36
|
+
},
|
|
31
37
|
// Shared secret Clients present to self-register (see api/resource.js).
|
|
32
38
|
// null disables auto-registration entirely — set it explicitly to turn it on.
|
|
33
39
|
clientJoinKey: null,
|
|
@@ -46,19 +52,16 @@ const DEFAULTS = {
|
|
|
46
52
|
defaultTimeoutSec: 1800,
|
|
47
53
|
maxTimeoutSec: 14400
|
|
48
54
|
},
|
|
55
|
+
// logRetentionDays, artifactRetentionDays and the `artifacts` section of
|
|
56
|
+
// older configs are no longer read: a job's log lines live as long as the
|
|
57
|
+
// job, and the Coordinator stores no artifacts (README §9).
|
|
49
58
|
retention: {
|
|
50
|
-
|
|
51
|
-
artifactRetentionDays: 30,
|
|
52
|
-
// How long a finished job (with its logs, artifacts and events) is kept
|
|
59
|
+
// How long a finished job (with its logs and events) is kept
|
|
53
60
|
// before the hourly retention task wipes it (services/cleanup.js):
|
|
54
61
|
// 1w | 2w | 1m | 3m | 6m | forever. `forever` keeps everything until a
|
|
55
62
|
// manual cleanup (dashboard Clean up database / thub-admin jobs clean).
|
|
56
63
|
jobRetention: 'forever'
|
|
57
64
|
},
|
|
58
|
-
artifacts: {
|
|
59
|
-
maxUploadMb: 512,
|
|
60
|
-
linkTtlHours: 168
|
|
61
|
-
},
|
|
62
65
|
// New-version check of the Coordinator/Agent/Client packages (README
|
|
63
66
|
// §10.2), every 15 minutes so a new Coordinator's "Update app" button
|
|
64
67
|
// shows up promptly. 0 disables the periodic check; registry null = the
|
|
@@ -122,6 +125,12 @@ function loadConfig(configPath = process.env.THUB_COORDINATOR_CONFIG){
|
|
|
122
125
|
config.clientJoinKey = process.env.THUB_CLIENT_JOIN_KEY;
|
|
123
126
|
}
|
|
124
127
|
|
|
128
|
+
if (![true, false, 'auto'].includes(config.session.secureCookie)){
|
|
129
|
+
throw new Error(
|
|
130
|
+
`session.secureCookie must be "auto", true or false (got ${JSON.stringify(config.session.secureCookie)}) in ${candidate || 'the config'}`
|
|
131
|
+
);
|
|
132
|
+
}
|
|
133
|
+
|
|
125
134
|
if (!Object.hasOwn(JOB_RETENTION, config.retention.jobRetention)){
|
|
126
135
|
throw new Error(
|
|
127
136
|
`retention.jobRetention must be one of: ${Object.keys(JOB_RETENTION).join(', ')} ` +
|
|
@@ -136,12 +145,10 @@ function loadConfig(configPath = process.env.THUB_COORDINATOR_CONFIG){
|
|
|
136
145
|
config.host = host;
|
|
137
146
|
config.port = Number(port);
|
|
138
147
|
config.dbPath = path.join(config.dataDir, 'thub.db');
|
|
139
|
-
config.artifactsDir = path.join(config.dataDir, 'artifacts');
|
|
140
148
|
config.workDir = path.join(config.dataDir, 'work');
|
|
141
149
|
config.avatarsDir = path.join(config.dataDir, 'avatars');
|
|
142
150
|
|
|
143
151
|
ensureWritableDir(config.dataDir, config.configPath);
|
|
144
|
-
ensureWritableDir(config.artifactsDir, config.configPath);
|
|
145
152
|
ensureWritableDir(config.avatarsDir, config.configPath);
|
|
146
153
|
|
|
147
154
|
return config;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
-- Dashboard sessions (services/session-store.js), instead of express-session's
|
|
2
|
+
-- in-memory store: they survive a restart, and expired ones are pruned.
|
|
3
|
+
-- `expires` is epoch milliseconds (the session cookie's own expiry).
|
|
4
|
+
CREATE TABLE sessions (
|
|
5
|
+
sid TEXT PRIMARY KEY,
|
|
6
|
+
sess TEXT NOT NULL,
|
|
7
|
+
expires INTEGER NOT NULL
|
|
8
|
+
);
|
|
9
|
+
|
|
10
|
+
CREATE INDEX idx_sessions_expires ON sessions(expires);
|
package/src/dev/virtual.js
CHANGED
|
@@ -170,7 +170,7 @@ function seedHistory(services, agentId, resourceIds){
|
|
|
170
170
|
// stay IDLE instead of going OUT_OF_SERVICE) and play out any job the
|
|
171
171
|
// scheduler assigns them — accept, PREPARING, RUNNING with streamed log
|
|
172
172
|
// lines, then a PASSED (mostly) or FAILED result — through the same
|
|
173
|
-
// services the Client API uses, so SSE
|
|
173
|
+
// services the Client API uses, so SSE and durations all work.
|
|
174
174
|
function startVirtualClients(services, config){
|
|
175
175
|
const { registry, jobs, logs, bus } = services,
|
|
176
176
|
intervalMs = config.heartbeat.intervalSec * 1000,
|
package/src/server.js
CHANGED
|
@@ -15,7 +15,8 @@
|
|
|
15
15
|
|
|
16
16
|
'use strict';
|
|
17
17
|
|
|
18
|
-
const
|
|
18
|
+
const fs = require('node:fs'),
|
|
19
|
+
path = require('node:path'),
|
|
19
20
|
express = require('express'),
|
|
20
21
|
session = require('express-session'),
|
|
21
22
|
|
|
@@ -31,19 +32,31 @@ const path = require('node:path'),
|
|
|
31
32
|
{ createScheduler } = require('./services/scheduler'),
|
|
32
33
|
{ createHeartbeatMonitor } = require('./services/heartbeat'),
|
|
33
34
|
{ createLogsService } = require('./services/logs'),
|
|
34
|
-
{ createArtifactsService } = require('./services/artifacts'),
|
|
35
35
|
{ createCommandsService } = require('./services/commands'),
|
|
36
36
|
{ createRetentionService } = require('./services/retention'),
|
|
37
37
|
{ createCleanupService } = require('./services/cleanup'),
|
|
38
38
|
{ startDevMode } = require('./dev/virtual'),
|
|
39
39
|
{ createUpdatesService } = require('./services/updates'),
|
|
40
|
+
{ createLiveService } = require('./services/live'),
|
|
41
|
+
{ SqliteSessionStore } = require('./services/session-store'),
|
|
40
42
|
|
|
41
43
|
{ createAgentRouter } = require('./api/agent'),
|
|
42
44
|
{ createResourceRouter } = require('./api/resource'),
|
|
43
45
|
{ createAdminRouter } = require('./api/admin'),
|
|
44
46
|
{ createWebRouter } = require('./web/routes');
|
|
45
47
|
|
|
48
|
+
// Artifacts are no longer stored on the Coordinator (README §9): an
|
|
49
|
+
// upgraded one frees the space the old <dataDir>/artifacts took, once.
|
|
50
|
+
function removeLegacyArtifacts(config){
|
|
51
|
+
const dir = path.join(config.dataDir, 'artifacts');
|
|
52
|
+
if (fs.existsSync(dir)){
|
|
53
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
54
|
+
console.log(`Removed ${dir}: the Coordinator no longer stores job artifacts`);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
46
58
|
function buildServices(config){
|
|
59
|
+
removeLegacyArtifacts(config);
|
|
47
60
|
const db = openDb(config.dbPath),
|
|
48
61
|
events = createEventsService(db),
|
|
49
62
|
registry = createRegistryService(db, { bus, events }),
|
|
@@ -52,13 +65,14 @@ function buildServices(config){
|
|
|
52
65
|
adminUsers = createAdminUsersService(db),
|
|
53
66
|
commands = createCommandsService({ bus }),
|
|
54
67
|
logs = createLogsService(db, { bus }),
|
|
55
|
-
|
|
56
|
-
jobs = createJobsService(db, { bus, events, registry, artifacts, config }),
|
|
68
|
+
jobs = createJobsService(db, { bus, events, registry, config }),
|
|
57
69
|
scheduler = createScheduler(db, { bus, events, registry, config }),
|
|
58
70
|
heartbeatMonitor = createHeartbeatMonitor(db, { bus, events, registry, jobs, config }),
|
|
59
|
-
retention = createRetentionService(db, {
|
|
71
|
+
retention = createRetentionService(db, { config }),
|
|
60
72
|
cleanup = createCleanupService(db, { jobs, config }),
|
|
61
|
-
updates = createUpdatesService({ config })
|
|
73
|
+
updates = createUpdatesService({ config }),
|
|
74
|
+
live = createLiveService(db, { bus, updates }),
|
|
75
|
+
sessionStore = new SqliteSessionStore(db);
|
|
62
76
|
|
|
63
77
|
jobs.reconcileOnStartup();
|
|
64
78
|
updates.start();
|
|
@@ -81,12 +95,13 @@ function buildServices(config){
|
|
|
81
95
|
commands,
|
|
82
96
|
jobs,
|
|
83
97
|
logs,
|
|
84
|
-
artifacts,
|
|
85
98
|
scheduler,
|
|
86
99
|
heartbeatMonitor,
|
|
87
100
|
retention,
|
|
88
101
|
cleanup,
|
|
89
|
-
updates
|
|
102
|
+
updates,
|
|
103
|
+
live,
|
|
104
|
+
sessionStore
|
|
90
105
|
};
|
|
91
106
|
}
|
|
92
107
|
|
|
@@ -98,6 +113,18 @@ function commonVersion(){
|
|
|
98
113
|
return require('@andrian.yablonskyy/thub-common/package.json').version;
|
|
99
114
|
}
|
|
100
115
|
|
|
116
|
+
// The session cookie's Secure flag (config session.secureCookie, §12).
|
|
117
|
+
// "auto": always Secure for an https:// publicUrl — the cookie must never
|
|
118
|
+
// travel over plain HTTP there — else express-session's per-request 'auto'
|
|
119
|
+
// (Secure on HTTPS connections), so plain-HTTP development still works.
|
|
120
|
+
function cookieSecure(config){
|
|
121
|
+
const setting = config.session?.secureCookie ?? 'auto';
|
|
122
|
+
if (setting === true || setting === false){
|
|
123
|
+
return setting;
|
|
124
|
+
}
|
|
125
|
+
return /^https:\/\//i.test(config.publicUrl || '') ? true : 'auto';
|
|
126
|
+
}
|
|
127
|
+
|
|
101
128
|
function createApp(config, services){
|
|
102
129
|
const app = express();
|
|
103
130
|
app.set('view engine', 'pug');
|
|
@@ -116,6 +143,9 @@ function createApp(config, services){
|
|
|
116
143
|
app.use(
|
|
117
144
|
session({
|
|
118
145
|
secret: config.sessionSecret,
|
|
146
|
+
// In the SQLite database (services/session-store.js), not in memory:
|
|
147
|
+
// survives restarts, and expired sessions are pruned.
|
|
148
|
+
store: services.sessionStore,
|
|
119
149
|
resave: false,
|
|
120
150
|
saveUninitialized: false,
|
|
121
151
|
// Idle timeout, not a fixed one (§10.1): `rolling` re-sends a fresh
|
|
@@ -124,9 +154,24 @@ function createApp(config, services){
|
|
|
124
154
|
// (routes.js) after their *last* request. Each user's own preference
|
|
125
155
|
// is applied per-request in web/routes.js, since it isn't known here.
|
|
126
156
|
rolling: true,
|
|
127
|
-
cookie: { httpOnly: true, sameSite: 'lax' }
|
|
157
|
+
cookie: { httpOnly: true, sameSite: 'lax', secure: cookieSecure(config) }
|
|
128
158
|
})
|
|
129
159
|
);
|
|
160
|
+
// Background requests of the dashboard's live updates (public/js/live.js):
|
|
161
|
+
// the GET /live stream and the page re-fetches it triggers (header
|
|
162
|
+
// X-Thub-Live: 1). They must not count as activity — otherwise an open
|
|
163
|
+
// tab would keep the session alive forever and the idle timeout (§10.1)
|
|
164
|
+
// would never fire — so they don't extend it (`touch` is what `rolling`
|
|
165
|
+
// uses to push the expiry out) and web/routes.js leaves the session alone.
|
|
166
|
+
app.use((req, res, next) => {
|
|
167
|
+
if (req.session && (req.path === '/live' || req.get('X-Thub-Live') === '1')){
|
|
168
|
+
req.thubPassive = true;
|
|
169
|
+
req.session.touch = function (){
|
|
170
|
+
return this;
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
next();
|
|
174
|
+
});
|
|
130
175
|
app.use(express.static(path.join(__dirname, '..', 'public')));
|
|
131
176
|
app.use('/avatars', express.static(config.avatarsDir));
|
|
132
177
|
|
|
@@ -134,17 +179,6 @@ function createApp(config, services){
|
|
|
134
179
|
app.use('/api/v1', createResourceRouter({ services, config }));
|
|
135
180
|
app.use('/api/v1/admin', createAdminRouter({ services }));
|
|
136
181
|
|
|
137
|
-
// HMAC-signed artifact downloads (§3.1, §12) — no session or bearer token.
|
|
138
|
-
app.get('/artifacts/download/:token', (req, res, next) => {
|
|
139
|
-
try {
|
|
140
|
-
const artifact = services.artifacts.verify(req.params.token);
|
|
141
|
-
res.download(artifact.path, artifact.name);
|
|
142
|
-
}
|
|
143
|
-
catch (err){
|
|
144
|
-
next(err);
|
|
145
|
-
}
|
|
146
|
-
});
|
|
147
|
-
|
|
148
182
|
app.use('/', createWebRouter({ services, config }));
|
|
149
183
|
|
|
150
184
|
app.use((err, req, res, next) => {
|
|
@@ -207,4 +241,4 @@ if (require.main === module){
|
|
|
207
241
|
start();
|
|
208
242
|
}
|
|
209
243
|
|
|
210
|
-
module.exports = { start, buildServices, createApp };
|
|
244
|
+
module.exports = { start, buildServices, createApp, cookieSecure };
|
package/src/services/cleanup.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @file packages/coordinator/src/services/cleanup.js
|
|
3
|
-
* @description Database cleanup: wipe finished jobs (logs,
|
|
3
|
+
* @description Database cleanup: wipe finished jobs (logs, events) and old events before a cutoff, then
|
|
4
4
|
* VACUUM to shrink the file — on demand from the dashboard, and hourly per retention.jobRetention (README §9)
|
|
5
5
|
*
|
|
6
6
|
* @author Andrian Yablonskyy
|
|
@@ -81,7 +81,7 @@ function createCleanupService(db, { jobs, config, now = () => new Date() }){
|
|
|
81
81
|
}
|
|
82
82
|
|
|
83
83
|
// Wipe everything older than `before` (a Date; default now): finished
|
|
84
|
-
// jobs that ended before it, with their logs
|
|
84
|
+
// jobs that ended before it, with their logs and events, and
|
|
85
85
|
// the other audit events (resource/agent/group) logged before it. Events
|
|
86
86
|
// of jobs that stay (still active, or newer) are kept. `vacuum`: true,
|
|
87
87
|
// false, or 'auto' (only if something was deleted and the last VACUUM is
|
package/src/services/jobs.js
CHANGED
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
|
|
16
16
|
const { validateJobSpec, maskEnv, JOB_STATES, ACTIVE_JOB_STATES, TERMINAL_JOB_STATES, RESOURCE_STATES } = require('@andrian.yablonskyy/thub-common'),
|
|
17
17
|
{ paginate } = require('./list-prefs'),
|
|
18
|
+
{ parseTerms, likeClause } = require('./search'),
|
|
18
19
|
// Named in spec errors: a field the Agent sends but this thub-common
|
|
19
20
|
// doesn't know means the Coordinator needs an update.
|
|
20
21
|
VERSIONS = `Coordinator v${require('../../package.json').version}, ` +
|
|
@@ -55,7 +56,7 @@ function nextJobId(db, source){
|
|
|
55
56
|
return `${prefix}-${String(n).padStart(5, '0')}`;
|
|
56
57
|
}
|
|
57
58
|
|
|
58
|
-
function createJobsService(db, { bus, events, registry,
|
|
59
|
+
function createJobsService(db, { bus, events, registry, config }){
|
|
59
60
|
function get(id){
|
|
60
61
|
return rowToJob(db.prepare('SELECT * FROM jobs WHERE id = ?').get(id));
|
|
61
62
|
}
|
|
@@ -87,16 +88,26 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
|
|
|
87
88
|
// Dashboard /jobs (§10): one sorted page plus the total, both in SQL so a
|
|
88
89
|
// long history never has to be loaded whole. `sort` is a list-prefs key.
|
|
89
90
|
const PAGE_SORT_SQL = {
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
91
|
+
id: 'jobs.id',
|
|
92
|
+
source: 'jobs.source',
|
|
93
|
+
user: 'json_extract(jobs.spec, \'$.user\')',
|
|
94
|
+
state: 'jobs.state',
|
|
95
|
+
resource: 'COALESCE(r.name, jobs.resource_name)',
|
|
96
|
+
created: 'jobs.created_at',
|
|
97
|
+
duration: 'jobs.duration_sec'
|
|
98
|
+
},
|
|
99
|
+
|
|
100
|
+
// What the Jobs page's search box (?q=) looks in. Named spec fields
|
|
101
|
+
// only — never the whole spec: while a job is active its `env` still
|
|
102
|
+
// holds the real --env values (§7.2), and a LIKE over them would let
|
|
103
|
+
// anyone who can see the list guess a secret one character at a time.
|
|
104
|
+
SEARCH_SQL = [
|
|
105
|
+
'jobs.id', 'jobs.state', 'jobs.source', 'jobs.message', 'r.name', 'jobs.resource_name', 'a.name',
|
|
106
|
+
...['user', 'command', 'suite', 'image', 'git.url', 'git.ref', 'target.group', 'target.labels', 'downloads', 'args', 'meta']
|
|
107
|
+
.map((path) => `json_extract(jobs.spec, '$.${path}')`)
|
|
108
|
+
];
|
|
109
|
+
|
|
110
|
+
function page({ state, source, q, sort = 'created', dir = 'desc', size = 25, page: pageNo = 1 } = {}){
|
|
100
111
|
const where = [],
|
|
101
112
|
params = [];
|
|
102
113
|
if (state){
|
|
@@ -107,15 +118,21 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
|
|
|
107
118
|
where.push('jobs.source = ?');
|
|
108
119
|
params.push(source);
|
|
109
120
|
}
|
|
121
|
+
const search = likeClause(SEARCH_SQL, parseTerms(q));
|
|
122
|
+
if (search.sql){
|
|
123
|
+
where.push(search.sql);
|
|
124
|
+
params.push(...search.params);
|
|
125
|
+
}
|
|
110
126
|
const whereSql = where.length ? `WHERE ${where.join(' AND ')}` : '',
|
|
127
|
+
fromSql = 'FROM jobs LEFT JOIN resources r ON r.id = jobs.resource_id LEFT JOIN agents a ON a.id = jobs.agent_id',
|
|
111
128
|
column = PAGE_SORT_SQL[sort] || PAGE_SORT_SQL.created,
|
|
112
129
|
order = dir === 'asc' ? 'ASC' : 'DESC',
|
|
113
|
-
total = db.prepare(`SELECT COUNT(*) AS n
|
|
130
|
+
total = db.prepare(`SELECT COUNT(*) AS n ${fromSql} ${whereSql}`).get(...params).n,
|
|
114
131
|
pagination = paginate(total, size, pageNo),
|
|
115
132
|
// Empty values (no user, never ran, …) last either way; created_at
|
|
116
133
|
// breaks ties so paging is stable.
|
|
117
134
|
rows = db.prepare(
|
|
118
|
-
`SELECT jobs.*
|
|
135
|
+
`SELECT jobs.* ${fromSql} ${whereSql}
|
|
119
136
|
ORDER BY ${column} IS NULL, ${column} ${order}, jobs.created_at DESC
|
|
120
137
|
LIMIT ? OFFSET ?`
|
|
121
138
|
).all(...params, pagination.limit, pagination.offset);
|
|
@@ -466,7 +483,7 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
|
|
|
466
483
|
}
|
|
467
484
|
|
|
468
485
|
// Admin: "clean the queue" — permanently delete finished jobs (and their
|
|
469
|
-
// logs
|
|
486
|
+
// logs, in the DB), for when the history itself,
|
|
470
487
|
// not just active work, needs clearing out. Unlike the nightly retention
|
|
471
488
|
// sweep (§9), this runs on demand and isn't limited to old jobs.
|
|
472
489
|
function cleanHistory(){
|
|
@@ -474,7 +491,7 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
|
|
|
474
491
|
}
|
|
475
492
|
|
|
476
493
|
// Finished jobs that ended before `beforeIso` (null: all of them), with
|
|
477
|
-
// their logs
|
|
494
|
+
// their logs and events — manual cleanup and the
|
|
478
495
|
// job retention task (services/cleanup.js). Active jobs are never touched.
|
|
479
496
|
function purgeFinishedBefore(beforeIso){
|
|
480
497
|
const placeholders = [...TERMINAL_JOB_STATES].map(() => '?').join(','),
|
|
@@ -487,14 +504,11 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
|
|
|
487
504
|
return 0;
|
|
488
505
|
}
|
|
489
506
|
|
|
490
|
-
const
|
|
491
|
-
deleteLogs = db.prepare('DELETE FROM job_logs WHERE job_id = ?'),
|
|
507
|
+
const deleteLogs = db.prepare('DELETE FROM job_logs WHERE job_id = ?'),
|
|
492
508
|
deleteEvents = db.prepare('DELETE FROM events WHERE entity = \'job\' AND entity_id = ?'),
|
|
493
509
|
deleteJob = db.prepare('DELETE FROM jobs WHERE id = ?'),
|
|
494
510
|
tx = db.transaction(() => {
|
|
495
511
|
for (const id of ids){
|
|
496
|
-
artifacts.deleteJobArtifacts(id);
|
|
497
|
-
deleteArtifacts.run(id);
|
|
498
512
|
deleteLogs.run(id);
|
|
499
513
|
deleteEvents.run(id);
|
|
500
514
|
deleteJob.run(id);
|
|
@@ -30,6 +30,10 @@ const PAGE_SIZES = [10, 25, 50, 100, 'all'],
|
|
|
30
30
|
resources: {
|
|
31
31
|
sorts: ['name', 'type', 'version', 'status', 'heartbeat'],
|
|
32
32
|
defaults: { size: DEFAULT_PAGE_SIZE, sort: 'name', dir: 'asc' }
|
|
33
|
+
},
|
|
34
|
+
agents: {
|
|
35
|
+
sorts: ['name', 'kind', 'version', 'created', 'used', 'status'],
|
|
36
|
+
defaults: { size: DEFAULT_PAGE_SIZE, sort: 'created', dir: 'desc' }
|
|
33
37
|
}
|
|
34
38
|
};
|
|
35
39
|
|