@andrian.yablonskyy/thub-coordinator 1.0.38 → 1.0.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/thub-admin.js +3 -5
- package/config.json +1 -6
- package/package.json +1 -1
- package/public/js/log-viewer.js +188 -56
- package/scripts/install-default-config.js +1 -1
- package/scripts/install-target.js +2 -2
- package/src/api/admin.js +1 -1
- package/src/api/agent.js +4 -10
- package/src/api/resource.js +6 -11
- package/src/api/sse.js +13 -10
- package/src/config.js +16 -9
- package/src/db/migrations/022_add_sessions.sql +10 -0
- package/src/db/migrations/023_drop_artifacts.sql +4 -0
- package/src/dev/virtual.js +1 -1
- package/src/server.js +37 -21
- package/src/services/cleanup.js +2 -2
- package/src/services/jobs.js +4 -7
- package/src/services/logs.js +12 -14
- package/src/services/retention.js +5 -14
- package/src/services/session-store.js +127 -0
- package/src/web/routes.js +75 -13
- package/test/cleanup.test.js +1 -5
- package/test/logs.test.js +150 -0
- package/test/scheduler.test.js +10 -12
- package/test/session.test.js +132 -0
- package/views/help/_agent-cli.pug +1 -1
- package/views/help/_coordinator.pug +9 -6
- package/views/help/_docker.pug +1 -1
- package/views/help/_env.pug +1 -1
- package/views/help/_git.pug +3 -2
- package/views/help/_overview.pug +2 -2
- package/views/help/_troubleshooting.pug +2 -0
- package/views/jobs/list.pug +2 -2
- package/views/jobs/show.pug +0 -10
- package/views/mixins/log-viewer.pug +14 -0
- package/src/services/artifacts.js +0 -149
package/src/server.js
CHANGED
|
@@ -15,7 +15,8 @@
|
|
|
15
15
|
|
|
16
16
|
'use strict';
|
|
17
17
|
|
|
18
|
-
const
|
|
18
|
+
const fs = require('node:fs'),
|
|
19
|
+
path = require('node:path'),
|
|
19
20
|
express = require('express'),
|
|
20
21
|
session = require('express-session'),
|
|
21
22
|
|
|
@@ -31,20 +32,31 @@ const path = require('node:path'),
|
|
|
31
32
|
{ createScheduler } = require('./services/scheduler'),
|
|
32
33
|
{ createHeartbeatMonitor } = require('./services/heartbeat'),
|
|
33
34
|
{ createLogsService } = require('./services/logs'),
|
|
34
|
-
{ createArtifactsService } = require('./services/artifacts'),
|
|
35
35
|
{ createCommandsService } = require('./services/commands'),
|
|
36
36
|
{ createRetentionService } = require('./services/retention'),
|
|
37
37
|
{ createCleanupService } = require('./services/cleanup'),
|
|
38
38
|
{ startDevMode } = require('./dev/virtual'),
|
|
39
39
|
{ createUpdatesService } = require('./services/updates'),
|
|
40
40
|
{ createLiveService } = require('./services/live'),
|
|
41
|
+
{ SqliteSessionStore } = require('./services/session-store'),
|
|
41
42
|
|
|
42
43
|
{ createAgentRouter } = require('./api/agent'),
|
|
43
44
|
{ createResourceRouter } = require('./api/resource'),
|
|
44
45
|
{ createAdminRouter } = require('./api/admin'),
|
|
45
46
|
{ createWebRouter } = require('./web/routes');
|
|
46
47
|
|
|
48
|
+
// Artifacts are no longer stored on the Coordinator (README §9): an
|
|
49
|
+
// upgraded one frees the space the old <dataDir>/artifacts took, once.
|
|
50
|
+
function removeLegacyArtifacts(config){
|
|
51
|
+
const dir = path.join(config.dataDir, 'artifacts');
|
|
52
|
+
if (fs.existsSync(dir)){
|
|
53
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
54
|
+
console.log(`Removed ${dir}: the Coordinator no longer stores job artifacts`);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
47
58
|
function buildServices(config){
|
|
59
|
+
removeLegacyArtifacts(config);
|
|
48
60
|
const db = openDb(config.dbPath),
|
|
49
61
|
events = createEventsService(db),
|
|
50
62
|
registry = createRegistryService(db, { bus, events }),
|
|
@@ -53,14 +65,14 @@ function buildServices(config){
|
|
|
53
65
|
adminUsers = createAdminUsersService(db),
|
|
54
66
|
commands = createCommandsService({ bus }),
|
|
55
67
|
logs = createLogsService(db, { bus }),
|
|
56
|
-
|
|
57
|
-
jobs = createJobsService(db, { bus, events, registry, artifacts, config }),
|
|
68
|
+
jobs = createJobsService(db, { bus, events, registry, config }),
|
|
58
69
|
scheduler = createScheduler(db, { bus, events, registry, config }),
|
|
59
70
|
heartbeatMonitor = createHeartbeatMonitor(db, { bus, events, registry, jobs, config }),
|
|
60
|
-
retention = createRetentionService(db, {
|
|
71
|
+
retention = createRetentionService(db, { config }),
|
|
61
72
|
cleanup = createCleanupService(db, { jobs, config }),
|
|
62
73
|
updates = createUpdatesService({ config }),
|
|
63
|
-
live = createLiveService(db, { bus, updates })
|
|
74
|
+
live = createLiveService(db, { bus, updates }),
|
|
75
|
+
sessionStore = new SqliteSessionStore(db);
|
|
64
76
|
|
|
65
77
|
jobs.reconcileOnStartup();
|
|
66
78
|
updates.start();
|
|
@@ -83,13 +95,13 @@ function buildServices(config){
|
|
|
83
95
|
commands,
|
|
84
96
|
jobs,
|
|
85
97
|
logs,
|
|
86
|
-
artifacts,
|
|
87
98
|
scheduler,
|
|
88
99
|
heartbeatMonitor,
|
|
89
100
|
retention,
|
|
90
101
|
cleanup,
|
|
91
102
|
updates,
|
|
92
|
-
live
|
|
103
|
+
live,
|
|
104
|
+
sessionStore
|
|
93
105
|
};
|
|
94
106
|
}
|
|
95
107
|
|
|
@@ -101,6 +113,18 @@ function commonVersion(){
|
|
|
101
113
|
return require('@andrian.yablonskyy/thub-common/package.json').version;
|
|
102
114
|
}
|
|
103
115
|
|
|
116
|
+
// The session cookie's Secure flag (config session.secureCookie, §12).
|
|
117
|
+
// "auto": always Secure for an https:// publicUrl — the cookie must never
|
|
118
|
+
// travel over plain HTTP there — else express-session's per-request 'auto'
|
|
119
|
+
// (Secure on HTTPS connections), so plain-HTTP development still works.
|
|
120
|
+
function cookieSecure(config){
|
|
121
|
+
const setting = config.session?.secureCookie ?? 'auto';
|
|
122
|
+
if (setting === true || setting === false){
|
|
123
|
+
return setting;
|
|
124
|
+
}
|
|
125
|
+
return /^https:\/\//i.test(config.publicUrl || '') ? true : 'auto';
|
|
126
|
+
}
|
|
127
|
+
|
|
104
128
|
function createApp(config, services){
|
|
105
129
|
const app = express();
|
|
106
130
|
app.set('view engine', 'pug');
|
|
@@ -119,6 +143,9 @@ function createApp(config, services){
|
|
|
119
143
|
app.use(
|
|
120
144
|
session({
|
|
121
145
|
secret: config.sessionSecret,
|
|
146
|
+
// In the SQLite database (services/session-store.js), not in memory:
|
|
147
|
+
// survives restarts, and expired sessions are pruned.
|
|
148
|
+
store: services.sessionStore,
|
|
122
149
|
resave: false,
|
|
123
150
|
saveUninitialized: false,
|
|
124
151
|
// Idle timeout, not a fixed one (§10.1): `rolling` re-sends a fresh
|
|
@@ -127,7 +154,7 @@ function createApp(config, services){
|
|
|
127
154
|
// (routes.js) after their *last* request. Each user's own preference
|
|
128
155
|
// is applied per-request in web/routes.js, since it isn't known here.
|
|
129
156
|
rolling: true,
|
|
130
|
-
cookie: { httpOnly: true, sameSite: 'lax' }
|
|
157
|
+
cookie: { httpOnly: true, sameSite: 'lax', secure: cookieSecure(config) }
|
|
131
158
|
})
|
|
132
159
|
);
|
|
133
160
|
// Background requests of the dashboard's live updates (public/js/live.js):
|
|
@@ -152,17 +179,6 @@ function createApp(config, services){
|
|
|
152
179
|
app.use('/api/v1', createResourceRouter({ services, config }));
|
|
153
180
|
app.use('/api/v1/admin', createAdminRouter({ services }));
|
|
154
181
|
|
|
155
|
-
// HMAC-signed artifact downloads (§3.1, §12) — no session or bearer token.
|
|
156
|
-
app.get('/artifacts/download/:token', (req, res, next) => {
|
|
157
|
-
try {
|
|
158
|
-
const artifact = services.artifacts.verify(req.params.token);
|
|
159
|
-
res.download(artifact.path, artifact.name);
|
|
160
|
-
}
|
|
161
|
-
catch (err){
|
|
162
|
-
next(err);
|
|
163
|
-
}
|
|
164
|
-
});
|
|
165
|
-
|
|
166
182
|
app.use('/', createWebRouter({ services, config }));
|
|
167
183
|
|
|
168
184
|
app.use((err, req, res, next) => {
|
|
@@ -225,4 +241,4 @@ if (require.main === module){
|
|
|
225
241
|
start();
|
|
226
242
|
}
|
|
227
243
|
|
|
228
|
-
module.exports = { start, buildServices, createApp };
|
|
244
|
+
module.exports = { start, buildServices, createApp, cookieSecure };
|
package/src/services/cleanup.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @file packages/coordinator/src/services/cleanup.js
|
|
3
|
-
* @description Database cleanup: wipe finished jobs (logs,
|
|
3
|
+
* @description Database cleanup: wipe finished jobs (logs, events) and old events before a cutoff, then
|
|
4
4
|
* VACUUM to shrink the file — on demand from the dashboard, and hourly per retention.jobRetention (README §9)
|
|
5
5
|
*
|
|
6
6
|
* @author Andrian Yablonskyy
|
|
@@ -81,7 +81,7 @@ function createCleanupService(db, { jobs, config, now = () => new Date() }){
|
|
|
81
81
|
}
|
|
82
82
|
|
|
83
83
|
// Wipe everything older than `before` (a Date; default now): finished
|
|
84
|
-
// jobs that ended before it, with their logs
|
|
84
|
+
// jobs that ended before it, with their logs and events, and
|
|
85
85
|
// the other audit events (resource/agent/group) logged before it. Events
|
|
86
86
|
// of jobs that stay (still active, or newer) are kept. `vacuum`: true,
|
|
87
87
|
// false, or 'auto' (only if something was deleted and the last VACUUM is
|
package/src/services/jobs.js
CHANGED
|
@@ -56,7 +56,7 @@ function nextJobId(db, source){
|
|
|
56
56
|
return `${prefix}-${String(n).padStart(5, '0')}`;
|
|
57
57
|
}
|
|
58
58
|
|
|
59
|
-
function createJobsService(db, { bus, events, registry,
|
|
59
|
+
function createJobsService(db, { bus, events, registry, config }){
|
|
60
60
|
function get(id){
|
|
61
61
|
return rowToJob(db.prepare('SELECT * FROM jobs WHERE id = ?').get(id));
|
|
62
62
|
}
|
|
@@ -483,7 +483,7 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
|
|
|
483
483
|
}
|
|
484
484
|
|
|
485
485
|
// Admin: "clean the queue" — permanently delete finished jobs (and their
|
|
486
|
-
// logs
|
|
486
|
+
// logs, in the DB), for when the history itself,
|
|
487
487
|
// not just active work, needs clearing out. Unlike the nightly retention
|
|
488
488
|
// sweep (§9), this runs on demand and isn't limited to old jobs.
|
|
489
489
|
function cleanHistory(){
|
|
@@ -491,7 +491,7 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
|
|
|
491
491
|
}
|
|
492
492
|
|
|
493
493
|
// Finished jobs that ended before `beforeIso` (null: all of them), with
|
|
494
|
-
// their logs
|
|
494
|
+
// their logs and events — manual cleanup and the
|
|
495
495
|
// job retention task (services/cleanup.js). Active jobs are never touched.
|
|
496
496
|
function purgeFinishedBefore(beforeIso){
|
|
497
497
|
const placeholders = [...TERMINAL_JOB_STATES].map(() => '?').join(','),
|
|
@@ -504,14 +504,11 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
|
|
|
504
504
|
return 0;
|
|
505
505
|
}
|
|
506
506
|
|
|
507
|
-
const
|
|
508
|
-
deleteLogs = db.prepare('DELETE FROM job_logs WHERE job_id = ?'),
|
|
507
|
+
const deleteLogs = db.prepare('DELETE FROM job_logs WHERE job_id = ?'),
|
|
509
508
|
deleteEvents = db.prepare('DELETE FROM events WHERE entity = \'job\' AND entity_id = ?'),
|
|
510
509
|
deleteJob = db.prepare('DELETE FROM jobs WHERE id = ?'),
|
|
511
510
|
tx = db.transaction(() => {
|
|
512
511
|
for (const id of ids){
|
|
513
|
-
artifacts.deleteJobArtifacts(id);
|
|
514
|
-
deleteArtifacts.run(id);
|
|
515
512
|
deleteLogs.run(id);
|
|
516
513
|
deleteEvents.run(id);
|
|
517
514
|
deleteJob.run(id);
|
package/src/services/logs.js
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
|
|
16
16
|
// §3.1 Log service: accepts batched log lines, assigns a monotonically
|
|
17
17
|
// increasing seq per job, stores them, and fans them out over the bus for
|
|
18
|
-
// SSE subscribers (§6.4).
|
|
18
|
+
// SSE subscribers (§6.4). Lines are deleted with their job (§9).
|
|
19
19
|
function createLogsService(db, { bus }){
|
|
20
20
|
const insert = db.prepare(
|
|
21
21
|
'INSERT INTO job_logs (job_id, seq, ts, stream, line) VALUES (?, ?, ?, ?, ?)'
|
|
@@ -47,24 +47,22 @@ function createLogsService(db, { bus }){
|
|
|
47
47
|
.all(jobId, afterSeq, limit);
|
|
48
48
|
}
|
|
49
49
|
|
|
50
|
-
//
|
|
51
|
-
|
|
50
|
+
// The dashboard log viewer's pages (views/mixins/log-viewer.pug): the
|
|
51
|
+
// `limit` lines just before `beforeSeq` (the end of the log when omitted),
|
|
52
|
+
// oldest first, and whether there are earlier ones to scroll up to.
|
|
53
|
+
function pageBefore(jobId, beforeSeq, limit){
|
|
52
54
|
const rows = db
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
55
|
+
.prepare('SELECT seq, ts, stream, line FROM job_logs WHERE job_id = ? AND seq < ? ORDER BY seq DESC LIMIT ?')
|
|
56
|
+
.all(jobId, beforeSeq ?? Number.MAX_SAFE_INTEGER, limit + 1),
|
|
57
|
+
hasMore = rows.length > limit;
|
|
58
|
+
return { lines: rows.slice(0, limit).reverse(), hasMore };
|
|
56
59
|
}
|
|
57
60
|
|
|
58
|
-
function
|
|
59
|
-
|
|
60
|
-
db.prepare(
|
|
61
|
-
`DELETE FROM job_logs WHERE job_id IN (
|
|
62
|
-
SELECT id FROM jobs WHERE finished_at IS NOT NULL AND finished_at <= ?
|
|
63
|
-
)`
|
|
64
|
-
).run(cutoff);
|
|
61
|
+
function count(jobId){
|
|
62
|
+
return db.prepare('SELECT COUNT(*) AS n FROM job_logs WHERE job_id = ?').get(jobId).n;
|
|
65
63
|
}
|
|
66
64
|
|
|
67
|
-
return { appendBatch, listSince,
|
|
65
|
+
return { appendBatch, listSince, pageBefore, count };
|
|
68
66
|
}
|
|
69
67
|
|
|
70
68
|
module.exports = { createLogsService };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @file packages/coordinator/src/services/retention.js
|
|
3
|
-
* @description
|
|
3
|
+
* @description Nightly consistent database backup (VACUUM INTO, README §9)
|
|
4
4
|
*
|
|
5
5
|
* @author Andrian Yablonskyy
|
|
6
6
|
* @copyright Copyright (c) 2026 Andrian Yablonskyy. All rights reserved.
|
|
@@ -17,20 +17,11 @@ const path = require('node:path');
|
|
|
17
17
|
|
|
18
18
|
const DAY_MS = 86400 * 1000;
|
|
19
19
|
|
|
20
|
-
// §9:
|
|
21
|
-
//
|
|
22
|
-
//
|
|
23
|
-
function createRetentionService(db, {
|
|
24
|
-
bus.on('job.finished', ({ jobId }) => {
|
|
25
|
-
const text = logs.renderConsoleLog(jobId);
|
|
26
|
-
if (text){
|
|
27
|
-
artifacts.storeGenerated(jobId, 'console.log', Buffer.from(text, 'utf8'), 'text/plain');
|
|
28
|
-
}
|
|
29
|
-
});
|
|
30
|
-
|
|
20
|
+
// §9: a nightly consistent backup via VACUUM INTO. (A job's log lines live
|
|
21
|
+
// as long as the job itself — retention.jobRetention / Clean up database,
|
|
22
|
+
// services/cleanup.js — and there are no artifacts to purge any more.)
|
|
23
|
+
function createRetentionService(db, { config }){
|
|
31
24
|
function runDaily(){
|
|
32
|
-
logs.purgeOlderThan(config.retention.logRetentionDays);
|
|
33
|
-
artifacts.purgeOlderThan(config.retention.artifactRetentionDays);
|
|
34
25
|
const backupPath = path.join(config.dataDir, `backup-${new Date().toISOString().slice(0, 10)}.db`);
|
|
35
26
|
db.exec(`VACUUM INTO '${backupPath.replace(/'/g, '\'\'')}'`);
|
|
36
27
|
}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file packages/coordinator/src/services/session-store.js
|
|
3
|
+
* @description Dashboard session store in the Coordinator's SQLite database: survives restarts, prunes expired sessions
|
|
4
|
+
*
|
|
5
|
+
* @author Andrian Yablonskyy
|
|
6
|
+
* @copyright Copyright (c) 2026 Andrian Yablonskyy. All rights reserved.
|
|
7
|
+
*
|
|
8
|
+
* This file is part of TestHub and is proprietary and confidential.
|
|
9
|
+
* Unauthorized copying, modification, distribution, or use of this file,
|
|
10
|
+
* via any medium, is strictly prohibited without prior written permission
|
|
11
|
+
* from AdSystem.PRO.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
'use strict';
|
|
15
|
+
|
|
16
|
+
const session = require('express-session');
|
|
17
|
+
|
|
18
|
+
// express-session's default MemoryStore loses every session on a restart,
|
|
19
|
+
// never drops expired ones (memory only grows) and warns against itself in
|
|
20
|
+
// production. This one keeps them in the `sessions` table (migration 022),
|
|
21
|
+
// so signing in survives a Coordinator restart or self-update, and an
|
|
22
|
+
// expired session is deleted when read and by a periodic sweep.
|
|
23
|
+
const PRUNE_INTERVAL_MS = 15 * 60 * 1000,
|
|
24
|
+
// A session without a cookie expiry (never the case here: routes.js sets
|
|
25
|
+
// each user's idle timeout) is kept this long.
|
|
26
|
+
FALLBACK_TTL_MS = 24 * 60 * 60 * 1000;
|
|
27
|
+
|
|
28
|
+
class SqliteSessionStore extends session.Store{
|
|
29
|
+
constructor(db, { pruneIntervalMs = PRUNE_INTERVAL_MS, now = Date.now } = {}){
|
|
30
|
+
super();
|
|
31
|
+
this.now = now;
|
|
32
|
+
this.stmts = {
|
|
33
|
+
get: db.prepare('SELECT sess, expires FROM sessions WHERE sid = ?'),
|
|
34
|
+
set: db.prepare(
|
|
35
|
+
`INSERT INTO sessions (sid, sess, expires) VALUES (?, ?, ?)
|
|
36
|
+
ON CONFLICT(sid) DO UPDATE SET sess = excluded.sess, expires = excluded.expires`
|
|
37
|
+
),
|
|
38
|
+
touch: db.prepare('UPDATE sessions SET sess = ?, expires = ? WHERE sid = ?'),
|
|
39
|
+
destroy: db.prepare('DELETE FROM sessions WHERE sid = ?'),
|
|
40
|
+
prune: db.prepare('DELETE FROM sessions WHERE expires <= ?'),
|
|
41
|
+
length: db.prepare('SELECT COUNT(*) AS n FROM sessions WHERE expires > ?'),
|
|
42
|
+
clear: db.prepare('DELETE FROM sessions')
|
|
43
|
+
};
|
|
44
|
+
if (pruneIntervalMs > 0){
|
|
45
|
+
this.pruneTimer = setInterval(() => this.prune(), pruneIntervalMs);
|
|
46
|
+
this.pruneTimer.unref?.();
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
expiresOf(sess){
|
|
51
|
+
const at = sess?.cookie?.expires ? new Date(sess.cookie.expires).getTime() : NaN;
|
|
52
|
+
return Number.isFinite(at) ? at : this.now() + FALLBACK_TTL_MS;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// Each method answers through `cb`, as express-session expects; errors
|
|
56
|
+
// (a locked or broken database) go there too instead of throwing.
|
|
57
|
+
run(cb, fn){
|
|
58
|
+
let result;
|
|
59
|
+
try {
|
|
60
|
+
result = fn();
|
|
61
|
+
}
|
|
62
|
+
catch (err){
|
|
63
|
+
return cb?.(err);
|
|
64
|
+
}
|
|
65
|
+
cb?.(null, result);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
get(sid, cb){
|
|
69
|
+
this.run(cb, () => {
|
|
70
|
+
const row = this.stmts.get.get(sid);
|
|
71
|
+
if (!row){
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
|
+
if (row.expires <= this.now()){
|
|
75
|
+
this.stmts.destroy.run(sid);
|
|
76
|
+
return null;
|
|
77
|
+
}
|
|
78
|
+
return JSON.parse(row.sess);
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
set(sid, sess, cb){
|
|
83
|
+
this.run(cb, () => {
|
|
84
|
+
this.stmts.set.run(sid, JSON.stringify(sess), this.expiresOf(sess));
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// Called for a request that didn't change the session: moves its expiry
|
|
89
|
+
// (rolling idle timeout, §10.1). A passive live-update request leaves the
|
|
90
|
+
// cookie's expiry as it was (server.js), so this writes the same value.
|
|
91
|
+
touch(sid, sess, cb){
|
|
92
|
+
this.run(cb, () => {
|
|
93
|
+
this.stmts.touch.run(JSON.stringify(sess), this.expiresOf(sess), sid);
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
destroy(sid, cb){
|
|
98
|
+
this.run(cb, () => {
|
|
99
|
+
this.stmts.destroy.run(sid);
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
length(cb){
|
|
104
|
+
this.run(cb, () => this.stmts.length.get(this.now()).n);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
clear(cb){
|
|
108
|
+
this.run(cb, () => {
|
|
109
|
+
this.stmts.clear.run();
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
prune(){
|
|
114
|
+
try {
|
|
115
|
+
return this.stmts.prune.run(this.now()).changes;
|
|
116
|
+
}
|
|
117
|
+
catch {
|
|
118
|
+
return 0; // the next sweep tries again
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
close(){
|
|
123
|
+
clearInterval(this.pruneTimer);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
module.exports = { SqliteSessionStore };
|
package/src/web/routes.js
CHANGED
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
const fs = require('node:fs'),
|
|
17
17
|
os = require('node:os'),
|
|
18
18
|
path = require('node:path'),
|
|
19
|
+
{ once } = require('node:events'),
|
|
19
20
|
express = require('express'),
|
|
20
21
|
multer = require('multer'),
|
|
21
22
|
{ requireAdminSession, requireAdminRole } = require('../auth'),
|
|
@@ -28,9 +29,13 @@ const fs = require('node:fs'),
|
|
|
28
29
|
listPrefs = require('../services/list-prefs'),
|
|
29
30
|
search = require('../services/search');
|
|
30
31
|
|
|
31
|
-
//
|
|
32
|
-
|
|
33
|
-
|
|
32
|
+
// Job log viewer: lines per page it loads (and the most one request may ask for).
|
|
33
|
+
const LOG_PAGE = 1000,
|
|
34
|
+
LOG_PAGE_MAX = 5000,
|
|
35
|
+
|
|
36
|
+
// Resources page sort keys (list-prefs.js) -> comparators. Empty values
|
|
37
|
+
// (never heartbeated, no version reported) sort last in both directions.
|
|
38
|
+
RESOURCE_SORT_VALUE = {
|
|
34
39
|
name: (r) => r.name,
|
|
35
40
|
type: (r) => r.type,
|
|
36
41
|
version: (r) => r.client_version,
|
|
@@ -203,14 +208,35 @@ function createWebRouter({ services, config }){
|
|
|
203
208
|
next();
|
|
204
209
|
});
|
|
205
210
|
|
|
211
|
+
// A Secure-only session cookie (server.js cookieSecure, §12) is never
|
|
212
|
+
// sent back over plain HTTP, so signing in would just loop back here.
|
|
213
|
+
// Say why instead — usually a proxy not passing X-Forwarded-Proto.
|
|
214
|
+
function httpsProblem(req){
|
|
215
|
+
if (req.session.cookie.secure !== true || req.secure){
|
|
216
|
+
return null;
|
|
217
|
+
}
|
|
218
|
+
return {
|
|
219
|
+
type: 'danger',
|
|
220
|
+
sticky: true,
|
|
221
|
+
text: `Signing in needs HTTPS: this Coordinator's session cookie is sent only over HTTPS (publicUrl is ${config.publicUrl}), ` +
|
|
222
|
+
'but this request reached it as plain HTTP. Open the dashboard through that https:// address; behind a reverse proxy, ' +
|
|
223
|
+
'make it send "X-Forwarded-Proto: https" and check trustProxy. To allow plain HTTP, set session.secureCookie to false.'
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
|
|
206
227
|
router.get('/login', (req, res) => {
|
|
207
228
|
if (req.session.user){
|
|
208
229
|
return res.redirect('/');
|
|
209
230
|
}
|
|
210
|
-
|
|
231
|
+
const problem = httpsProblem(req);
|
|
232
|
+
res.render('login', { title: 'Sign in', ...(problem ? { messages: [problem] } : {}) });
|
|
211
233
|
});
|
|
212
234
|
|
|
213
235
|
router.post('/login', (req, res) => {
|
|
236
|
+
const problem = httpsProblem(req);
|
|
237
|
+
if (problem){
|
|
238
|
+
return res.status(400).render('login', { title: 'Sign in', messages: [problem] });
|
|
239
|
+
}
|
|
214
240
|
const { username, password } = req.body,
|
|
215
241
|
user = services.adminUsers.verify(username, password);
|
|
216
242
|
if (!user){
|
|
@@ -786,7 +812,7 @@ function createWebRouter({ services, config }){
|
|
|
786
812
|
});
|
|
787
813
|
|
|
788
814
|
// "Clean up database" (§9): everything before the given time — finished
|
|
789
|
-
// jobs with their logs
|
|
815
|
+
// jobs with their logs, and older history — then VACUUM. The
|
|
790
816
|
// time is dd/mm/yyyy HH:MM:SS in the user's own time zone.
|
|
791
817
|
router.post('/jobs/clean-history', requireAdminRole, (req, res) => {
|
|
792
818
|
const tz = req.session.user.timezone || 'UTC',
|
|
@@ -818,8 +844,7 @@ function createWebRouter({ services, config }){
|
|
|
818
844
|
return res.status(404).send('Not found');
|
|
819
845
|
}
|
|
820
846
|
job.resource = job.resource_id ? services.registry.get(job.resource_id) : null;
|
|
821
|
-
const
|
|
822
|
-
jobActive = ACTIVE_JOB_STATES.has(job.state);
|
|
847
|
+
const jobActive = ACTIVE_JOB_STATES.has(job.state);
|
|
823
848
|
res.render('jobs/show', {
|
|
824
849
|
title: job.id,
|
|
825
850
|
active: 'jobs',
|
|
@@ -829,7 +854,6 @@ function createWebRouter({ services, config }){
|
|
|
829
854
|
? services.registry.waitingReason(job.spec.target.type, job.spec.target.labels || [], job.spec.target.group,
|
|
830
855
|
job.spec.target.client)
|
|
831
856
|
: null,
|
|
832
|
-
artifacts,
|
|
833
857
|
jobActive,
|
|
834
858
|
canCancel: req.session.user.role === 'admin' && jobActive
|
|
835
859
|
});
|
|
@@ -847,15 +871,53 @@ function createWebRouter({ services, config }){
|
|
|
847
871
|
res.redirect(returnTo(req, `/jobs/${req.params.id}`));
|
|
848
872
|
});
|
|
849
873
|
|
|
850
|
-
//
|
|
851
|
-
//
|
|
852
|
-
//
|
|
874
|
+
// The log viewer's pages (public/js/log-viewer.js): the last `limit`
|
|
875
|
+
// lines first, then earlier ones (`before` = the oldest seq it has) as the
|
|
876
|
+
// reader scrolls up. `total` is how many lines the job has in all.
|
|
853
877
|
router.get('/jobs/:id/logs', (req, res) => {
|
|
854
|
-
|
|
878
|
+
const before = Number.parseInt(req.query.before, 10),
|
|
879
|
+
limit = Math.min(Math.max(Number.parseInt(req.query.limit, 10) || LOG_PAGE, 1), LOG_PAGE_MAX),
|
|
880
|
+
page = services.logs.pageBefore(req.params.id, before > 0 ? before : undefined, limit);
|
|
881
|
+
res.json({ ...page, total: services.logs.count(req.params.id) });
|
|
882
|
+
});
|
|
883
|
+
|
|
884
|
+
// RAW (log viewer): the whole log as plain text, for a new tab, a
|
|
885
|
+
// download or grep. Streamed in pages, waiting for the socket to drain,
|
|
886
|
+
// so a huge log never sits in memory. Log lines live as long as their job
|
|
887
|
+
// (§9), so this is always the complete log.
|
|
888
|
+
router.get('/jobs/:id/log.txt', async (req, res, next) => {
|
|
889
|
+
try {
|
|
890
|
+
const job = services.jobs.get(req.params.id);
|
|
891
|
+
if (!job){
|
|
892
|
+
return res.status(404).type('text/plain').send('Unknown job');
|
|
893
|
+
}
|
|
894
|
+
res.set({
|
|
895
|
+
'Content-Type': 'text/plain; charset=utf-8',
|
|
896
|
+
// The log is whatever the job printed: never let a browser sniff it
|
|
897
|
+
// into HTML.
|
|
898
|
+
'X-Content-Type-Options': 'nosniff',
|
|
899
|
+
'Content-Disposition': `inline; filename="${job.id}.log"`,
|
|
900
|
+
'Cache-Control': 'no-store'
|
|
901
|
+
});
|
|
902
|
+
for (let seq = 0, page; (page = services.logs.listSince(job.id, seq, LOG_PAGE_MAX)).length;){
|
|
903
|
+
const chunk = page.map((l) => `[${l.ts}] [${l.stream}] ${l.line}\n`).join('');
|
|
904
|
+
seq = page.at(-1).seq;
|
|
905
|
+
if (!res.write(chunk)){
|
|
906
|
+
await once(res, 'drain');
|
|
907
|
+
}
|
|
908
|
+
if (res.destroyed){
|
|
909
|
+
return;
|
|
910
|
+
}
|
|
911
|
+
}
|
|
912
|
+
res.end();
|
|
913
|
+
}
|
|
914
|
+
catch (err){
|
|
915
|
+
next(err);
|
|
916
|
+
}
|
|
855
917
|
});
|
|
856
918
|
|
|
857
919
|
router.get('/jobs/:id/stream', (req, res) => {
|
|
858
|
-
attachJobStream(req, res, { jobId: req.params.id, services
|
|
920
|
+
attachJobStream(req, res, { jobId: req.params.id, services });
|
|
859
921
|
});
|
|
860
922
|
|
|
861
923
|
// Resource groups (§13.1) — their own page since they're a distinct
|
package/test/cleanup.test.js
CHANGED
|
@@ -23,7 +23,6 @@ const test = require('node:test'),
|
|
|
23
23
|
{ createEventsService } = require('../src/services/events'),
|
|
24
24
|
{ createRegistryService } = require('../src/services/registry'),
|
|
25
25
|
{ createAgentsService } = require('../src/services/agents'),
|
|
26
|
-
{ createArtifactsService } = require('../src/services/artifacts'),
|
|
27
26
|
{ createJobsService } = require('../src/services/jobs'),
|
|
28
27
|
{ createCleanupService, retentionCutoff } = require('../src/services/cleanup'),
|
|
29
28
|
{ JOB_STATES } = require('@andrian.yablonskyy/thub-common');
|
|
@@ -35,8 +34,6 @@ function setup(jobRetention = 'forever'){
|
|
|
35
34
|
config = {
|
|
36
35
|
dataDir,
|
|
37
36
|
dbPath: path.join(dataDir, 'thub.db'),
|
|
38
|
-
artifactsDir: path.join(dataDir, 'artifacts'),
|
|
39
|
-
artifacts: { linkTtlHours: 1 },
|
|
40
37
|
sessionSecret: 's',
|
|
41
38
|
publicUrl: 'http://x',
|
|
42
39
|
scheduler: { maxQueuedPerAgent: 1000 },
|
|
@@ -48,8 +45,7 @@ function setup(jobRetention = 'forever'){
|
|
|
48
45
|
events = createEventsService(db),
|
|
49
46
|
registry = createRegistryService(db, { bus, events }),
|
|
50
47
|
{ agent } = createAgentsService(db, { events }).create({ name: 'ci', kind: 'ci' }),
|
|
51
|
-
|
|
52
|
-
jobs = createJobsService(db, { bus, events, registry, artifacts, config }),
|
|
48
|
+
jobs = createJobsService(db, { bus, events, registry, config }),
|
|
53
49
|
cleanup = createCleanupService(db, { jobs, config, now: () => NOW });
|
|
54
50
|
registry.registerAuto({ clientId: 'c', name: 'lab', type: 'sw', labels: [] });
|
|
55
51
|
|