@andrian.yablonskyy/thub-coordinator 1.0.37 → 1.0.39

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/bin/thub-admin.js +3 -5
  2. package/config.json +1 -6
  3. package/package.json +2 -2
  4. package/public/css/thub.css +129 -0
  5. package/public/js/config-import.js +37 -33
  6. package/public/js/copy-to-clipboard.js +21 -18
  7. package/public/js/help.js +93 -0
  8. package/public/js/list-search.js +51 -0
  9. package/public/js/live.js +287 -0
  10. package/public/js/log-viewer.js +289 -45
  11. package/public/js/remove-resource.js +5 -3
  12. package/public/js/resource-card.js +14 -13
  13. package/public/js/tooltips.js +3 -1
  14. package/scripts/install-default-config.js +1 -1
  15. package/scripts/install-target.js +2 -2
  16. package/src/api/admin.js +1 -1
  17. package/src/api/agent.js +4 -10
  18. package/src/api/resource.js +6 -11
  19. package/src/api/sse.js +13 -10
  20. package/src/config.js +16 -9
  21. package/src/db/migrations/022_add_sessions.sql +10 -0
  22. package/src/db/migrations/023_drop_artifacts.sql +4 -0
  23. package/src/dev/virtual.js +1 -1
  24. package/src/server.js +55 -21
  25. package/src/services/cleanup.js +2 -2
  26. package/src/services/jobs.js +33 -19
  27. package/src/services/list-prefs.js +4 -0
  28. package/src/services/live.js +124 -0
  29. package/src/services/logs.js +12 -14
  30. package/src/services/retention.js +5 -14
  31. package/src/services/search.js +59 -0
  32. package/src/services/session-store.js +127 -0
  33. package/src/web/routes.js +205 -34
  34. package/test/cleanup.test.js +1 -5
  35. package/test/list-prefs.test.js +6 -0
  36. package/test/live.test.js +153 -0
  37. package/test/logs.test.js +150 -0
  38. package/test/scheduler.test.js +10 -12
  39. package/test/search.test.js +97 -0
  40. package/test/session.test.js +132 -0
  41. package/test/views.test.js +30 -0
  42. package/views/admin/agents.pug +28 -10
  43. package/views/groups/list.pug +14 -1
  44. package/views/help/_agent-cli.pug +103 -0
  45. package/views/help/_agent-setup.pug +72 -0
  46. package/views/help/_ci.pug +44 -0
  47. package/views/help/_client-machines.pug +87 -0
  48. package/views/help/_client-setup.pug +122 -0
  49. package/views/help/_coordinator.pug +137 -0
  50. package/views/help/_docker.pug +124 -0
  51. package/views/help/_env.pug +127 -0
  52. package/views/help/_git.pug +130 -0
  53. package/views/help/_mixins.pug +24 -0
  54. package/views/help/_overview.pug +118 -0
  55. package/views/help/_quick-start.pug +37 -0
  56. package/views/help/_troubleshooting.pug +44 -0
  57. package/views/help/index.pug +67 -0
  58. package/views/index.pug +3 -2
  59. package/views/jobs/list.pug +8 -5
  60. package/views/jobs/show.pug +0 -10
  61. package/views/layout.pug +24 -1
  62. package/views/mixins/list-controls.pug +36 -0
  63. package/views/mixins/log-viewer.pug +39 -3
  64. package/views/mixins/resource.pug +1 -1
  65. package/views/resources/list.pug +7 -4
  66. package/src/services/artifacts.js +0 -149
@@ -21,18 +21,19 @@
21
21
  }
22
22
  }
23
23
 
24
- document.querySelectorAll('[data-resource-card]').forEach((el) => {
25
- el.addEventListener('click', (e) => {
26
- // The Resources table's own action buttons stay plain actions.
27
- if (!e.target.closest('form, button, a, input')){
28
- open(el);
29
- }
30
- });
31
- el.addEventListener('keydown', (e) => {
32
- if ((e.key === 'Enter' || e.key === ' ') && e.target === el){
33
- e.preventDefault();
34
- open(el);
35
- }
36
- });
24
+ // Delegated, so rows and cards swapped in by live updates (live.js) work.
25
+ document.addEventListener('click', (e) => {
26
+ const el = e.target.closest('[data-resource-card]');
27
+ // The Resources table's own action buttons stay plain actions.
28
+ if (el && !e.target.closest('form, button, a, input')){
29
+ open(el);
30
+ }
31
+ });
32
+ document.addEventListener('keydown', (e) => {
33
+ const el = e.target.closest?.('[data-resource-card]');
34
+ if (el && (e.key === 'Enter' || e.key === ' ') && e.target === el){
35
+ e.preventDefault();
36
+ open(el);
37
+ }
37
38
  });
38
39
  })();
@@ -13,6 +13,8 @@
13
13
 
14
14
  'use strict';
15
15
 
16
+ // Delegated (one instance per element, created on first hover/focus), so
17
+ // content swapped in by live updates (public/js/live.js) gets tooltips too.
16
18
  (function (){
17
- document.querySelectorAll('[data-bs-toggle="tooltip"]').forEach((el) => new bootstrap.Tooltip(el));
19
+ new bootstrap.Tooltip(document.body, { selector: '[data-bs-toggle="tooltip"]' });
18
20
  })();
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * @file scripts/install-default-config.js
3
3
  * @description npm postinstall: on a real global install, creates the Coordinator's directory layout
4
- * (~/.config/thub, ~/.thub and its artifacts/avatars/work subdirs) and, if it
4
+ * (~/.config/thub, ~/.thub and its avatars/work subdirs) and, if it
5
5
  * doesn't already exist, ~/.config/thub/coordinator.json with every default option and
6
6
  * a freshly generated sessionSecret (README §12, §13)
7
7
  *
@@ -52,7 +52,7 @@ function resolveTargetUser(){
52
52
  }
53
53
 
54
54
  // Default layout (README §13): config under ~/.config/thub, runtime data
55
- // (SQLite DB, uploaded artifacts, avatars, work files) under ~/.thub.
55
+ // (SQLite DB, avatars, work files) under ~/.thub.
56
56
  // An existing coordinator.json's own dataDir wins over the default, so a
57
57
  // re-install prepares (and the systemd unit allows writes to) the
58
58
  // directory the Coordinator will really use.
@@ -69,7 +69,7 @@ function targetPaths(user){
69
69
  // Written by the dashboard's "Update app" button, watched by
70
70
  // thub-coordinator-update.path (README §10.2).
71
71
  updateRequestFile: path.join(dataDir, 'update-request.json'),
72
- dataSubdirs: ['artifacts', 'avatars', 'work'].map((d) => path.join(dataDir, d))
72
+ dataSubdirs: ['avatars', 'work'].map((d) => path.join(dataDir, d))
73
73
  };
74
74
  }
75
75
 
package/src/api/admin.js CHANGED
@@ -134,7 +134,7 @@ function createAdminRouter({ services }){
134
134
  res.json({ canceled });
135
135
  });
136
136
 
137
- // Permanently deletes finished jobs and their logs/artifacts (§13.1).
137
+ // Permanently deletes finished jobs and their logs (§13.1).
138
138
  // Database cleanup (§9): optional `before` (ISO 8601; default now) —
139
139
  // finished jobs that ended before it, older history, then VACUUM.
140
140
  router.post('/jobs/clean-history', (req, res) => {
package/src/api/agent.js CHANGED
@@ -80,19 +80,13 @@ function createAgentRouter({ services, config }){
80
80
  });
81
81
 
82
82
  router.get('/jobs/:id/logs/stream', auth, (req, res) => {
83
- attachJobStream(req, res, { jobId: req.params.id, services, config });
83
+ attachJobStream(req, res, { jobId: req.params.id, services });
84
84
  });
85
85
 
86
+ // No artifacts are stored any more (README §9); an Agent older than that
87
+ // still asks after a finished job's `thub status`, so answer "none".
86
88
  router.get('/jobs/:id/artifacts', auth, (req, res) => {
87
- const artifacts = services.artifacts.listForJob(req.params.id).map((a) => ({
88
- id: a.id,
89
- name: a.name,
90
- size: a.size,
91
- sha256: a.sha256,
92
- contentType: a.content_type,
93
- url: services.artifacts.signedUrl(a)
94
- }));
95
- res.json({ artifacts });
89
+ res.json({ artifacts: [] });
96
90
  });
97
91
 
98
92
  router.get('/resources', auth, (req, res) => {
@@ -14,12 +14,9 @@
14
14
  'use strict';
15
15
 
16
16
  const express = require('express'),
17
- multer = require('multer'),
18
17
  { requireRole, requireJoinKey, appVersion } = require('../auth'),
19
18
  { JOB_STATES } = require('@andrian.yablonskyy/thub-common');
20
19
 
21
- const upload = multer({ dest: require('node:os').tmpdir() });
22
-
23
20
  function requireOwnResource(req, res, next){
24
21
  if (req.resource.id !== req.params.id){
25
22
  return res.status(403).json({ error: 'Token does not match resource' });
@@ -211,14 +208,12 @@ function createResourceRouter({ services, config }){
211
208
  }
212
209
  });
213
210
 
214
- router.post('/jobs/:id/artifacts', auth, requireOwnJob(services), upload.any(), (req, res, next) => {
215
- try {
216
- const stored = services.artifacts.storeUploaded(req.params.id, req.files || []);
217
- res.status(201).json({ artifacts: stored.map((a) => ({ id: a.id, name: a.name, size: a.size })) });
218
- }
219
- catch (err){
220
- next(err);
221
- }
211
+ // Artifacts are no longer stored on the Coordinator (README §9). A Client
212
+ // older than that still uploads its results after each job and would end
213
+ // the job as ERROR on a 404, so accept the upload and discard it unread.
214
+ router.post('/jobs/:id/artifacts', auth, requireOwnJob(services), (req, res) => {
215
+ req.resume();
216
+ req.on('end', () => res.status(201).json({ artifacts: [] }));
222
217
  });
223
218
 
224
219
  router.post('/jobs/:id/result', auth, requireOwnJob(services), (req, res, next) => {
package/src/api/sse.js CHANGED
@@ -15,11 +15,12 @@
15
15
 
16
16
  const { TERMINAL_JOB_STATES, JOB_STATES } = require('@andrian.yablonskyy/thub-common');
17
17
 
18
- const WAITING_CHECK_MS = 15_000;
18
+ const WAITING_CHECK_MS = 15_000,
19
+ REPLAY_PAGE = 2000;
19
20
 
20
21
  // §6.4: log / state / end events, resumable via Last-Event-ID (§7: "the
21
22
  // Agent reconnects with exponential backoff and resumes from the last seq").
22
- function attachJobStream(req, res, { jobId, services, config }){
23
+ function attachJobStream(req, res, { jobId, services }){
23
24
  const { jobs, logs, registry, bus } = services,
24
25
 
25
26
  job = jobs.get(jobId);
@@ -47,17 +48,19 @@ function attachJobStream(req, res, { jobId, services, config }){
47
48
  res.write(`data: ${JSON.stringify(data)}\n\n`);
48
49
  }
49
50
 
50
- for (const line of logs.listSince(jobId, afterSeq)){
51
- writeEvent('log', { ts: line.ts, stream: line.stream, line: line.line }, line.seq);
52
- }
53
-
54
- function artifactsUrl(){
55
- return `${config.publicUrl}/jobs/${jobId}#artifacts`;
51
+ // Replay everything after `afterSeq`, in pages (listSince stops at 500 a
52
+ // call — replaying one page used to skip lines 501.. of a longer log). All
53
+ // synchronous, so no live line can slip in between replay and subscribing.
54
+ for (let seq = afterSeq, page; (page = logs.listSince(jobId, seq, REPLAY_PAGE)).length;){
55
+ for (const line of page){
56
+ writeEvent('log', { ts: line.ts, stream: line.stream, line: line.line }, line.seq);
57
+ }
58
+ seq = page.at(-1).seq;
56
59
  }
57
60
 
58
61
  const current = jobs.get(jobId);
59
62
  if (TERMINAL_JOB_STATES.has(current.state)){
60
- writeEvent('end', { state: current.state, artifactsUrl: artifactsUrl() });
63
+ writeEvent('end', { state: current.state });
61
64
  return res.end();
62
65
  }
63
66
 
@@ -97,7 +100,7 @@ function attachJobStream(req, res, { jobId, services, config }){
97
100
  if (evt.jobId !== jobId){
98
101
  return;
99
102
  }
100
- writeEvent('end', { state: evt.state, artifactsUrl: artifactsUrl() });
103
+ writeEvent('end', { state: evt.state });
101
104
  cleanup();
102
105
  res.end();
103
106
  };
package/src/config.js CHANGED
@@ -28,6 +28,12 @@ const DEFAULTS = {
28
28
  trustProxy: 'loopback',
29
29
  dataDir: path.join(process.cwd(), '.data'),
30
30
  sessionSecret: 'dev-only-change-me',
31
+ session: {
32
+ // The dashboard session cookie's Secure flag (server.js cookieSecure):
33
+ // "auto" = Secure whenever publicUrl is https:// (else per request:
34
+ // Secure only on a connection that is itself HTTPS); true / false force it.
35
+ secureCookie: 'auto'
36
+ },
31
37
  // Shared secret Clients present to self-register (see api/resource.js).
32
38
  // null disables auto-registration entirely — set it explicitly to turn it on.
33
39
  clientJoinKey: null,
@@ -46,19 +52,16 @@ const DEFAULTS = {
46
52
  defaultTimeoutSec: 1800,
47
53
  maxTimeoutSec: 14400
48
54
  },
55
+ // logRetentionDays, artifactRetentionDays and the `artifacts` section of
56
+ // older configs are no longer read: a job's log lines live as long as the
57
+ // job, and the Coordinator stores no artifacts (README §9).
49
58
  retention: {
50
- logRetentionDays: 14,
51
- artifactRetentionDays: 30,
52
- // How long a finished job (with its logs, artifacts and events) is kept
59
+ // How long a finished job (with its logs and events) is kept
53
60
  // before the hourly retention task wipes it (services/cleanup.js):
54
61
  // 1w | 2w | 1m | 3m | 6m | forever. `forever` keeps everything until a
55
62
  // manual cleanup (dashboard Clean up database / thub-admin jobs clean).
56
63
  jobRetention: 'forever'
57
64
  },
58
- artifacts: {
59
- maxUploadMb: 512,
60
- linkTtlHours: 168
61
- },
62
65
  // New-version check of the Coordinator/Agent/Client packages (README
63
66
  // §10.2), every 15 minutes so a new Coordinator's "Update app" button
64
67
  // shows up promptly. 0 disables the periodic check; registry null = the
@@ -122,6 +125,12 @@ function loadConfig(configPath = process.env.THUB_COORDINATOR_CONFIG){
122
125
  config.clientJoinKey = process.env.THUB_CLIENT_JOIN_KEY;
123
126
  }
124
127
 
128
+ if (![true, false, 'auto'].includes(config.session.secureCookie)){
129
+ throw new Error(
130
+ `session.secureCookie must be "auto", true or false (got ${JSON.stringify(config.session.secureCookie)}) in ${candidate || 'the config'}`
131
+ );
132
+ }
133
+
125
134
  if (!Object.hasOwn(JOB_RETENTION, config.retention.jobRetention)){
126
135
  throw new Error(
127
136
  `retention.jobRetention must be one of: ${Object.keys(JOB_RETENTION).join(', ')} ` +
@@ -136,12 +145,10 @@ function loadConfig(configPath = process.env.THUB_COORDINATOR_CONFIG){
136
145
  config.host = host;
137
146
  config.port = Number(port);
138
147
  config.dbPath = path.join(config.dataDir, 'thub.db');
139
- config.artifactsDir = path.join(config.dataDir, 'artifacts');
140
148
  config.workDir = path.join(config.dataDir, 'work');
141
149
  config.avatarsDir = path.join(config.dataDir, 'avatars');
142
150
 
143
151
  ensureWritableDir(config.dataDir, config.configPath);
144
- ensureWritableDir(config.artifactsDir, config.configPath);
145
152
  ensureWritableDir(config.avatarsDir, config.configPath);
146
153
 
147
154
  return config;
@@ -0,0 +1,10 @@
1
+ -- Dashboard sessions (services/session-store.js), instead of express-session's
2
+ -- in-memory store: they survive a restart, and expired ones are pruned.
3
+ -- `expires` is epoch milliseconds (the session cookie's own expiry).
4
+ CREATE TABLE sessions (
5
+ sid TEXT PRIMARY KEY,
6
+ sess TEXT NOT NULL,
7
+ expires INTEGER NOT NULL
8
+ );
9
+
10
+ CREATE INDEX idx_sessions_expires ON sessions(expires);
@@ -0,0 +1,4 @@
1
+ -- Job artifacts are no longer uploaded to or stored on the Coordinator
2
+ -- (README §9): Clients keep a job's files only in its workspace, which they
3
+ -- delete when the job ends. server.js removes <dataDir>/artifacts on startup.
4
+ DROP TABLE IF EXISTS artifacts;
@@ -170,7 +170,7 @@ function seedHistory(services, agentId, resourceIds){
170
170
  // stay IDLE instead of going OUT_OF_SERVICE) and play out any job the
171
171
  // scheduler assigns them — accept, PREPARING, RUNNING with streamed log
172
172
  // lines, then a PASSED (mostly) or FAILED result — through the same
173
- // services the Client API uses, so SSE, durations and artifacts all work.
173
+ // services the Client API uses, so SSE and durations all work.
174
174
  function startVirtualClients(services, config){
175
175
  const { registry, jobs, logs, bus } = services,
176
176
  intervalMs = config.heartbeat.intervalSec * 1000,
package/src/server.js CHANGED
@@ -15,7 +15,8 @@
15
15
 
16
16
  'use strict';
17
17
 
18
- const path = require('node:path'),
18
+ const fs = require('node:fs'),
19
+ path = require('node:path'),
19
20
  express = require('express'),
20
21
  session = require('express-session'),
21
22
 
@@ -31,19 +32,31 @@ const path = require('node:path'),
31
32
  { createScheduler } = require('./services/scheduler'),
32
33
  { createHeartbeatMonitor } = require('./services/heartbeat'),
33
34
  { createLogsService } = require('./services/logs'),
34
- { createArtifactsService } = require('./services/artifacts'),
35
35
  { createCommandsService } = require('./services/commands'),
36
36
  { createRetentionService } = require('./services/retention'),
37
37
  { createCleanupService } = require('./services/cleanup'),
38
38
  { startDevMode } = require('./dev/virtual'),
39
39
  { createUpdatesService } = require('./services/updates'),
40
+ { createLiveService } = require('./services/live'),
41
+ { SqliteSessionStore } = require('./services/session-store'),
40
42
 
41
43
  { createAgentRouter } = require('./api/agent'),
42
44
  { createResourceRouter } = require('./api/resource'),
43
45
  { createAdminRouter } = require('./api/admin'),
44
46
  { createWebRouter } = require('./web/routes');
45
47
 
48
+ // Artifacts are no longer stored on the Coordinator (README §9): an
49
+ // upgraded one frees the space the old <dataDir>/artifacts took, once.
50
+ function removeLegacyArtifacts(config){
51
+ const dir = path.join(config.dataDir, 'artifacts');
52
+ if (fs.existsSync(dir)){
53
+ fs.rmSync(dir, { recursive: true, force: true });
54
+ console.log(`Removed ${dir}: the Coordinator no longer stores job artifacts`);
55
+ }
56
+ }
57
+
46
58
  function buildServices(config){
59
+ removeLegacyArtifacts(config);
47
60
  const db = openDb(config.dbPath),
48
61
  events = createEventsService(db),
49
62
  registry = createRegistryService(db, { bus, events }),
@@ -52,13 +65,14 @@ function buildServices(config){
52
65
  adminUsers = createAdminUsersService(db),
53
66
  commands = createCommandsService({ bus }),
54
67
  logs = createLogsService(db, { bus }),
55
- artifacts = createArtifactsService(db, { config }),
56
- jobs = createJobsService(db, { bus, events, registry, artifacts, config }),
68
+ jobs = createJobsService(db, { bus, events, registry, config }),
57
69
  scheduler = createScheduler(db, { bus, events, registry, config }),
58
70
  heartbeatMonitor = createHeartbeatMonitor(db, { bus, events, registry, jobs, config }),
59
- retention = createRetentionService(db, { bus, logs, artifacts, config }),
71
+ retention = createRetentionService(db, { config }),
60
72
  cleanup = createCleanupService(db, { jobs, config }),
61
- updates = createUpdatesService({ config });
73
+ updates = createUpdatesService({ config }),
74
+ live = createLiveService(db, { bus, updates }),
75
+ sessionStore = new SqliteSessionStore(db);
62
76
 
63
77
  jobs.reconcileOnStartup();
64
78
  updates.start();
@@ -81,12 +95,13 @@ function buildServices(config){
81
95
  commands,
82
96
  jobs,
83
97
  logs,
84
- artifacts,
85
98
  scheduler,
86
99
  heartbeatMonitor,
87
100
  retention,
88
101
  cleanup,
89
- updates
102
+ updates,
103
+ live,
104
+ sessionStore
90
105
  };
91
106
  }
92
107
 
@@ -98,6 +113,18 @@ function commonVersion(){
98
113
  return require('@andrian.yablonskyy/thub-common/package.json').version;
99
114
  }
100
115
 
116
+ // The session cookie's Secure flag (config session.secureCookie, §12).
117
+ // "auto": always Secure for an https:// publicUrl — the cookie must never
118
+ // travel over plain HTTP there — else express-session's per-request 'auto'
119
+ // (Secure on HTTPS connections), so plain-HTTP development still works.
120
+ function cookieSecure(config){
121
+ const setting = config.session?.secureCookie ?? 'auto';
122
+ if (setting === true || setting === false){
123
+ return setting;
124
+ }
125
+ return /^https:\/\//i.test(config.publicUrl || '') ? true : 'auto';
126
+ }
127
+
101
128
  function createApp(config, services){
102
129
  const app = express();
103
130
  app.set('view engine', 'pug');
@@ -116,6 +143,9 @@ function createApp(config, services){
116
143
  app.use(
117
144
  session({
118
145
  secret: config.sessionSecret,
146
+ // In the SQLite database (services/session-store.js), not in memory:
147
+ // survives restarts, and expired sessions are pruned.
148
+ store: services.sessionStore,
119
149
  resave: false,
120
150
  saveUninitialized: false,
121
151
  // Idle timeout, not a fixed one (§10.1): `rolling` re-sends a fresh
@@ -124,9 +154,24 @@ function createApp(config, services){
124
154
  // (routes.js) after their *last* request. Each user's own preference
125
155
  // is applied per-request in web/routes.js, since it isn't known here.
126
156
  rolling: true,
127
- cookie: { httpOnly: true, sameSite: 'lax' }
157
+ cookie: { httpOnly: true, sameSite: 'lax', secure: cookieSecure(config) }
128
158
  })
129
159
  );
160
+ // Background requests of the dashboard's live updates (public/js/live.js):
161
+ // the GET /live stream and the page re-fetches it triggers (header
162
+ // X-Thub-Live: 1). They must not count as activity — otherwise an open
163
+ // tab would keep the session alive forever and the idle timeout (§10.1)
164
+ // would never fire — so they don't extend it (`touch` is what `rolling`
165
+ // uses to push the expiry out) and web/routes.js leaves the session alone.
166
+ app.use((req, res, next) => {
167
+ if (req.session && (req.path === '/live' || req.get('X-Thub-Live') === '1')){
168
+ req.thubPassive = true;
169
+ req.session.touch = function (){
170
+ return this;
171
+ };
172
+ }
173
+ next();
174
+ });
130
175
  app.use(express.static(path.join(__dirname, '..', 'public')));
131
176
  app.use('/avatars', express.static(config.avatarsDir));
132
177
 
@@ -134,17 +179,6 @@ function createApp(config, services){
134
179
  app.use('/api/v1', createResourceRouter({ services, config }));
135
180
  app.use('/api/v1/admin', createAdminRouter({ services }));
136
181
 
137
- // HMAC-signed artifact downloads (§3.1, §12) — no session or bearer token.
138
- app.get('/artifacts/download/:token', (req, res, next) => {
139
- try {
140
- const artifact = services.artifacts.verify(req.params.token);
141
- res.download(artifact.path, artifact.name);
142
- }
143
- catch (err){
144
- next(err);
145
- }
146
- });
147
-
148
182
  app.use('/', createWebRouter({ services, config }));
149
183
 
150
184
  app.use((err, req, res, next) => {
@@ -207,4 +241,4 @@ if (require.main === module){
207
241
  start();
208
242
  }
209
243
 
210
- module.exports = { start, buildServices, createApp };
244
+ module.exports = { start, buildServices, createApp, cookieSecure };
@@ -1,6 +1,6 @@
1
1
  /**
2
2
  * @file packages/coordinator/src/services/cleanup.js
3
- * @description Database cleanup: wipe finished jobs (logs, artifacts, events) and old events before a cutoff, then
3
+ * @description Database cleanup: wipe finished jobs (logs, events) and old events before a cutoff, then
4
4
  * VACUUM to shrink the file — on demand from the dashboard, and hourly per retention.jobRetention (README §9)
5
5
  *
6
6
  * @author Andrian Yablonskyy
@@ -81,7 +81,7 @@ function createCleanupService(db, { jobs, config, now = () => new Date() }){
81
81
  }
82
82
 
83
83
  // Wipe everything older than `before` (a Date; default now): finished
84
- // jobs that ended before it, with their logs, artifacts and events, and
84
+ // jobs that ended before it, with their logs and events, and
85
85
  // the other audit events (resource/agent/group) logged before it. Events
86
86
  // of jobs that stay (still active, or newer) are kept. `vacuum`: true,
87
87
  // false, or 'auto' (only if something was deleted and the last VACUUM is
@@ -15,6 +15,7 @@
15
15
 
16
16
  const { validateJobSpec, maskEnv, JOB_STATES, ACTIVE_JOB_STATES, TERMINAL_JOB_STATES, RESOURCE_STATES } = require('@andrian.yablonskyy/thub-common'),
17
17
  { paginate } = require('./list-prefs'),
18
+ { parseTerms, likeClause } = require('./search'),
18
19
  // Named in spec errors: a field the Agent sends but this thub-common
19
20
  // doesn't know means the Coordinator needs an update.
20
21
  VERSIONS = `Coordinator v${require('../../package.json').version}, ` +
@@ -55,7 +56,7 @@ function nextJobId(db, source){
55
56
  return `${prefix}-${String(n).padStart(5, '0')}`;
56
57
  }
57
58
 
58
- function createJobsService(db, { bus, events, registry, artifacts, config }){
59
+ function createJobsService(db, { bus, events, registry, config }){
59
60
  function get(id){
60
61
  return rowToJob(db.prepare('SELECT * FROM jobs WHERE id = ?').get(id));
61
62
  }
@@ -87,16 +88,26 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
87
88
  // Dashboard /jobs (§10): one sorted page plus the total, both in SQL so a
88
89
  // long history never has to be loaded whole. `sort` is a list-prefs key.
89
90
  const PAGE_SORT_SQL = {
90
- id: 'jobs.id',
91
- source: 'jobs.source',
92
- user: 'json_extract(jobs.spec, \'$.user\')',
93
- state: 'jobs.state',
94
- resource: 'COALESCE(r.name, jobs.resource_name)',
95
- created: 'jobs.created_at',
96
- duration: 'jobs.duration_sec'
97
- };
98
-
99
- function page({ state, source, sort = 'created', dir = 'desc', size = 25, page: pageNo = 1 } = {}){
91
+ id: 'jobs.id',
92
+ source: 'jobs.source',
93
+ user: 'json_extract(jobs.spec, \'$.user\')',
94
+ state: 'jobs.state',
95
+ resource: 'COALESCE(r.name, jobs.resource_name)',
96
+ created: 'jobs.created_at',
97
+ duration: 'jobs.duration_sec'
98
+ },
99
+
100
+ // What the Jobs page's search box (?q=) looks in. Named spec fields
101
+ // only — never the whole spec: while a job is active its `env` still
102
+ // holds the real --env values (§7.2), and a LIKE over them would let
103
+ // anyone who can see the list guess a secret one character at a time.
104
+ SEARCH_SQL = [
105
+ 'jobs.id', 'jobs.state', 'jobs.source', 'jobs.message', 'r.name', 'jobs.resource_name', 'a.name',
106
+ ...['user', 'command', 'suite', 'image', 'git.url', 'git.ref', 'target.group', 'target.labels', 'downloads', 'args', 'meta']
107
+ .map((path) => `json_extract(jobs.spec, '$.${path}')`)
108
+ ];
109
+
110
+ function page({ state, source, q, sort = 'created', dir = 'desc', size = 25, page: pageNo = 1 } = {}){
100
111
  const where = [],
101
112
  params = [];
102
113
  if (state){
@@ -107,15 +118,21 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
107
118
  where.push('jobs.source = ?');
108
119
  params.push(source);
109
120
  }
121
+ const search = likeClause(SEARCH_SQL, parseTerms(q));
122
+ if (search.sql){
123
+ where.push(search.sql);
124
+ params.push(...search.params);
125
+ }
110
126
  const whereSql = where.length ? `WHERE ${where.join(' AND ')}` : '',
127
+ fromSql = 'FROM jobs LEFT JOIN resources r ON r.id = jobs.resource_id LEFT JOIN agents a ON a.id = jobs.agent_id',
111
128
  column = PAGE_SORT_SQL[sort] || PAGE_SORT_SQL.created,
112
129
  order = dir === 'asc' ? 'ASC' : 'DESC',
113
- total = db.prepare(`SELECT COUNT(*) AS n FROM jobs ${whereSql}`).get(...params).n,
130
+ total = db.prepare(`SELECT COUNT(*) AS n ${fromSql} ${whereSql}`).get(...params).n,
114
131
  pagination = paginate(total, size, pageNo),
115
132
  // Empty values (no user, never ran, …) last either way; created_at
116
133
  // breaks ties so paging is stable.
117
134
  rows = db.prepare(
118
- `SELECT jobs.* FROM jobs LEFT JOIN resources r ON r.id = jobs.resource_id ${whereSql}
135
+ `SELECT jobs.* ${fromSql} ${whereSql}
119
136
  ORDER BY ${column} IS NULL, ${column} ${order}, jobs.created_at DESC
120
137
  LIMIT ? OFFSET ?`
121
138
  ).all(...params, pagination.limit, pagination.offset);
@@ -466,7 +483,7 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
466
483
  }
467
484
 
468
485
  // Admin: "clean the queue" — permanently delete finished jobs (and their
469
- // logs/artifacts, on disk and in the DB), for when the history itself,
486
+ // logs, in the DB), for when the history itself,
470
487
  // not just active work, needs clearing out. Unlike the nightly retention
471
488
  // sweep (§9), this runs on demand and isn't limited to old jobs.
472
489
  function cleanHistory(){
@@ -474,7 +491,7 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
474
491
  }
475
492
 
476
493
  // Finished jobs that ended before `beforeIso` (null: all of them), with
477
- // their logs, artifacts (files too) and events — manual cleanup and the
494
+ // their logs and events — manual cleanup and the
478
495
  // job retention task (services/cleanup.js). Active jobs are never touched.
479
496
  function purgeFinishedBefore(beforeIso){
480
497
  const placeholders = [...TERMINAL_JOB_STATES].map(() => '?').join(','),
@@ -487,14 +504,11 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
487
504
  return 0;
488
505
  }
489
506
 
490
- const deleteArtifacts = db.prepare('DELETE FROM artifacts WHERE job_id = ?'),
491
- deleteLogs = db.prepare('DELETE FROM job_logs WHERE job_id = ?'),
507
+ const deleteLogs = db.prepare('DELETE FROM job_logs WHERE job_id = ?'),
492
508
  deleteEvents = db.prepare('DELETE FROM events WHERE entity = \'job\' AND entity_id = ?'),
493
509
  deleteJob = db.prepare('DELETE FROM jobs WHERE id = ?'),
494
510
  tx = db.transaction(() => {
495
511
  for (const id of ids){
496
- artifacts.deleteJobArtifacts(id);
497
- deleteArtifacts.run(id);
498
512
  deleteLogs.run(id);
499
513
  deleteEvents.run(id);
500
514
  deleteJob.run(id);
@@ -30,6 +30,10 @@ const PAGE_SIZES = [10, 25, 50, 100, 'all'],
30
30
  resources: {
31
31
  sorts: ['name', 'type', 'version', 'status', 'heartbeat'],
32
32
  defaults: { size: DEFAULT_PAGE_SIZE, sort: 'name', dir: 'asc' }
33
+ },
34
+ agents: {
35
+ sorts: ['name', 'kind', 'version', 'created', 'used', 'status'],
36
+ defaults: { size: DEFAULT_PAGE_SIZE, sort: 'created', dir: 'desc' }
33
37
  }
34
38
  };
35
39