@andrian.yablonskyy/thub-coordinator 1.0.30 → 1.0.32

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andrian.yablonskyy/thub-coordinator",
-  "version": "1.0.30",
+  "version": "1.0.32",
   "description": "TestHub Coordinator — job queue, resource registry, scheduler, heartbeat monitor, log/artifact store and web dashboard",
   "main": "src/server.js",
   "engines": {
@@ -19,7 +19,7 @@
     "lint:fix": "eslint . --fix"
   },
   "dependencies": {
-    "@andrian.yablonskyy/thub-common": "*",
+    "@andrian.yablonskyy/thub-common": "^1.0.27",
     "better-sqlite3": "^12.4.1",
     "express": "^5.1.0",
     "express-session": "^1.18.2",

package/src/api/agent.js

@@ -47,7 +47,7 @@ function createAgentRouter({ services, config }){
     }
     const resource = job.resource_id ? services.registry.get(job.resource_id) : null;
     res.json({
-      ...job,
+      ...services.jobs.publicJob(job),
       resource: resource ? { id: resource.id, name: resource.name } : job.resource_name ? { id: null, name: job.resource_name } : null
     });
   });
@@ -60,13 +60,13 @@ function createAgentRouter({ services, config }){
       mine: req.query.mine === 'true' || req.query.mine === '1',
       limit: req.query.limit ? Number(req.query.limit) : undefined
     });
-    res.json({ jobs });
+    res.json({ jobs: jobs.map(services.jobs.publicJob) });
   });
 
   router.post('/jobs/:id/cancel', auth, (req, res, next) => {
     try {
       const job = services.jobs.cancel(req.params.id, { agentId: req.agent.id, isAdmin: false });
-      res.json(job);
+      res.json(services.jobs.publicJob(job));
     }
     catch (err){
       next(err);

package/src/api/sse.js

@@ -13,7 +13,9 @@
 
 'use strict';
 
-const { TERMINAL_JOB_STATES } = require('@andrian.yablonskyy/thub-common');
+const { TERMINAL_JOB_STATES, JOB_STATES } = require('@andrian.yablonskyy/thub-common');
+
+const WAITING_CHECK_MS = 15_000;
 
 // §6.4: log / state / end events, resumable via Last-Event-ID (§7: "the
 // Agent reconnects with exponential backoff and resumes from the last seq").
@@ -62,7 +64,26 @@ function attachJobStream(req, res, { jobId, services, config }){
   const resourceName = (id) => (id ? registry.get(id)?.name : undefined);
   writeEvent('state', { state: current.state, resource: resourceName(current.resource_id) });
 
-  const onLog = (entry) => {
+  // While QUEUED: why no Client has taken it yet (`waiting`), re-checked
+  // as Clients come and go; sent again only when the reason changes.
+  let lastReason = null;
+  function checkWaiting(){
+    const job = jobs.get(jobId);
+    if (job?.state !== JOB_STATES.QUEUED){
+      lastReason = null;
+      return;
+    }
+    const { type, labels = [], group, client } = job.spec.target,
+      reason = registry.waitingReason(type, labels, group, client, { jobImage: Boolean(job.spec.image) });
+    if (reason && reason !== lastReason){
+      writeEvent('waiting', { reason });
+    }
+    lastReason = reason;
+  }
+  checkWaiting();
+  const waitingTimer = setInterval(checkWaiting, WAITING_CHECK_MS),
+
+    onLog = (entry) => {
       if (entry.jobId === jobId){
         writeEvent('log', { ts: entry.ts, stream: entry.stream, line: entry.line }, entry.seq);
       }
@@ -82,6 +103,7 @@ function attachJobStream(req, res, { jobId, services, config }){
     };
 
   function cleanup(){
+    clearInterval(waitingTimer);
     bus.off('job.log', onLog);
     bus.off('job.state', onState);
     bus.off('job.finished', onFinished);

package/src/server.js

@@ -90,6 +90,14 @@ function buildServices(config){
   };
 }
 
+function commonPath(){
+  return path.dirname(require.resolve('@andrian.yablonskyy/thub-common/package.json'));
+}
+
+function commonVersion(){
+  return require('@andrian.yablonskyy/thub-common/package.json').version;
+}
+
 function createApp(config, services){
   const app = express();
   app.set('view engine', 'pug');
@@ -98,6 +106,10 @@ function createApp(config, services){
   app.locals.services = services;
   // Shown in every dashboard page's footer (layout.pug).
   app.locals.coordinatorVersion = require('../package.json').version;
+  // The thub-common this process actually loaded (it validates job specs):
+  // installs keep the one they got, so it can lag behind — shown on the
+  // dashboard and logged at startup to make that visible.
+  app.locals.commonVersion = commonVersion();
 
   app.use(express.json({ limit: '2mb' }));
   app.use(express.urlencoded({ extended: true }));
@@ -162,6 +174,7 @@ function start(configPath){
   const server = app.listen(config.port, config.host, () => {
     console.log(`Config: ${config.configPath || '(built-in defaults)'}; data: ${config.dataDir}`);
     console.log(`TestHub Coordinator listening on http://${config.host}:${config.port}`);
+    console.log(`Coordinator v${require('../package.json').version}, thub-common v${commonVersion()} (${commonPath()})`);
   });
 
   return { app, server, services, config };

package/src/services/jobs.js

@@ -13,8 +13,12 @@
 
 'use strict';
 
-const { validateJobSpec, JOB_STATES, ACTIVE_JOB_STATES, TERMINAL_JOB_STATES, RESOURCE_STATES } = require('@andrian.yablonskyy/thub-common'),
-  { paginate } = require('./list-prefs');
+const { validateJobSpec, maskEnv, JOB_STATES, ACTIVE_JOB_STATES, TERMINAL_JOB_STATES, RESOURCE_STATES } = require('@andrian.yablonskyy/thub-common'),
+  { paginate } = require('./list-prefs'),
+  // Named in spec errors: a field the Agent sends but this thub-common
+  // doesn't know means the Coordinator needs an update.
+  VERSIONS = `Coordinator v${require('../../package.json').version}, ` +
+    `thub-common v${require('@andrian.yablonskyy/thub-common/package.json').version}`;
 
 function rowToJob(row){
   if (!row){
@@ -131,7 +135,7 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
       rawSpec?.priority === undefined ? { ...rawSpec, priority: DEFAULT_PRIORITY[source] } : rawSpec
     );
     if (!valid){
-      throw Object.assign(new Error(`Invalid job spec: ${errors.join('; ')}`), { status: 400 });
+      throw Object.assign(new Error(`Invalid job spec (${VERSIONS}): ${errors.join('; ')}`), { status: 400 });
     }
     spec.source = source;
 
@@ -244,6 +248,10 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
       bus.emit('job.queued', { jobId: id });
     }
     if (TERMINAL_JOB_STATES.has(state)){
+      // A LOST job may be requeued (markLost), and needs its env for that.
+      if (state !== JOB_STATES.LOST){
+        forgetEnv(id);
+      }
       bus.emit('job.finished', { jobId: id, state });
       // LOST means the resource is presumed OUT_OF_SERVICE (that's why the
       // job went LOST) — it only comes back via a real heartbeat (§4.1),
@@ -374,14 +382,30 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
     if (job.state === JOB_STATES.LOST || TERMINAL_JOB_STATES.has(job.state)){
       return job;
     }
-    const spec = job.spec;
     setState(id, JOB_STATES.LOST);
     if (config.scheduler.requeueOnLost && job.attempt < 2){
       return setState(id, JOB_STATES.QUEUED);
     }
+    forgetEnv(id);
     return get(id);
   }
 
+  // A finished job's `--env` values (registry passwords etc.) aren't kept:
+  // only the names stay, masked.
+  function forgetEnv(id){
+    const row = db.prepare('SELECT spec FROM jobs WHERE id = ?').get(id),
+      spec = row && JSON.parse(row.spec);
+    if (spec?.env){
+      db.prepare('UPDATE jobs SET spec = ? WHERE id = ?').run(JSON.stringify({ ...spec, env: maskEnv(spec.env) }), id);
+    }
+  }
+
+  // A job as the Agent API shows it: `--env` values masked (any Agent token
+  // can read any job; only the Client running it gets them).
+  function publicJob(job){
+    return job?.spec?.env ? { ...job, spec: { ...job.spec, env: maskEnv(job.spec.env) } } : job;
+  }
+
   function checkTimeouts(){
     const now = Date.now(),
       active = db
@@ -509,6 +533,7 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
 
   return {
     get,
+    publicJob,
     list,
     page,
     create,

package/src/services/registry.js

@@ -701,6 +701,40 @@ function createRegistryService(db, { bus, events }){
     return rows.filter((r) => matchesTarget(r, labels, groupId, resourceId, needs));
   }
 
+  // Why a queued job isn't assigned yet, in words — for the Agent and the
+  // dashboard (null when an idle Client fits: the next scheduler pass takes
+  // it, or higher-priority jobs are ahead). Names each matching Client that
+  // isn't free and what it's doing, and — for a job's own Docker image —
+  // the idle ones that don't allow that (sw.allowJobImages).
+  function waitingReason(type, labels, groupId, resourceId, needs = {}){
+    const matching = db.prepare('SELECT * FROM resources WHERE type = ?').all(type).map(rowToResource)
+        .filter((r) => matchesTarget(r, labels, groupId, resourceId)),
+      free = (r) => r.status === RESOURCE_STATES.IDLE && !r.remove_requested_at,
+      what = (r) => (r.remove_requested_at
+        ? 'being removed'
+        : { [RESOURCE_STATES.BUSY]: 'busy', [RESOURCE_STATES.OUT_OF_SERVICE]: 'offline', [RESOURCE_STATES.MAINTENANCE]: 'in maintenance',
+          [RESOURCE_STATES.REGISTERED]: 'not connected yet' }[r.status] || r.status.toLowerCase()),
+      names = (rs) => rs.map((r) => `${r.name} (${what(r)})`).join(', '),
+      kind = type.toUpperCase();
+    if (!matching.length){
+      return `no ${kind} Client matches this job's target`;
+    }
+    if (needs.jobImage){
+      const allowing = matching.filter((r) => matchesTarget(r, labels, groupId, resourceId, needs)),
+        refusing = matching.filter((r) => !allowing.includes(r) && free(r));
+      if (!allowing.some(free)){
+        return `no idle ${kind} Client runs job-supplied Docker images — ` +
+          (allowing.length ? `the ones that do: ${names(allowing)}` : 'none does') +
+          (refusing.length
+            ? `; idle, but not allowing them: ${refusing.map((r) => r.name).join(', ')} ` +
+              '(enable "Run jobs\' own images" on its Emulator config tab, or sw.allowJobImages in its config)'
+            : '');
+      }
+      return null;
+    }
+    return matching.some(free) ? null : `every matching ${kind} Client is taken: ${names(matching)}`;
+  }
+
   function everSatisfiable(type, labels, groupId, resourceId, needs){
     const rows = db.prepare('SELECT * FROM resources WHERE type = ?').all(type).map(rowToResource);
     return rows.some((r) => matchesTarget(r, labels, groupId, resourceId, needs));
@@ -734,7 +768,8 @@ function createRegistryService(db, { bus, events }){
     requestRemoval,
     pendingRemovals,
     findIdleCandidates,
-    everSatisfiable
+    everSatisfiable,
+    waitingReason
   };
 }
 

package/src/web/routes.js

@@ -714,6 +714,10 @@ function createWebRouter({ services, config }){
       active: 'jobs',
       job,
       pinnedClient: job.spec.target.client ? services.registry.get(job.spec.target.client) : null,
+      waitingReason: job.state === JOB_STATES.QUEUED
+        ? services.registry.waitingReason(job.spec.target.type, job.spec.target.labels || [], job.spec.target.group,
+          job.spec.target.client, { jobImage: Boolean(job.spec.image) })
+        : null,
       artifacts,
       jobActive,
       canCancel: req.session.user.role === 'admin' && jobActive

package/test/scheduler.test.js

@@ -546,6 +546,52 @@ test('a job-supplied Docker image only goes to SW Clients that allow it; otherwi
   assert.equal(jobs.get(fileJob.id).resource_id, strict.id); // jobs without an image still go anywhere
 });
 
+test('waiting reason: says why a queued job has no Client — a job image nobody idle allows, or every match taken', () => {
+  const { registry, db } = buildTestServices(),
+    reason = (needs) => registry.waitingReason('sw', [], undefined, undefined, needs),
+    strict = registerResource(registry, { name: 'sw-strict', type: 'sw' }),
+    { resourceId: open } = registry.registerAuto({
+      clientId: 'c-open', name: 'sw-open', type: 'sw', labels: [], capabilities: { sw: { image: 'emu', allowJobImages: true } }
+    });
+  registry.heartbeat(strict.id, { state: 'idle' });
+  db.prepare('UPDATE resources SET status = ? WHERE id = ?').run(RESOURCE_STATES.OUT_OF_SERVICE, open);
+
+  assert.equal(
+    reason({ jobImage: true }),
+    'no idle SW Client runs job-supplied Docker images — the ones that do: sw-open (offline); idle, but not allowing them: sw-strict ' +
+      '(enable "Run jobs\' own images" on its Emulator config tab, or sw.allowJobImages in its config)'
+  );
+  assert.equal(reason({}), null); // sw-strict can take a job without an image
+
+  registry.heartbeat(open, { state: 'idle' });
+  assert.equal(reason({ jobImage: true }), null);
+
+  registry.assignToJob(open, 'cli');
+  registry.assignToJob(strict.id, 'cli');
+  assert.equal(reason({}), 'every matching SW Client is taken: sw-strict (busy), sw-open (busy)');
+  assert.equal(registry.waitingReason('hw', [], undefined, undefined, {}), 'no HW Client matches this job\'s target');
+});
+
+test('--env: the Client gets the values; the Agent API sees them masked; they\'re dropped when the job ends (not on a requeued LOST)', () => {
+  const { registry, agents, jobs, scheduler } = buildTestServices(),
+    { agent } = agents.create({ name: 'dev', kind: 'cli' }),
+    r = registerResource(registry, { name: 'sw-env', type: 'sw' }),
+    env = { DOCKER_REGISTRY: 'registry.lab:5000', DOCKER_USERNAME: 'ci', DOCKER_PASSWORD: 's3cret' };
+  registry.heartbeat(r.id, { state: 'idle' });
+
+  const job = jobs.create({ agentId: agent.id, source: 'cli', spec: makeSpec({ env }) });
+  assert.deepEqual(jobs.get(job.id).spec.env, env); // what the Client is handed
+  assert.deepEqual(jobs.publicJob(jobs.get(job.id)).spec.env, { DOCKER_REGISTRY: '***', DOCKER_USERNAME: '***', DOCKER_PASSWORD: '***' });
+
+  scheduler.runPass();
+  jobs.markLost(job.id); // requeued: still needs them
+  assert.equal(jobs.get(job.id).state, JOB_STATES.QUEUED);
+  assert.deepEqual(jobs.get(job.id).spec.env, env);
+
+  jobs.cancel(job.id, { isAdmin: true });
+  assert.deepEqual(jobs.get(job.id).spec.env, { DOCKER_REGISTRY: '***', DOCKER_USERNAME: '***', DOCKER_PASSWORD: '***' });
+});
+
 test('every job carries its command and runs on any matching Client; an old-Agent spec is refused', () => {
   const { registry, agents, jobs, scheduler } = buildTestServices(),
     { agent } = agents.create({ name: 'dev', kind: 'cli' }),

package/views/jobs/show.pug

@@ -19,6 +19,10 @@ block content
       )
         button.btn.btn-outline-danger.btn-sm(type="submit") Cancel
 
+  if waitingReason
+    .alert.alert-warning.py-2.small
+      i.bi.bi-hourglass-split.me-1
+      | Waiting: #{waitingReason}
   dl.row
     dt.col-sm-2 Source
     dd.col-sm-10.text-uppercase= job.source
@@ -32,6 +36,10 @@ block content
     if job.spec.target.client
       dt.col-sm-2 Client
       dd.col-sm-10= pinnedClient ? pinnedClient.name : job.spec.target.client
+    if job.spec.env && Object.keys(job.spec.env).length
+      dt.col-sm-2 Environment
+      dd.col-sm-10.font-monospace(data-bs-toggle="tooltip" data-bs-title="Set by --env; values are only given to the Client that runs the job")
+        = Object.keys(job.spec.env).join(', ')
     dt.col-sm-2 Created
     dd.col-sm-10= fmtDate(job.created_at)
     if job.duration_sec != null

package/views/layout.pug

@@ -12,7 +12,12 @@ html(lang="en" data-bs-theme=(user && user.theme) || "auto")
       .container-fluid
         a.navbar-brand.d-flex.flex-column.lh-1.py-1(href="/")
           span TestHub
-          small.text-body-secondary.fw-normal.mt-1(style="font-size: .7rem;")
+          small.text-body-secondary.fw-normal.mt-1(
+            style="font-size: .7rem;"
+            data-bs-toggle="tooltip"
+            data-bs-placement="bottom"
+            data-bs-title=`Coordinator v${coordinatorVersion} · thub-common v${commonVersion} (validates job specs)`
+          )
             | ver. #{coordinatorVersion}
             if user && user.role !== 'admin' && updates.coordinatorUpdate
               span.badge.text-bg-warning.ms-1(