@andrian.yablonskyy/thub-coordinator 1.0.31 → 1.0.32

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andrian.yablonskyy/thub-coordinator",
-  "version": "1.0.31",
+  "version": "1.0.32",
   "description": "TestHub Coordinator — job queue, resource registry, scheduler, heartbeat monitor, log/artifact store and web dashboard",
   "main": "src/server.js",
   "engines": {
@@ -19,7 +19,7 @@
     "lint:fix": "eslint . --fix"
   },
   "dependencies": {
-    "@andrian.yablonskyy/thub-common": "^1.0.26",
+    "@andrian.yablonskyy/thub-common": "^1.0.27",
     "better-sqlite3": "^12.4.1",
     "express": "^5.1.0",
     "express-session": "^1.18.2",

package/src/api/agent.js

@@ -47,7 +47,7 @@ function createAgentRouter({ services, config }){
     }
     const resource = job.resource_id ? services.registry.get(job.resource_id) : null;
     res.json({
-      ...job,
+      ...services.jobs.publicJob(job),
       resource: resource ? { id: resource.id, name: resource.name } : job.resource_name ? { id: null, name: job.resource_name } : null
     });
   });
@@ -60,13 +60,13 @@ function createAgentRouter({ services, config }){
       mine: req.query.mine === 'true' || req.query.mine === '1',
       limit: req.query.limit ? Number(req.query.limit) : undefined
     });
-    res.json({ jobs });
+    res.json({ jobs: jobs.map(services.jobs.publicJob) });
   });
 
   router.post('/jobs/:id/cancel', auth, (req, res, next) => {
     try {
       const job = services.jobs.cancel(req.params.id, { agentId: req.agent.id, isAdmin: false });
-      res.json(job);
+      res.json(services.jobs.publicJob(job));
     }
     catch (err){
       next(err);

package/src/api/sse.js

@@ -13,7 +13,9 @@
 
 'use strict';
 
-const { TERMINAL_JOB_STATES } = require('@andrian.yablonskyy/thub-common');
+const { TERMINAL_JOB_STATES, JOB_STATES } = require('@andrian.yablonskyy/thub-common');
+
+const WAITING_CHECK_MS = 15_000;
 
 // §6.4: log / state / end events, resumable via Last-Event-ID (§7: "the
 // Agent reconnects with exponential backoff and resumes from the last seq").
@@ -62,7 +64,26 @@ function attachJobStream(req, res, { jobId, services, config }){
   const resourceName = (id) => (id ? registry.get(id)?.name : undefined);
   writeEvent('state', { state: current.state, resource: resourceName(current.resource_id) });
 
-  const onLog = (entry) => {
+  // While QUEUED: why no Client has taken it yet (`waiting`), re-checked
+  // as Clients come and go; sent again only when the reason changes.
+  let lastReason = null;
+  function checkWaiting(){
+    const job = jobs.get(jobId);
+    if (job?.state !== JOB_STATES.QUEUED){
+      lastReason = null;
+      return;
+    }
+    const { type, labels = [], group, client } = job.spec.target,
+      reason = registry.waitingReason(type, labels, group, client, { jobImage: Boolean(job.spec.image) });
+    if (reason && reason !== lastReason){
+      writeEvent('waiting', { reason });
+    }
+    lastReason = reason;
+  }
+  checkWaiting();
+  const waitingTimer = setInterval(checkWaiting, WAITING_CHECK_MS),
+
+    onLog = (entry) => {
       if (entry.jobId === jobId){
         writeEvent('log', { ts: entry.ts, stream: entry.stream, line: entry.line }, entry.seq);
       }
@@ -82,6 +103,7 @@ function attachJobStream(req, res, { jobId, services, config }){
     };
 
   function cleanup(){
+    clearInterval(waitingTimer);
     bus.off('job.log', onLog);
     bus.off('job.state', onState);
     bus.off('job.finished', onFinished);

package/src/services/jobs.js

@@ -13,7 +13,7 @@
 
 'use strict';
 
-const { validateJobSpec, JOB_STATES, ACTIVE_JOB_STATES, TERMINAL_JOB_STATES, RESOURCE_STATES } = require('@andrian.yablonskyy/thub-common'),
+const { validateJobSpec, maskEnv, JOB_STATES, ACTIVE_JOB_STATES, TERMINAL_JOB_STATES, RESOURCE_STATES } = require('@andrian.yablonskyy/thub-common'),
   { paginate } = require('./list-prefs'),
   // Named in spec errors: a field the Agent sends but this thub-common
   // doesn't know means the Coordinator needs an update.
@@ -248,6 +248,10 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
       bus.emit('job.queued', { jobId: id });
     }
     if (TERMINAL_JOB_STATES.has(state)){
+      // A LOST job may be requeued (markLost), and needs its env for that.
+      if (state !== JOB_STATES.LOST){
+        forgetEnv(id);
+      }
       bus.emit('job.finished', { jobId: id, state });
       // LOST means the resource is presumed OUT_OF_SERVICE (that's why the
       // job went LOST) — it only comes back via a real heartbeat (§4.1),
@@ -378,14 +382,30 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
     if (job.state === JOB_STATES.LOST || TERMINAL_JOB_STATES.has(job.state)){
       return job;
     }
-    const spec = job.spec;
     setState(id, JOB_STATES.LOST);
     if (config.scheduler.requeueOnLost && job.attempt < 2){
       return setState(id, JOB_STATES.QUEUED);
     }
+    forgetEnv(id);
     return get(id);
   }
 
+  // A finished job's `--env` values (registry passwords etc.) aren't kept:
+  // only the names stay, masked.
+  function forgetEnv(id){
+    const row = db.prepare('SELECT spec FROM jobs WHERE id = ?').get(id),
+      spec = row && JSON.parse(row.spec);
+    if (spec?.env){
+      db.prepare('UPDATE jobs SET spec = ? WHERE id = ?').run(JSON.stringify({ ...spec, env: maskEnv(spec.env) }), id);
+    }
+  }
+
+  // A job as the Agent API shows it: `--env` values masked (any Agent token
+  // can read any job; only the Client running it gets them).
+  function publicJob(job){
+    return job?.spec?.env ? { ...job, spec: { ...job.spec, env: maskEnv(job.spec.env) } } : job;
+  }
+
   function checkTimeouts(){
     const now = Date.now(),
       active = db
@@ -513,6 +533,7 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
 
   return {
     get,
+    publicJob,
     list,
     page,
     create,

package/src/services/registry.js

@@ -701,6 +701,40 @@ function createRegistryService(db, { bus, events }){
     return rows.filter((r) => matchesTarget(r, labels, groupId, resourceId, needs));
   }
 
+  // Why a queued job isn't assigned yet, in words — for the Agent and the
+  // dashboard (null when an idle Client fits: the next scheduler pass takes
+  // it, or higher-priority jobs are ahead). Names each matching Client that
+  // isn't free and what it's doing, and — for a job's own Docker image —
+  // the idle ones that don't allow that (sw.allowJobImages).
+  function waitingReason(type, labels, groupId, resourceId, needs = {}){
+    const matching = db.prepare('SELECT * FROM resources WHERE type = ?').all(type).map(rowToResource)
+        .filter((r) => matchesTarget(r, labels, groupId, resourceId)),
+      free = (r) => r.status === RESOURCE_STATES.IDLE && !r.remove_requested_at,
+      what = (r) => (r.remove_requested_at
+        ? 'being removed'
+        : { [RESOURCE_STATES.BUSY]: 'busy', [RESOURCE_STATES.OUT_OF_SERVICE]: 'offline', [RESOURCE_STATES.MAINTENANCE]: 'in maintenance',
+          [RESOURCE_STATES.REGISTERED]: 'not connected yet' }[r.status] || r.status.toLowerCase()),
+      names = (rs) => rs.map((r) => `${r.name} (${what(r)})`).join(', '),
+      kind = type.toUpperCase();
+    if (!matching.length){
+      return `no ${kind} Client matches this job's target`;
+    }
+    if (needs.jobImage){
+      const allowing = matching.filter((r) => matchesTarget(r, labels, groupId, resourceId, needs)),
+        refusing = matching.filter((r) => !allowing.includes(r) && free(r));
+      if (!allowing.some(free)){
+        return `no idle ${kind} Client runs job-supplied Docker images — ` +
+          (allowing.length ? `the ones that do: ${names(allowing)}` : 'none does') +
+          (refusing.length
+            ? `; idle, but not allowing them: ${refusing.map((r) => r.name).join(', ')} ` +
+              '(enable "Run jobs\' own images" on its Emulator config tab, or sw.allowJobImages in its config)'
+            : '');
+      }
+      return null;
+    }
+    return matching.some(free) ? null : `every matching ${kind} Client is taken: ${names(matching)}`;
+  }
+
   function everSatisfiable(type, labels, groupId, resourceId, needs){
     const rows = db.prepare('SELECT * FROM resources WHERE type = ?').all(type).map(rowToResource);
     return rows.some((r) => matchesTarget(r, labels, groupId, resourceId, needs));
@@ -734,7 +768,8 @@ function createRegistryService(db, { bus, events }){
     requestRemoval,
     pendingRemovals,
     findIdleCandidates,
-    everSatisfiable
+    everSatisfiable,
+    waitingReason
   };
 }
 

package/src/web/routes.js

@@ -714,6 +714,10 @@ function createWebRouter({ services, config }){
       active: 'jobs',
       job,
       pinnedClient: job.spec.target.client ? services.registry.get(job.spec.target.client) : null,
+      waitingReason: job.state === JOB_STATES.QUEUED
+        ? services.registry.waitingReason(job.spec.target.type, job.spec.target.labels || [], job.spec.target.group,
+          job.spec.target.client, { jobImage: Boolean(job.spec.image) })
+        : null,
       artifacts,
       jobActive,
       canCancel: req.session.user.role === 'admin' && jobActive

package/test/scheduler.test.js

@@ -546,6 +546,52 @@ test('a job-supplied Docker image only goes to SW Clients that allow it; otherwi
   assert.equal(jobs.get(fileJob.id).resource_id, strict.id); // jobs without an image still go anywhere
 });
 
+test('waiting reason: says why a queued job has no Client — a job image nobody idle allows, or every match taken', () => {
+  const { registry, db } = buildTestServices(),
+    reason = (needs) => registry.waitingReason('sw', [], undefined, undefined, needs),
+    strict = registerResource(registry, { name: 'sw-strict', type: 'sw' }),
+    { resourceId: open } = registry.registerAuto({
+      clientId: 'c-open', name: 'sw-open', type: 'sw', labels: [], capabilities: { sw: { image: 'emu', allowJobImages: true } }
+    });
+  registry.heartbeat(strict.id, { state: 'idle' });
+  db.prepare('UPDATE resources SET status = ? WHERE id = ?').run(RESOURCE_STATES.OUT_OF_SERVICE, open);
+
+  assert.equal(
+    reason({ jobImage: true }),
+    'no idle SW Client runs job-supplied Docker images — the ones that do: sw-open (offline); idle, but not allowing them: sw-strict ' +
+      '(enable "Run jobs\' own images" on its Emulator config tab, or sw.allowJobImages in its config)'
+  );
+  assert.equal(reason({}), null); // sw-strict can take a job without an image
+
+  registry.heartbeat(open, { state: 'idle' });
+  assert.equal(reason({ jobImage: true }), null);
+
+  registry.assignToJob(open, 'cli');
+  registry.assignToJob(strict.id, 'cli');
+  assert.equal(reason({}), 'every matching SW Client is taken: sw-strict (busy), sw-open (busy)');
+  assert.equal(registry.waitingReason('hw', [], undefined, undefined, {}), 'no HW Client matches this job\'s target');
+});
+
+test('--env: the Client gets the values; the Agent API sees them masked; they\'re dropped when the job ends (not on a requeued LOST)', () => {
+  const { registry, agents, jobs, scheduler } = buildTestServices(),
+    { agent } = agents.create({ name: 'dev', kind: 'cli' }),
+    r = registerResource(registry, { name: 'sw-env', type: 'sw' }),
+    env = { DOCKER_REGISTRY: 'registry.lab:5000', DOCKER_USERNAME: 'ci', DOCKER_PASSWORD: 's3cret' };
+  registry.heartbeat(r.id, { state: 'idle' });
+
+  const job = jobs.create({ agentId: agent.id, source: 'cli', spec: makeSpec({ env }) });
+  assert.deepEqual(jobs.get(job.id).spec.env, env); // what the Client is handed
+  assert.deepEqual(jobs.publicJob(jobs.get(job.id)).spec.env, { DOCKER_REGISTRY: '***', DOCKER_USERNAME: '***', DOCKER_PASSWORD: '***' });
+
+  scheduler.runPass();
+  jobs.markLost(job.id); // requeued: still needs them
+  assert.equal(jobs.get(job.id).state, JOB_STATES.QUEUED);
+  assert.deepEqual(jobs.get(job.id).spec.env, env);
+
+  jobs.cancel(job.id, { isAdmin: true });
+  assert.deepEqual(jobs.get(job.id).spec.env, { DOCKER_REGISTRY: '***', DOCKER_USERNAME: '***', DOCKER_PASSWORD: '***' });
+});
+
 test('every job carries its command and runs on any matching Client; an old-Agent spec is refused', () => {
   const { registry, agents, jobs, scheduler } = buildTestServices(),
     { agent } = agents.create({ name: 'dev', kind: 'cli' }),

package/views/jobs/show.pug

@@ -19,6 +19,10 @@ block content
       )
         button.btn.btn-outline-danger.btn-sm(type="submit") Cancel
 
+  if waitingReason
+    .alert.alert-warning.py-2.small
+      i.bi.bi-hourglass-split.me-1
+      | Waiting: #{waitingReason}
   dl.row
     dt.col-sm-2 Source
     dd.col-sm-10.text-uppercase= job.source
@@ -32,6 +36,10 @@ block content
     if job.spec.target.client
       dt.col-sm-2 Client
       dd.col-sm-10= pinnedClient ? pinnedClient.name : job.spec.target.client
+    if job.spec.env && Object.keys(job.spec.env).length
+      dt.col-sm-2 Environment
+      dd.col-sm-10.font-monospace(data-bs-toggle="tooltip" data-bs-title="Set by --env; values are only given to the Client that runs the job")
+        = Object.keys(job.spec.env).join(', ')
     dt.col-sm-2 Created
     dd.col-sm-10= fmtDate(job.created_at)
     if job.duration_sec != null