@andrian.yablonskyy/thub-coordinator 1.0.17 → 1.0.18

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andrian.yablonskyy/thub-coordinator",
-  "version": "1.0.17",
+  "version": "1.0.18",
   "description": "TestHub Coordinator — job queue, resource registry, scheduler, heartbeat monitor, log/artifact store and web dashboard",
   "main": "src/server.js",
   "engines": {

package/public/js/remove-resource.js

@@ -0,0 +1,45 @@
+/**
+ * @file        packages/coordinator/public/js/remove-resource.js
+ * @description Dashboard: warning modal for removing a resource that is running a job (stop job, then remove)
+ *
+ * @author      Andrian Yablonskyy
+ * @copyright   Copyright (c) 2026 Andrian Yablonskyy. All rights reserved.
+ *
+ * This file is part of TestHub and is proprietary and confidential.
+ * Unauthorized copying, modification, distribution, or use of this file,
+ * via any medium, is strictly prohibited without prior written permission
+ * from AdSystem.PRO.
+ */
+
+'use strict';
+
+(function (){
+  const modalEl = document.getElementById('remove-resource-modal');
+  if (!modalEl){
+    return;
+  }
+  const field = (name) => modalEl.querySelector(`[data-field="${name}"]`);
+
+  document.querySelectorAll('[data-remove-resource]').forEach((btn) => {
+    btn.addEventListener('click', () => {
+      const { removeResource: id, removeName: name, removeJob: job, removeReturn: returnTo } = btn.dataset;
+      field('name').textContent = name;
+      field('job').textContent = job;
+      field('job').href = `/jobs/${encodeURIComponent(job)}`;
+      field('job-text').textContent = job;
+      field('form').action = `/resources/${encodeURIComponent(id)}/remove`;
+      field('return').value = returnTo || '/resources';
+
+      // Clicked from inside a resource card: close that first — Bootstrap
+      // doesn't stack modals.
+      const card = btn.closest('.modal');
+      if (card){
+        card.addEventListener('hidden.bs.modal', () => bootstrap.Modal.getOrCreateInstance(modalEl).show(), { once: true });
+        bootstrap.Modal.getInstance(card)?.hide();
+      }
+      else {
+        bootstrap.Modal.getOrCreateInstance(modalEl).show();
+      }
+    });
+  });
+})();

package/src/api/admin.js

@@ -60,6 +60,21 @@ function createAdminRouter({ services }){
     res.json({ resourceToken: token });
   });
 
+  // Removes a resource; 409 while it runs a job unless ?stopJob=1, which
+  // cancels the job and removes the resource once the Client confirms
+  // (`pending: true`). Queued jobs pinned to it are canceled on removal.
+  router.delete('/resources/:id', (req, res, next) => {
+    try {
+      const { resource, pending, stoppedJob, canceledJobs } = services.jobs.removeResource(req.params.id, {
+        stopJob: req.query.stopJob === '1'
+      });
+      res.json({ removed: resource.id, pending, stoppedJob, canceledJobs });
+    }
+    catch (err){
+      next(err);
+    }
+  });
+
   // Cancels every QUEUED/ASSIGNED/PREPARING/RUNNING job (§13.1).
   router.post('/jobs/reset-queue', (req, res) => {
     const canceled = services.jobs.resetQueue();

package/src/api/agent.js

@@ -28,7 +28,11 @@ function createAgentRouter({ services, config }){
 
   router.post('/jobs', auth, (req, res, next) => {
     try {
-      const job = services.jobs.create({ agentId: req.agent.id, source: req.body.source || 'cli', spec: req.body });
+      // The job source (A-/M- id series, default priority) is the agent
+      // token's admin-assigned kind alone — never taken from the request,
+      // so it can't be spoofed or mis-detected on the Agent side.
+      const source = req.agent.kind,
+        job = services.jobs.create({ agentId: req.agent.id, source, spec: req.body });
       res.status(201).json({ jobId: job.id, state: job.state, webUrl: `${config.publicUrl}/jobs/${job.id}` });
     }
     catch (err){
@@ -42,7 +46,10 @@ function createAgentRouter({ services, config }){
       return res.status(404).json({ error: 'Unknown job' });
     }
     const resource = job.resource_id ? services.registry.get(job.resource_id) : null;
-    res.json({ ...job, resource: resource ? { id: resource.id, name: resource.name } : null });
+    res.json({
+      ...job,
+      resource: resource ? { id: resource.id, name: resource.name } : job.resource_name ? { id: null, name: job.resource_name } : null
+    });
   });
 
   router.get('/jobs', auth, (req, res) => {

package/src/api/resource.js

@@ -103,6 +103,12 @@ function createResourceRouter({ services, config }){
       if (updateTo){
         commands.push({ command: 'self-update', version: updateTo });
       }
+      // Removal pending (jobs.removeResource): once the Client reports its
+      // job stopped, finish it. This response still goes out; the next
+      // heartbeat gets 401, the token being gone with the resource.
+      if (req.resource.remove_requested_at && !activeJobId){
+        services.jobs.completeRemoval(req.params.id, { by: 'removal-confirmed' });
+      }
       res.json({ serverTime: new Date().toISOString(), commands });
     }
     catch (err){

package/src/db/migrations/013_add_job_resource_name.sql

@@ -0,0 +1,4 @@
+-- Name of the resource a job ran on, kept once that resource is removed
+-- from the dashboard (registry.remove): jobs.resource_id is then NULLed
+-- (it's a foreign key), and this is what the job's history shows instead.
+ALTER TABLE jobs ADD COLUMN resource_name TEXT;

package/src/db/migrations/014_add_remove_requested_at.sql

@@ -0,0 +1,5 @@
+-- Set when an admin removes a resource that is running a job: the job is
+-- canceled right away, and the resource itself is deleted once its Client
+-- confirms the job stopped (next heartbeat with no active job) or after a
+-- timeout (heartbeat.js). Meanwhile the scheduler never assigns it work.
+ALTER TABLE resources ADD COLUMN remove_requested_at TEXT;

package/src/db/migrations/015_add_job_duration.sql

@@ -0,0 +1,9 @@
+-- How long a finished job ran (README §9): seconds from started_at (RUNNING)
+-- to finished_at, computed once when it reaches a terminal state
+-- (jobs.setState) and shown instead of the finish time. NULL for a job
+-- that never ran (e.g. canceled while queued). finished_at itself stays:
+-- log/artifact retention is keyed on it. Backfilled for existing jobs.
+ALTER TABLE jobs ADD COLUMN duration_sec INTEGER;
+UPDATE jobs
+   SET duration_sec = MAX(0, CAST(ROUND((julianday(finished_at) - julianday(started_at)) * 86400) AS INTEGER))
+ WHERE finished_at IS NOT NULL AND started_at IS NOT NULL;

package/src/services/heartbeat.js

@@ -47,10 +47,31 @@ function createHeartbeatMonitor(db, { bus, events, registry, jobs, config }){
     }
   }
 
-  const timer = setInterval(sweepOnce, config.heartbeat.sweepIntervalSec * 1000);
+  // Resources an admin removed while they ran a job (jobs.removeResource):
+  // normally deleted by the heartbeat route once the Client reports the job
+  // stopped; this covers a Client that's gone (OUT_OF_SERVICE) or never
+  // confirms — after the time a cancel plus the runner's kill grace needs.
+  function sweepRemovals(){
+    const timeoutMs = Math.max(60, 6 * config.heartbeat.intervalSec) * 1000;
+    for (const r of registry.pendingRemovals()){
+      if (r.status === RESOURCE_STATES.OUT_OF_SERVICE || Date.now() - new Date(r.remove_requested_at).getTime() >= timeoutMs){
+        try {
+          jobs.completeRemoval(r.id, { by: 'removal-timeout' });
+        }
+        catch (err){
+          console.error(`removing ${r.name}: ${err.message}`);
+        }
+      }
+    }
+  }
+
+  const timer = setInterval(() => {
+    sweepOnce();
+    sweepRemovals();
+  }, config.heartbeat.sweepIntervalSec * 1000);
   timer.unref?.();
 
-  return { sweepOnce, stop: () => clearInterval(timer) };
+  return { sweepOnce, sweepRemovals, stop: () => clearInterval(timer) };
 }
 
 module.exports = { createHeartbeatMonitor };

package/src/services/jobs.js

@@ -30,7 +30,10 @@ function rowToJob(row){
 // developer's own machine — separate series (each own counter row) rather
 // than a shared counter with just a different letter, so e.g. A-00042
 // doesn't imply 41 manual runs happened first.
-const JOB_ID_PREFIXES = { ci: 'A', cli: 'M' };
+const JOB_ID_PREFIXES = { ci: 'A', cli: 'M' },
+  // Default priority when the spec doesn't set one (§7.1): a developer's
+  // manual run jumps ahead of CI's queue.
+  DEFAULT_PRIORITY = { ci: 50, cli: 60 };
 
 function nextJobId(db, source){
   const prefix = JOB_ID_PREFIXES[source];
@@ -85,7 +88,9 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
   // Agent: POST /jobs (§6.1, §4.3). Rejects unsatisfiable label requests
   // immediately (§5.4) instead of letting them starve in the queue.
   function create({ agentId, source, spec: rawSpec }){
-    const { valid, spec, errors } = validateJobSpec(rawSpec);
+    const { valid, spec, errors } = validateJobSpec(
+      rawSpec?.priority === undefined ? { ...rawSpec, priority: DEFAULT_PRIORITY[source] } : rawSpec
+    );
     if (!valid){
       throw Object.assign(new Error(`Invalid job spec: ${errors.join('; ')}`), { status: 400 });
     }
@@ -152,8 +157,10 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
       params.push(now);
     }
     if (TERMINAL_JOB_STATES.has(state)){
-      fields.push('finished_at = ?');
-      params.push(now);
+      // Duration is computed once, here, and stored — never derived at
+      // display time. Measured from started_at (RUNNING), like the timeout.
+      fields.push('finished_at = ?', 'duration_sec = ?');
+      params.push(now, job.started_at ? Math.max(0, Math.round((Date.parse(now) - Date.parse(job.started_at)) / 1000)) : null);
     }
     if (extra.message !== undefined){
       fields.push('message = ?');
@@ -168,9 +175,11 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
       params.push(JSON.stringify(extra.summary));
     }
     if (state === JOB_STATES.QUEUED){
-      // Requeue: drop the previous assignment so the scheduler treats it fresh.
-      fields.push('resource_id = ?', 'assigned_at = ?', 'attempt = attempt + 1');
-      params.push(null, null);
+      // Requeue: drop the previous assignment (and its start time, so the
+      // next attempt's duration/timeout count from its own start) so the
+      // scheduler treats it fresh.
+      fields.push('resource_id = ?', 'assigned_at = ?', 'started_at = ?', 'attempt = attempt + 1');
+      params.push(null, null, null);
     }
 
     params.push(id);
@@ -213,6 +222,44 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
     return setState(id, JOB_STATES.CANCELED);
   }
 
+  // Queued jobs pinned to a resource (`target.client`) can never run once it's
+  // removed, so they're canceled rather than left in the queue forever.
+  function cancelPinnedTo(resourceId, message){
+    const queued = db.prepare('SELECT id, spec FROM jobs WHERE state = ?').all(JOB_STATES.QUEUED)
+      .filter((row) => JSON.parse(row.spec).target?.client === resourceId);
+    for (const { id }of queued){
+      setState(id, JOB_STATES.CANCELED, { message });
+    }
+    return queued.map((row) => row.id);
+  }
+
+  // Admin "Remove" (dashboard, DELETE /admin/resources/:id). A resource
+  // that's running a job needs `stopJob`: the job is canceled at once, but
+  // the Client only hears that in its next heartbeat's commands — so the
+  // resource (and with it the Client's token) is deleted only after that
+  // (completeRemoval, from the heartbeat route or the sweeper's timeout).
+  function removeResource(resourceId, { by, stopJob = false } = {}){
+    const r = registry.get(resourceId);
+    if (!r){
+      throw Object.assign(new Error('Unknown resource'), { status: 404 });
+    }
+    const job = activeForResource(resourceId);
+    if (job && !stopJob){
+      throw Object.assign(new Error(`${r.name} is running job ${job.id}`), { status: 409, jobId: job.id });
+    }
+    if (job){
+      cancel(job.id, { isAdmin: true });
+      registry.requestRemoval(resourceId, { by });
+      return { resource: r, pending: true, stoppedJob: job.id, canceledJobs: [] };
+    }
+    return { resource: r, pending: false, stoppedJob: null, canceledJobs: completeRemoval(resourceId, { by }) };
+  }
+
+  function completeRemoval(resourceId, { by } = {}){
+    const r = registry.remove(resourceId, { by });
+    return cancelPinnedTo(resourceId, `Client ${r.name} was removed`);
+  }
+
   // The job a resource is working on (ASSIGNED through RUNNING), if any —
   // for the resource card's Cancel button.
   function activeForResource(resourceId){
@@ -362,6 +409,9 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
     create,
     setState,
     cancel,
+    cancelPinnedTo,
+    removeResource,
+    completeRemoval,
     activeForResource,
     applyResult,
     requeueUnacked,

package/src/services/registry.js

@@ -14,7 +14,7 @@
 'use strict';
 
 const { v4: uuid } = require('uuid'),
-  { RESOURCE_STATES, BUSY_SOURCES, compareVersions } = require('@andrian.yablonskyy/thub-common'),
+  { RESOURCE_STATES, BUSY_SOURCES, ACTIVE_JOB_STATES, compareVersions } = require('@andrian.yablonskyy/thub-common'),
   { generateToken, hashToken } = require('./tokens');
 
 function rowToResource(row){
@@ -474,9 +474,44 @@ function createRegistryService(db, { bus, events }){
       (!resourceId || r.id === resourceId);
   }
 
+  // Admin: remove a resource from the registry (dashboard "Remove"). Refused
+  // while it holds an active job — cancel that first. Its finished jobs stay,
+  // keeping the resource's name (jobs.resource_name) in place of the id.
+  // A Client that's still running reappears when it next re-registers.
+  function remove(resourceId, { by } = {}){
+    const r = get(resourceId);
+    if (!r){
+      throw Object.assign(new Error('Unknown resource'), { status: 404 });
+    }
+    const placeholders = [...ACTIVE_JOB_STATES].map(() => '?').join(','),
+      active = db
+        .prepare(`SELECT id FROM jobs WHERE resource_id = ? AND state IN (${placeholders}) LIMIT 1`)
+        .get(resourceId, ...ACTIVE_JOB_STATES);
+    if (active){
+      throw Object.assign(new Error(`${r.name} is working on job ${active.id} — cancel it first`), { status: 409 });
+    }
+    db.transaction(() => {
+      db.prepare('UPDATE jobs SET resource_name = ?, resource_id = NULL WHERE resource_id = ?').run(r.name, resourceId);
+      db.prepare('DELETE FROM resources WHERE id = ?').run(resourceId);
+    })();
+    events.record('resource', resourceId, 'resource.removed', { name: r.name, by });
+    return r;
+  }
+
+  // Step one of removing a resource that's running a job (jobs.removeResource):
+  // it takes no new work from here on and is deleted once the job stopped.
+  function requestRemoval(resourceId, { by } = {}){
+    db.prepare('UPDATE resources SET remove_requested_at = ? WHERE id = ?').run(new Date().toISOString(), resourceId);
+    events.record('resource', resourceId, 'resource.removal_requested', { by });
+  }
+
+  function pendingRemovals(){
+    return db.prepare('SELECT * FROM resources WHERE remove_requested_at IS NOT NULL').all().map(rowToResource);
+  }
+
   function findIdleCandidates(type, labels, groupId, resourceId){
     const rows = db
-      .prepare('SELECT * FROM resources WHERE status = ? AND type = ?')
+      .prepare('SELECT * FROM resources WHERE status = ? AND type = ? AND remove_requested_at IS NULL')
       .all(RESOURCE_STATES.IDLE, type)
       .map(rowToResource);
     return rows.filter((r) => matchesTarget(r, labels, groupId, resourceId));
@@ -504,6 +539,9 @@ function createRegistryService(db, { bus, events }){
     requestUpdateAll,
     pendingUpdate,
     assignToJob,
+    remove,
+    requestRemoval,
+    pendingRemovals,
     findIdleCandidates,
     everSatisfiable
   };

package/src/web/routes.js

@@ -284,6 +284,26 @@ function createWebRouter({ services, config }){
     res.redirect(returnTo(req, '/resources'));
   });
 
+  router.post('/resources/:id/remove', requireAdminRole, (req, res) => {
+    try {
+      const { resource: r, pending, stoppedJob, canceledJobs } = services.jobs.removeResource(req.params.id, {
+        by: req.session.user.username,
+        stopJob: req.body.stopJob === '1'
+      });
+      flash(
+        req,
+        'warning',
+        pending
+          ? `Stopped job ${stoppedJob} on ${r.name}; ${r.name} is removed as soon as its Client confirms the job ended.`
+          : `Removed ${r.name}.` + (canceledJobs.length ? ` Canceled job(s) queued for it: ${canceledJobs.join(', ')}.` : '')
+      );
+    }
+    catch (err){
+      flash(req, 'danger', err.message);
+    }
+    res.redirect(returnTo(req, '/resources'));
+  });
+
   router.post('/resources/:id/rotate-token', requireAdminRole, (req, res) => {
     const { generateToken, hashToken } = require('../services/tokens');
     const token = generateToken('res');
@@ -395,13 +415,19 @@ function createWebRouter({ services, config }){
     const filters = { state: req.query.state || undefined, source: req.query.source || undefined },
       jobs = services.jobs.list({ ...filters, limit: 200 }).map((j) => ({
         ...j,
-        resource: j.resource_id ? services.registry.get(j.resource_id) : null
+        resource: j.resource_id ? services.registry.get(j.resource_id) : null,
+        active: ACTIVE_JOB_STATES.has(j.state)
       }));
     res.render('jobs/list', {
       title: 'Jobs',
       active: 'jobs',
       jobs,
       filters,
+      // Keeps the list's filters across a row's Cancel.
+      filterQuery: (() => {
+        const q = new URLSearchParams(Object.entries(filters).filter(([, v]) => v)).toString();
+        return q ? `?${q}` : '';
+      })(),
       canManage: req.session.user.role === 'admin'
     });
   });
@@ -440,9 +466,16 @@ function createWebRouter({ services, config }){
     });
   });
 
+  // From the job page or a row on /jobs (returnTo says which).
   router.post('/jobs/:id/cancel', requireAdminRole, (req, res) => {
-    services.jobs.cancel(req.params.id, { isAdmin: true });
-    res.redirect(`/jobs/${req.params.id}`);
+    try {
+      services.jobs.cancel(req.params.id, { isAdmin: true });
+      flash(req, 'warning', `Canceled job ${req.params.id}.`);
+    }
+    catch (err){
+      flash(req, 'danger', err.message);
+    }
+    res.redirect(returnTo(req, `/jobs/${req.params.id}`));
   });
 
   // Plain, one-shot fetch — used by the log viewer for a job that's

package/test/scheduler.test.js

@@ -157,6 +157,34 @@ test('job ids are split by source: A-##### for ci, M-##### for cli', () => {
   assert.equal(Number(ciJob2.id.slice(2)), Number(ciJob.id.slice(2)) + 1);
 });
 
+test('default priority follows the job source; an explicit one is kept', () => {
+  const { agents, registry, jobs } = buildTestServices(),
+    { agent } = agents.create({ name: 'ci', kind: 'ci' });
+  registerResource(registry, { name: 'lab-sw-01', type: 'sw' });
+
+  assert.equal(jobs.create({ agentId: agent.id, source: 'ci', spec: makeSpec() }).priority, 50);
+  assert.equal(jobs.create({ agentId: agent.id, source: 'cli', spec: makeSpec() }).priority, 60);
+  assert.equal(jobs.create({ agentId: agent.id, source: 'ci', spec: makeSpec({ priority: 90 }) }).priority, 90);
+});
+
+test('duration is stored once when a job finishes, from its start; none if it never ran', () => {
+  const { registry, agents, jobs, scheduler, db } = buildTestServices(),
+    { agent } = agents.create({ name: 'ci', kind: 'ci' }),
+    r = registerResource(registry, { name: 'lab-sw-01', type: 'sw' });
+  registry.heartbeat(r.id, { state: 'idle' });
+
+  const ran = jobs.create({ agentId: agent.id, source: 'cli', spec: makeSpec() });
+  scheduler.runPass();
+  jobs.setState(ran.id, JOB_STATES.RUNNING);
+  db.prepare('UPDATE jobs SET started_at = ? WHERE id = ?').run(new Date(Date.now() - 125e3).toISOString(), ran.id);
+  jobs.setState(ran.id, JOB_STATES.PASSED);
+  assert.equal(jobs.get(ran.id).duration_sec, 125);
+
+  const never = jobs.create({ agentId: agent.id, source: 'cli', spec: makeSpec() });
+  jobs.cancel(never.id, { isAdmin: true });
+  assert.equal(jobs.get(never.id).duration_sec, null);
+});
+
 test('admin resetQueue cancels every active job and leaves finished ones alone', () => {
   const { registry, agents, jobs, scheduler } = buildTestServices(),
     { agent } = agents.create({ name: 'ci', kind: 'ci' }),
@@ -419,6 +447,84 @@ test('a job with target.client is rejected (422) when the client is unknown or c
   );
 });
 
+test('removing a resource keeps its job history by name and cancels jobs pinned to it', () => {
+  const { registry, agents, jobs, scheduler } = buildTestServices(),
+    { agent } = agents.create({ name: 'ci', kind: 'ci' }),
+    r = registerResource(registry, { name: 'lab-old', type: 'sw' });
+  registry.heartbeat(r.id, { state: 'idle' });
+
+  const done = jobs.create({ agentId: agent.id, source: 'cli', spec: makeSpec() });
+  scheduler.runPass();
+  assert.throws(() => registry.remove(r.id), /working on job/); // ASSIGNED: refused
+  jobs.setState(done.id, JOB_STATES.PASSED);
+  registry.heartbeat(r.id, { state: 'busy' }); // keep it from taking the pinned job below
+
+  const pinned = jobs.create({ agentId: agent.id, source: 'cli', spec: makeSpec({ target: { type: 'sw', client: r.id } }) });
+  registry.remove(r.id);
+  assert.deepEqual(jobs.cancelPinnedTo(r.id, 'removed'), [pinned.id]);
+
+  assert.equal(registry.get(r.id), undefined);
+  const kept = jobs.get(done.id);
+  assert.equal(kept.resource_id, null);
+  assert.equal(kept.resource_name, 'lab-old');
+  assert.equal(jobs.get(pinned.id).state, JOB_STATES.CANCELED);
+  assert.throws(() => registry.remove(r.id), /Unknown resource/);
+});
+
+test('removing a resource running a job: stops the job now, removes once the Client confirms', () => {
+  const { registry, agents, jobs, scheduler } = buildTestServices(),
+    { agent } = agents.create({ name: 'ci', kind: 'ci' }),
+    r = registerResource(registry, { name: 'lab-busy', type: 'sw' });
+  registry.heartbeat(r.id, { state: 'idle' });
+  const running = jobs.create({ agentId: agent.id, source: 'cli', spec: makeSpec() });
+  scheduler.runPass();
+  jobs.setState(running.id, JOB_STATES.RUNNING);
+
+  assert.throws(() => jobs.removeResource(r.id), /running job/); // needs stopJob
+
+  const res = jobs.removeResource(r.id, { stopJob: true });
+  assert.equal(res.pending, true);
+  assert.equal(res.stoppedJob, running.id);
+  assert.equal(jobs.get(running.id).state, JOB_STATES.CANCELED);
+  assert.ok(registry.get(r.id).remove_requested_at); // still there: the Client hasn't heard yet
+
+  // Pending removal takes no new work, even once it reports idle.
+  registry.heartbeat(r.id, { state: 'idle' });
+  const next = jobs.create({ agentId: agent.id, source: 'cli', spec: makeSpec() });
+  scheduler.runPass();
+  assert.equal(jobs.get(next.id).state, JOB_STATES.QUEUED);
+
+  jobs.completeRemoval(r.id); // what the heartbeat route does on the Client's "no active job"
+  assert.equal(registry.get(r.id), undefined);
+  assert.equal(jobs.get(running.id).resource_name, 'lab-busy');
+});
+
+test('the sweeper finishes a pending removal whose Client never confirms', () => {
+  const { registry, agents, jobs, scheduler, heartbeatMonitor, db } = buildTestServices(),
+    { agent } = agents.create({ name: 'ci', kind: 'ci' }),
+    r = registerResource(registry, { name: 'lab-silent', type: 'sw' });
+  registry.heartbeat(r.id, { state: 'idle' });
+  jobs.create({ agentId: agent.id, source: 'cli', spec: makeSpec() });
+  scheduler.runPass();
+  jobs.removeResource(r.id, { stopJob: true });
+
+  heartbeatMonitor.sweepRemovals();
+  assert.ok(registry.get(r.id)); // not timed out yet
+
+  db.prepare('UPDATE resources SET remove_requested_at = ? WHERE id = ?').run(new Date(Date.now() - 3600e3).toISOString(), r.id);
+  heartbeatMonitor.sweepRemovals();
+  assert.equal(registry.get(r.id), undefined);
+});
+
+test('a removed Client re-registers as a fresh resource under the same name', () => {
+  const { registry } = buildTestServices(),
+    r = registerResource(registry, { name: 'lab-back', type: 'sw', clientId: 'c-back' });
+  registry.remove(r.id);
+  const again = registerResource(registry, { name: 'lab-back', type: 'sw', clientId: 'c-back' });
+  assert.notEqual(again.id, r.id);
+  assert.equal(again.name, 'lab-back');
+});
+
 test('a resource can belong to several groups at once', () => {
   const { registry, groups } = buildTestServices(),
     g1 = groups.create({ name: 'g1' }),

package/views/index.pug

@@ -30,7 +30,7 @@ block content
           .card-body
             h5.card-title= r.name
             p.card-text.mb-1
-              +resourceBadge(r.status)
+              +resourceBadge(r.status, r.remove_requested_at)
               |
               = ' ' + r.type.toUpperCase()
             if r.busy_source
@@ -41,5 +41,9 @@ block content
   each r in resources
     +resourceCard(r, groupsById, '/')
 
+  if user && user.role === 'admin'
+    +removeResourceModal()
+
 block scripts
   script(src="/js/resource-card.js")
+  script(src="/js/remove-resource.js")

package/views/jobs/list.pug

@@ -32,7 +32,9 @@ block content
         th State
         th Resource
         th Created
-        th Finished
+        th Duration
+        if canManage
+          th
     tbody
       each j in jobs
         tr
@@ -40,6 +42,18 @@ block content
           td.text-uppercase= j.source
           td= j.spec.user || '—'
           td: +jobBadge(j.state)
-          td= j.resource ? j.resource.name : '—'
+          td= j.resource ? j.resource.name : j.resource_name ? `${j.resource_name} (removed)` : '—'
           td= fmtDate(j.created_at)
-          td= fmtDate(j.finished_at)
+          td(title=j.finished_at ? `Finished ${fmtDate(j.finished_at)}` : null)= j.duration_sec != null ? fmtDuration(j.duration_sec * 1000) : '—'
+          if canManage
+            td.text-end
+              if j.active
+                form.d-inline(
+                  method="post"
+                  action=`/jobs/${j.id}/cancel`
+                  data-confirm=`Cancel job ${j.id}?`
+                  onsubmit="return confirm(this.dataset.confirm)"
+                )
+                  input(type="hidden" name="returnTo" value=`/jobs${filterQuery}`)
+                  button.btn.btn-sm.btn-outline-danger(type="submit" aria-label="Cancel" title="Cancel this job")
+                    i.bi.bi-x-circle

package/views/jobs/show.pug

@@ -9,7 +9,7 @@ block content
       |
       +jobBadge(job.state)
     if canCancel
-      form(method="post" action=`/jobs/${job.id}/cancel`)
+      form(method="post" action=`/jobs/${job.id}/cancel` onsubmit=`return confirm('Cancel job ${job.id}?')`)
         button.btn.btn-outline-danger.btn-sm(type="submit") Cancel
 
   dl.row
@@ -19,7 +19,7 @@ block content
       dt.col-sm-2 User
       dd.col-sm-10= job.spec.user
     dt.col-sm-2 Resource
-    dd.col-sm-10= job.resource ? job.resource.name : '—'
+    dd.col-sm-10= job.resource ? job.resource.name : job.resource_name ? `${job.resource_name} (removed)` : '—'
     dt.col-sm-2 Target
     dd.col-sm-10= `${job.spec.target.type} — ${(job.spec.target.labels || []).join(', ')}`
     if job.spec.target.client
@@ -27,6 +27,9 @@ block content
       dd.col-sm-10= pinnedClient ? pinnedClient.name : job.spec.target.client
     dt.col-sm-2 Created
     dd.col-sm-10= fmtDate(job.created_at)
+    if job.duration_sec != null
+      dt.col-sm-2 Duration
+      dd.col-sm-10(title=`Finished ${fmtDate(job.finished_at)}`)= fmtDuration(job.duration_sec * 1000)
     if job.message
       dt.col-sm-2 Message
       dd.col-sm-10= job.message

package/views/mixins/resource.pug

@@ -32,6 +32,82 @@ mixin resourceActions(r, returnTo)
     )
       i.bi.bi-arrow-repeat
   +updateButton(`/resources/${r.id}/update`, r.update_to, returnTo)
+  +removeResourceButton(r, returnTo)
+
+//- Remove: a plain confirm when the resource is idle; with a running job,
+//- the shared warning modal (+removeResourceModal) instead, which stops the
+//- job first (stopJob=1). Disabled once a removal is already pending.
+mixin removeResourceButton(r, returnTo)
+  if r.remove_requested_at
+    span.d-inline-block.ms-1(tabindex="0" data-bs-toggle="tooltip" data-bs-title="Removal pending — waiting for the Client to confirm its job stopped")
+      button.btn.btn-sm.btn-outline-danger(type="button" disabled style="pointer-events: none;" aria-label="Remove")
+        i.bi.bi-hourglass-split
+  else if r.activeJob
+    button.btn.btn-sm.btn-outline-danger.ms-1(
+      type="button"
+      aria-label="Remove"
+      data-bs-toggle="tooltip"
+      data-bs-title="Remove this resource from the list — it's running a job"
+      data-remove-resource=r.id
+      data-remove-name=r.name
+      data-remove-job=r.activeJob.id
+      data-remove-return=returnTo
+    )
+      i.bi.bi-trash
+  else
+    form.d-inline.ms-1(
+      method="post"
+      action=`/resources/${r.id}/remove`
+      data-confirm=`Remove ${r.name} from the list? Its job history is kept. A Client that's still running shows up again when it restarts and re-registers.`
+      onsubmit="return confirm(this.dataset.confirm)"
+    )
+      input(type="hidden" name="returnTo" value=returnTo)
+      button.btn.btn-sm.btn-outline-danger(
+        type="submit"
+        aria-label="Remove"
+        data-bs-toggle="tooltip"
+        data-bs-title="Remove this resource from the list"
+      )
+        i.bi.bi-trash
+
+//- One per page (admins): filled in by public/js/remove-resource.js from
+//- the clicked Remove button's data-remove-* attributes.
+mixin removeResourceModal()
+  .modal.fade#remove-resource-modal(tabindex="-1" aria-labelledby="remove-resource-title" aria-hidden="true")
+    .modal-dialog.modal-dialog-centered
+      .modal-content
+        .modal-header.text-bg-warning
+          h5.modal-title#remove-resource-title
+            i.bi.bi-exclamation-triangle-fill.me-2
+            | Client is running a job
+          button.btn-close(type="button" data-bs-dismiss="modal" aria-label="Close")
+        .modal-body
+          p
+            strong(data-field="name")
+            |  is running job
+            |
+            a(data-field="job")
+            | .
+          p.mb-2 Removing it will:
+          ul.mb-2
+            li
+              | stop job
+              |
+              strong(data-field="job-text")
+              |  immediately (it ends as
+              |
+              code CANCELED
+              | ), then
+            li remove the resource as soon as its Client confirms the job stopped (at most about a minute if the Client doesn't answer).
+          p.small.text-body-secondary.mb-0 Its job history is kept. A Client that's still running shows up again when it restarts and re-registers.
+        .modal-footer
+          button.btn.btn-secondary(type="button" data-bs-dismiss="modal") Keep it
+          form(method="post" data-field="form")
+            input(type="hidden" name="stopJob" value="1")
+            input(type="hidden" name="returnTo" data-field="return")
+            button.btn.btn-danger(type="submit")
+              i.bi.bi-stop-circle.me-1
+              | Stop job and remove
 
 //- One udev device line: path (or udev index / ST-Link serial), extra
 //- detail, and whether the node was present when the Client registered.
@@ -156,7 +232,7 @@ mixin resourceCard(r, groupsById, returnTo)
             dt.col-sm-4 Type
             dd.col-sm-8.text-uppercase= r.type
             dt.col-sm-4 Status
-            dd.col-sm-8: +resourceBadge(r.status)
+            dd.col-sm-8: +resourceBadge(r.status, r.remove_requested_at)
             +uptimeRows(r)
             dt.col-sm-4 Client version
             dd.col-sm-8: +versionCell(r.client_version, updates.latest.client, r.update_to)

package/views/mixins/status-badge.pug

@@ -1,6 +1,13 @@
-mixin resourceBadge(status)
+//- `removing`: the resource's remove_requested_at — removal is pending
+//- until its Client confirms its stopped job ended (jobs.removeResource).
+mixin resourceBadge(status, removing)
   - const map = { IDLE: 'success', BUSY: 'primary', OUT_OF_SERVICE: 'danger', MAINTENANCE: 'secondary', REGISTERED: 'secondary' }
   span.badge(class=`text-bg-${map[status] || 'secondary'}`)= status
+  if removing
+    span.badge.text-bg-warning.ms-1(
+      data-bs-toggle="tooltip"
+      data-bs-title="Its job was stopped; removed as soon as the Client confirms"
+    ) REMOVING
 
 mixin jobBadge(state)
   - const map = { QUEUED: 'secondary', ASSIGNED: 'info', PREPARING: 'info', RUNNING: 'primary', PASSED: 'success', FAILED: 'danger', ERROR: 'danger', TIMEOUT: 'warning', CANCELED: 'secondary', LOST: 'danger' }

package/views/resources/list.pug

@@ -30,7 +30,7 @@ block content
           td= r.name
           td.text-uppercase= r.type
           td: +versionCell(r.client_version, updates.latest.client, r.update_to)
-          td: +resourceBadge(r.status)
+          td: +resourceBadge(r.status, r.remove_requested_at)
           td
             each l in r.labels
               span.badge.text-bg-light.border.me-1= l
@@ -49,5 +49,9 @@ block content
   each r in resources
     +resourceCard(r, groupsById, '/resources')
 
+  if user && user.role === 'admin'
+    +removeResourceModal()
+
 block scripts
   script(src="/js/resource-card.js")
+  script(src="/js/remove-resource.js")