@andrian.yablonskyy/thub-coordinator 1.0.17 → 1.0.19

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andrian.yablonskyy/thub-coordinator",
-  "version": "1.0.17",
+  "version": "1.0.19",
   "description": "TestHub Coordinator — job queue, resource registry, scheduler, heartbeat monitor, log/artifact store and web dashboard",
   "main": "src/server.js",
   "engines": {

package/public/js/remove-resource.js

@@ -0,0 +1,45 @@
+/**
+ * @file        packages/coordinator/public/js/remove-resource.js
+ * @description Dashboard: warning modal for removing a resource that is running a job (stop job, then remove)
+ *
+ * @author      Andrian Yablonskyy
+ * @copyright   Copyright (c) 2026 Andrian Yablonskyy. All rights reserved.
+ *
+ * This file is part of TestHub and is proprietary and confidential.
+ * Unauthorized copying, modification, distribution, or use of this file,
+ * via any medium, is strictly prohibited without prior written permission
+ * from AdSystem.PRO.
+ */
+
+'use strict';
+
+(function (){
+  const modalEl = document.getElementById('remove-resource-modal');
+  if (!modalEl){
+    return;
+  }
+  const field = (name) => modalEl.querySelector(`[data-field="${name}"]`);
+
+  document.querySelectorAll('[data-remove-resource]').forEach((btn) => {
+    btn.addEventListener('click', () => {
+      const { removeResource: id, removeName: name, removeJob: job, removeReturn: returnTo } = btn.dataset;
+      field('name').textContent = name;
+      field('job').textContent = job;
+      field('job').href = `/jobs/${encodeURIComponent(job)}`;
+      field('job-text').textContent = job;
+      field('form').action = `/resources/${encodeURIComponent(id)}/remove`;
+      field('return').value = returnTo || '/resources';
+
+      // Clicked from inside a resource card: close that first — Bootstrap
+      // doesn't stack modals.
+      const card = btn.closest('.modal');
+      if (card){
+        card.addEventListener('hidden.bs.modal', () => bootstrap.Modal.getOrCreateInstance(modalEl).show(), { once: true });
+        bootstrap.Modal.getInstance(card)?.hide();
+      }
+      else {
+        bootstrap.Modal.getOrCreateInstance(modalEl).show();
+      }
+    });
+  });
+})();

package/src/api/admin.js

@@ -60,6 +60,21 @@ function createAdminRouter({ services }){
     res.json({ resourceToken: token });
   });
 
+  // Removes a resource; 409 while it runs a job unless ?stopJob=1, which
+  // cancels the job and removes the resource once the Client confirms
+  // (`pending: true`). Queued jobs pinned to it are canceled on removal.
+  router.delete('/resources/:id', (req, res, next) => {
+    try {
+      const { resource, pending, stoppedJob, canceledJobs } = services.jobs.removeResource(req.params.id, {
+        stopJob: req.query.stopJob === '1'
+      });
+      res.json({ removed: resource.id, pending, stoppedJob, canceledJobs });
+    }
+    catch (err){
+      next(err);
+    }
+  });
+
   // Cancels every QUEUED/ASSIGNED/PREPARING/RUNNING job (§13.1).
   router.post('/jobs/reset-queue', (req, res) => {
     const canceled = services.jobs.resetQueue();

package/src/api/agent.js

@@ -28,7 +28,11 @@ function createAgentRouter({ services, config }){
 
   router.post('/jobs', auth, (req, res, next) => {
     try {
-      const job = services.jobs.create({ agentId: req.agent.id, source: req.body.source || 'cli', spec: req.body });
+      // The job source (A-/M- id series, default priority) is the agent
+      // token's admin-assigned kind alone — never taken from the request,
+      // so it can't be spoofed or mis-detected on the Agent side.
+      const source = req.agent.kind,
+        job = services.jobs.create({ agentId: req.agent.id, source, spec: req.body });
       res.status(201).json({ jobId: job.id, state: job.state, webUrl: `${config.publicUrl}/jobs/${job.id}` });
     }
     catch (err){
@@ -42,7 +46,10 @@ function createAgentRouter({ services, config }){
       return res.status(404).json({ error: 'Unknown job' });
     }
     const resource = job.resource_id ? services.registry.get(job.resource_id) : null;
-    res.json({ ...job, resource: resource ? { id: resource.id, name: resource.name } : null });
+    res.json({
+      ...job,
+      resource: resource ? { id: resource.id, name: resource.name } : job.resource_name ? { id: null, name: job.resource_name } : null
+    });
   });
 
   router.get('/jobs', auth, (req, res) => {

package/src/api/resource.js

@@ -103,6 +103,12 @@ function createResourceRouter({ services, config }){
       if (updateTo){
         commands.push({ command: 'self-update', version: updateTo });
       }
+      // Removal pending (jobs.removeResource): once the Client reports its
+      // job stopped, finish it. This response still goes out; the next
+      // heartbeat gets 401, the token being gone with the resource.
+      if (req.resource.remove_requested_at && !activeJobId){
+        services.jobs.completeRemoval(req.params.id, { by: 'removal-confirmed' });
+      }
       res.json({ serverTime: new Date().toISOString(), commands });
     }
     catch (err){

package/src/db/migrations/013_add_job_resource_name.sql

@@ -0,0 +1,4 @@
+-- Name of the resource a job ran on, kept once that resource is removed
+-- from the dashboard (registry.remove): jobs.resource_id is then NULLed
+-- (it's a foreign key), and this is what the job's history shows instead.
+ALTER TABLE jobs ADD COLUMN resource_name TEXT;

package/src/db/migrations/014_add_remove_requested_at.sql

@@ -0,0 +1,5 @@
+-- Set when an admin removes a resource that is running a job: the job is
+-- canceled right away, and the resource itself is deleted once its Client
+-- confirms the job stopped (next heartbeat with no active job) or after a
+-- timeout (heartbeat.js). Meanwhile the scheduler never assigns it work.
+ALTER TABLE resources ADD COLUMN remove_requested_at TEXT;

package/src/db/migrations/015_add_job_duration.sql

@@ -0,0 +1,9 @@
+-- How long a finished job ran (README §9): seconds from started_at (RUNNING)
+-- to finished_at, computed once when it reaches a terminal state
+-- (jobs.setState) and shown instead of the finish time. NULL for a job
+-- that never ran (e.g. canceled while queued). finished_at itself stays:
+-- log/artifact retention is keyed on it. Backfilled for existing jobs.
+ALTER TABLE jobs ADD COLUMN duration_sec INTEGER;
+UPDATE jobs
+   SET duration_sec = MAX(0, CAST(ROUND((julianday(finished_at) - julianday(started_at)) * 86400) AS INTEGER))
+ WHERE finished_at IS NOT NULL AND started_at IS NOT NULL;

package/src/db/migrations/016_add_list_prefs.sql

@@ -0,0 +1,5 @@
+-- Per-user list view preferences (README §10.1): page size and sort
+-- column/direction for each dashboard list, as JSON keyed by page —
+-- {"jobs": {"size": 25, "sort": "created", "dir": "desc"}, "resources": {...}}.
+-- NULL/missing keys fall back to each page's defaults (services/list-prefs.js).
+ALTER TABLE admin_users ADD COLUMN list_prefs TEXT;

package/src/services/admin-users.js

@@ -14,7 +14,8 @@
 'use strict';
 
 const crypto = require('node:crypto'),
-  { v4: uuid } = require('uuid');
+  { v4: uuid } = require('uuid'),
+  { normalize: normalizeListPrefs } = require('./list-prefs');
 
 function hashPassword(password){
   const salt = crypto.randomBytes(16).toString('hex'),
@@ -44,10 +45,21 @@ function toProfile(row){
     avatarPath: row.avatar_path,
     timezone: row.timezone,
     theme: row.theme,
-    sessionTimeoutMin: row.session_timeout_min
+    sessionTimeoutMin: row.session_timeout_min,
+    listPrefs: parseListPrefs(row.list_prefs)
   };
 }
 
+function parseListPrefs(json){
+  try {
+    const prefs = JSON.parse(json || '{}');
+    return prefs && typeof prefs === 'object' && !Array.isArray(prefs) ? prefs : {};
+  }
+  catch {
+    return {};
+  }
+}
+
 // §10: dashboard login uses its own admin/viewer account table with
 // session cookies — deliberately separate from the API bearer tokens.
 function createAdminUsersService(db){
@@ -157,6 +169,18 @@ function createAdminUsersService(db){
     return getById(id);
   }
 
+  // Remembers one list page's view (size/sort/dir, §10.1) for this user;
+  // stored normalized, so only valid values ever reach the profile.
+  function setListPrefs(id, list, prefs){
+    const row = db.prepare('SELECT list_prefs FROM admin_users WHERE id = ?').get(id);
+    if (!row){
+      throw Object.assign(new Error('Unknown user'), { status: 404 });
+    }
+    const all = { ...parseListPrefs(row.list_prefs), [list]: normalizeListPrefs(list, prefs) };
+    db.prepare('UPDATE admin_users SET list_prefs = ? WHERE id = ?').run(JSON.stringify(all), id);
+    return getById(id);
+  }
+
   return {
     getByUsername,
     getById,
@@ -166,6 +190,7 @@ function createAdminUsersService(db){
     changePassword,
     verify,
     updateProfile,
+    setListPrefs,
     SESSION_TIMEOUT_OPTIONS_MIN,
     THEMES
   };

package/src/services/heartbeat.js

@@ -47,10 +47,31 @@ function createHeartbeatMonitor(db, { bus, events, registry, jobs, config }){
     }
   }
 
-  const timer = setInterval(sweepOnce, config.heartbeat.sweepIntervalSec * 1000);
+  // Resources an admin removed while they ran a job (jobs.removeResource):
+  // normally deleted by the heartbeat route once the Client reports the job
+  // stopped; this covers a Client that's gone (OUT_OF_SERVICE) or never
+  // confirms — after the time a cancel plus the runner's kill grace needs.
+  function sweepRemovals(){
+    const timeoutMs = Math.max(60, 6 * config.heartbeat.intervalSec) * 1000;
+    for (const r of registry.pendingRemovals()){
+      if (r.status === RESOURCE_STATES.OUT_OF_SERVICE || Date.now() - new Date(r.remove_requested_at).getTime() >= timeoutMs){
+        try {
+          jobs.completeRemoval(r.id, { by: 'removal-timeout' });
+        }
+        catch (err){
+          console.error(`removing ${r.name}: ${err.message}`);
+        }
+      }
+    }
+  }
+
+  const timer = setInterval(() => {
+    sweepOnce();
+    sweepRemovals();
+  }, config.heartbeat.sweepIntervalSec * 1000);
   timer.unref?.();
 
-  return { sweepOnce, stop: () => clearInterval(timer) };
+  return { sweepOnce, sweepRemovals, stop: () => clearInterval(timer) };
 }
 
 module.exports = { createHeartbeatMonitor };

package/src/services/jobs.js

@@ -13,7 +13,8 @@
 
 'use strict';
 
-const { validateJobSpec, JOB_STATES, ACTIVE_JOB_STATES, TERMINAL_JOB_STATES } = require('@andrian.yablonskyy/thub-common');
+const { validateJobSpec, JOB_STATES, ACTIVE_JOB_STATES, TERMINAL_JOB_STATES } = require('@andrian.yablonskyy/thub-common'),
+  { paginate } = require('./list-prefs');
 
 function rowToJob(row){
   if (!row){
@@ -30,7 +31,10 @@ function rowToJob(row){
 // developer's own machine — separate series (each own counter row) rather
 // than a shared counter with just a different letter, so e.g. A-00042
 // doesn't imply 41 manual runs happened first.
-const JOB_ID_PREFIXES = { ci: 'A', cli: 'M' };
+const JOB_ID_PREFIXES = { ci: 'A', cli: 'M' },
+  // Default priority when the spec doesn't set one (§7.1): a developer's
+  // manual run jumps ahead of CI's queue.
+  DEFAULT_PRIORITY = { ci: 50, cli: 60 };
 
 function nextJobId(db, source){
   const prefix = JOB_ID_PREFIXES[source];
@@ -76,6 +80,44 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
     return db.prepare(sql).all(...params).map(rowToJob);
   }
 
+  // Dashboard /jobs (§10): one sorted page plus the total, both in SQL so a
+  // long history never has to be loaded whole. `sort` is a list-prefs key.
+  const PAGE_SORT_SQL = {
+    id: 'jobs.id',
+    source: 'jobs.source',
+    user: 'json_extract(jobs.spec, \'$.user\')',
+    state: 'jobs.state',
+    resource: 'COALESCE(r.name, jobs.resource_name)',
+    created: 'jobs.created_at',
+    duration: 'jobs.duration_sec'
+  };
+
+  function page({ state, source, sort = 'created', dir = 'desc', size = 25, page: pageNo = 1 } = {}){
+    const where = [],
+      params = [];
+    if (state){
+      where.push('jobs.state = ?');
+      params.push(state);
+    }
+    if (source){
+      where.push('jobs.source = ?');
+      params.push(source);
+    }
+    const whereSql = where.length ? `WHERE ${where.join(' AND ')}` : '',
+      column = PAGE_SORT_SQL[sort] || PAGE_SORT_SQL.created,
+      order = dir === 'asc' ? 'ASC' : 'DESC',
+      total = db.prepare(`SELECT COUNT(*) AS n FROM jobs ${whereSql}`).get(...params).n,
+      pagination = paginate(total, size, pageNo),
+      // Empty values (no user, never ran, …) last either way; created_at
+      // breaks ties so paging is stable.
+      rows = db.prepare(
+        `SELECT jobs.* FROM jobs LEFT JOIN resources r ON r.id = jobs.resource_id ${whereSql}
+         ORDER BY ${column} IS NULL, ${column} ${order}, jobs.created_at DESC
+         LIMIT ? OFFSET ?`
+      ).all(...params, pagination.limit, pagination.offset);
+    return { rows: rows.map(rowToJob), pagination };
+  }
+
   function countQueuedForAgent(agentId){
     return db
       .prepare('SELECT COUNT(*) AS n FROM jobs WHERE agent_id = ? AND state IN (\'QUEUED\',\'ASSIGNED\')')
@@ -85,7 +127,9 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
   // Agent: POST /jobs (§6.1, §4.3). Rejects unsatisfiable label requests
   // immediately (§5.4) instead of letting them starve in the queue.
   function create({ agentId, source, spec: rawSpec }){
-    const { valid, spec, errors } = validateJobSpec(rawSpec);
+    const { valid, spec, errors } = validateJobSpec(
+      rawSpec?.priority === undefined ? { ...rawSpec, priority: DEFAULT_PRIORITY[source] } : rawSpec
+    );
     if (!valid){
       throw Object.assign(new Error(`Invalid job spec: ${errors.join('; ')}`), { status: 400 });
     }
@@ -152,8 +196,10 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
       params.push(now);
     }
     if (TERMINAL_JOB_STATES.has(state)){
-      fields.push('finished_at = ?');
-      params.push(now);
+      // Duration is computed once, here, and stored — never derived at
+      // display time. Measured from started_at (RUNNING), like the timeout.
+      fields.push('finished_at = ?', 'duration_sec = ?');
+      params.push(now, job.started_at ? Math.max(0, Math.round((Date.parse(now) - Date.parse(job.started_at)) / 1000)) : null);
     }
     if (extra.message !== undefined){
       fields.push('message = ?');
@@ -168,9 +214,11 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
       params.push(JSON.stringify(extra.summary));
     }
     if (state === JOB_STATES.QUEUED){
-      // Requeue: drop the previous assignment so the scheduler treats it fresh.
-      fields.push('resource_id = ?', 'assigned_at = ?', 'attempt = attempt + 1');
-      params.push(null, null);
+      // Requeue: drop the previous assignment (and its start time, so the
+      // next attempt's duration/timeout count from its own start) so the
+      // scheduler treats it fresh.
+      fields.push('resource_id = ?', 'assigned_at = ?', 'started_at = ?', 'attempt = attempt + 1');
+      params.push(null, null, null);
     }
 
     params.push(id);
@@ -213,6 +261,44 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
     return setState(id, JOB_STATES.CANCELED);
   }
 
+  // Queued jobs pinned to a resource (`target.client`) can never run once it's
+  // removed, so they're canceled rather than left in the queue forever.
+  function cancelPinnedTo(resourceId, message){
+    const queued = db.prepare('SELECT id, spec FROM jobs WHERE state = ?').all(JOB_STATES.QUEUED)
+      .filter((row) => JSON.parse(row.spec).target?.client === resourceId);
+    for (const { id }of queued){
+      setState(id, JOB_STATES.CANCELED, { message });
+    }
+    return queued.map((row) => row.id);
+  }
+
+  // Admin "Remove" (dashboard, DELETE /admin/resources/:id). A resource
+  // that's running a job needs `stopJob`: the job is canceled at once, but
+  // the Client only hears that in its next heartbeat's commands — so the
+  // resource (and with it the Client's token) is deleted only after that
+  // (completeRemoval, from the heartbeat route or the sweeper's timeout).
+  function removeResource(resourceId, { by, stopJob = false } = {}){
+    const r = registry.get(resourceId);
+    if (!r){
+      throw Object.assign(new Error('Unknown resource'), { status: 404 });
+    }
+    const job = activeForResource(resourceId);
+    if (job && !stopJob){
+      throw Object.assign(new Error(`${r.name} is running job ${job.id}`), { status: 409, jobId: job.id });
+    }
+    if (job){
+      cancel(job.id, { isAdmin: true });
+      registry.requestRemoval(resourceId, { by });
+      return { resource: r, pending: true, stoppedJob: job.id, canceledJobs: [] };
+    }
+    return { resource: r, pending: false, stoppedJob: null, canceledJobs: completeRemoval(resourceId, { by }) };
+  }
+
+  function completeRemoval(resourceId, { by } = {}){
+    const r = registry.remove(resourceId, { by });
+    return cancelPinnedTo(resourceId, `Client ${r.name} was removed`);
+  }
+
   // The job a resource is working on (ASSIGNED through RUNNING), if any —
   // for the resource card's Cancel button.
   function activeForResource(resourceId){
@@ -316,6 +402,15 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
     return active.length;
   }
 
+  // How many jobs Reset queue / Clean history would touch — shown in their
+  // confirmation modals on /jobs.
+  function countActiveAndFinished(){
+    const count = (states) => db
+      .prepare(`SELECT COUNT(*) AS n FROM jobs WHERE state IN (${[...states].map(() => '?').join(',')})`)
+      .get(...states).n;
+    return { active: count(ACTIVE_JOB_STATES), finished: count(TERMINAL_JOB_STATES) };
+  }
+
   // Admin: "clean the queue" — permanently delete finished jobs (and their
   // logs/artifacts, on disk and in the DB), for when the history itself,
   // not just active work, needs clearing out. Unlike the nightly retention
@@ -359,9 +454,13 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
   return {
     get,
     list,
+    page,
     create,
     setState,
     cancel,
+    cancelPinnedTo,
+    removeResource,
+    completeRemoval,
     activeForResource,
     applyResult,
     requeueUnacked,
@@ -370,6 +469,7 @@ function createJobsService(db, { bus, events, registry, artifacts, config }){
     checkAssignAcks,
     reconcileOnStartup,
     resetQueue,
+    countActiveAndFinished,
     cleanHistory
   };
 }

package/src/services/list-prefs.js

@@ -0,0 +1,92 @@
+/**
+ * @file        packages/coordinator/src/services/list-prefs.js
+ * @description Per-user list view preferences (page size, sort column/direction) for the dashboard's
+ *              Jobs and Resources pages (README §10.1), plus the pagination math they share
+ *
+ * @author      Andrian Yablonskyy
+ * @copyright   Copyright (c) 2026 Andrian Yablonskyy. All rights reserved.
+ *
+ * This file is part of TestHub and is proprietary and confidential.
+ * Unauthorized copying, modification, distribution, or use of this file,
+ * via any medium, is strictly prohibited without prior written permission
+ * from AdSystem.PRO.
+ */
+
+'use strict';
+
+// 'all' shows every record on one page.
+const PAGE_SIZES = [10, 25, 50, 100, 'all'],
+  DEFAULT_PAGE_SIZE = 25,
+  DIRS = ['asc', 'desc'],
+
+  // Sortable columns per page, and each page's default order. The keys are
+  // what the URL (?sort=) and the stored profile carry; the pages map them
+  // to actual columns (jobs.page's SQL, the resources route's comparators).
+  LISTS = {
+    jobs: {
+      sorts: ['id', 'source', 'user', 'state', 'resource', 'created', 'duration'],
+      defaults: { size: DEFAULT_PAGE_SIZE, sort: 'created', dir: 'desc' }
+    },
+    resources: {
+      sorts: ['name', 'type', 'version', 'status', 'heartbeat'],
+      defaults: { size: DEFAULT_PAGE_SIZE, sort: 'name', dir: 'asc' }
+    }
+  };
+
+function parseSize(value){
+  if (value === 'all'){
+    return 'all';
+  }
+  const n = Number(value);
+  return PAGE_SIZES.includes(n) ? n : undefined;
+}
+
+// A complete, valid {size, sort, dir} for `list`: each field from `prefs`
+// when valid there, else that page's default.
+function normalize(list, prefs = {}){
+  const def = LISTS[list];
+  if (!def){
+    throw Object.assign(new Error(`Unknown list "${list}"`), { status: 400 });
+  }
+  return {
+    size: parseSize(prefs.size) ?? def.defaults.size,
+    sort: def.sorts.includes(prefs.sort) ? prefs.sort : def.defaults.sort,
+    dir: DIRS.includes(prefs.dir) ? prefs.dir : def.defaults.dir
+  };
+}
+
+// What a request asks for: the stored prefs, overridden by whichever of
+// ?size / ?sort / ?dir the URL carries (invalid values are ignored).
+function fromQuery(list, query, stored){
+  const base = normalize(list, stored),
+    wanted = { ...base };
+  if (query.size !== undefined && parseSize(query.size) !== undefined){
+    wanted.size = parseSize(query.size);
+  }
+  if (LISTS[list].sorts.includes(query.sort)){
+    wanted.sort = query.sort;
+  }
+  if (DIRS.includes(query.dir)){
+    wanted.dir = query.dir;
+  }
+  return { prefs: wanted, changed: ['size', 'sort', 'dir'].some((k) => wanted[k] !== base[k]) };
+}
+
+// Page window for `total` records: `page` is 1-based and clamped into range.
+function paginate(total, size, page){
+  const limit = size === 'all' ? Math.max(total, 1) : size,
+    pages = Math.max(1, Math.ceil(total / limit)),
+    current = Math.min(Math.max(1, Number.parseInt(page, 10) || 1), pages),
+    offset = (current - 1) * limit;
+  return {
+    page: current,
+    pages,
+    total,
+    offset,
+    limit,
+    from: total ? offset + 1 : 0,
+    to: Math.min(offset + limit, total)
+  };
+}
+
+module.exports = { PAGE_SIZES, DEFAULT_PAGE_SIZE, LISTS, normalize, fromQuery, paginate };

package/src/services/registry.js

@@ -14,7 +14,7 @@
 'use strict';
 
 const { v4: uuid } = require('uuid'),
-  { RESOURCE_STATES, BUSY_SOURCES, compareVersions } = require('@andrian.yablonskyy/thub-common'),
+  { RESOURCE_STATES, BUSY_SOURCES, ACTIVE_JOB_STATES, compareVersions } = require('@andrian.yablonskyy/thub-common'),
   { generateToken, hashToken } = require('./tokens');
 
 function rowToResource(row){
@@ -474,9 +474,44 @@ function createRegistryService(db, { bus, events }){
       (!resourceId || r.id === resourceId);
   }
 
+  // Admin: remove a resource from the registry (dashboard "Remove"). Refused
+  // while it holds an active job — cancel that first. Its finished jobs stay,
+  // keeping the resource's name (jobs.resource_name) in place of the id.
+  // A Client that's still running reappears when it next re-registers.
+  function remove(resourceId, { by } = {}){
+    const r = get(resourceId);
+    if (!r){
+      throw Object.assign(new Error('Unknown resource'), { status: 404 });
+    }
+    const placeholders = [...ACTIVE_JOB_STATES].map(() => '?').join(','),
+      active = db
+        .prepare(`SELECT id FROM jobs WHERE resource_id = ? AND state IN (${placeholders}) LIMIT 1`)
+        .get(resourceId, ...ACTIVE_JOB_STATES);
+    if (active){
+      throw Object.assign(new Error(`${r.name} is working on job ${active.id} — cancel it first`), { status: 409 });
+    }
+    db.transaction(() => {
+      db.prepare('UPDATE jobs SET resource_name = ?, resource_id = NULL WHERE resource_id = ?').run(r.name, resourceId);
+      db.prepare('DELETE FROM resources WHERE id = ?').run(resourceId);
+    })();
+    events.record('resource', resourceId, 'resource.removed', { name: r.name, by });
+    return r;
+  }
+
+  // Step one of removing a resource that's running a job (jobs.removeResource):
+  // it takes no new work from here on and is deleted once the job stopped.
+  function requestRemoval(resourceId, { by } = {}){
+    db.prepare('UPDATE resources SET remove_requested_at = ? WHERE id = ?').run(new Date().toISOString(), resourceId);
+    events.record('resource', resourceId, 'resource.removal_requested', { by });
+  }
+
+  function pendingRemovals(){
+    return db.prepare('SELECT * FROM resources WHERE remove_requested_at IS NOT NULL').all().map(rowToResource);
+  }
+
   function findIdleCandidates(type, labels, groupId, resourceId){
     const rows = db
-      .prepare('SELECT * FROM resources WHERE status = ? AND type = ?')
+      .prepare('SELECT * FROM resources WHERE status = ? AND type = ? AND remove_requested_at IS NULL')
       .all(RESOURCE_STATES.IDLE, type)
       .map(rowToResource);
     return rows.filter((r) => matchesTarget(r, labels, groupId, resourceId));
@@ -504,6 +539,9 @@ function createRegistryService(db, { bus, events }){
     requestUpdateAll,
     pendingUpdate,
     assignToJob,
+    remove,
+    requestRemoval,
+    pendingRemovals,
     findIdleCandidates,
     everSatisfiable
   };