@aywengo/mercury-fleet 0.0.1-bootstrap

package/dist/registry.js

@@ -0,0 +1,197 @@
+/**
+ * The host registry: Fleet's only piece of truth about what it may talk to.
+ *
+ * Phase 0 scope from docs/fleet-design.md section 12 is registry plus probe, with no dispatch. Routing
+ * does not exist yet, so nothing here decides where a Run goes.
+ */
+export class RegistryError extends Error {
+}
+/** Operator-assigned and stable: it appears in output and becomes part of every future binding. */
+const ID_RE = /^[a-z0-9][a-z0-9._-]{0,62}[a-z0-9]$|^[a-z0-9]$/;
+/**
+ * Validate a base URL.
+ *
+ * Credentials embedded in the URL are rejected outright. Allowing https://user:token@host would store the
+ * secret in the hosts table, which is exactly what credential_ref exists to prevent -- and the table is not
+ * written to with 0600 the way the credential file is.
+ */
+export function normalizeBaseUrl(raw) {
+    let url;
+    try {
+        url = new URL(raw);
+    }
+    catch {
+        throw new RegistryError(`base url is not a valid URL: ${JSON.stringify(raw)}`);
+    }
+    if (url.protocol !== 'http:' && url.protocol !== 'https:') {
+        throw new RegistryError(`base url must be http or https, got ${url.protocol}`);
+    }
+    if (url.username || url.password) {
+        throw new RegistryError('base url must not embed credentials. Put the token in the credential file and pass its name ' +
+            'with --credential; a URL secret lands in the database, where the file has 0600 and the table does not.');
+    }
+    if (url.search || url.hash) {
+        throw new RegistryError('base url must not carry a query or fragment');
+    }
+    // Strip a trailing slash so callers can append paths without producing //api/agents.
+    return url.href.replace(/\/+$/, '');
+}
+function validateId(id) {
+    if (!ID_RE.test(id)) {
+        throw new RegistryError(`host id ${JSON.stringify(id)} is not usable. Use a stable slug of lowercase letters, digits, ` +
+            `dot, dash and underscore, e.g. "mac-studio" or "box-lan-2".`);
+    }
+}
+function parseJson(raw, fallback) {
+    try {
+        return JSON.parse(raw);
+    }
+    catch {
+        // A corrupt row must not make the whole registry unreadable; fall back and let the next probe rewrite it.
+        return fallback;
+    }
+}
+function rowToHost(row) {
+    return {
+        id: row.id,
+        baseUrl: row.base_url,
+        credentialRef: row.credential_ref,
+        enabled: row.enabled === 1,
+        labels: parseJson(row.labels, {}),
+        localPaths: parseJson(row.local_paths, []),
+        agentsCache: parseJson(row.agents_cache, []),
+        addedAt: row.added_at,
+        lastSeenAt: row.last_seen_at,
+    };
+}
+function rowToProbe(row) {
+    return {
+        hostId: row.host_id,
+        outcome: row.outcome,
+        detail: row.detail,
+        activeRuns: row.active_runs,
+        queueDepth: row.queue_depth,
+        workerCount: row.worker_count,
+        workerId: row.worker_id,
+        agents: row.agents === null ? null : parseJson(row.agents, null),
+        probedAt: row.probed_at,
+        lastError: row.last_error,
+    };
+}
+export class HostRegistry {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    add(input) {
+        validateId(input.id);
+        const baseUrl = normalizeBaseUrl(input.baseUrl);
+        if (!input.credentialRef || !input.credentialRef.trim()) {
+            throw new RegistryError('credential_ref is required; pass the NAME of a credential, not the secret');
+        }
+        for (const p of input.localPaths ?? []) {
+            if (!p.startsWith('/')) {
+                throw new RegistryError(`local path ${JSON.stringify(p)} is not absolute. These paths are declared as they exist on the ` +
+                    `WORKER, so a relative path has no meaning to resolve against.`);
+            }
+        }
+        const exists = this.db.prepare('SELECT 1 FROM hosts WHERE id = ?').get(input.id);
+        if (exists) {
+            throw new RegistryError(`host ${input.id} already exists. Ids are stable and operator-assigned; remove it first if you ` +
+                `really mean to replace it, because a binding refers to it by id.`);
+        }
+        this.db
+            .prepare(`INSERT INTO hosts (id, base_url, credential_ref, enabled, labels, local_paths, agents_cache, added_at)
+         VALUES (?, ?, ?, ?, ?, ?, '[]', ?)`)
+            .run(input.id, baseUrl, input.credentialRef.trim(), input.enabled === false ? 0 : 1, JSON.stringify(input.labels ?? {}), JSON.stringify(input.localPaths ?? []), new Date().toISOString());
+        return this.get(input.id);
+    }
+    get(id) {
+        const row = this.db.prepare('SELECT * FROM hosts WHERE id = ?').get(id);
+        return row ? rowToHost(row) : null;
+    }
+    list() {
+        const rows = this.db.prepare('SELECT * FROM hosts ORDER BY id').all();
+        return rows.map(rowToHost);
+    }
+    /** Registry plus the cached probe snapshot. This is what `fleet hosts list` renders. */
+    listWithProbe() {
+        return this.list().map((host) => ({ ...host, probe: this.probeFor(host.id) }));
+    }
+    probeFor(id) {
+        const row = this.db.prepare('SELECT * FROM host_probe WHERE host_id = ?').get(id);
+        return row ? rowToProbe(row) : null;
+    }
+    setEnabled(id, enabled) {
+        const res = this.db.prepare('UPDATE hosts SET enabled = ? WHERE id = ?').run(enabled ? 1 : 0, id);
+        if (Number(res.changes) === 0)
+            throw new RegistryError(`no such host: ${id}`);
+        return this.get(id);
+    }
+    /**
+     * Forget a host and its cached probe.
+     *
+     * Refused while the host still owns Runs. `host_probe` cascades because it is cache and a cache row for a
+     * host that no longer exists describes nothing. `fleet_runs` must NOT cascade: deleting it would orphan
+     * Runs that are executing right now, on a machine Fleet can no longer name. The caller has to deal with
+     * those Runs first, or accept the loss explicitly.
+     */
+    remove(id, opts = {}) {
+        if (!opts.force) {
+            const owned = this.db
+                .prepare('SELECT COUNT(*) AS n, SUM(child_run_id IS NULL) AS unresolved FROM fleet_runs WHERE host_id = ?')
+                .get(id);
+            const count = Number(owned.n);
+            if (count > 0) {
+                const unresolved = Number(owned.unresolved ?? 0);
+                throw new RegistryError(`host ${id} still owns ${count} Fleet Run(s)${unresolved ? `, ${unresolved} of them with no ` +
+                    `recorded child answer` : ''}. Removing it would delete the only record of those Runs and leave ` +
+                    `them running on a machine Fleet cannot name. Let them finish, or pass force to accept the loss.`);
+            }
+        }
+        else {
+            // Explicit force: drop the bindings first so the FK does not reject the host row, and say nothing
+            // here -- the caller asked for this and the route records it.
+            this.db.prepare('DELETE FROM fleet_runs WHERE host_id = ?').run(id);
+        }
+        const res = this.db.prepare('DELETE FROM hosts WHERE id = ?').run(id);
+        return Number(res.changes) > 0;
+    }
+    /**
+     * Record a probe. The cache row is always written, including for failures, because "we tried and it
+     * refused" is information the operator needs; last_seen_at moves only on success, because it answers
+     * "when did we last know this host was alive", and a 401 does not establish that.
+     */
+    recordProbe(rec) {
+        const host = this.get(rec.hostId);
+        if (!host)
+            throw new RegistryError(`cannot record probe for unknown host ${rec.hostId}`);
+        this.db
+            .prepare(`INSERT INTO host_probe
+           (host_id, outcome, detail, active_runs, queue_depth, worker_count, worker_id, agents, probed_at, last_error)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+         ON CONFLICT(host_id) DO UPDATE SET
+           outcome = excluded.outcome, detail = excluded.detail,
+           active_runs = excluded.active_runs, queue_depth = excluded.queue_depth,
+           worker_count = excluded.worker_count, worker_id = excluded.worker_id,
+           agents = excluded.agents, probed_at = excluded.probed_at, last_error = excluded.last_error`)
+            .run(rec.hostId, rec.outcome, rec.detail, rec.activeRuns, rec.queueDepth, rec.workerCount, rec.workerId, rec.agents === null ? null : JSON.stringify(rec.agents), rec.probedAt, rec.lastError);
+        if (rec.outcome === 'ok') {
+            this.db
+                .prepare('UPDATE hosts SET last_seen_at = ?, agents_cache = ? WHERE id = ?')
+                .run(rec.probedAt, JSON.stringify(rec.agents ?? []), rec.hostId);
+        }
+        else if (rec.agents !== null) {
+            // A reachable host that failed a deeper check still told us something about its agents; keep the
+            // advisory cache honest without claiming the host was seen healthy.
+            this.db.prepare('UPDATE hosts SET agents_cache = ? WHERE id = ?').run(JSON.stringify(rec.agents), rec.hostId);
+        }
+    }
+    /** Drop cached probe rows for hosts that no longer exist. Cheap hygiene after an interrupted delete. */
+    pruneProbeCache() {
+        const res = this.db
+            .prepare('DELETE FROM host_probe WHERE host_id NOT IN (SELECT id FROM hosts)')
+            .run();
+        return Number(res.changes);
+    }
+}

package/dist/routing.js

@@ -0,0 +1,219 @@
+/**
+ * Routing (docs/fleet-design.md section 6).
+ *
+ * A pure function over declared facts. It is pure deliberately: a router tested only by "a Run got placed"
+ * cannot tell a working filter from an inert one, because both place the Run. Every exclusion here is returned
+ * as data, so a test can assert not just that a host was chosen but that the WRONG hosts were rejected and
+ * for the stated reason.
+ */
+import { readFileSync } from 'node:fs';
+export class RoutingError extends Error {
+    status;
+    exclusions;
+    constructor(message, status, exclusions) {
+        super(message);
+        this.name = 'RoutingError';
+        this.status = status;
+        this.exclusions = exclusions;
+    }
+}
+/**
+ * Normalise a path for comparison without touching the filesystem.
+ *
+ * Fleet cannot see a child's disk (section 4.2), so this is string work on declared values only -- it must not
+ * resolve symlinks or stat anything, because the path exists on another machine.
+ */
+function normalizePath(p) {
+    const trimmed = p.replace(/\/+/g, '/').replace(/\/+$/, '') || '/';
+    return trimmed;
+}
+function declaresPath(host, localPath) {
+    const want = normalizePath(localPath);
+    return host.localPaths.some((declared) => {
+        const have = normalizePath(declared);
+        // A declared path covers itself and anything beneath it: declaring a repo root is the common case and
+        // requiring operators to list every subdirectory would push them toward declaring '/' instead.
+        return have === want || want.startsWith(`${have}/`);
+    });
+}
+/**
+ * Score a host for preference. Lower is better.
+ *
+ * Only reached when several hosts survive the hard filters, and it can never override one: capacity is a
+ * preference, locality is a constraint. A null capacity means the host has not been probed recently, which is
+ * treated as neutral rather than free -- an unprobed host must not look like the emptiest machine in the fleet.
+ */
+function score(host) {
+    const p = host.probe;
+    if (!p || p.outcome !== 'ok')
+        return 1_000;
+    const active = p.activeRuns ?? 0;
+    const queued = p.queueDepth ?? 0;
+    const workers = p.workerCount ?? 1;
+    // One unit of headroom per worker keeps a big machine from being penalised for its capacity.
+    const headroom = Math.max(0, active + queued - workers);
+    return active * 2 + queued + headroom;
+}
+function bad(field, value) {
+    throw new RoutingError(`${field} must be ${'a non-empty string'}, got ${JSON.stringify(value) ?? typeof value}`, 400, []);
+}
+function validateRequest(request) {
+    if (request.host !== undefined && (typeof request.host !== 'string' || !request.host))
+        bad('host', request.host);
+    if (request.agent !== undefined && (typeof request.agent !== 'string' || !request.agent))
+        bad('agent', request.agent);
+    if (request.repository !== undefined) {
+        if (typeof request.repository !== 'object' || request.repository === null || Array.isArray(request.repository)) {
+            bad('repository', request.repository);
+        }
+        for (const key of ['url', 'localPath', 'ref']) {
+            const v = request.repository[key];
+            if (v !== undefined && (typeof v !== 'string' || !v))
+                bad(`repository.${key}`, v);
+        }
+    }
+    if (request.labels !== undefined) {
+        if (typeof request.labels !== 'object' || request.labels === null || Array.isArray(request.labels)) {
+            bad('labels', request.labels);
+        }
+        for (const [k, v] of Object.entries(request.labels)) {
+            if (typeof v !== 'string')
+                bad(`labels.${k}`, v);
+        }
+    }
+}
+export function routeRun(hosts, request, opts = {}) {
+    // Types are checked here rather than trusted. A body like {"repository":{"localPath":123}} would otherwise
+    // reach a .replace() on a number and surface as a 500 -- sending the operator to the service logs for what
+    // is a mistake in their own request.
+    validateRequest(request);
+    const considered = hosts.map((h) => h.id);
+    const exclusions = [];
+    if (hosts.length === 0) {
+        throw new RoutingError('no hosts are registered with Fleet', 400, []);
+    }
+    // Explicit placement first: an operator who names a host has already made the decision.
+    if (request.host) {
+        const chosen = hosts.find((h) => h.id === request.host);
+        if (!chosen) {
+            throw new RoutingError(`host ${JSON.stringify(request.host)} is not registered. Known hosts: ${considered.join(', ') || '(none)'}`, 404, []);
+        }
+        if (!chosen.enabled) {
+            throw new RoutingError(`host ${chosen.id} is disabled`, 409, [{ hostId: chosen.id, reason: 'disabled' }]);
+        }
+        // The rewrite still applies. Naming a host removes locality as a FILTER, but a localPath handed to a child
+        // that does not have it still fails there -- and it fails as a Run failure rather than as a routing
+        // decision, which is the worse place to learn it.
+        const explicitUrl = !request.repository?.url && request.repository?.localPath
+            ? opts.resolveCloneUrl?.(normalizePath(request.repository.localPath)) ?? null
+            : null;
+        return {
+            hostId: chosen.id,
+            score: 0,
+            rewroteLocalPath: explicitUrl !== null,
+            repository: explicitUrl ? { ...request.repository, url: explicitUrl, localPath: undefined } : request.repository,
+            considered,
+        };
+    }
+    const repo = request.repository;
+    const localPath = repo?.localPath;
+    // A caller who supplies a clone URL has already removed the constraint themselves; the design calls the
+    // rewrite the most valuable thing in this section precisely because it turns the hardest rule into a non-issue.
+    const rewritten = !repo?.url && localPath ? opts.resolveCloneUrl?.(normalizePath(localPath)) ?? null : null;
+    const localityConstrained = Boolean(localPath) && !repo?.url && !rewritten;
+    const survivors = [];
+    for (const host of hosts) {
+        if (!host.enabled) {
+            exclusions.push({ hostId: host.id, reason: 'disabled' });
+            continue;
+        }
+        if (localityConstrained && !declaresPath(host, localPath)) {
+            exclusions.push({
+                hostId: host.id,
+                reason: `does not declare ${normalizePath(localPath)} among its local paths `
+                    + `[${host.localPaths.join(', ') || 'none declared'}]`,
+            });
+            continue;
+        }
+        if (request.agent) {
+            // Prefer a fresh probe over the cached list; the cache is advisory (section 5) and only used when
+            // nothing fresher exists.
+            const agents = host.probe?.agents ?? host.agentsCache;
+            if (!agents.includes(request.agent)) {
+                exclusions.push({
+                    hostId: host.id,
+                    reason: `does not offer agent ${JSON.stringify(request.agent)} `
+                        + `(known: ${agents.join(', ') || 'none'})`,
+                });
+                continue;
+            }
+        }
+        const unmatched = Object.entries(request.labels ?? {}).filter(([k, v]) => host.labels[k] !== v);
+        if (unmatched.length > 0) {
+            exclusions.push({
+                hostId: host.id,
+                reason: `labels do not match: needs ${unmatched.map(([k, v]) => `${k}=${v}`).join(', ')}`
+                    + `; has ${Object.keys(host.labels).length ? Object.entries(host.labels).map(([k, v]) => `${k}=${v}`).join(', ') : 'no labels'}`,
+            });
+            continue;
+        }
+        survivors.push({ host, score: score(host) });
+    }
+    if (survivors.length === 0) {
+        // Section 6: a localPath with no matching host is a submission error, not a scheduling wait. Silently
+        // queueing it, or quietly rewriting it to a URL nobody asked for, turns a five-second mistake into an hour
+        // of confusion -- so the answer names every host that was considered and why each was ruled out.
+        const lines = exclusions.map((e) => `  - ${e.hostId}: ${e.reason}`).join('\n');
+        throw new RoutingError(`no host can run this task. ${hosts.length} considered:\n${lines}`
+            + (localityConstrained
+                ? '\nDeclare the path on a host (fleet hosts edit --local-path), or give a git url so locality stops mattering.'
+                : ''), 400, exclusions);
+    }
+    survivors.sort((a, b) => a.score - b.score || a.host.id.localeCompare(b.host.id));
+    const chosen = survivors[0];
+    const repository = rewritten
+        ? { ...repo, url: rewritten, localPath: undefined }
+        : repo;
+    return {
+        hostId: chosen.host.id,
+        score: chosen.score,
+        rewroteLocalPath: rewritten !== null,
+        repository,
+        considered,
+    };
+}
+/**
+ * Load the local-path -> clone-URL map.
+ *
+ * Returns null when unset so callers keep locality as a hard constraint rather than silently routing
+ * everywhere. A malformed file is an operator error worth hearing about, so it throws rather than degrading to
+ * "no mapping", which would look like a routing bug.
+ */
+export function loadRepoUrlMap(file) {
+    if (!file)
+        return undefined;
+    let raw;
+    try {
+        raw = readFileSync(file, 'utf8');
+    }
+    catch (err) {
+        throw new Error(`FLEET_REPO_URLS_FILE ${JSON.stringify(file)} could not be read: ${err.message}`);
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch (err) {
+        throw new Error(`FLEET_REPO_URLS_FILE is not valid JSON: ${err.message}`);
+    }
+    if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
+        throw new Error('FLEET_REPO_URLS_FILE must contain a JSON object of localPath -> git URL');
+    }
+    const map = {};
+    for (const [k, v] of Object.entries(parsed)) {
+        if (typeof v !== 'string' || !v)
+            throw new Error(`repo url for ${JSON.stringify(k)} must be a non-empty string`);
+        map[k.replace(/\/+$/, '') || '/'] = v;
+    }
+    return (localPath) => map[localPath.replace(/\/+$/, '') || '/'] ?? null;
+}