omp-conductor 0.20.1 → 0.20.3

package/src/graph-health.ts

@@ -52,7 +52,10 @@ export type CodeGraphHealth =
         active: "active" | "inactive" | "unknown";
       };
       refresh: {
-        result: "success" | "failed" | "unknown";
+        /** "running": the oneshot is executing right now (#1090). systemd has
+         *  already cleared the previous exit timestamp mid-run, so this is a
+         *  progress report — never a claim about staleness or absence. */
+        result: "success" | "failed" | "running" | "unknown";
         fresh: boolean | null;
         lastSuccessAt?: string;
         ageMs?: number;
@@ -208,6 +211,12 @@ function refreshResult(result: ReadOnlyCommandResult, now: number): RefreshResul
   const serviceResult = fields.get("Result");
   const exitStatus = fields.get("ExecMainStatus");
   const completedAt = fields.get("ExecMainExitTimestamp");
+  // #1090: an empty ExecMainExitTimestamp says "no completed run yet", which
+  // is equally true of a unit that has never run and of one executing right
+  // now — systemd clears it when the next run starts. ActiveState comes back
+  // from the same single `systemctl show` and separates the two; evidence
+  // captured without the property keeps the conservative pre-#1090 reading.
+  const activeState = fields.get("ActiveState");
   if (serviceResult === undefined || exitStatus === undefined || completedAt === undefined) {
     return unknown("refresh service returned an unrecognized result", true);
   }
@@ -219,6 +228,12 @@ function refreshResult(result: ReadOnlyCommandResult, now: number): RefreshResul
     };
   }
   if (completedAt === "" || completedAt === "n/a") {
+    if (activeState === "activating" || activeState === "active") {
+      // There is no completed timestamp to measure staleness against yet, but
+      // this is progress, not absence: the reindex is keeping the graph fresh
+      // at this moment, so neither the row nor the previous message degrades.
+      return { health: { result: "running", fresh: null }, uncertain: false };
+    }
     return unknown("no successful refresh has been recorded", false);
   }
   const completedMs = Date.parse(completedAt);
@@ -290,6 +305,7 @@ export async function probeCodeGraph(
       "--property=Result",
       "--property=ExecMainStatus",
       "--property=ExecMainExitTimestamp",
+      "--property=ActiveState",
     ]),
   ]);
 

package/src/groom.ts

@@ -114,6 +114,17 @@ export interface ReadyGateRejection {
  * {@link toSpecDurableVerdict} still recovers the verdict and the candidate is
  * not re-groomed — the spec is rejected, not ungroomed.
  *
+ * That durability is bounded by the verdict itself (#1060). A rejection
+ * beside a verdict the contract still accepts is not re-groomable, because
+ * the fault lies in the issue — missing criteria, a missing lane — and only
+ * an edit to the issue can fix it. A rejection beside a verdict the contract
+ * refuses — veltro#731's prose lane entry, persisted before parse-time lane
+ * validation — reads as not-durable through {@link toSpecDurableVerdict},
+ * and the candidate becomes re-groomable: no issue edit can ever satisfy a
+ * gate whose refusal is baked into the verdict, and complying with the
+ * refusal's remedy would write the unusable entry into the issue itself, so
+ * a fresh pass is the only repair.
+ *
  * Fails closed on anything it cannot read: an empty `missing` array is not a
  * rejection (a gate that found nothing missing passed), and a non-string entry
  * would render as `undefined` in a digest line an operator is asked to act on.

package/src/knowledge.ts

@@ -19,6 +19,10 @@
  * accumulated knowledge is worse than one fewer fact, because a worker cannot
  * tell a clipped warning from a complete one.
  *
+ * What the cap never drops is structure: an operator distillation's headings,
+ * paragraphs and blank separators survive every append verbatim and in place —
+ * the facts rotate, the shape of the knowledge stays.
+ *
  * Writes are best-effort. This is an optimisation, and a run must never fail
  * because the overlay could not be written: the callers are settlement paths.
  */
@@ -74,31 +78,69 @@ function encodeRepoKey(repo: string): string {
 }
 
 /**
- * The overlay's entry lines, oldest first, with blank lines and any hand-added
- * prose dropped.
+ * One line of the overlay document. An `entry` is a fleet-learned fact — the
+ * `- `-prefixed line the writer produces and attributes; anything else, a
+ * heading, a paragraph or a blank separator, is `structure`: hand-added
+ * scaffolding from an operator distillation. The distinction is the format:
+ * entries are appendable, deduplicated and evictable, structure is none of
+ * those things, and it survives every write verbatim and in place.
+ */
+type OverlayBlock = { kind: "entry" | "structure"; line: string };
+
+/**
+ * The overlay document parsed into blocks, in file order. Lossless: joining
+ * the lines back with "\n" reproduces the input, which is what makes keeping
+ * the structure through a write a property of the parse rather than a hope.
  *
  * One parse, shared by the writer and the brief renderer on purpose: they must
  * agree on what counts as an entry, or the cap one enforces is not the cap the
  * other renders and a "16 KB" section arrives at 20.
  */
-function entriesOf(text: string): string[] {
-  return text
-    .split("\n")
-    .map((line) => line.trimEnd())
-    .filter((line) => line.startsWith(ENTRY_PREFIX) && line.slice(ENTRY_PREFIX.length).trim() !== "");
+function blocksOf(text: string): OverlayBlock[] {
+  if (text === "") return [];
+  const lines = text.split("\n");
+  // The final newline is the file's terminator, not a blank last line: pop it
+  // here so appended entries join the document itself, and let renderDoc put
+  // exactly one back.
+  if (lines.at(-1) === "") lines.pop();
+  return lines.map((raw) => {
+    const line = raw.trimEnd();
+    return line.startsWith(ENTRY_PREFIX) && line.slice(ENTRY_PREFIX.length).trim() !== ""
+      ? { kind: "entry", line }
+      : { kind: "structure", line: raw };
+  });
+}
+
+/**
+ * The document text for a block list: a blank run at the EOF seam is
+ * whitespace, not content, so it is trimmed and exactly one newline terminates
+ * the document — the shape every writer of this format has always produced.
+ * Interior blanks pass through untouched.
+ */
+function renderDoc(blocks: readonly OverlayBlock[]): string {
+  const kept = [...blocks];
+  while (kept.at(-1)?.kind === "structure" && kept.at(-1)?.line.trim() === "") kept.pop();
+  return `${kept.map((b) => b.line).join("\n")}\n`;
 }
 
 /**
- * The newest entries that fit under {@link KNOWLEDGE_MAX_BYTES}, oldest dropped
- * first and only ever whole.
+ * The newest content that fits under {@link KNOWLEDGE_MAX_BYTES}, oldest
+ * entries dropped first and only ever whole.
+ *
+ * Only entries evict. A distilled file keeps its headings and paragraphs even
+ * when the facts beneath them have been rotated out, and when only non-entry
+ * text remains the document simply stays over the ceiling: operator prose is
+ * not ours to delete, and distilling it back under the cap is the remedy the
+ * format itself names.
  *
  * Byte lengths, not character counts: the cap is a context-budget promise, and
  * one accented identifier or box-drawing character is several bytes.
  */
-function withinCap(entries: readonly string[]): string[] {
-  const kept = [...entries];
-  while (kept.length > 0 && Buffer.byteLength(`${kept.join("\n")}\n`, "utf8") > KNOWLEDGE_MAX_BYTES) {
-    kept.shift();
+function withinCap(blocks: readonly OverlayBlock[]): OverlayBlock[] {
+  const kept = [...blocks];
+  const bytes = (): number => Buffer.byteLength(renderDoc(kept), "utf8");
+  while (kept.some((b) => b.kind === "entry") && bytes() > KNOWLEDGE_MAX_BYTES) {
+    kept.splice(kept.findIndex((b) => b.kind === "entry"), 1);
   }
   return kept;
 }
@@ -139,6 +181,10 @@ export function readKnowledge(repo: string): string | undefined {
  * `proofCommands` for every issue in a repo — cannot inflate the file until the
  * cap evicts something real.
  *
+ * The rewrite preserves every non-entry line the file carries — the headings,
+ * paragraphs and blank separators of an operator distillation — exactly where
+ * they were: entries rotate under the cap, structure does not.
+ *
  * Best-effort by contract: the callers are settlement paths, and a failed write
  * here must never turn a finished run into a failed one.
  */
@@ -157,25 +203,33 @@ export function appendKnowledge(
   if (rendered.length === 0) return;
   const path = knowledgePath(repo);
   try {
-    const existing = entriesOf(readKnowledge(repo) ?? "");
-    // Dynamic membership over the file's own lines, grown as this batch is
+    const existing = blocksOf(readKnowledge(repo) ?? "");
+    // Dynamic membership over the file's own entries, grown as this batch is
     // folded in, so one call cannot append the same fact twice either.
-    const seen = new Set(existing);
-    const added: string[] = [];
+    const seen = new Set(existing.filter((b) => b.kind === "entry").map((b) => b.line));
+    const added: OverlayBlock[] = [];
     for (const line of rendered) {
       if (seen.has(line)) continue;
       seen.add(line);
-      added.push(line);
+      added.push({ kind: "entry", line });
     }
     if (added.length === 0) return;
-    const kept = withinCap([...existing, ...added]);
+    // A batch appended under a distilled paragraph or heading takes one blank
+    // separator, so it never glues onto the prose above it; after an entry or
+    // a blank none is needed.
+    const tail = existing.at(-1);
+    const gap: OverlayBlock[] =
+      tail !== undefined && tail.kind === "structure" && tail.line.trim() !== ""
+        ? [{ kind: "structure", line: "" }]
+        : [];
+    const kept = withinCap([...existing, ...gap, ...added]);
     mkdirSync(dirname(path), { recursive: true });
     // Atomic like every other durable conductor state file: a brief renderer
     // reading while a settlement writes sees the old file or the new one, never
     // a half-written entry.
     const tmp = `${path}.${process.pid.toString(36)}.${Date.now().toString(36)}.tmp`;
     try {
-      writeFileSync(tmp, `${kept.join("\n")}\n`, "utf8");
+      writeFileSync(tmp, renderDoc(kept), "utf8");
       renameSync(tmp, path);
     } catch (err) {
       rmSync(tmp, { force: true });
@@ -209,7 +263,7 @@ export function appendKnowledge(
 export function knowledgeSection(repo: string): string {
   const text = readKnowledge(repo);
   if (text === undefined) return "";
-  const entries = withinCap(entriesOf(text));
+  const entries = withinCap(blocksOf(text)).filter((b) => b.kind === "entry").map((b) => b.line);
   if (entries.length === 0) return "";
   return [
     KNOWLEDGE_HEADING,

package/src/lifecycle.ts

@@ -194,6 +194,103 @@ export function livingDaemon(): DaemonRecord | undefined {
   return undefined;
 }
 
+/**
+ * `DaemonRecord.logFile` for a daemon nobody spawned — which is how systemd
+ * runs it (`ExecStart … omp-conductor daemon`) and how a hand-run foreground
+ * `daemon` works too. The field is required and `status` prints it, so it has
+ * to say something true: a foreground daemon opened no log of its own —
+ * whoever started it owns its stdout, be that the journal, a terminal, or a
+ * pane.
+ */
+const FOREGROUND_LOG = "<inherited stdout — started in the foreground>";
+
+/** What this process would publish as its own boot record, given its flags. */
+function foregroundRecord(o: { port?: number; project?: string }): DaemonRecord {
+  return {
+    pid: process.pid,
+    port: o.port ?? DEFAULT_PORT,
+    startedAt: Date.now(),
+    logFile: FOREGROUND_LOG,
+    ...(o.project === undefined ? {} : { project: o.project }),
+  };
+}
+
+export type ForegroundBoot =
+  | { kind: "claimed"; record: DaemonRecord }
+  | { kind: "foreign-live"; pid: number };
+
+/**
+ * The boot half of pidfile ownership (#1114): a daemon that starts without a
+ * managed spawn — systemd's `ExecStart`, a reboot, a bare hand-run — publishes
+ * its own record here, so the single-owner guard (`startDaemon`'s live-record
+ * refusal), `wake`, `status` and orphan reconciliation are armed no matter who
+ * started the process.
+ *
+ * A foreign live record is refused, never stolen; a stale one is cleared by
+ * {@link livingDaemon} on the way past; and a record already naming this pid
+ * was written by whoever spawned us and knows the real log file — replacing
+ * it with a guess would be a downgrade.
+ */
+export function publishForegroundRecord(
+  o: { port?: number; project?: string } = {},
+): ForegroundBoot {
+  const running = livingDaemon();
+  if (running !== undefined && running.pid !== process.pid) {
+    return { kind: "foreign-live", pid: running.pid };
+  }
+  if (running !== undefined) return { kind: "claimed", record: running };
+  const record = foregroundRecord(o);
+  writeRecord(record);
+  return { kind: "claimed", record };
+}
+
+/**
+ * The running half of pidfile ownership (#1114): the record can vanish under
+ * a perfectly healthy daemon — an external sweep of the shared runtime
+ * directory, a tmp cleaner — and nothing else re-arms the single-owner guard
+ * until the next restart. The daemon itself repairs that: absent or stale
+ * means it publishes again; a live foreign record is left alone (a `--once`
+ * drill may legitimately own visibility mid-tick).
+ */
+export function republishSelfRecord(o: { port?: number; project?: string } = {}): void {
+  const current = livingDaemon();
+  // Ours is already published, or a live foreign process owns visibility:
+  // either way there is nothing to repair.
+  if (current !== undefined) return;
+  writeRecord(foregroundRecord(o));
+}
+
+/** How often a running daemon re-asserts its record ({@link republishSelfRecord}). */
+export const RECORD_REPAIR_INTERVAL_MS = 60_000;
+
+/** Stops the periodic repair started by {@link startRecordRepair}. */
+export interface RecordRepair {
+  stop(): void;
+}
+
+/**
+ * Re-assert the record on an interval while the daemon runs. A failed repair
+ * never propagates: the pidfile is visibility, not correctness, and a failed
+ * attempt retries on the next tick rather than killing the dispatcher over it.
+ */
+export function startRecordRepair(
+  o: { port?: number; project?: string } = {},
+  intervalMs = RECORD_REPAIR_INTERVAL_MS,
+): RecordRepair {
+  const timer = setInterval(() => {
+    try {
+      republishSelfRecord(o);
+    } catch (err) {
+      process.stderr.write(
+        `omp-conductor: could not re-assert the daemon pidfile: ${(err as Error).message}\n`,
+      );
+    }
+  }, intervalMs);
+  // Bookkeeping must never hold the event loop open at shutdown.
+  timer.unref?.();
+  return { stop: () => clearInterval(timer) };
+}
+
 /**
  * The `daemon --once` lease file, one per process. Created `O_EXCL` so two
  * simultaneous `--once` runs cannot both win ownership, and it names the
@@ -1186,6 +1283,10 @@ export type SystemctlResult = {
   stderr: string;
   /** Binary not found (ENOENT). Distinct from a failed invocation of a present binary. */
   missing?: boolean;
+  /** The client bound expired (spawnSync ETIMEDOUT). A state-changing verb
+   *  that reports this is re-judged from the unit's observed state before it
+   *  becomes a failure (#1112). */
+  timedOut?: boolean;
 };
 
 export type SystemctlFn = (args: string[]) => SystemctlResult;
@@ -1251,14 +1352,169 @@ function testSystemctl(args: string[]): SystemctlResult {
   }
 }
 
-function defaultSystemctl(args: string[]): SystemctlResult {
-  if (process.env["NODE_ENV"] === "test") return testSystemctl(args);
+/** Verbs whose completion is a unit job: systemd holds the client until the
+ * stop/start finishes, which legitimately takes as long as the unit's own
+ * graceful-shutdown budget. Everything else answers from manager state and
+ * keeps the short probe bound (#1112). */
+const STATE_CHANGING_VERBS: Record<string, true> = {
+  start: true,
+  stop: true,
+  restart: true,
+  "try-restart": true,
+  reload: true,
+  "reload-or-restart": true,
+};
+
+/**
+ * Bound for probe verbs (`show`, `is-active`, `is-enabled`, …): a hung dbus
+ * becomes `unknown` ownership instead of a hang. Callers must NOT treat that
+ * timeout as "no unit". State-changing verbs get their own budget below — a
+ * 15s bound on `restart` is what declared a healthy restart failed and rolled
+ * a good release back (#1112).
+ */
+const SYSTEMCTL_PROBE_TIMEOUT_MS = 15_000;
+
+/**
+ * systemd's built-in `TimeoutStopSec`: how long any unit without an explicit
+ * override may legitimately spend stopping before systemd itself escalates.
+ * A client bound for a state-changing verb never sits below this floor —
+ * smaller is structurally able to lose against a healthy, graceful stop.
+ */
+const SYSTEMD_DEFAULT_TIMEOUT_STOP_MS = 90_000;
+
+/** Slack over the unit's own stop budget: the start half of a restart plus
+ * scheduling delay, before the client's stopwatch means anything at all. */
+const STATE_VERB_SLACK_MS = 30_000;
+
+type RawSystemctl = (args: string[], timeoutMs: number) => SystemctlResult;
+
+/** One unit-state reading, in the shape the post-timeout verdict needs. */
+interface UnitStateSample {
+  activeState?: string;
+  /** `ExecMainStartTimestampMonotonic` — changes on every fresh start of the
+   *  unit, which is what makes "the old instance" distinguishable. */
+  startStampMonotonic?: string;
+  /** `TimeoutStopUSec` — integer microseconds, or `infinity`. */
+  timeoutStopUSec?: string;
+}
+
+const STATE_SAMPLE_PROPERTIES = [
+  "--property=ActiveState",
+  "--property=ExecMainStartTimestampMonotonic",
+  "--property=TimeoutStopUSec",
+] as const;
+
+function sampleUnitState(unit: string, raw: RawSystemctl): UnitStateSample | undefined {
+  const ran = raw(
+    ["show", unit, "--value", ...STATE_SAMPLE_PROPERTIES],
+    SYSTEMCTL_PROBE_TIMEOUT_MS,
+  );
+  if (!ran.ok) return undefined;
+  const values = ran.stdout.split("\n");
+  const pick = (index: number): string | undefined => {
+    const value = values[index]?.trim();
+    return value === undefined || value === "" ? undefined : value;
+  };
+  return {
+    activeState: pick(0),
+    startStampMonotonic: pick(1),
+    timeoutStopUSec: pick(2),
+  };
+}
+
+/**
+ * The client bound for one state-changing verb: at least the unit's own
+ * `TimeoutStopUSec` — read live, because any fixed constant re-creates this
+ * bug on whatever host drains slower than it — and never below systemd's own
+ * default stop budget. An unreadable budget gets the documented fallback.
+ */
+function stateVerbTimeoutMs(sample: UnitStateSample | undefined): number {
+  let stopMs = SYSTEMD_DEFAULT_TIMEOUT_STOP_MS;
+  const usec =
+    sample?.timeoutStopUSec === undefined ? Number.NaN : Number.parseInt(sample.timeoutStopUSec, 10);
+  if (Number.isFinite(usec) && usec > 0) {
+    stopMs = Math.max(Math.ceil(usec / 1000), stopMs);
+  }
+  return stopMs + STATE_VERB_SLACK_MS;
+}
+
+/**
+ * The stopwatch lost; the unit's own state decides. A state-changing verb that
+ * outlived the client bound succeeded exactly when the manager shows the
+ * verb's goal reached: not-active for `stop`, active with a FRESH start
+ * timestamp for the start/restart family — an unchanged timestamp means the
+ * old instance is still the running one. Anything else stays a failure that
+ * carries both facts.
+ */
+function stateVerbTimeoutVerdict(
+  verb: string,
+  unit: string,
+  before: UnitStateSample | undefined,
+  after: UnitStateSample | undefined,
+  timedOut: SystemctlResult,
+  timeoutMs: number,
+): SystemctlResult {
+  const base =
+    `systemctl ${verb} ${unit} exceeded its ${Math.round(timeoutMs / 1000)}s client bound ` +
+    `(${timedOut.stderr.trim() || "client timeout"})`;
+  if (after?.activeState === undefined) {
+    return {
+      ...timedOut,
+      stderr:
+        `${base}, and the unit's state could not be re-read afterwards — ` +
+        `the outcome is unproven, not failed`,
+    };
+  }
+  if (verb === "stop") {
+    if (after.activeState !== "active" && after.activeState !== "deactivating") {
+      // Goal state reached: the stop completed, whatever the stopwatch said.
+      return { ok: true, stdout: "", stderr: "" };
+    }
+    return {
+      ...timedOut,
+      stderr: `${base}; the unit is still ${after.activeState} — the stop did not complete`,
+    };
+  }
+  const freshStart =
+    before !== undefined &&
+    after.startStampMonotonic !== undefined &&
+    after.startStampMonotonic !== before.startStampMonotonic;
+  if (after.activeState === "active" && freshStart) {
+    // The unit restarted; the client simply gave up first.
+    return { ok: true, stdout: "", stderr: "" };
+  }
+  const why =
+    after.activeState !== "active"
+      ? `the unit is ${after.activeState}`
+      : before === undefined
+        ? "the unit is active but no pre-call sample exists to prove the start is fresh"
+        : "its start timestamp is unchanged, so the running instance predates the call";
+  return { ...timedOut, stderr: `${base}; ${why} — the ${verb} did not complete` };
+}
+
+/**
+ * Run one systemctl invocation with #1112 semantics: probes keep the short
+ * hung-dbus bound, state-changing verbs wait out the unit's own stop budget,
+ * and a state-changing verb that outlives even that is judged by the unit's
+ * observed state rather than the client's stopwatch. Exported for tests,
+ * which inject their own {@link RawSystemctl}.
+ */
+export function runSystemctlInvocation(args: string[], raw: RawSystemctl): SystemctlResult {
+  const verb = args[0] ?? "";
+  if (STATE_CHANGING_VERBS[verb] !== true) return raw(args, SYSTEMCTL_PROBE_TIMEOUT_MS);
+  const unit = args.length > 1 && !args[1]!.startsWith("-") ? args[1]! : SYSTEMD_UNIT;
+  const before = sampleUnitState(unit, raw);
+  const timeoutMs = stateVerbTimeoutMs(before);
+  const ran = raw(args, timeoutMs);
+  if (ran.ok || !ran.timedOut) return ran;
+  return stateVerbTimeoutVerdict(verb, unit, before, sampleUnitState(unit, raw), ran, timeoutMs);
+}
+
+function spawnSystemctl(args: string[], timeoutMs: number): SystemctlResult {
   try {
     const res = spawnSync("systemctl", args, {
       encoding: "utf8",
-      // Bound the wait: a hung dbus becomes `unknown` ownership, not a hang.
-      // Callers must NOT treat that timeout as "no unit".
-      timeout: 15_000,
+      timeout: timeoutMs,
       env: process.env,
     });
     if (res.error) {
@@ -1268,6 +1524,7 @@ function defaultSystemctl(args: string[]): SystemctlResult {
         stdout: "",
         stderr: err.message,
         ...(err.code === "ENOENT" ? { missing: true } : {}),
+        ...(err.code === "ETIMEDOUT" ? { timedOut: true } : {}),
       };
     }
     return {
@@ -1280,6 +1537,11 @@ function defaultSystemctl(args: string[]): SystemctlResult {
   }
 }
 
+function defaultSystemctl(args: string[]): SystemctlResult {
+  if (process.env["NODE_ENV"] === "test") return testSystemctl(args);
+  return runSystemctlInvocation(args, spawnSystemctl);
+}
+
 let systemctl: SystemctlFn = defaultSystemctl;
 
 /** Shared manager runner. Production reaches the hardcoded binary; test

package/src/orchestrator-tick.ts

@@ -3642,14 +3642,20 @@ export async function sendArmNotice(text: string, surface: ArmNoticeSurface): Pr
     throw new Error("no Telegram bot token readable — the arm notice cannot send");
   }
   let topicId: number | undefined;
+  let project: { name: string; workspaceRoot?: string } | undefined;
   if (surface.project !== undefined) {
     try {
-      topicId = resolveProjectTopicId(findProject(loadConfig(), surface.project));
+      const p = findProject(loadConfig(), surface.project);
+      topicId = resolveProjectTopicId(p);
+      project = { name: p.name, workspaceRoot: p.workspaceRoot };
     } catch {
       /* no project config — flat chat, exactly like armTicks */
     }
   }
-  await sendTelegram(token, channel.owner, text, { topicId });
+  await sendTelegram(token, channel.owner, text, {
+    topicId,
+    ...(project === undefined ? {} : { project }),
+  });
 }
 
 /**