beadcyte 0.4.0

package/src/estimator.mjs

@@ -0,0 +1,314 @@
+// gantt-estimator.mjs — historical-lookback duration estimator for beads-gantt.
+//
+// v0 (this file): for each shipped bead we compute the calendar duration
+// `closed_at - started_at` (raw wall-clock, keeps blocked/in_review waits —
+// which is real economic time, and the only signal available without
+// agent-reported actuals). We bucket by (issue_type, priority) and take the
+// median of the last N in bucket, falling back (type,priority) → (type) →
+// global. The number of samples that landed in each fallback layer is
+// reported so the SVG legend can be honest about the model.
+//
+// v1 (future, blocked on agents logging metadata.economics.total_hours at
+// close): prefer that number where present, fall back to calendar duration
+// otherwise. The plumbing is here already — see estimateBead()'s comment.
+//
+// A bead only counts as a sample if it shipped — see ship.mjs for what that
+// means and why. The distinction is doing real work here: it separates
+// ship-record closes from procedural ones (superseded, deferred-then-closed,
+// admin cleanup) which would otherwise poison the median with a five-minute
+// wall-clock.
+
+import { isShipped } from "./ship.mjs";
+
+const MS_PER_DAY = 86_400_000;
+const MIN_SAMPLES_FOR_BUCKET = 3; // below this, fall back a layer
+
+/**
+ * Hours in a day, for turning reported work-hours into the unit every
+ * consumer of this model actually means (bp-67g.39).
+ *
+ * This was 8 — an office workday — while wall-clock samples were 24-hour
+ * calendar days, and both went into the same bucket median. The scheduler
+ * then read that median as calendar days: it adds `est.days` to a date, and
+ * the Gantt draws it against a 24-hour axis. So the same duration measured
+ * two ways came out 3x apart, and a bucket could average the two.
+ *
+ * Measured on this repo's 28 beads carrying both numbers, `agent_hours`
+ * divided by wall-clock `started_at → closed_at` has a **median of 1.05**
+ * (range 0.25-3.60). An hour of reported work really is about an hour of
+ * calendar here, so one constant of 24 is not a compromise between the two
+ * definitions — it is what the data says.
+ *
+ * The cost, stated plainly: for a HUMAN seat, eight logged hours spans at
+ * least one working day and this will under-project it. Two things make that
+ * the right trade anyway — the alternative is inventing a working-hours model
+ * and calling a guess a measurement, and the scheduler already clamps every
+ * cap against observed throughput (Little's Law), which is where "this team
+ * does not actually finish that much in a day" belongs. If a seat-aware unit
+ * is ever needed, `metadata.economics.by_seat` is where the seat is.
+ */
+const HOURS_PER_DAY = 24;
+
+/**
+ * Fallback ratio from a bucket median to its p90, used only when a bead has
+ * its own recorded hours and no bucket exists to measure spread from.
+ *
+ * The observed ratios on this repo are 1.38x (task), 1.51x (bug), 2.21x
+ * (feature) and 1.76x globally, so this is the middle of a real range rather
+ * than the invented 1.5 it replaces — but it is still a fallback, and
+ * `estimateBead` prefers a measured spread whenever it has one.
+ */
+const FALLBACK_P90_RATIO = 1.76;
+
+/**
+ * Smallest estimate we will report, in days — about seven minutes.
+ *
+ * This replaces a 0.5-day floor that was destroying the model rather than
+ * protecting it. On a 577-bead repo, 6 of the 9 qualifying (type, priority)
+ * buckets had medians below half a day — 215, 342, 209, 622, 706 and 272
+ * minutes — and every one of them was reported as 0.50d, so buckets three
+ * times apart came out identical and 96% of all beads got the same number.
+ *
+ * The samples were not the problem: only 1 of 143 was under fifteen
+ * minutes, and the global median was 437 minutes of real work. The floor
+ * was simply an order of magnitude larger than anything it needed to guard.
+ *
+ * Nothing downstream needs half a day either. BeadsGantt already floors a
+ * bar at `Math.max(2, ...)` pixels so a short bead stays visible;
+ * scheduleGreedy floors its own `remaining` at 0.5 and advances in whole-day
+ * timesteps, so a sub-day estimate behaves identically to a half-day one in
+ * the plan. All the old floor changed was the number on screen, and it
+ * changed it into a constant.
+ *
+ * What remains is a guard against zero: a zero-length bar has no start and
+ * the scheduler would never retire it.
+ */
+const MIN_ESTIMATE_DAYS = 0.005;
+
+/**
+ * Build a duration model from the full bead list.
+ * Returns { estimate(bead) → days, stats } where stats explains the model
+ * for the SVG legend footer.
+ */
+export function buildEstimator(allBeads, { lookback = 60 } = {}) {
+  const shipped = allBeads
+    .filter(
+      (b) =>
+        b.status === "closed" &&
+        b.started_at &&
+        b.closed_at &&
+        isShipped(b),
+    )
+    .map((b) => {
+      // Prefer agent-reported work-hours when available — many beads have
+      // `bd close` run immediately after `bd update --claim`, so calendar
+      // duration is near-zero even for real work. metadata.economics.totals
+      // .agent_hours is the honest number when it's there.
+      const agentHours = b.metadata?.economics?.totals?.agent_hours;
+      const wallClockDays =
+        (new Date(b.closed_at).getTime() -
+          new Date(b.started_at).getTime()) /
+        MS_PER_DAY;
+      const days =
+        typeof agentHours === "number" && agentHours > 0
+          ? agentHours / HOURS_PER_DAY
+          : wallClockDays;
+      return {
+        id: b.id,
+        type: b.issue_type,
+        priority: b.priority ?? 4,
+        days,
+        _source: typeof agentHours === "number" && agentHours > 0 ? "agent" : "wallclock",
+        closed: new Date(b.closed_at).getTime(),
+      };
+    })
+    .filter((s) => s.days > 0 && s.days < 90) // trim outliers (deferred>90d)
+    .sort((a, b) => b.closed - a.closed)
+    .slice(0, lookback * 4); // pull a wide window; buckets trim further
+
+  const median = (arr) => {
+    if (!arr.length) return null;
+    const s = [...arr].sort((a, b) => a - b);
+    const m = Math.floor(s.length / 2);
+    return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2;
+  };
+  const p90 = (arr) => {
+    if (!arr.length) return null;
+    const s = [...arr].sort((a, b) => a - b);
+    return s[Math.min(s.length - 1, Math.floor(s.length * 0.9))];
+  };
+
+  // Bucket maps
+  const byTypeAndPri = new Map();
+  const byType = new Map();
+  for (const s of shipped) {
+    const k = `${s.type}|${s.priority}`;
+    if (!byTypeAndPri.has(k)) byTypeAndPri.set(k, []);
+    byTypeAndPri.get(k).push(s.days);
+    if (!byType.has(s.type)) byType.set(s.type, []);
+    byType.get(s.type).push(s.days);
+  }
+  const globalDays = shipped.map((s) => s.days);
+  const globalMed = median(globalDays) ?? 5;
+  const globalP90 = p90(globalDays) ?? 10;
+
+  // Lead time — created_at → closed_at — is a separate signal from
+  // work-time. Wall-clock lead includes queue wait, review, context
+  // switches, etc. — much more useful for Little's-Law scheduling than
+  // per-bead work duration. Sample the same shipped set as the duration
+  // buckets, but keep it in its own median.
+  const leadSamples = allBeads
+    .filter(
+      (b) =>
+        b.status === "closed" &&
+        b.created_at &&
+        b.closed_at &&
+        isShipped(b),
+    )
+    .map(
+      (b) =>
+        (new Date(b.closed_at).getTime() -
+          new Date(b.created_at).getTime()) /
+        MS_PER_DAY,
+    )
+    .filter((d) => d > 0 && d < 180);
+  const leadMed = median(leadSamples) ?? globalMed;
+  const leadP90 = p90(leadSamples) ?? globalP90;
+
+  const stats = {
+    shipped_sample_size: shipped.length,
+    // Three decimals, not one. At one, every sub-day median reports as 0.0 —
+    // this repo's global median of 0.0118d printed as `0`, which made the
+    // legend claim no history and hid bp-67g.39 entirely. bp-67g.24 removed
+    // the half-day floor precisely so sub-day durations survive the model;
+    // rounding them away on the way out undid that at the last step.
+    // `median_lead_days` also feeds the Little's Law cap calibration, so the
+    // precision is load-bearing and not only cosmetic.
+    global_median_days: round(globalMed, 3),
+    global_p90_days: round(globalP90, 3),
+    // Lead-time medians for scheduler calibration. Populated when there's
+    // enough shipped history; falls back to work-time medians otherwise.
+    median_lead_days: round(leadMed, 3),
+    p90_lead_days: round(leadP90, 3),
+    lead_sample_size: leadSamples.length,
+    used_agent_hours: 0,
+    used_spent_floor: 0,
+    used_type_pri: 0,
+    used_type: 0,
+    used_global: 0,
+  };
+
+  /**
+   * Spread of the tightest bucket that has samples, as a p90/median ratio.
+   * Used to put an honest p90 on a bead estimated from its own recorded
+   * hours, which has no distribution of its own.
+   */
+  function spreadFor(type, pri) {
+    for (const bucket of [byTypeAndPri.get(`${type}|${pri}`), byType.get(type), globalDays]) {
+      if (!bucket || bucket.length < MIN_SAMPLES_FOR_BUCKET) continue;
+      const m = median(bucket);
+      const p = p90(bucket);
+      if (m > 0 && p !== null && p >= m) return p / m;
+    }
+    return FALLBACK_P90_RATIO;
+  }
+
+  function estimateBead(bead) {
+    const pri = bead.priority ?? 4;
+    const type = bead.issue_type ?? "task";
+    const agentHours = bead.metadata?.economics?.totals?.agent_hours;
+    const hasHours = typeof agentHours === "number" && agentHours > 0;
+    const hoursAsDays = hasHours ? agentHours / HOURS_PER_DAY : 0;
+
+    // A CLOSED bead's recorded hours are the answer, not an estimate: we know
+    // what it took. Reported hours beat the bucketed median here because a
+    // bead closed shortly after being claimed has a near-zero wall-clock even
+    // when real work happened.
+    if (hasHours && bead.status === "closed") {
+      stats.used_agent_hours = (stats.used_agent_hours ?? 0) + 1;
+      const days = Math.max(MIN_ESTIMATE_DAYS, hoursAsDays);
+      return {
+        days,
+        // No distribution of its own, so borrow the spread of the beads it
+        // sits with rather than asserting a made-up multiplier.
+        p90: days * spreadFor(type, pri),
+        source: `agent, ${agentHours.toFixed(2)}h`,
+      };
+    }
+
+    const kTP = `${type}|${pri}`;
+    const bucketTP = byTypeAndPri.get(kTP);
+    if (bucketTP && bucketTP.length >= MIN_SAMPLES_FOR_BUCKET) {
+      stats.used_type_pri++;
+      return fromBucket(median(bucketTP), p90(bucketTP),
+        `(${type}, P${pri}), n=${bucketTP.length}`);
+    }
+    const bucketT = byType.get(type);
+    if (bucketT && bucketT.length >= MIN_SAMPLES_FOR_BUCKET) {
+      stats.used_type++;
+      return fromBucket(median(bucketT), p90(bucketT), `(${type}), n=${bucketT.length}`);
+    }
+    stats.used_global++;
+    return fromBucket(globalMed, globalP90, `global, n=${shipped.length}`);
+
+    /**
+     * A bucketed estimate, floored at whatever this bead has ALREADY
+     * consumed.
+     *
+     * Live beads are the ones being scheduled, and `agent_hours` on a live
+     * bead is work spent, not work needed — taking it as the estimate says a
+     * half-finished bead is done. But it is a hard lower bound: a bead cannot
+     * end up having taken less than it already has. So the recorded hours
+     * become a floor under the bucket median rather than a replacement for
+     * it, and the source says when that floor is what answered.
+     */
+    function fromBucket(med, hi, source) {
+      const bucketDays = Math.max(MIN_ESTIMATE_DAYS, med);
+      if (hasHours && hoursAsDays > bucketDays) {
+        stats.used_spent_floor = (stats.used_spent_floor ?? 0) + 1;
+        return {
+          days: hoursAsDays,
+          p90: hoursAsDays * spreadFor(type, pri),
+          source: `${source}, ≥ ${agentHours.toFixed(2)}h spent`,
+        };
+      }
+      return { days: bucketDays, p90: hi, source };
+    }
+  }
+
+  return { estimate: estimateBead, stats };
+}
+
+function round(n, d) {
+  const f = 10 ** d;
+  return Math.round(n * f) / f;
+}
+
+/**
+ * What an estimate's `source` string means, for a reader who meets
+ * "(feature), n=8" in the drawer and asks what n is (bp-67g.47). Lives here
+ * because this file is what writes those strings: add a layer or rename one
+ * and both change together. Null for a string this build does not write.
+ */
+export function describeEstimateSource(source) {
+  if (typeof source !== "string" || !source) return null;
+  const floor = /, ≥ ([\d.]+)h spent$/.exec(source);
+  const base = floor ? source.slice(0, floor.index) : source;
+  let text;
+  let m;
+  if ((m = /^agent, ([\d.]+)h$/.exec(base))) {
+    text = `Not an estimate: this bead is closed and ${m[1]} hours of work were recorded on it, so that is what it took.`;
+  } else if ((m = /^\(([\w-]+), P(\d)\), n=(\d+)$/.exec(base))) {
+    text = `The median time of the ${m[3]} shipped ${m[1]} beads at P${m[2]} — the tightest bucket with at least ${MIN_SAMPLES_FOR_BUCKET} samples. n is that sample count; a small n deserves less trust.`;
+  } else if ((m = /^\(([\w-]+)\), n=(\d+)$/.exec(base))) {
+    text = `The median time of the ${m[2]} shipped ${m[1]} beads at any priority — fewer than ${MIN_SAMPLES_FOR_BUCKET} shipped at this type and priority, so the estimate fell back a layer. n is that sample count.`;
+  } else if ((m = /^global, n=(\d+)$/.exec(base))) {
+    text = `The median time of all ${m[1]} shipped beads — fewer than ${MIN_SAMPLES_FOR_BUCKET} shipped of this type, so the estimate fell back to the whole repo. n is that sample count.`;
+  } else {
+    return null;
+  }
+  if (floor) {
+    text += ` Floored at the ${floor[1]} hours already spent on it: a bead cannot end up having taken less than it already has.`;
+  }
+  return text;
+}

package/src/format.mjs

@@ -0,0 +1,22 @@
+// format.mjs — how a duration is said out loud.
+//
+// One ladder, because the alternative is three. The web had it in
+// economics.ts as `formatHours`, bead-detail.ts wrapped it as
+// `formatEstimate(days)`, and the SVG CLI had nothing and printed raw days —
+// which is how the legend came to read `Global median = 0.012d` (bp-67g.39).
+//
+// The model's native unit is days and every consumer means calendar days by
+// it, but on an agent-driven repo the medians are minutes. A number has to be
+// printed in a unit a person can hold.
+
+/** Hours as a short human string: "17m", "5.5h", "3.2d". */
+export function formatHours(h) {
+  if (h < 1) return `${(h * 60).toFixed(0)}m`;
+  if (h < 24) return `${h.toFixed(1)}h`;
+  return `${(h / 24).toFixed(1)}d`;
+}
+
+/** Days — the estimator's own unit — as the same short string. */
+export function formatDays(d) {
+  return formatHours(d * 24);
+}

package/src/history-walk.mjs

@@ -0,0 +1,170 @@
+// history-walk.mjs — the background walk over `bd history` (bp-67g.57).
+//
+// One shell-out per bead is ~0.4s and ~0.4MB on this repo, so nothing here
+// runs in the request path — and bd serialises its own calls, so the walk
+// must also stay OUT OF THE WAY of the request path: measured, a `bd list`
+// issued while two `bd history` calls ran took 1.9s instead of 0.56s, and a
+// first request that raced a walk took 4.9–7.4s instead of 1.0s. Hence one
+// call at a time (two concurrent history calls take as long as two in a
+// row), a `beforeEach` gate the server uses to hold the walk while a
+// `bd list` is in flight, and a start delay the server adds so the boot-time
+// prefetch is answered before the first history call is made.
+//
+// `refresh()` returns at once; the walk proceeds one bead at a time, writes
+// each answer into a per-project cache keyed
+// by bead id and `updated_at`, persists the cache next to the server's own
+// state file (tmpdir/beadcyte/history-<project>.json) so a restart does not
+// repeat the first pass, and tells the server when something changed so the
+// cached payload can be re-merged and re-stamped for the next poll.
+//
+// `runHistory` is injected so the walker is testable without bd.
+
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { firstInReviewAt, firstAssignedAt, planWalk } from "./review-hours.mjs";
+import { projectKey, stateDir } from "./server-state.mjs";
+
+const execFileAsync = promisify(execFile);
+
+/** Default runner: `bd history --json <id>` in the project directory. */
+export async function runBdHistory(project, id) {
+  const { stdout } = await execFileAsync("bd", ["history", "--json", id], {
+    cwd: project,
+    maxBuffer: 64 * 1024 * 1024,
+  });
+  const parsed = JSON.parse(stdout || "null");
+  return Array.isArray(parsed) ? parsed : [];
+}
+
+export function historyCacheFile(project, base) {
+  return join(stateDir(base), `history-${projectKey(project)}.json`);
+}
+
+function readCache(file) {
+  try {
+    const parsed = JSON.parse(readFileSync(file, "utf8"));
+    return parsed && typeof parsed === "object" ? parsed : {};
+  } catch {
+    return {};
+  }
+}
+
+function writeCache(file, cache) {
+  try {
+    mkdirSync(join(file, ".."), { recursive: true });
+    writeFileSync(file, JSON.stringify(cache));
+  } catch {
+    // A read-only tmpdir costs a repeat of the first pass next start; the
+    // walk itself still works from memory.
+  }
+}
+
+/**
+ * @param {object} opts
+ * @param {(project: string, id: string) => Promise<unknown[]>} [opts.runHistory]
+ * @param {number} [opts.concurrency]  default 1; see the header
+ * @param {() => Promise<void>} [opts.beforeEach]  awaited before every history call — the server hands in "no bd list in flight"
+ * @param {(project: string) => string} [opts.cacheFileFor]
+ * @param {(msg: string) => void} [opts.log]
+ * @param {() => number} [opts.now]
+ */
+export function createHistoryWalker(opts = {}) {
+  const runHistory = opts.runHistory ?? runBdHistory;
+  const concurrency = Math.max(1, opts.concurrency ?? 1);
+  const beforeEach = opts.beforeEach ?? (() => Promise.resolve());
+  const cacheFileFor = opts.cacheFileFor ?? ((project) => historyCacheFile(project));
+  const log = opts.log ?? (() => {});
+  const now = opts.now ?? (() => Date.now());
+
+  /** project -> { cache, file, running: Promise|null, pending: beads|null, listeners } */
+  const projects = new Map();
+
+  function state(project) {
+    let s = projects.get(project);
+    if (!s) {
+      const file = cacheFileFor(project);
+      s = { cache: readCache(file), file, running: null, pending: null, listeners: new Set() };
+      projects.set(project, s);
+    }
+    return s;
+  }
+
+  async function walk(project, beads) {
+    const s = state(project);
+    const ids = planWalk(beads, s.cache);
+    if (ids.length === 0) return { walked: 0, changed: false, ms: 0 };
+    const byId = new Map(beads.map((b) => [b.id, b]));
+    const started = now();
+    let changed = false;
+    let i = 0;
+    const worker = async () => {
+      while (i < ids.length) {
+        const id = ids[i++];
+        const b = byId.get(id);
+        try {
+          await beforeEach();
+          const entries = await runHistory(project, id);
+          const inReviewAt = firstInReviewAt(entries);
+          const assignedAt = firstAssignedAt(entries);
+          const prev = s.cache[id];
+          s.cache[id] = { updated_at: b?.updated_at, in_review_at: inReviewAt, assigned_at: assignedAt, walked_at: new Date(now()).toISOString() };
+          if (!prev || prev.in_review_at !== inReviewAt || prev.assigned_at !== assignedAt || prev.updated_at !== b?.updated_at) changed = true;
+        } catch (e) {
+          log(`history walk: ${id} failed: ${e?.message ?? e}`);
+        }
+      }
+    };
+    await Promise.all(Array.from({ length: Math.min(concurrency, ids.length) }, worker));
+    writeCache(s.file, s.cache);
+    const ms = now() - started;
+    log(`history walk: ${ids.length} bead${ids.length === 1 ? "" : "s"} in ${ms}ms (${concurrency} at a time)`);
+    if (changed) for (const fn of s.listeners) fn(project);
+    return { walked: ids.length, changed, ms };
+  }
+
+  return {
+    /** What the walk knows for this project right now. */
+    cache(project) {
+      return state(project).cache;
+    },
+
+    /**
+     * Start (or queue) a walk for these beads and return immediately. A walk
+     * already running finishes first; the newest bead list replaces any
+     * earlier pending one, so a burst of polls costs one extra walk at most.
+     */
+    refresh(project, beads) {
+      const s = state(project);
+      if (s.running) {
+        s.pending = beads;
+        return s.running;
+      }
+      const run = async () => {
+        let next = beads;
+        while (next) {
+          const current = next;
+          next = null;
+          s.pending = null;
+          await walk(project, current);
+          next = s.pending;
+        }
+      };
+      s.running = run().finally(() => {
+        s.running = null;
+      });
+      return s.running;
+    },
+
+    /** Called after a walk that changed something for the project. */
+    onChange(project, fn) {
+      state(project).listeners.add(fn);
+    },
+
+    /** For tests and diagnostics: await the in-flight walk, if any. */
+    idle(project) {
+      return state(project).running ?? Promise.resolve();
+    },
+  };
+}

package/src/index.mjs

@@ -0,0 +1,5 @@
+// Library entry — expose the estimator and scheduler so other tools can compose
+// them (dashboards, seat-load views, wins-over-time). The CLI is at
+// `src/cli.mjs` and mounted at `bin/beadcyte.mjs`.
+export { buildEstimator } from "./estimator.mjs";
+export { scheduleGreedy } from "./scheduler.mjs";