loki-mode 10.6.0 → 10.6.6

package/packages/control-plane/src/server/runs.ts

@@ -0,0 +1,116 @@
+import { and, asc, desc, eq, gte, lt, sql } from "drizzle-orm";
+import { fold, partialCost } from "../../../../loki-ts/src/engine10/events.ts";
+import type { EventEnvelope } from "../../../../loki-ts/src/engine10/types.ts";
+import type { Db } from "../db/migrate.ts";
+import { events, runs } from "../db/schema.ts";
+
+const str = (x: unknown): string | null => (typeof x === "string" ? x : null);
+
+export function loadEvents(db: Db, sourceId: string, runId: string): EventEnvelope[] {
+  return db.select().from(events).where(and(eq(events.sourceId, sourceId), eq(events.runId, runId))).orderBy(asc(events.seq)).all()
+    .map((r) => ({ v: 1, seq: r.seq, ts: r.ts, run: r.runId, type: r.type, stage: r.stage, data: r.data }) as EventEnvelope);
+}
+
+/** Rebuilds the runs projection row from stored events with the real fold(). The only writer of `runs`. */
+export function rebuildRun(db: Db, sourceId: string, runId: string): void {
+  const evs = loadEvents(db, sourceId, runId);
+  if (evs.length === 0) return;
+  const f = fold(evs);
+  const pc = partialCost(evs, f.run.tampered);
+  const sd = f.run.started?.data ?? {};
+  const done = f.run.completed?.data ?? {};
+  const pr = evs.find((e) => e.type === "pr.opened")?.data;
+  const row = {
+    sourceId, runId,
+    originRepo: str(sd.origin_repo), issueRef: str(sd.issue_ref), taskSource: str(sd.task_source),
+    provider: str(sd.provider), model: str(sd.model),
+    startedAt: f.run.started?.ts ?? evs[0]?.ts ?? null, endedAt: f.run.completed?.ts ?? null,
+    verdict: f.run.verdict,
+    prUrl: str(done.pr_url) ?? str(pr?.url), prDraft: typeof pr?.draft === "boolean" ? Number(pr.draft) : null,
+    costUsd: f.cost.usd, partialUsd: pc.usd, measuredSessions: pc.measured, totalSessions: pc.total,
+    inputTokens: f.cost.inputTokens, outputTokens: f.cost.outputTokens,
+    wallS: typeof done.wall_s === "number" ? done.wall_s : null,
+    lastSeq: f.lastSeq, lastEventAt: evs[evs.length - 1]?.ts ?? null, tampered: Number(f.run.tampered),
+  };
+  // conflict is owned by ingest, so it is left out of the update set and survives a rebuild
+  db.insert(runs).values(row).onConflictDoUpdate({ target: [runs.sourceId, runs.runId], set: row }).run();
+}
+
+const parseRun = (r: typeof runs.$inferSelect) => ({
+  source_id: r.sourceId, run_id: r.runId, origin_repo: r.originRepo, issue_ref: r.issueRef, task_source: r.taskSource,
+  provider: r.provider, model: r.model, started_at: r.startedAt, ended_at: r.endedAt, verdict: r.verdict,
+  pr_url: r.prUrl, pr_draft: r.prDraft === null ? null : r.prDraft === 1,
+  cost_usd: r.costUsd, partial_usd: r.partialUsd, measured_sessions: r.measuredSessions, total_sessions: r.totalSessions,
+  input_tokens: r.inputTokens, output_tokens: r.outputTokens, wall_s: r.wallS, last_seq: r.lastSeq,
+  last_event_at: r.lastEventAt, tampered: r.tampered === 1, conflict: r.conflict === 1,
+  status: r.endedAt ? "completed" : "running", elapsed_s: elapsedS(r),
+});
+// A run with no run.completed yet is running: add its live fields (stage, files) from the stored events.
+const withLive = (db: Db, r: typeof runs.$inferSelect) => {
+  const p = parseRun(r);
+  return r.endedAt ? { ...p, current_stage: null, files_touched: [] as string[] } : { ...p, ...liveInfo(loadEvents(db, r.sourceId, r.runId)) };
+};
+
+/** Live view of one run from its events so far: current stage, files touched. Elapsed is computed per request. */
+export function liveInfo(evs: EventEnvelope[]) {
+  const open: string[] = [];
+  for (const e of evs) {
+    if (e.stage === null || !e.type.startsWith("stage.")) continue;
+    const i = open.indexOf(e.stage);
+    if (e.type === "stage.started") { if (i < 0) open.push(e.stage); } else if (i >= 0) open.splice(i, 1);
+  }
+  const lastStage = [...evs].reverse().find((e) => e.stage !== null && e.type.startsWith("stage."))?.stage ?? null;
+  const files = new Set<string>();
+  for (const e of evs) for (const k of ["changed_files", "files"]) {
+    const v = e.data[k];
+    if (Array.isArray(v)) for (const f of v) if (typeof f === "string") files.add(f);
+  }
+  return { current_stage: open[open.length - 1] ?? lastStage, files_touched: [...files] };
+}
+
+const elapsedS = (r: typeof runs.$inferSelect): number | null => {
+  if (!r.startedAt) return null;
+  const end = r.endedAt ? Date.parse(r.endedAt) : Date.now();
+  return Math.max(0, (end - Date.parse(r.startedAt)) / 1000);
+};
+
+export interface ListQuery { verdict?: string; repo?: string; since?: string; until?: string; limit?: number; cursor?: string }
+
+export function listRuns(db: Db, q: ListQuery) {
+  const limit = Math.min(Math.max(q.limit ?? 50, 1), 200);
+  const offset = Math.max(Number.parseInt(q.cursor ?? "0", 10) || 0, 0); // ponytail: cursor is an opaque offset, keyset when write rate makes pages shift
+  const where = and(
+    q.verdict ? eq(runs.verdict, q.verdict) : undefined,
+    q.repo ? eq(runs.originRepo, q.repo) : undefined,
+    q.since ? gte(runs.startedAt, q.since) : undefined,
+    q.until ? lt(runs.startedAt, q.until) : undefined,
+  );
+  const total = db.select({ n: sql<number>`count(*)` }).from(runs).where(where).get()?.n ?? 0;
+  const rows = db.select().from(runs).where(where).orderBy(desc(runs.startedAt), asc(runs.runId)).limit(limit).offset(offset).all();
+  return { runs: rows.map((r) => withLive(db, r)), total, next_cursor: offset + rows.length < total ? String(offset + rows.length) : null };
+}
+
+/** Summary row plus the folded detail: stage timeline, receipt, not_proven. Null when the run is unknown. */
+export function runDetail(db: Db, sourceId: string, runId: string) {
+  const r = db.select().from(runs).where(and(eq(runs.sourceId, sourceId), eq(runs.runId, runId))).get();
+  if (!r) return null;
+  const evs = loadEvents(db, sourceId, runId);
+  const stages: { stage: string; started_at: string | null; ended_at: string | null; status: string }[] = [];
+  for (const e of evs) {
+    if (e.stage === null || !e.type.startsWith("stage.")) continue;
+    let s = stages.find((x) => x.stage === e.stage);
+    if (!s) stages.push((s = { stage: e.stage, started_at: null, ended_at: null, status: "started" }));
+    if (e.type === "stage.started") s.started_at = e.ts;
+    else { s.ended_at = e.ts; s.status = e.type.slice("stage.".length); }
+  }
+  const f = fold(evs);
+  const done = f.run.completed?.data;
+  const sealed = evs.find((e) => e.type === "receipt.sealed")?.data;
+  return {
+    ...withLive(db, r),
+    stages,
+    stages_completed: f.completed,
+    receipt: sealed ? { sha256: str(sealed.receipt_sha256), signed: sealed.signed === true, verdict: str(sealed.verdict), path: str(sealed.path) } : null,
+    not_proven: Array.isArray(done?.not_proven) ? (done.not_proven as string[]) : [],
+  };
+}

package/packages/control-plane/src/server/serve.ts

@@ -0,0 +1,15 @@
+// Entry for `loki control serve` (bundled to dist/server.js). Config comes from env: PORT, LOKI_CONTROL_DB.
+// Binds loopback only. LOKI_CONTROL_PORT_FALLBACK=1 (set for the default port, never for an explicit --port)
+// retries on any free port when PORT is taken; the URL printed below is always the real one.
+import { createApp } from "./app.ts";
+
+const { app } = createApp({ dbPath: process.env.LOKI_CONTROL_DB ?? "control.db" });
+const serve = (port: number) => Bun.serve({ port, hostname: "127.0.0.1", fetch: app.fetch });
+let s: ReturnType<typeof serve>;
+try {
+  s = serve(Number(process.env.PORT ?? 0));
+} catch (e) {
+  if ((e as { code?: string }).code !== "EADDRINUSE" || process.env.LOKI_CONTROL_PORT_FALLBACK !== "1") throw e;
+  s = serve(0);
+}
+console.log(`loki-control listening on ${s.url}`);

package/packages/control-plane/src/shipper/backfill.ts

@@ -0,0 +1,40 @@
+// CP-02 backfill: ship every .loki/runs/*/events.jsonl of a repo through the same shipRun path as the live hook.
+import { readdirSync } from "node:fs";
+import { join } from "node:path";
+import { backoffMs, hasSpool, shipEnabled, shipRun, sourceId, type ShipResult } from "./ship.ts";
+
+export interface BackfillOpts {
+  repoDir: string;
+  env?: NodeJS.ProcessEnv;
+  attempts?: number; // per run, with backoff between
+  sleep?: (ms: number) => Promise<void>;
+  fetchImpl?: typeof fetch;
+}
+
+export async function backfill(o: BackfillOpts): Promise<{ runs: number; sent: number; failed: string[] }> {
+  const env = o.env ?? process.env;
+  const url = shipEnabled(env);
+  if (!url) return { runs: 0, sent: 0, failed: [] };
+  const sleep = o.sleep ?? ((ms) => new Promise<void>((r) => setTimeout(r, ms)));
+  const attempts = o.attempts ?? 3;
+  const root = join(o.repoDir, ".loki", "runs");
+  let ids: string[] = [];
+  try { ids = readdirSync(root).sort(); } catch { return { runs: 0, sent: 0, failed: [] }; }
+  const source = sourceId(o.repoDir);
+  let runs = 0, sent = 0;
+  const failed: string[] = [];
+  for (const id of ids) {
+    const runDir = join(root, id);
+    if (!hasSpool(runDir)) continue;
+    runs++;
+    let res: ShipResult = { ok: false, sent: 0, acked_seq: -1, pending: 0 };
+    for (let a = 1; a <= attempts; a++) {
+      res = await shipRun({ runDir, url, source, token: env.LOKI_CONTROL_TOKEN, fetchImpl: o.fetchImpl });
+      sent += res.sent;
+      if (res.ok) break;
+      if (a < attempts) await sleep(backoffMs(a));
+    }
+    if (!res.ok) failed.push(id);
+  }
+  return { runs, sent, failed };
+}

package/packages/control-plane/src/shipper/ship.ts

@@ -0,0 +1,95 @@
+// CP-02 shipper (docs/v10/CONTROL-PLANE.md section 5). Reads events.jsonl (the spool, never written here) and POSTs
+// batches to /v1/ingest. Progress lives in ship.json next to the log. Never throws: a failure returns ok:false.
+import { createHash } from "node:crypto";
+import { existsSync, readFileSync, realpathSync, renameSync, writeFileSync } from "node:fs";
+import { hostname } from "node:os";
+import { join } from "node:path";
+import { readEvents } from "../../../../loki-ts/src/engine10/events.ts";
+import type { EventEnvelope } from "../../../../loki-ts/src/engine10/types.ts";
+import { redactSecrets } from "../../../../loki-ts/src/util/redact.ts";
+
+export const BATCH = 200;
+const MAX_BYTES = 800_000; // the service rejects a body over 1 MB
+
+/** sha256(hostname NUL realpath(repo)), 16 hex. No raw path or hostname leaves the machine. */
+export function sourceId(repoDir: string): string {
+  let real = repoDir;
+  try { real = realpathSync(repoDir); } catch { /* keep as given */ }
+  return createHash("sha256").update(`${hostname()}\0${real}`).digest("hex").slice(0, 16);
+}
+
+const redactDeep = (x: unknown): unknown =>
+  typeof x === "string" ? redactSecrets(x)
+  : Array.isArray(x) ? x.map(redactDeep)
+  : x && typeof x === "object" ? Object.fromEntries(Object.entries(x).map(([k, v]) => [k, redactDeep(v)]))
+  : x;
+
+export const redactEvent = (e: EventEnvelope): EventEnvelope => ({ ...e, data: redactDeep(e.data) as EventEnvelope["data"] });
+
+/** Backoff 1, 2, 4 ... 60 s with up to 25% jitter. */
+export const backoffMs = (failures: number, rnd: () => number = Math.random): number =>
+  Math.min(60_000, 1000 * 2 ** Math.max(0, failures - 1)) * (1 + 0.25 * rnd());
+
+export interface ShipState { acked_seq: number; url: string; skipped?: number }
+export function readShip(runDir: string, url: string): ShipState {
+  try {
+    const s = JSON.parse(readFileSync(join(runDir, "ship.json"), "utf8")) as Partial<ShipState>;
+    if (s.url === url && Number.isInteger(s.acked_seq)) return { acked_seq: s.acked_seq as number, url };
+  } catch { /* none or corrupt: start over; the server dedupes */ }
+  return { acked_seq: -1, url };
+}
+function writeShip(runDir: string, s: ShipState): void {
+  const tmp = join(runDir, `ship.json.${process.pid}.tmp`);
+  writeFileSync(tmp, JSON.stringify(s) + "\n");
+  renameSync(tmp, join(runDir, "ship.json"));
+}
+
+export interface ShipOpts {
+  runDir: string; // <repo>/.loki/runs/<id>
+  url: string; // base URL of the control plane
+  source: string;
+  token?: string | undefined;
+  timeoutMs?: number;
+  fetchImpl?: typeof fetch | undefined; // tests
+}
+export interface ShipResult { ok: boolean; sent: number; acked_seq: number; pending: number }
+
+/** One pass: ship every event above acked_seq. ship.json is written only after a 2xx (or a 409, which a retry cannot fix). */
+export async function shipRun(o: ShipOpts): Promise<ShipResult> {
+  const st = readShip(o.runDir, o.url);
+  const all = readEvents(join(o.runDir, "events.jsonl"));
+  let lines = 0;
+  try { lines = readFileSync(join(o.runDir, "events.jsonl"), "utf8").split("\n").filter((l) => l.trim() !== "").length; } catch { /* none */ }
+  const skipped = lines - all.length; // invalid lines (for example a tamper forgery) are never sent: the service 400s a whole batch on one
+  const todo = all.filter((e) => e.seq > st.acked_seq).sort((a, b) => a.seq - b.seq);
+  if (todo.length === 0) return { ok: true, sent: 0, acked_seq: st.acked_seq, pending: 0 };
+  const run_id = todo[0]!.run;
+  const f = o.fetchImpl ?? fetch;
+  let sent = 0;
+  const fail = (): ShipResult => ({ ok: false, sent, acked_seq: st.acked_seq, pending: todo.length - sent });
+  for (let i = 0; i < todo.length;) {
+    const batch: EventEnvelope[] = [];
+    for (let bytes = 0; i < todo.length && batch.length < BATCH; i++) {
+      bytes += JSON.stringify(todo[i]).length;
+      if (batch.length > 0 && bytes > MAX_BYTES) break;
+      batch.push(redactEvent(todo[i]!));
+    }
+    try {
+      const r = await f(`${o.url.replace(/\/+$/, "")}/v1/ingest`, {
+        method: "POST",
+        headers: { "content-type": "application/json", ...(o.token ? { authorization: `Bearer ${o.token}` } : {}) },
+        body: JSON.stringify({ source: o.source, run_id, events: batch }),
+        signal: AbortSignal.timeout(o.timeoutMs ?? 10_000),
+      });
+      if (!r.ok && r.status !== 409) return fail();
+    } catch { return fail(); }
+    st.acked_seq = batch[batch.length - 1]!.seq;
+    st.skipped = skipped;
+    sent += batch.length;
+    try { writeShip(o.runDir, st); } catch { return fail(); }
+  }
+  return { ok: true, sent, acked_seq: st.acked_seq, pending: 0 };
+}
+
+export const shipEnabled = (env: NodeJS.ProcessEnv): string | null => env.LOKI_CONTROL_URL || null;
+export const hasSpool = (runDir: string): boolean => existsSync(join(runDir, "events.jsonl"));