@wrenautomation/aerie 0.8.5 → 0.8.6

package/dist/bin.js

@@ -30266,7 +30266,7 @@ function oneOf(name, col, values) {
   const list2 = values.map((v) => `'${v.replaceAll("'", "''")}'`).join(", ");
   return check2(name, sql.raw(`"${col.name}" IN (${list2})`));
 }
-var aerie, ACTOR_KINDS, AREA_ROLES, TASK_ROLES, LAYOUTS, PLACES, at, area, actor, areaRole, task, taskArea, taskActor, taskHolder, view, tsvector, taskEvent, milestone, mirrorFile, setting, coordGen, coordLease, coordClosing, coordTask, coordQueue, decision, routeRule, intake2, remoteCall;
+var aerie, ACTOR_KINDS, AREA_ROLES, TASK_ROLES, LAYOUTS, PLACES, at, area, actor, areaRole, task, taskArea, taskActor, taskHolder, view, tsvector, taskEvent, milestone, mirrorFile, setting, coordGen, coordLease, coordClosing, coordTask, coordLeaseEvent, coordQueue, decision, routeRule, intake2, remoteCall;
 var init_schema2 = __esm({
   "src/schema.ts"() {
     "use strict";
@@ -30565,6 +30565,27 @@ var init_schema2 = __esm({
       },
       (t) => [primaryKey({ columns: [t.ns, t.taskId], name: "pk_coord_task" })]
     );
+    coordLeaseEvent = aerie.table(
+      "coord_lease_event",
+      {
+        id: bigserial({ mode: "number" }).notNull(),
+        ns: text2().notNull(),
+        taskId: text2("task_id").notNull(),
+        holder: text2().notNull(),
+        gen: bigint4({ mode: "number" }).notNull(),
+        /** claim | take (from the queue) | reclaim | release | done | expire (freed by `by`) | lost (lapsed, seen at renew) | reap (lapsed run lane) */
+        kind: text2().notNull(),
+        queued: boolean4().notNull(),
+        expires: at("expires").notNull(),
+        by: text2().notNull(),
+        at: at("at").notNull()
+      },
+      (t) => [
+        primaryKey({ columns: [t.id], name: "pk_coord_lease_event" }),
+        index("coord_lease_event_task_idx").on(t.ns, t.taskId),
+        index("coord_lease_event_at_idx").on(t.ns, t.at)
+      ]
+    );
     coordQueue = aerie.table(
       "coord_queue",
       {
@@ -33024,6 +33045,7 @@ function pgCoord(o) {
   const C = coordClosing;
   const T = coordTask;
   const Q = coordQueue;
+  const E = coordLeaseEvent;
   const lockQueue = (tx) => tx.execute(
     sql`select pg_advisory_xact_lock(hashtextextended(${`aerie.coord:${ns}:queue`}, 0))`
   );
@@ -33043,7 +33065,7 @@ function pgCoord(o) {
     if (!hit[0] || hit[0].status === name) return;
     await aerieStore({ db: tx }).edit(id, { status: name }, by);
   };
-  const lease = async (tx, id, holder, leaseMs, queued, key) => {
+  const lease = async (tx, id, holder, leaseMs, queued, key, how = "claim") => {
     const t = clock();
     const others = await rows2(
       tx,
@@ -33056,13 +33078,17 @@ function pgCoord(o) {
     );
     const gen = num(g?.gen);
     await tx.execute(sql`
-      insert into ${L} (ns, task_id, holder, gen, expires, queued, lane, pin, take_key)
+      with up as (insert into ${L} (ns, task_id, holder, gen, expires, queued, lane, pin, take_key)
       values (${ns}, ${id}, ${holder}, ${gen}, ${plus(t, leaseMs)}, ${!!queued}, ${queued?.lane ?? null}, ${queued?.pin ?? null}, ${key ?? null})
       on conflict (ns, task_id, holder) do update set gen = excluded.gen, expires = excluded.expires,
         take_key = coalesce(excluded.take_key, ${L}.take_key),
         queued = excluded.queued or ${L}.queued,
         lane = case when excluded.queued then excluded.lane else ${L}.lane end,
-        pin = case when excluded.queued then excluded.pin else ${L}.pin end`);
+        pin = case when excluded.queued then excluded.pin else ${L}.pin end
+      returning task_id, holder, gen, queued, expires, (xmax = 0) as fresh)
+      insert into ${E} (ns, task_id, holder, gen, kind, queued, expires, by, at)
+      select ${ns}, task_id, holder, gen, case when fresh then ${how} else 'reclaim' end, queued, expires, holder, ${t}
+      from up`);
     await touch(tx, id, t);
     await setStatus(tx, id, "doing", holder);
     return { gen, others: others.map((r) => r.holder) };
@@ -33075,8 +33101,13 @@ function pgCoord(o) {
     if (!has.length) return { last: true, closeNow: false, status: "ready" };
     const gone = await rows2(
       tx,
-      sql`delete from ${L} where ns = ${ns} and task_id = ${id} and holder = any(${textArr(holders)})
-          returning holder, queued, lane, pin`
+      sql`with gone as (delete from ${L} where ns = ${ns} and task_id = ${id} and holder = any(${textArr(holders)})
+            returning holder, gen, queued, lane, pin, expires),
+          logged as (insert into ${E} (ns, task_id, holder, gen, kind, queued, expires, by, at)
+            select ${ns}, ${id}, holder, gen,
+              case when holder <> ${x.by} then 'expire' else ${x.done ? "done" : x.why ?? "release"} end,
+              queued, expires, ${x.by}, ${t} from gone)
+          select holder, queued, lane, pin from gone`
     );
     if (x.done && gone.some((l) => l.holder === x.by))
       await tx.execute(sql`insert into ${C} (ns, task_id, holder, reason, at)
@@ -33156,7 +33187,7 @@ function pgCoord(o) {
           );
           if (!still.length) continue;
           lost.push({ id: l.id, holder: l.holder });
-          await drop(tx, l.id, [l.holder], { queue: l.queued, by: l.holder });
+          await drop(tx, l.id, [l.holder], { queue: l.queued, by: l.holder, why: "lost" });
         }
       }
       await tx.execute(sql`update ${L} set expires = ${plus(t, leaseMs)}
@@ -33284,7 +33315,8 @@ function pgCoord(o) {
               sql`select 1 from ${L} where ns = ${ns} and task_id = ${l.id} and holder = ${l.holder}
                     and expires < ${t}`
             );
-            if (still.length) await drop(tx, l.id, [l.holder], { queue: l.queued, by: l.holder });
+            if (still.length)
+              await drop(tx, l.id, [l.holder], { queue: l.queued, by: l.holder, why: "reap" });
           }
         }
         if (x.cap !== void 0) {
@@ -33314,10 +33346,28 @@ function pgCoord(o) {
         holder,
         leaseMs,
         { lane: item.lane, pin: item.pin },
-        x.key
+        x.key,
+        "take"
       );
       return { id: item.id, gen: r.gen };
     }),
+    leaseEvents: async ({ since, limit = 1e4 }) => (await rows2(
+      db,
+      sql`select id::float8 as id, task_id, holder, gen::float8 as gen, kind, queued, ${ms(sql`expires`)} as expires,
+              by, ${ms(sql`at`)} as at from ${E} where ns = ${ns} and at >= ${at3(since)} order by id limit ${limit}`
+    )).map(
+      (r) => ({
+        id: num(r.id),
+        task: String(r.task_id),
+        holder: String(r.holder),
+        gen: num(r.gen),
+        kind: r.kind,
+        queued: !!r.queued,
+        expires: num(r.expires),
+        by: String(r.by),
+        at: num(r.at)
+      })
+    ),
     fence: async (id, holder, gen) => (await rows2(
       db,
       sql`select 1 from ${L} where ns = ${ns} and task_id = ${id} and holder = ${holder}
@@ -33876,7 +33926,13 @@ var init_remote = __esm({
         HOLDER,
         MS
       ]),
-      fence: read([ID2, HOLDER, external_exports.int().nonnegative()])
+      fence: read([ID2, HOLDER, external_exports.int().nonnegative()]),
+      leaseEvents: read([
+        external_exports.strictObject({
+          since: external_exports.number().nonnegative(),
+          limit: external_exports.int().positive().max(1e4).optional()
+        })
+      ])
     };
     OPS = { store: STORE_OPS, coord: COORD_OPS };
     CALL = external_exports.object({
@@ -40739,6 +40795,69 @@ function registerIntakeCommands(program3, ctx) {
   );
 }
 
+// src/coord/audit.ts
+var START = /* @__PURE__ */ new Set(["claim", "take", "reclaim"]);
+function leaseAudit(events, held, o) {
+  const starts = {};
+  const ends = {};
+  const doubleDelivery = [];
+  const orphans = [];
+  const mins = (ms2) => Math.round(ms2 / 6e3) / 10;
+  const open3 = /* @__PURE__ */ new Map();
+  for (const e of [...events].sort((a, b) => a.id - b.id)) {
+    const byTask = open3.get(e.task) ?? /* @__PURE__ */ new Map();
+    open3.set(e.task, byTask);
+    if (START.has(e.kind)) {
+      starts[e.kind] = (starts[e.kind] ?? 0) + 1;
+      if (!e.queued) continue;
+      const had = byTask.get(e.holder);
+      if (e.kind !== "reclaim" || !had) {
+        const others = [...byTask].filter(([h]) => h !== e.holder);
+        if (others.length)
+          doubleDelivery.push({
+            task: e.task,
+            holder: e.holder,
+            at: e.at,
+            open: others.map(([h, l]) => ({ holder: h, since: l.since }))
+          });
+      }
+      byTask.set(e.holder, { since: had?.since ?? e.at });
+      continue;
+    }
+    ends[e.kind] = (ends[e.kind] ?? 0) + 1;
+    byTask.delete(e.holder);
+    if ((e.kind === "expire" || e.kind === "reap") && e.at - e.expires > o.graceMs)
+      orphans.push({
+        task: e.task,
+        holder: e.holder,
+        expires: e.expires,
+        lapsedMin: mins(e.at - e.expires),
+        freed: { kind: e.kind, by: e.by, at: e.at }
+      });
+  }
+  for (const t of held)
+    for (const l of t.holders)
+      if (o.now - l.expires > o.graceMs)
+        orphans.push({
+          task: t.id,
+          holder: l.holder,
+          expires: l.expires,
+          lapsedMin: mins(o.now - l.expires),
+          freed: null
+        });
+  return {
+    since: o.since,
+    until: o.now,
+    graceMin: mins(o.graceMs),
+    events: events.length,
+    starts,
+    ends,
+    doubleDelivery,
+    orphans,
+    ok: !doubleDelivery.length && !orphans.length
+  };
+}
+
 // src/cli-run.ts
 init_core3();
 
@@ -41496,6 +41615,13 @@ async function work(o) {
 
 // src/cli-run.ts
 var each = (v, prev = []) => [...prev, v];
+function sinceOf(v, now) {
+  const m = /^(\d+(?:\.\d+)?)([dh])$/.exec(v);
+  if (m) return now - Number(m[1]) * (m[2] === "d" ? 864e5 : 36e5);
+  const t = Date.parse(v);
+  if (Number.isNaN(t)) throw new Error(`aerie: --since ${v}: give an ISO time or Nd / Nh`);
+  return t;
+}
 var print2 = (x) => void process.stdout.write(typeof x === "string" ? x : `${JSON.stringify(x, null, 2)}
 `);
 async function pickerOf(a) {
@@ -41727,6 +41853,27 @@ function registerRunCommands(program3, ctx) {
 ` : "No pool sessions.\n"
     );
   });
+  program3.command("coord").description("the coord store").command("audit").description("double delivery and orphan holders from the lease log (WA-1364.3 AC #5)").option("--since <when>", "ISO time, or Nd / Nh back", "3d").option("--grace <min>", "minutes past expiry before a lease counts as orphaned", "60").option("--json", "JSON out").action(
+    (o) => board(async (_a3, c) => {
+      if (!c.leaseEvents) throw new Error(`aerie: coord store ${c.name} keeps no lease log`);
+      const now = Date.now();
+      const since = sinceOf(String(o.since), now);
+      const [events, held] = await Promise.all([c.leaseEvents({ since }), c.held()]);
+      const r = leaseAudit(events, held, { since, now, graceMs: Number(o.grace) * 6e4 });
+      print2(
+        o.json ? r : `${r.ok ? "ok" : "FAIL"}: ${r.events} lease events since ${new Date(since).toISOString()}; starts ${JSON.stringify(r.starts)}, ends ${JSON.stringify(r.ends)}; ${r.doubleDelivery.length} double deliveries, ${r.orphans.length} orphans
+${[
+          ...r.doubleDelivery.map(
+            (d) => `double  ${d.task}  ${d.holder} at ${new Date(d.at).toISOString()}; open: ${d.open.map((x) => x.holder).join(", ")}`
+          ),
+          ...r.orphans.map(
+            (x) => `orphan  ${x.task}  ${x.holder}  ${x.lapsedMin} min past expiry${x.freed ? `, ${x.freed.kind} by ${x.freed.by}` : ", still held"}`
+          )
+        ].map((l) => `${l}
+`).join("")}`
+      );
+    })
+  );
   program3.command("events").description("the event log across tasks").command("search").argument("<words>", 'web-search words: "a phrase", -not, or').option("--task <id>", "one task").option("--kind <kind>", "one kind, e.g. run.tool or routed").option("--limit <n>", "at most n", "50").option("--json", "JSON out").action(
     (words, o) => ctx.withStore(async (a) => {
       const rs = await a.searchEvents(words, {