omp-fabric 1.13.1 → 1.14.0

package/dist/compaction/lcm-runtime.js

@@ -867,6 +867,20 @@ function migrateSessions(options) {
 import crypto3 from "node:crypto";
 var DEFAULT_LCM_BUDGET = { calls: Number.POSITIVE_INFINITY, inputTokens: Number.POSITIVE_INFINITY, outputTokens: Number.POSITIVE_INFINITY, cost: Number.POSITIVE_INFINITY, wallMs: Number.POSITIVE_INFINITY, sessionCalls: Number.POSITIVE_INFINITY };
 var DEFAULT_LEAF_ENTRIES = 32;
+var DEFAULT_MAINTENANCE_CONCURRENCY = 3;
+var LEASE_MS = 3e4;
+var LEASE_SWEEP_GRACE_MS = 6e4;
+var LcmRejection = class extends Error {
+  reason;
+  constructor(reason) {
+    super(reason);
+    this.name = "LcmRejection";
+    this.reason = reason;
+  }
+};
+var isLcmRejection = (error) => error instanceof LcmRejection;
+var isLcmCapacityRejection = (error) => isLcmRejection(error) && (error.reason === "project lease held" || error.reason === "budget exhausted");
+var LEGACY_CONTENTION_RETIREMENTS = /* @__PURE__ */ new Set(["job lease held", "project lease held", "LcmRejection: job lease held", "LcmRejection: project lease held"]);
 var hash3 = (v) => hashLcmPayload(v);
 var parse = (v) => JSON.parse(String(v));
 var rawHash = (e) => e.payloadHash;
@@ -881,9 +895,14 @@ AND CAST(json_extract(j.payload,'$.nextRetryAt') AS INTEGER)<=?
 AND (? IS NULL OR json_extract(n.payload,'$.sessionId')=?)
 ORDER BY CAST(json_extract(j.payload,'$.priority') AS INTEGER) DESC, j.created_at, j.job_id
 LIMIT ?`;
+var LIVE_LEASES_SQL = `SELECT coalesce(json_extract(n.payload,'$.sessionId'),'') sessionId
+FROM maintenance_jobs j
+LEFT JOIN summary_nodes n ON n.node_id=json_extract(j.payload,'$.nodeId') AND n.project_key=j.project_key
+WHERE j.project_key=? AND j.status='running' AND j.job_id<>? AND CAST(coalesce(json_extract(j.payload,'$.leaseUntil'),0) AS INTEGER)>?`;
 var LcmMaintenance = class {
   constructor(ledger, options = {}) {
     this.ledger = ledger;
+    this.maxConcurrentJobs = options.maxConcurrentJobs ?? DEFAULT_MAINTENANCE_CONCURRENCY;
     this.projectKey = ledger.project.key;
     this.now = options.now ?? Date.now;
     this.ownerId = options.ownerId ?? crypto3.randomUUID();
@@ -905,9 +924,27 @@ var LcmMaintenance = class {
   maxOutputChars;
   budget;
   modelTimeoutMs;
+  maxConcurrentJobs;
   budgetPolicy() {
     return { ...this.budget };
   }
+  get concurrencyLimit() {
+    const raw = typeof this.maxConcurrentJobs === "function" ? this.maxConcurrentJobs() : this.maxConcurrentJobs;
+    return Number.isFinite(raw) ? Math.max(1, Math.floor(raw)) : DEFAULT_MAINTENANCE_CONCURRENCY;
+  }
+  liveLeases(db, at, excludeJobId) {
+    return db.prepare(LIVE_LEASES_SQL).all(this.projectKey, excludeJobId ?? "", at - LEASE_SWEEP_GRACE_MS);
+  }
+  admits(db, at, sessionId, leases) {
+    const day = this.day(at);
+    const project = this.usage(db, day);
+    const session = this.usage(db, day, sessionId);
+    let sessionReserved = 0;
+    for (const lease of leases) if (lease.sessionId === sessionId) sessionReserved += 1;
+    const reserved = leases.length + 1;
+    const fits = (used, cap) => used < cap && used + (project.calls > 0 ? used / project.calls * reserved : 0) <= cap;
+    return project.calls + leases.length < this.budget.calls && fits(project.inputTokens, this.budget.inputTokens) && fits(project.outputTokens, this.budget.outputTokens) && fits(project.cost, this.budget.cost) && fits(project.wallMs, this.budget.wallMs) && session.calls + sessionReserved < this.budget.sessionCalls;
+  }
   recordRevision(db, node) {
     if (!node.text) return;
     const row = db.prepare("SELECT coalesce(max(revision),0) n FROM summary_node_revisions WHERE node_id=?").get(node.nodeId);
@@ -990,7 +1027,7 @@ var LcmMaintenance = class {
   claimableJobs(sessionId, limit = 64) {
     const t = this.now();
     const scope = sessionId ?? null;
-    return this.ledger.readOnly((db) => db.prepare(CLAIMABLE_JOBS_SQL).all(this.projectKey, t, t, t, scope, scope, limit).map((r) => {
+    return this.ledger.readOnly((db) => db.prepare(CLAIMABLE_JOBS_SQL).all(this.projectKey, t - LEASE_SWEEP_GRACE_MS, t, t, scope, scope, limit).map((r) => {
       const job = parse(r.payload);
       if (job.projectKey !== this.projectKey) throw new Error("job project does not match ledger project");
       return job;
@@ -1001,25 +1038,43 @@ var LcmMaintenance = class {
   }
   isClaimable(job) {
     const t = this.now();
-    return (job.state === "pending" || job.state === "running" && (job.leaseUntil ?? 0) <= t) && job.eligibleAt <= t && job.nextRetryAt <= t;
+    return (job.state === "pending" || job.state === "running" && (job.leaseUntil ?? 0) + LEASE_SWEEP_GRACE_MS <= t) && job.eligibleAt <= t && job.nextRetryAt <= t;
   }
-  sweepExpiredLeases(graceMs = 0) {
+  sweepExpiredLeases() {
     return this.ledger.transaction((db) => {
-      const t = this.now();
+      const t = this.now() - LEASE_SWEEP_GRACE_MS;
       const rows = db.prepare("SELECT payload FROM maintenance_jobs WHERE project_key=? AND status=?").all(this.projectKey, "running");
       const swept = [];
       for (const row of rows) {
         const old = parse(row.payload);
         if (old.projectKey !== this.projectKey) throw new Error("job project does not match ledger project");
-        if ((old.leaseUntil ?? 0) + graceMs > t) continue;
+        if ((old.leaseUntil ?? 0) > t) continue;
+        const now = this.now();
         const { ownerId: _ownerId, leaseToken: _leaseToken, leaseUntil: _leaseUntil, ...base } = old;
-        const job = { ...base, state: "pending", error: "lease expired", eligibleAt: t, nextRetryAt: t, updatedAt: t };
+        const job = { ...base, state: "pending", error: "lease expired", eligibleAt: now, nextRetryAt: now, updatedAt: now };
         db.prepare("UPDATE maintenance_jobs SET status=?,payload=?,updated_at=? WHERE job_id=? AND project_key=?").run(job.state, JSON.stringify(job), job.updatedAt, job.jobId, this.projectKey);
         swept.push(job);
       }
       return swept;
     });
   }
+  recoverLegacyContentionRetirements() {
+    return this.ledger.transaction((db) => {
+      const rows = db.prepare("SELECT payload FROM maintenance_jobs WHERE project_key=? AND status=?").all(this.projectKey, "failed");
+      const recovered = [];
+      for (const row of rows) {
+        const old = parse(row.payload);
+        if (old.projectKey !== this.projectKey) throw new Error("job project does not match ledger project");
+        if (old.legacyRecovery || !LEGACY_CONTENTION_RETIREMENTS.has(old.error ?? "")) continue;
+        const t = this.now();
+        const { error: _error, ...base } = old;
+        const job = { ...base, state: "pending", attempts: 0, legacyRecovery: true, eligibleAt: t, nextRetryAt: t, updatedAt: t };
+        db.prepare("UPDATE maintenance_jobs SET status=?,payload=?,updated_at=? WHERE job_id=? AND project_key=?").run(job.state, JSON.stringify(job), job.updatedAt, job.jobId, this.projectKey);
+        recovered.push(job);
+      }
+      return recovered;
+    });
+  }
   recordFailure(job, error) {
     return this.ledger.transaction((db) => {
       const row = db.prepare("SELECT payload FROM maintenance_jobs WHERE job_id=? AND project_key=?").get(job.jobId, this.projectKey);
@@ -1132,18 +1187,15 @@ var LcmMaintenance = class {
       const old = parse(row.payload);
       if (old.projectKey !== this.projectKey) throw new Error("job project does not match ledger project");
       const t = this.now();
-      const expired = old.state === "running" && (old.leaseUntil ?? 0) <= t;
-      if (old.state === "running" && !expired) throw new Error("job lease held");
-      if (old.state !== "pending" && !expired || old.eligibleAt > t || old.nextRetryAt > t) throw new Error("job not eligible");
-      const active = db.prepare("SELECT payload FROM maintenance_jobs WHERE project_key=? AND status=?").all(this.projectKey, "running").map((r) => parse(r.payload)).filter((j) => j.jobId !== old.jobId && (j.leaseUntil ?? 0) > t);
-      if (active.length) throw new Error("project lease held");
+      const expired = old.state === "running" && (old.leaseUntil ?? 0) + LEASE_SWEEP_GRACE_MS <= t;
+      if (old.state === "running" && !expired) throw new LcmRejection("job lease held");
+      if (old.state !== "pending" && !expired || old.eligibleAt > t || old.nextRetryAt > t) throw new LcmRejection("job not eligible");
+      const leases = this.liveLeases(db, t, old.jobId);
+      if (leases.length >= this.concurrencyLimit) throw new LcmRejection("project lease held");
       const nodeRow = db.prepare("SELECT payload FROM summary_nodes WHERE node_id=? AND project_key=?").get(old.nodeId, this.projectKey);
       const sessionId = nodeRow?.payload ? parse(nodeRow.payload).sessionId : "";
-      const day = this.day(t);
-      const projectUsage = this.usage(db, day);
-      const sessionUsage = this.usage(db, day, sessionId);
-      if (projectUsage.calls >= this.budget.calls || projectUsage.inputTokens >= this.budget.inputTokens || projectUsage.outputTokens >= this.budget.outputTokens || projectUsage.cost >= this.budget.cost || projectUsage.wallMs >= this.budget.wallMs || sessionUsage.calls >= this.budget.sessionCalls) throw new Error("budget exhausted");
-      const out = { ...old, state: "running", ownerId, leaseToken: token(), leaseUntil: t + 3e4, updatedAt: t };
+      if (!this.admits(db, t, sessionId, leases)) throw new LcmRejection("budget exhausted");
+      const out = { ...old, state: "running", ownerId, leaseToken: token(), leaseUntil: t + LEASE_MS, updatedAt: t };
       db.prepare("UPDATE maintenance_jobs SET status=?,payload=?,updated_at=? WHERE job_id=? AND project_key=?").run(out.state, JSON.stringify(out), t, jobId, this.projectKey);
       return out;
     });
@@ -1156,15 +1208,15 @@ var LcmMaintenance = class {
       if (old.projectKey !== this.projectKey) throw new Error("job project does not match ledger project");
       const t = this.now();
       const expired = old.state === "running" && (old.leaseUntil ?? 0) <= t;
-      if (old.state === "running" && !expired) throw new Error("job lease held");
+      if (old.state === "running" && !expired) throw new LcmRejection("job lease held");
       if (old.state === "completed") throw new Error("job already completed");
-      const out = { ...old, state: "running", ownerId, leaseToken: token(), leaseUntil: t + 3e4, updatedAt: t };
+      const out = { ...old, state: "running", ownerId, leaseToken: token(), leaseUntil: t + LEASE_MS, updatedAt: t };
       db.prepare("UPDATE maintenance_jobs SET status=?,payload=?,updated_at=? WHERE job_id=? AND project_key=?").run(out.state, JSON.stringify(out), t, jobId, this.projectKey);
       return out;
     });
   }
   renew(job) {
-    return this.fenced(job, (old) => ({ ...old, leaseUntil: this.now() + 3e4, updatedAt: this.now() }));
+    return this.fenced(job, (old) => ({ ...old, leaseUntil: this.now() + LEASE_MS, updatedAt: this.now() }));
   }
   validateText(text) {
     if (typeof text !== "string" || text.length === 0) throw new Error("invalid summary text");
@@ -1178,12 +1230,10 @@ var LcmMaintenance = class {
     if (!Number.isSafeInteger(inputBytes) || inputBytes < 0) throw new Error("invalid input bound");
     if (utf8Bytes(text) >= inputBytes) throw new Error("summary does not shrink its input");
   }
-  withinBudget(sessionId) {
+  withinBudget(sessionId, excludeJobId) {
     return this.ledger.readOnly((db) => {
-      const day = this.day(this.now());
-      const project = this.usage(db, day);
-      const session = this.usage(db, day, sessionId);
-      return project.calls < this.budget.calls && project.inputTokens < this.budget.inputTokens && project.outputTokens < this.budget.outputTokens && project.cost < this.budget.cost && project.wallMs < this.budget.wallMs && session.calls < this.budget.sessionCalls;
+      const at = this.now();
+      return this.admits(db, at, sessionId, this.liveLeases(db, at, excludeJobId));
     });
   }
   accountRejected(sessionId, result) {
@@ -1277,7 +1327,7 @@ var LcmMaintenance = class {
       let lastError = new Error("LCM summarization did not converge");
       for (const [index, level] of LCM_MODEL_LEVELS.entries()) {
         if (controller.signal.aborted) break;
-        if (index > 0 && !this.withinBudget(sessionId)) break;
+        if (index > 0 && !this.withinBudget(sessionId, job.jobId)) break;
         const target = Math.max(1, Math.min(Math.floor(this.maxOutputChars * level.share), inputBytes - 1));
         let result;
         try {
@@ -1302,7 +1352,8 @@ var LcmMaintenance = class {
       const fallback = emergencyReduce(input, this.maxOutputChars, sources);
       try {
         return this.completeEmergency(claimed, fallback);
-      } catch {
+      } catch (fenced) {
+        if (isLcmRejection(fenced) && fenced.reason === "lease fenced") throw fenced;
         try {
           this.fail(claimed, String(error));
         } catch {
@@ -1335,7 +1386,7 @@ var LcmMaintenance = class {
     return job;
   }
   assertLease(current, expected) {
-    if (current.ownerId !== expected.ownerId || current.leaseToken !== expected.leaseToken || current.state !== "running" || (current.leaseUntil ?? 0) < this.now()) throw new Error("lease fenced");
+    if (current.ownerId !== expected.ownerId || current.leaseToken !== expected.leaseToken || current.state !== "running" || (current.leaseUntil ?? 0) < this.now()) throw new LcmRejection("lease fenced");
   }
   fenced(job, update) {
     return this.ledger.transaction((db) => {
@@ -1349,7 +1400,6 @@ var LcmMaintenance = class {
 };
 
 // src/compaction/lcm-runtime.ts
-var LEASE_SWEEP_GRACE_MS = 6e4;
 var MAX_RECONCILE_RECOVERIES = 3;
 var CLAIMABLE_JOB_LIMIT = 256;
 var FAILED_JOB_WINDOW_MS = 24 * 60 * 60 * 1e3;
@@ -1445,6 +1495,7 @@ var LcmRuntime = class {
       ...initial.lcmMaxInputChars === void 0 ? {} : { maxInputChars: initial.lcmMaxInputChars },
       ...initial.lcmMaxOutputChars === void 0 ? {} : { maxOutputChars: initial.lcmMaxOutputChars },
       ...initial.modelTimeoutSeconds === void 0 ? {} : { modelTimeoutMs: initial.modelTimeoutSeconds * 1e3 },
+      maxConcurrentJobs: () => this.resolveOptions().maintenanceConcurrency ?? DEFAULT_MAINTENANCE_CONCURRENCY,
       budget: {
         ...initial.maxDailyModelCalls ? { calls: initial.maxDailyModelCalls } : {},
         ...initial.maxSessionModelCalls ? { sessionCalls: initial.maxSessionModelCalls } : {},
@@ -1587,7 +1638,7 @@ var LcmRuntime = class {
   reclaimExpiredLeases() {
     if (this.closed) return 0;
     try {
-      return this.maintenance.sweepExpiredLeases(LEASE_SWEEP_GRACE_MS).length;
+      return this.maintenance.sweepExpiredLeases().length + this.maintenance.recoverLegacyContentionRetirements().length;
     } catch (error) {
       this.degradedError = error;
       return 0;
@@ -1680,24 +1731,49 @@ var LcmRuntime = class {
       try {
         const input = inputFor(node);
         await this.maintenance.run(job, model, input, this.signal);
+        return "done";
       } catch (error) {
+        if (isLcmCapacityRejection(error)) return "capacity";
+        if (isLcmRejection(error)) return "rejected";
         this.degradedError = error;
         try {
           this.maintenance.recordFailure(job, error);
         } catch {
         }
+        return "done";
       }
     };
+    const dispatch = async (items) => {
+      let next = 0;
+      const worker = async (slot) => {
+        while (!this.closed && Date.now() < runDeadline && slot < this.maintenance.concurrencyLimit) {
+          const item = items[next];
+          if (!item) return;
+          next += 1;
+          if (await runJob(item.job, item.node) === "capacity") return;
+        }
+      };
+      await Promise.all(Array.from({ length: Math.min(this.maintenance.concurrencyLimit, items.length) }, (_unused, slot) => worker(slot)));
+    };
     this.reclaimExpiredLeases();
     for (let pass = 0; pass < passes && !this.closed && Date.now() < runDeadline; pass += 1) {
       const leafCandidate = this.maintenance.createLeaf(this.maintenance.selectLeaf(raw, this.context.sessionManager.getLeafId()));
       const leaf = leafCandidate ? this.maintenance.getNode(leafCandidate.nodeId) : void 0;
+      const batch = [];
+      const queued = /* @__PURE__ */ new Set();
       if (leaf?.state === "pending") {
         const job = this.maintenance.jobForNode(leaf.nodeId);
-        if (job && this.maintenance.isClaimable(job)) await runJob(job, leaf);
+        if (job && this.maintenance.isClaimable(job)) {
+          batch.push({ job, node: leaf });
+          queued.add(job.jobId);
+        }
       }
       const pending = this.actionableJobs(sessionId, this.context.sessionManager.getLeafId(), passes);
-      for (const item of pending) await runJob(item.job, item.node);
+      for (const item of pending) if (!queued.has(item.job.jobId)) {
+        queued.add(item.job.jobId);
+        batch.push(item);
+      }
+      await dispatch(batch);
       const children = this.maintenance.selectCondensation(sessionId, this.activeSources, this.context.sessionManager.getLeafId());
       if (children.length >= fanIn) {
         const node = this.maintenance.createCondensed(children);
@@ -1766,8 +1842,13 @@ var LcmRuntime = class {
     }
     const job = this.maintenance.jobForNode(leaf.nodeId);
     if (!job) throw new Error("LCM emergency fallback job was not created");
-    const completed = this.maintenance.completeEmergency(this.maintenance.claimEmergency(job.jobId), fallback);
-    return { summary: completed.text ?? fallback, firstKeptEntryId: input.firstKeptEntryId, tokensBefore: input.tokensBefore, source: "emergency", branch: input.branch };
+    try {
+      const completed = this.maintenance.completeEmergency(this.maintenance.claimEmergency(job.jobId), fallback);
+      return { summary: completed.text ?? fallback, firstKeptEntryId: input.firstKeptEntryId, tokensBefore: input.tokensBefore, source: "emergency", branch: input.branch };
+    } catch (error) {
+      if (!isLcmRejection(error)) throw error;
+      return { summary: fallback, firstKeptEntryId: input.firstKeptEntryId, tokensBefore: input.tokensBefore, source: "emergency", branch: input.branch };
+    }
   }
   report() {
     const day = new Date(Date.now()).toISOString().slice(0, 10);