@graphrefly/graphrefly 0.44.0 → 0.46.0

package/dist/utils/job-queue/index.cjs

@@ -0,0 +1,717 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/utils/job-queue/index.ts
+var job_queue_exports = {};
+__export(job_queue_exports, {
+  JobFlowGraph: () => JobFlowGraph,
+  JobQueueGraph: () => JobQueueGraph,
+  jobEventKeyOf: () => jobEventKeyOf,
+  jobFlow: () => jobFlow,
+  jobQueue: () => jobQueue
+});
+module.exports = __toCommonJS(job_queue_exports);
+var import_core2 = require("@graphrefly/pure-ts/core");
+var import_extra2 = require("@graphrefly/pure-ts/extra");
+var import_graph2 = require("@graphrefly/pure-ts/graph");
+
+// src/base/meta/domain-meta.ts
+function domainMeta(domain, kind, extra) {
+  return {
+    [domain]: true,
+    [`${domain}_type`]: kind,
+    ...extra ?? {}
+  };
+}
+
+// src/base/mutation/index.ts
+var import_core = require("@graphrefly/pure-ts/core");
+var import_extra = require("@graphrefly/pure-ts/extra");
+var import_graph = require("@graphrefly/pure-ts/graph");
+var DEFAULT_AUDIT_GUARD = (0, import_core.policy)((allow, deny) => {
+  allow("observe");
+  allow("signal");
+  deny("write");
+});
+function createAuditLog(opts) {
+  const log = (0, import_extra.reactiveLog)([], {
+    name: opts.name,
+    maxSize: opts.retainedLimit ?? 1024,
+    guard: opts.guard ?? DEFAULT_AUDIT_GUARD,
+    ...opts.versioning != null ? { versioning: opts.versioning } : {}
+  });
+  log.withLatest();
+  if (opts.graph) {
+    opts.graph.add(log.entries, { name: opts.name });
+  }
+  return log;
+}
+function deepFreeze(value) {
+  if (value === null || typeof value !== "object" || Object.isFrozen(value)) return value;
+  for (const k of Object.keys(value)) {
+    deepFreeze(value[k]);
+  }
+  return Object.freeze(value);
+}
+function mutate(act, opts) {
+  const { up, down } = typeof act === "function" ? { up: act, down: void 0 } : act;
+  const freeze = opts.freeze ?? true;
+  if (opts.frame === "inline") {
+    return function wrapped(...args) {
+      const sealed = freeze ? args.map(deepFreeze) : args;
+      const t_ns = (0, import_core.wallClockNs)();
+      const seq = opts.seq ? bumpCursor(opts.seq) : void 0;
+      try {
+        const result = up(...sealed);
+        if (opts.log && opts.onSuccessRecord) {
+          appendAudit(
+            opts.log,
+            opts.onSuccessRecord,
+            sealed,
+            result,
+            { t_ns, seq },
+            opts.handlerVersion
+          );
+        }
+        return result;
+      } catch (err) {
+        if (opts.log && opts.onFailureRecord) {
+          const errorType = err instanceof Error ? err.name : typeof err;
+          appendAudit(
+            opts.log,
+            opts.onFailureRecord,
+            sealed,
+            err,
+            { t_ns, seq, errorType },
+            opts.handlerVersion
+          );
+        }
+        throw err;
+      }
+    };
+  }
+  return function wrapped(...args) {
+    const sealed = freeze ? args.map(deepFreeze) : args;
+    const t_ns = (0, import_core.wallClockNs)();
+    let result;
+    let captured;
+    let captureSet = false;
+    let seq;
+    try {
+      (0, import_core.batch)(() => {
+        if (opts.seq) seq = bumpCursor(opts.seq);
+        try {
+          result = up(...sealed);
+          if (opts.log && opts.onSuccessRecord) {
+            appendAudit(
+              opts.log,
+              opts.onSuccessRecord,
+              sealed,
+              result,
+              { t_ns, seq },
+              opts.handlerVersion
+            );
+          }
+        } catch (err) {
+          captured = err;
+          captureSet = true;
+          throw err;
+        }
+      });
+    } catch (outerErr) {
+      if (captureSet && down) {
+        try {
+          down(...sealed);
+        } catch (downErr) {
+          console.error(
+            `mutate: down hook threw \u2014 original action error preserved (${captured instanceof Error ? captured.name : typeof captured}). Down error:`,
+            downErr
+          );
+        }
+      }
+      if (captureSet && opts.log && opts.onFailureRecord) {
+        const errorType = captured instanceof Error ? captured.name : typeof captured;
+        appendAudit(
+          opts.log,
+          opts.onFailureRecord,
+          sealed,
+          captured,
+          { t_ns, seq, errorType },
+          opts.handlerVersion
+        );
+      }
+      throw captureSet ? captured : outerErr;
+    }
+    return result;
+  };
+}
+var _bumpCursorWarned = /* @__PURE__ */ new WeakSet();
+function bumpCursor(seq) {
+  const raw = seq.cache;
+  const valid = typeof raw === "number" && Number.isFinite(raw);
+  if (!valid && raw !== void 0 && !_bumpCursorWarned.has(seq)) {
+    _bumpCursorWarned.add(seq);
+    console.warn(
+      `bumpCursor: cursor cache held a non-numeric value (${String(raw)}); resetting to 0. Causes include: a snapshot codec round-tripping the cursor as a string / null / NaN, OR a malformed initial seed (e.g. state<number>(NaN)). Audit consumers may see colliding seq values after this point.`
+    );
+  }
+  const cur = valid ? raw : 0;
+  const next = cur + 1;
+  seq.down([[import_core.DIRTY], [import_core.DATA, next]]);
+  return next;
+}
+function appendAudit(audit, builder, args, value, meta, handlerVersion) {
+  const record = builder(args, value, meta);
+  if (record === void 0) return;
+  const stamped = handlerVersion != null ? { ...record, handlerVersion } : record;
+  audit.append(stamped);
+}
+function registerCursor(graph, name, initial = 0) {
+  const cursor = (0, import_core.node)([], { initial, name, describeKind: "state" });
+  graph.add(cursor, { name });
+  return cursor;
+}
+
+// src/utils/job-queue/index.ts
+var DEFAULT_MAX_PER_PUMP = 256;
+var DEFAULT_COMPLETED_RETAINED_LIMIT = 1024;
+function requireNonNegativeInt(value, label) {
+  if (!Number.isFinite(value) || !Number.isInteger(value) || value < 0) {
+    throw new Error(`${label} must be a non-negative integer`);
+  }
+  return value;
+}
+function jobQueueMeta(kind, extra) {
+  return domainMeta("job_queue", kind, extra);
+}
+var jobEventKeyOf = (e) => e.action;
+var JobQueueGraph = class extends import_graph2.Graph {
+  _pending;
+  _jobs;
+  _seqCursor;
+  pending;
+  jobs;
+  depth;
+  /** Audit log of every queue mutation (Audit 2). */
+  events;
+  /** Alias for {@link JobQueueGraph.events} — Audit 2 `.audit` duplication. */
+  audit;
+  // Tier 8 / COMPOSITION-GUIDE §35: mutate wrappers for the four
+  // single-record mutation methods. Assigned in the constructor (NOT via
+  // class-field initializers) because field initializers run before the
+  // constructor body — `this.events` and `this._seqCursor` aren't ready yet.
+  // `claim` stays inline because it emits one record per claimed job.
+  _enqueueImpl;
+  _ackImpl;
+  _nackImpl;
+  _removeByIdImpl;
+  constructor(name, opts = {}) {
+    super(name, opts.graph);
+    this._pending = (0, import_extra2.reactiveList)([], { name: "pending" });
+    this._jobs = (0, import_extra2.reactiveMap)({ name: "jobs" });
+    this.pending = this._pending.items;
+    this.jobs = this._jobs.entries;
+    this.add(this.pending, { name: "pending" });
+    this.add(this.jobs, { name: "jobs" });
+    this.depth = (0, import_core2.node)(
+      [this.pending],
+      (batchData, actions, ctx) => {
+        const data = batchData.map(
+          (batch3, i) => batch3 != null && batch3.length > 0 ? batch3.at(-1) : ctx.prevData[i]
+        );
+        actions.emit(data[0].length);
+      },
+      {
+        name: "depth",
+        describeKind: "derived",
+        meta: jobQueueMeta("queue_depth"),
+        initial: 0
+      }
+    );
+    this.add(this.depth, { name: "depth" });
+    this.addDisposer((0, import_extra2.keepalive)(this.depth));
+    this.events = createAuditLog({
+      name: "events",
+      retainedLimit: 1024,
+      graph: this
+    });
+    this.audit = this.events;
+    this._seqCursor = registerCursor(this, "seq", 0);
+    this._enqueueImpl = mutate(
+      (payload, enqueueOpts) => {
+        const seq = this._seqCursor.cache;
+        const id = enqueueOpts.id ?? `${this.name}-${seq}`;
+        if (this._jobs.get(id) !== void 0) {
+          throw new Error(`jobQueue("${this.name}"): duplicate job id "${id}"`);
+        }
+        const job = {
+          id,
+          payload,
+          attempts: 0,
+          metadata: Object.freeze({ ...enqueueOpts.metadata ?? {} }),
+          state: "queued"
+        };
+        this._jobs.set(id, job);
+        this._pending.append(id);
+        return id;
+      },
+      {
+        frame: "inline",
+        log: this.events,
+        seq: this._seqCursor,
+        freeze: false,
+        onSuccessRecord: ([payload], id, { t_ns, seq }) => ({
+          action: "enqueue",
+          id,
+          payload,
+          t_ns,
+          seq: seq ?? 0
+        })
+      }
+    );
+    this._ackImpl = mutate(
+      (id, _job) => {
+        this._jobs.delete(id);
+      },
+      {
+        frame: "inline",
+        log: this.events,
+        seq: this._seqCursor,
+        freeze: false,
+        onSuccessRecord: ([id, job], _r, { t_ns, seq }) => ({
+          action: "ack",
+          id,
+          attempts: job.attempts,
+          t_ns,
+          seq: seq ?? 0
+        })
+      }
+    );
+    this._nackImpl = mutate(
+      (id, job, requeue) => {
+        if (requeue) {
+          this._jobs.set(id, { ...job, state: "queued" });
+          this._pending.append(id);
+        } else {
+          this._jobs.delete(id);
+        }
+      },
+      {
+        frame: "inline",
+        log: this.events,
+        seq: this._seqCursor,
+        freeze: false,
+        onSuccessRecord: ([id, job], _r, { t_ns, seq }) => ({
+          action: "nack",
+          id,
+          attempts: job.attempts,
+          t_ns,
+          seq: seq ?? 0
+        })
+      }
+    );
+    this._removeByIdImpl = mutate(
+      (id, job) => {
+        if (job.state === "queued") {
+          const pending = this.pending.cache;
+          const idx = pending.indexOf(id);
+          if (idx >= 0) this._pending.pop(idx);
+        }
+        this._jobs.delete(id);
+      },
+      {
+        frame: "inline",
+        log: this.events,
+        seq: this._seqCursor,
+        freeze: false,
+        onSuccessRecord: ([id, job], _r, { t_ns, seq }) => ({
+          action: "remove",
+          id,
+          attempts: job.attempts,
+          t_ns,
+          seq: seq ?? 0
+        })
+      }
+    );
+  }
+  /**
+   * Wire append-log storage tiers (Audit 4). Returns a disposer.
+   *
+   * Named `attachEventStorage` to avoid colliding with {@link Graph.attachSnapshotStorage}.
+   */
+  attachEventStorage(tiers) {
+    return this.events.attachStorage(tiers);
+  }
+  enqueue(payload, opts = {}) {
+    return this._enqueueImpl(payload, opts);
+  }
+  claim(limit = 1) {
+    const max = requireNonNegativeInt(limit, "job queue claim limit");
+    if (max === 0) return [];
+    const out = [];
+    while (out.length < max) {
+      const ids = this.pending.cache;
+      if (ids.length === 0) break;
+      const id = this._pending.pop(0);
+      const job = this._jobs.get(id);
+      if (!job || job.state !== "queued") continue;
+      const inflight = {
+        ...job,
+        state: "inflight",
+        attempts: job.attempts + 1
+      };
+      this._jobs.set(id, inflight);
+      out.push(inflight);
+      this.events.append({
+        action: "claim",
+        id,
+        attempts: inflight.attempts,
+        t_ns: (0, import_core2.wallClockNs)(),
+        seq: bumpCursor(this._seqCursor)
+      });
+    }
+    return out;
+  }
+  ack(id) {
+    const job = this._jobs.get(id);
+    if (!job || job.state !== "inflight") return false;
+    this._ackImpl(id, job);
+    return true;
+  }
+  nack(id, opts = {}) {
+    const job = this._jobs.get(id);
+    if (!job || job.state !== "inflight") return false;
+    this._nackImpl(id, job, opts.requeue ?? true);
+    return true;
+  }
+  /**
+   * Remove a job by id regardless of its current state. Returns `true` if
+   * the job existed and was removed, `false` if no job has this id.
+   *
+   * `ack` only works on inflight; `nack` only works on inflight.
+   * `removeById` is the state-agnostic escape hatch — useful for
+   * audit/observability layers that enqueue but never claim, and need to
+   * finalize a job when an external decision (e.g. harness verify
+   * outcome) resolves it. Distinct name from the inherited
+   * {@link Graph.remove}, which removes a mounted child subgraph by path.
+   *
+   * When the job is in `queued` state, its id is also pulled from the
+   * `pending` list — depth + pending snapshot stay consistent.
+   */
+  removeById(id) {
+    const job = this._jobs.get(id);
+    if (!job) return false;
+    this._removeByIdImpl(id, job);
+    return true;
+  }
+  /**
+   * Subscribe to a Node and enqueue each DATA payload into this queue.
+   * Returns a disposer that stops consuming when called.
+   *
+   * Used internally by {@link JobFlowGraph} for stage-to-stage wiring but
+   * also useful for users wiring an external source into a job queue.
+   *
+   * @param source - Node whose DATA values are enqueued.
+   * @param opts - Optional enqueue options (id generator, metadata prefix).
+   */
+  consumeFrom(source, opts) {
+    return source.subscribe((msgs) => {
+      for (const m of msgs) {
+        if (m[0] !== import_core2.DATA) continue;
+        const payload = m[1];
+        this.enqueue(payload, opts ? { metadata: opts.metadata } : void 0);
+      }
+    });
+  }
+};
+var JobFlowGraph = class extends import_graph2.Graph {
+  _stageNames;
+  _stageWorkFns;
+  _queues = /* @__PURE__ */ new Map();
+  _completed;
+  completed;
+  completedCount;
+  constructor(name, opts = {}) {
+    super(name, opts.graph);
+    const rawStages = opts.stages ?? ["incoming", "processing", "done"];
+    const stageNames = [];
+    const stageWorkFns = /* @__PURE__ */ new Map();
+    const stageMaxPerPump = /* @__PURE__ */ new Map();
+    const stageMaxInflight = /* @__PURE__ */ new Map();
+    for (const raw of rawStages) {
+      const stageName = typeof raw === "string" ? raw.trim() : raw.name.trim();
+      if (typeof raw !== "string" && raw.work) {
+        stageWorkFns.set(stageName, raw.work);
+      }
+      if (typeof raw !== "string" && raw.maxPerPump != null) {
+        stageMaxPerPump.set(
+          stageName,
+          Math.max(
+            1,
+            requireNonNegativeInt(raw.maxPerPump, `job flow stage "${stageName}" maxPerPump`)
+          )
+        );
+      }
+      if (typeof raw !== "string" && raw.maxInflight != null) {
+        stageMaxInflight.set(
+          stageName,
+          Math.max(
+            1,
+            requireNonNegativeInt(raw.maxInflight, `job flow stage "${stageName}" maxInflight`)
+          )
+        );
+      }
+      stageNames.push(stageName);
+    }
+    if (stageNames.length < 2) {
+      throw new Error(`jobFlow("${name}"): requires at least 2 stages`);
+    }
+    const unique = new Set(stageNames);
+    if (unique.size !== stageNames.length) {
+      throw new Error(`jobFlow("${name}"): stage names must be unique`);
+    }
+    this._stageNames = Object.freeze([...stageNames]);
+    this._stageWorkFns = stageWorkFns;
+    for (const stage of this._stageNames) {
+      const q = jobQueue(`${name}-${stage}`);
+      this._queues.set(stage, q);
+      this.mount(stage, q);
+    }
+    this._completed = (0, import_extra2.reactiveLog)([], {
+      name: "completed",
+      maxSize: DEFAULT_COMPLETED_RETAINED_LIMIT
+    });
+    this.completed = this._completed.entries;
+    this.add(this.completed, { name: "completed" });
+    this.completedCount = (0, import_core2.node)(
+      [this.completed],
+      (batchData, actions, ctx) => {
+        const data = batchData.map(
+          (batch3, i) => batch3 != null && batch3.length > 0 ? batch3.at(-1) : ctx.prevData[i]
+        );
+        actions.emit(data[0].length);
+      },
+      {
+        name: "completedCount",
+        describeKind: "derived",
+        meta: jobQueueMeta("job_flow_completed_count"),
+        initial: 0
+      }
+    );
+    this.add(this.completedCount, { name: "completedCount" });
+    this.addDisposer((0, import_extra2.keepalive)(this.completedCount));
+    const defaultMaxPerPump = Math.max(
+      1,
+      requireNonNegativeInt(opts.maxPerPump ?? DEFAULT_MAX_PER_PUMP, "job flow maxPerPump")
+    );
+    for (let i = 0; i < this._stageNames.length; i += 1) {
+      const stage = this._stageNames[i];
+      const current = this.queue(stage);
+      const next = i + 1 < this._stageNames.length ? this.queue(this._stageNames[i + 1]) : null;
+      const workFn = this._stageWorkFns.get(stage);
+      const stagePerPump = stageMaxPerPump.get(stage) ?? defaultMaxPerPump;
+      const stageMaxInflightCap = stageMaxInflight.get(stage);
+      const inflightCounter = stageMaxInflightCap !== void 0 ? (0, import_core2.node)([], { name: `__inflight__/${stage}`, initial: 0 }) : null;
+      if (inflightCounter) {
+        this.add(inflightCounter, { name: `__inflight__/${stage}` });
+      }
+      const isTerminal = next === null;
+      if (workFn) {
+        const pumpDeps = inflightCounter != null ? [current.pending, inflightCounter] : [current.pending];
+        const pump = (0, import_core2.node)(
+          pumpDeps,
+          (_data, _actions, ctx) => {
+            if (!("inflight" in ctx.store)) {
+              ctx.store.inflight = {
+                entries: /* @__PURE__ */ new Set(),
+                terminated: false
+              };
+            }
+            const inflightStore = ctx.store.inflight;
+            const inflight = inflightStore.entries;
+            let processed = 0;
+            while (processed < stagePerPump) {
+              if (stageMaxInflightCap !== void 0 && inflight.size >= stageMaxInflightCap) {
+                break;
+              }
+              const claims = current.claim(1);
+              if (claims.length === 0) break;
+              const job = claims[0];
+              if (!job) break;
+              const prevPath = job.metadata.job_flow_path ?? [];
+              const newPath = [...prevPath, stage];
+              const ac = new AbortController();
+              const entry = { unsub: () => void 0, ac };
+              inflight.add(entry);
+              inflightCounter?.emit(inflight.size);
+              let result;
+              try {
+                result = workFn(job, { signal: ac.signal });
+              } catch {
+                inflight.delete(entry);
+                inflightCounter?.emit(inflight.size);
+                current.nack(job.id, { requeue: false });
+                processed += 1;
+                continue;
+              }
+              const resultNode = (0, import_extra2.fromAny)(result);
+              let settled = false;
+              let unsub;
+              const cleanupSub = () => {
+                if (unsub) {
+                  unsub();
+                } else {
+                  Promise.resolve().then(() => unsub?.());
+                }
+                inflight.delete(entry);
+                if (!inflightStore.terminated) {
+                  inflightCounter?.emit(inflight.size);
+                }
+              };
+              unsub = resultNode.subscribe((msgs) => {
+                if (settled) return;
+                for (const m of msgs) {
+                  if (m[0] === import_core2.DATA) {
+                    settled = true;
+                    cleanupSub();
+                    const newPayload = m[1];
+                    const newMetadata = {
+                      ...job.metadata,
+                      job_flow_path: newPath
+                    };
+                    if (isTerminal) {
+                      const completedJob = {
+                        ...job,
+                        payload: newPayload,
+                        metadata: Object.freeze(newMetadata)
+                      };
+                      (0, import_core2.batch)(() => {
+                        current.ack(job.id);
+                        this._completed.append(completedJob);
+                      });
+                    } else {
+                      (0, import_core2.batch)(() => {
+                        current.ack(job.id);
+                        next.enqueue(newPayload, {
+                          metadata: newMetadata
+                        });
+                      });
+                    }
+                    return;
+                  } else if (m[0] === import_core2.ERROR) {
+                    settled = true;
+                    cleanupSub();
+                    current.nack(job.id, { requeue: false });
+                    return;
+                  }
+                }
+              });
+              entry.unsub = () => unsub?.();
+              processed += 1;
+            }
+            return {
+              onDeactivation: () => {
+                inflightStore.terminated = true;
+                for (const e of inflight) {
+                  try {
+                    e.ac.abort();
+                  } catch {
+                  }
+                  try {
+                    e.unsub();
+                  } catch {
+                  }
+                }
+                inflight.clear();
+                delete ctx.store.inflight;
+              }
+            };
+          },
+          {
+            meta: jobQueueMeta("job_flow_pump", { stage, has_work: true })
+          }
+        );
+        this.addDisposer((0, import_extra2.keepalive)(pump));
+      } else {
+        const pump = this.effect(
+          `pump_${stage}`,
+          [`${stage}::pending`],
+          () => {
+            let moved = 0;
+            while (moved < stagePerPump) {
+              const claim = current.claim(1);
+              if (claim.length === 0) break;
+              const job = claim[0];
+              if (!job) break;
+              const prevPath = job.metadata.job_flow_path ?? [];
+              const newPath = [...prevPath, stage];
+              const newMetadata = { ...job.metadata, job_flow_path: newPath };
+              if (isTerminal) {
+                const completedJob = {
+                  ...job,
+                  metadata: Object.freeze(newMetadata)
+                };
+                (0, import_core2.batch)(() => {
+                  current.ack(job.id);
+                  this._completed.append(completedJob);
+                });
+              } else {
+                (0, import_core2.batch)(() => {
+                  current.ack(job.id);
+                  next.enqueue(job.payload, {
+                    metadata: newMetadata
+                  });
+                });
+              }
+              moved += 1;
+            }
+          },
+          {
+            meta: jobQueueMeta("job_flow_pump", { stage, has_work: false })
+          }
+        );
+        this.addDisposer((0, import_extra2.keepalive)(pump));
+      }
+    }
+  }
+  stages() {
+    return this._stageNames;
+  }
+  queue(stage) {
+    const q = this._queues.get(stage);
+    if (!q) throw new Error(`jobFlow("${this.name}"): unknown stage "${stage}"`);
+    return q;
+  }
+  enqueue(payload, opts = {}) {
+    return this.queue(this._stageNames[0]).enqueue(payload, opts);
+  }
+  retainedCompleted() {
+    return this.completed.cache;
+  }
+};
+function jobQueue(name, opts) {
+  return new JobQueueGraph(name, opts);
+}
+function jobFlow(name, opts) {
+  const g = new JobFlowGraph(name, opts);
+  const { factory: _f, factoryArgs: _fa, ...tagArgs } = opts ?? {};
+  g.tagFactory("jobFlow", (0, import_core2.placeholderArgs)(tagArgs));
+  return g;
+}
+//# sourceMappingURL=index.cjs.map