@dudousxd/nestjs-durable-telescope 0.9.2 → 0.10.0

package/dist/index.cjs

@@ -26,6 +26,9 @@ __export(index_exports, {
   durableDashboard: () => durableDashboard,
   durableDurationProvider: () => durableDurationProvider,
   durableExecutionScope: () => durableExecutionScope,
+  durableFailedTriggersProvider: () => durableFailedTriggersProvider,
+  durableLifecycleTriggersProvider: () => durableLifecycleTriggersProvider,
+  durableOwedLifecycleProvider: () => durableOwedLifecycleProvider,
   durableRecentFailuresProvider: () => durableRecentFailuresProvider,
   durableRunsOverTimeProvider: () => durableRunsOverTimeProvider,
   durableStateBreakdownProvider: () => durableStateBreakdownProvider,
@@ -37,6 +40,7 @@ __export(index_exports, {
   durableTraceContext: () => durableTraceContext,
   durableWorkerHealthProvider: () => durableWorkerHealthProvider,
   durableWorkerStatusProvider: () => durableWorkerStatusProvider,
+  runSpanContext: () => runSpanContext,
   runTraceContext: () => runTraceContext,
   runTraceId: () => runTraceId
 });
@@ -82,14 +86,18 @@ function runIds(runId) {
 }
 __name(runIds, "runIds");
 function runTraceContext(runId, parent = import_api.context.active()) {
-  return import_api.trace.setSpanContext(parent, {
+  return import_api.trace.setSpanContext(parent, runSpanContext(runId));
+}
+__name(runTraceContext, "runTraceContext");
+function runSpanContext(runId) {
+  return {
     traceId: runTraceId(runId),
     spanId: runSpanId(runId),
     traceFlags: import_api.TraceFlags.SAMPLED,
     isRemote: true
-  });
+  };
 }
-__name(runTraceContext, "runTraceContext");
+__name(runSpanContext, "runSpanContext");
 
 // src/durable-execution-scope.ts
 var activeScope = new import_node_async_hooks.AsyncLocalStorage();
@@ -176,10 +184,14 @@ var RUN_EVENT_TYPES = /* @__PURE__ */ new Set([
   "run.started",
   "run.completed",
   "run.failed",
-  "run.suspended"
+  "run.suspended",
+  "run.triggered",
+  "run.trigger_failed"
 ]);
 
 // src/durable-telescope.watcher.ts
+var REMOTE_EVENT = Symbol.for("@dudousxd/nestjs-durable:remote-event");
+var isRemote = /* @__PURE__ */ __name((event) => event[REMOTE_EVENT] === true, "isRemote");
 var DurableTelescopeWatcher = class {
   static {
     __name(this, "DurableTelescopeWatcher");
@@ -196,7 +208,9 @@ var DurableTelescopeWatcher = class {
     if (typeof engine?.subscribe !== "function") return;
     const tracer = import_api3.trace.getTracer("@dudousxd/nestjs-durable-telescope");
     engine.subscribe((event) => {
+      if (isRemote(event)) return;
       const family = runEventFamily(event.type);
+      const source = lifecycleSourceOfEvent(event);
       const record = /* @__PURE__ */ __name(() => ctx.record({
         type: this.type,
         ...family !== void 0 ? {
@@ -220,7 +234,13 @@ var DurableTelescopeWatcher = class {
           output: event.output,
           error: event.error,
           durationMs: event.durationMs,
-          reason: event.reason
+          reason: event.reason,
+          triggeredRunId: event.triggeredRunId,
+          ...source ? {
+            lifecycleSourceRunId: source.runId,
+            lifecycleKind: source.kind,
+            lifecycleSourceTraceId: runTraceId(source.runId)
+          } : {}
         },
         tags: this.tags(event)
       }), "record");
@@ -244,6 +264,21 @@ var DurableTelescopeWatcher = class {
         }
         import_api3.context.with(import_api3.trace.setSpan(base, step), record);
         step.end();
+      } else if (source) {
+        const started = tracer.startSpan(`run ${event.workflow ?? event.runId}`, {
+          attributes: {
+            "durable.run_id": event.runId,
+            "durable.lifecycle.source_run_id": source.runId,
+            "durable.lifecycle.kind": source.kind
+          },
+          links: [
+            {
+              context: runSpanContext(source.runId)
+            }
+          ]
+        }, base);
+        import_api3.context.with(import_api3.trace.setSpan(base, started), record);
+        started.end();
       } else {
         import_api3.context.with(base, record);
       }
@@ -274,9 +309,28 @@ var DurableTelescopeWatcher = class {
       tags.push(event.type === "run.routed" ? "routed" : "unrouted");
       tags.push(`onError:${event.error?.code ?? "*"}`);
     }
+    if (event.type === "run.triggered") {
+      tags.push("triggered");
+      if (event.triggeredRunId) tags.push(`run:${event.triggeredRunId}`);
+    }
+    if (event.type === "run.trigger_failed") tags.push("trigger_failed", `subscriber:${event.name}`);
+    const source = lifecycleSourceOfEvent(event);
+    if (source) {
+      tags.push("lifecycle", `source:${source.runId}`);
+      const traceId = (0, import_nestjs_durable_core.traceIdOfTraceparent)(event.sourceTraceparent);
+      if (traceId) tags.push(`sourceTrace:${traceId}`);
+    }
     return tags;
   }
 };
+function lifecycleSourceOfEvent(event) {
+  if (event.type !== "run.started" || !event.workflow) return void 0;
+  return (0, import_nestjs_durable_core.lifecycleSourceOf)({
+    id: event.runId,
+    workflow: event.workflow
+  });
+}
+__name(lifecycleSourceOfEvent, "lifecycleSourceOfEvent");
 
 // src/durable-telescope.extension.ts
 var import_nestjs_telescope2 = require("@dudousxd/nestjs-telescope");
@@ -398,6 +452,66 @@ function durableDashboard(opts = {}) {
               }
             ]
           },
+          {
+            kind: "table",
+            title: "Failed triggers",
+            data: {
+              provider: "durable.failedTriggers"
+            },
+            columns: [
+              {
+                key: "at",
+                label: "Time"
+              },
+              {
+                key: "workflow",
+                label: "Source workflow"
+              },
+              {
+                key: "runId",
+                label: "Source run",
+                link: {
+                  href: runHref
+                }
+              },
+              {
+                key: "subscriber",
+                label: "Subscriber"
+              },
+              {
+                key: "error",
+                label: "Error"
+              }
+            ]
+          },
+          {
+            kind: "table",
+            title: "Owed lifecycle events",
+            data: {
+              provider: "durable.owedLifecycle"
+            },
+            columns: [
+              {
+                key: "settledAt",
+                label: "Settled"
+              },
+              {
+                key: "workflow",
+                label: "Workflow"
+              },
+              {
+                key: "runId",
+                label: "Run",
+                link: {
+                  href: runHref
+                }
+              },
+              {
+                key: "owed",
+                label: "Owed to"
+              }
+            ]
+          },
           {
             kind: "table",
             // Lists ALL worker groups (starved ones sorted first); the Status column
@@ -425,6 +539,52 @@ function durableDashboard(opts = {}) {
                 label: "Status"
               }
             ]
+          },
+          {
+            kind: "stat",
+            title: "Owed runs",
+            data: {
+              provider: "durable.owedLifecycle",
+              query: {
+                metric: "count"
+              }
+            },
+            spark: false,
+            thresholds: {
+              warn: 1,
+              bad: 50,
+              direction: "up-bad"
+            }
+          }
+        ]
+      },
+      {
+        // Workflows started when another one settles (`run.completed:<wf>` / `run.failed:<wf>`).
+        title: "Lifecycle",
+        cols: 2,
+        panels: [
+          {
+            kind: "timeseries",
+            title: "Lifecycle triggers over time",
+            data: {
+              provider: "durable.lifecycleTriggers"
+            },
+            series: [
+              "triggered",
+              "failed"
+            ],
+            style: "stacked"
+          },
+          {
+            kind: "topN",
+            title: "Triggered by subscriber",
+            data: {
+              provider: "durable.lifecycleTriggers",
+              query: {
+                metric: "bySubscriber"
+              }
+            },
+            limit: 10
           }
         ]
       },
@@ -1073,6 +1233,148 @@ function durableStateBreakdownProvider() {
   };
 }
 __name(durableStateBreakdownProvider, "durableStateBreakdownProvider");
+function stamp(at) {
+  return at ? `${new Date(at).toISOString().replace("T", " ").slice(0, 16)}Z` : "";
+}
+__name(stamp, "stamp");
+async function triggerFailures(ctx, limit) {
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const entry of await fetchRunEntries(ctx, [
+    "run.trigger_failed"
+  ], limit)) {
+    const c = entry.content ?? {};
+    if (c.event !== "run.trigger_failed") continue;
+    const key = `${c.runId}\0${c.name}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(entry);
+  }
+  return out;
+}
+__name(triggerFailures, "triggerFailures");
+function durableFailedTriggersProvider() {
+  return {
+    name: "durable.failedTriggers",
+    async resolve(query, ctx) {
+      const limit = Math.min(200, Math.max(10, Number(query?.limit ?? 50)));
+      const windowMs = query?.windowMs === void 0 ? 24 * 60 * 60 * 1e3 : Number(query.windowMs);
+      const cutoff = windowMs > 0 ? Date.now() - windowMs : 0;
+      const rows = (await triggerFailures(ctx, RUN_EVENTS_LIMIT)).filter((entry) => entryTime(entry) >= cutoff).slice(0, limit).map((entry) => {
+        const c = entry.content ?? {};
+        return {
+          at: stamp(entry.createdAt),
+          workflow: c.workflow ?? "",
+          runId: c.runId ?? "",
+          subscriber: c.name ?? "",
+          error: c.error?.message ?? ""
+        };
+      });
+      return {
+        rows
+      };
+    }
+  };
+}
+__name(durableFailedTriggersProvider, "durableFailedTriggersProvider");
+var LIFECYCLE_GRACE_MS = 3e4;
+function durableOwedLifecycleProvider() {
+  return {
+    name: "durable.owedLifecycle",
+    async resolve(query, ctx) {
+      const store = ctx.moduleRef.get(import_nestjs_durable_core2.STATE_STORE_CANONICAL, {
+        strict: false
+      });
+      const limit = Math.min(200, Math.max(10, Number(query?.limit ?? 50)));
+      const cutoff = Date.now() - LIFECYCLE_GRACE_MS;
+      const runs = await store.listRuns({
+        tag: import_nestjs_durable_core2.LIFECYCLE_PENDING_TAG,
+        statuses: [
+          "completed",
+          "failed",
+          "cancelled",
+          "dead"
+        ],
+        limit: 1e3
+      });
+      const owed = runs.filter((run) => +new Date(run.updatedAt) <= cutoff);
+      if (query?.metric === "count") return {
+        value: owed.length
+      };
+      const rows = owed.sort((a, b) => +new Date(b.updatedAt) - +new Date(a.updatedAt)).slice(0, limit).map((run) => ({
+        settledAt: stamp(run.updatedAt),
+        workflow: run.workflow,
+        runId: run.id,
+        owed: (0, import_nestjs_durable_core2.owedSubscribersOf)(run.tags).join(", ") || "every subscriber"
+      }));
+      return {
+        rows
+      };
+    }
+  };
+}
+__name(durableOwedLifecycleProvider, "durableOwedLifecycleProvider");
+function durableLifecycleTriggersProvider() {
+  return {
+    name: "durable.lifecycleTriggers",
+    async resolve(query, ctx) {
+      const seen = /* @__PURE__ */ new Set();
+      const entries = (await fetchRunEntries(ctx, [
+        "run.triggered",
+        "run.trigger_failed"
+      ])).filter((entry) => {
+        const c = entry.content ?? {};
+        if (c.event !== "run.triggered" && c.event !== "run.trigger_failed") return false;
+        const key = `${c.event}\0${c.runId}\0${c.name}`;
+        if (seen.has(key)) return false;
+        seen.add(key);
+        return true;
+      });
+      if (query?.metric === "bySubscriber") {
+        const limit = Math.max(1, Number(query?.limit ?? 10));
+        const counts = /* @__PURE__ */ new Map();
+        for (const entry of entries) {
+          const c = entry.content ?? {};
+          if (c.event !== "run.triggered") continue;
+          const label = `${c.workflow ?? "unknown"} \u2192 ${c.name ?? "unknown"}`;
+          counts.set(label, (counts.get(label) ?? 0) + 1);
+        }
+        const items = [
+          ...counts
+        ].map(([label, value]) => ({
+          label,
+          value
+        })).sort((a, b) => b.value - a.value || a.label.localeCompare(b.label)).slice(0, limit);
+        return {
+          items
+        };
+      }
+      const n = Math.max(1, Number(query?.buckets ?? 24));
+      const now = Date.now();
+      let minT = now;
+      for (const entry of entries) if (entry.createdAt) minT = Math.min(minT, entryTime(entry));
+      const bucketSize = Math.max(now - minT, 1) / n;
+      const rows = Array.from({
+        length: n
+      }, (_, i) => ({
+        label: new Date(minT + i * bucketSize).toISOString().slice(11, 16),
+        triggered: 0,
+        failed: 0
+      }));
+      for (const entry of entries) {
+        const index = Math.floor((entryTime(entry) - minT) / bucketSize);
+        const row = rows[Math.min(n - 1, Math.max(0, index))];
+        if (!row) continue;
+        if (entry.content.event === "run.triggered") row.triggered += 1;
+        else row.failed += 1;
+      }
+      return {
+        rows
+      };
+    }
+  };
+}
+__name(durableLifecycleTriggersProvider, "durableLifecycleTriggersProvider");
 
 // src/durable-telescope.extension.ts
 function durableTelescopeExtension(opts = {}) {
@@ -1101,7 +1403,10 @@ function durableTelescopeExtension(opts = {}) {
       durableRunsOverTimeProvider(),
       durableSuccessRateProvider(),
       durableThroughputProvider(),
-      durableStateBreakdownProvider()
+      durableStateBreakdownProvider(),
+      durableFailedTriggersProvider(),
+      durableOwedLifecycleProvider(),
+      durableLifecycleTriggersProvider()
     ], "dataProviders")
   });
 }
@@ -1113,6 +1418,9 @@ __name(durableTelescopeExtension, "durableTelescopeExtension");
   durableDashboard,
   durableDurationProvider,
   durableExecutionScope,
+  durableFailedTriggersProvider,
+  durableLifecycleTriggersProvider,
+  durableOwedLifecycleProvider,
   durableRecentFailuresProvider,
   durableRunsOverTimeProvider,
   durableStateBreakdownProvider,
@@ -1124,6 +1432,7 @@ __name(durableTelescopeExtension, "durableTelescopeExtension");
   durableTraceContext,
   durableWorkerHealthProvider,
   durableWorkerStatusProvider,
+  runSpanContext,
   runTraceContext,
   runTraceId
 });